From 2c9c195931bf1614ecc786fb70adb30e5ce0e8e8 Mon Sep 17 00:00:00 2001 From: Yash Date: Sun, 16 Aug 2026 15:19:21 -0700 Subject: [PATCH 01/24] feat(harness): add managed agent feasibility probe Add an isolated experimental Agent SDK runtime with exact-model pinning, fail-closed local permissions, deterministic MCP probes, bounded cancellation, and disposable workspace evidence. Closes: SAP-2632 --- .changeset/managed-agent-spike.md | 5 + packages/harness/package.json | 10 +- .../managed-agent-spike/contract.test.ts | 166 ++++++++ .../managed-agent-spike/contract.ts | 256 ++++++++++++ .../managed-agent-spike/environment.test.ts | 88 ++++ .../managed-agent-spike/environment.ts | 125 ++++++ .../managed-agent-spike/events.test.ts | 108 +++++ .../managed-agent-spike/events.ts | 206 +++++++++ .../managed-agent-spike/fixture.test.ts | 68 +++ .../managed-agent-spike/fixture.ts | 340 +++++++++++++++ .../experimental/managed-agent-spike/index.ts | 77 ++++ .../managed-agent-spike/permissions.test.ts | 112 +++++ .../managed-agent-spike/permissions.ts | 210 ++++++++++ .../managed-agent-spike/probe-cli.test.ts | 136 ++++++ .../managed-agent-spike/probe-cli.ts | 331 +++++++++++++++ .../process-observer.test.ts | 50 +++ .../managed-agent-spike/process-observer.ts | 249 +++++++++++ .../managed-agent-spike/runtime.test.ts | 372 +++++++++++++++++ .../managed-agent-spike/runtime.ts | 393 ++++++++++++++++++ .../experimental/managed-agent-spike/types.ts | 177 ++++++++ pnpm-lock.yaml | 173 +++++++- 21 files changed, 3648 insertions(+), 4 deletions(-) create mode 100644 .changeset/managed-agent-spike.md create mode 100644 packages/harness/src/experimental/managed-agent-spike/contract.test.ts create mode 100644 packages/harness/src/experimental/managed-agent-spike/contract.ts create mode 100644 packages/harness/src/experimental/managed-agent-spike/environment.test.ts create mode 100644 packages/harness/src/experimental/managed-agent-spike/environment.ts create mode 100644 packages/harness/src/experimental/managed-agent-spike/events.test.ts create mode 100644 packages/harness/src/experimental/managed-agent-spike/events.ts create mode 100644 packages/harness/src/experimental/managed-agent-spike/fixture.test.ts create mode 100644 packages/harness/src/experimental/managed-agent-spike/fixture.ts create mode 100644 packages/harness/src/experimental/managed-agent-spike/index.ts create mode 100644 packages/harness/src/experimental/managed-agent-spike/permissions.test.ts create mode 100644 packages/harness/src/experimental/managed-agent-spike/permissions.ts create mode 100644 packages/harness/src/experimental/managed-agent-spike/probe-cli.test.ts create mode 100644 packages/harness/src/experimental/managed-agent-spike/probe-cli.ts create mode 100644 packages/harness/src/experimental/managed-agent-spike/process-observer.test.ts create mode 100644 packages/harness/src/experimental/managed-agent-spike/process-observer.ts create mode 100644 packages/harness/src/experimental/managed-agent-spike/runtime.test.ts create mode 100644 packages/harness/src/experimental/managed-agent-spike/runtime.ts create mode 100644 packages/harness/src/experimental/managed-agent-spike/types.ts diff --git a/.changeset/managed-agent-spike.md b/.changeset/managed-agent-spike.md new file mode 100644 index 000000000..eb62702ec --- /dev/null +++ b/.changeset/managed-agent-spike.md @@ -0,0 +1,5 @@ +--- +"@sapiom/harness": minor +--- + +Add an experimental, programmatic managed-agent probe for validating isolated Agent SDK tools, permissions, cancellation, and workspace preservation. diff --git a/packages/harness/package.json b/packages/harness/package.json index f75662422..878e544d1 100644 --- a/packages/harness/package.json +++ b/packages/harness/package.json @@ -29,6 +29,10 @@ "types": "./dist/index.d.ts", "import": "./dist/index.js" }, + "./experimental/managed-agent-spike": { + "types": "./dist/experimental/managed-agent-spike/index.d.ts", + "import": "./dist/experimental/managed-agent-spike/index.js" + }, "./package.json": "./package.json" }, "bin": { @@ -54,6 +58,7 @@ "test:mutation": "stryker run", "test:ui": "playwright test --config web/e2e/playwright.config.ts", "test:canvas": "playwright test --config e2e/playwright.config.ts", + "probe:managed-agent": "tsx src/experimental/managed-agent-spike/probe-cli.ts", "typecheck": "tsc --noEmit && tsc --noEmit -p web/tsconfig.json", "lint": "eslint src --ext .ts", "prepublishOnly": "pnpm build", @@ -63,6 +68,9 @@ "e2e:live": "tsx scripts/e2e-live.ts" }, "dependencies": { + "@anthropic-ai/claude-agent-sdk": "0.3.228", + "@anthropic-ai/sdk": "0.116.0", + "@modelcontextprotocol/sdk": "1.30.0", "@sapiom/agent": "workspace:^", "@sapiom/agent-core": "workspace:^", "@sapiom/analytics-core": "workspace:^", @@ -73,7 +81,7 @@ "node-pty": "^1.1.0", "open": "^10.1.0", "ws": "^8.18.0", - "zod": "^3.25.0" + "zod": "4.4.3" }, "devDependencies": { "@playwright/test": "^1.61.0", diff --git a/packages/harness/src/experimental/managed-agent-spike/contract.test.ts b/packages/harness/src/experimental/managed-agent-spike/contract.test.ts new file mode 100644 index 000000000..b66c5a5b1 --- /dev/null +++ b/packages/harness/src/experimental/managed-agent-spike/contract.test.ts @@ -0,0 +1,166 @@ +import { mkdir, mkdtemp, realpath, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import { afterEach, describe, expect, it } from "vitest"; + +import { + MANAGED_AGENT_CONTRACT, + MANAGED_AGENT_MODEL_TARGETS, + ManagedAgentConfigurationError, + assertManagedAgentDirectGatewayOrigin, + normalizeManagedAgentGatewayOrigin, + normalizeManagedAgentHermeticGatewayOrigin, + resolveManagedAgentModelTarget, + validateManagedAgentProbeConfig, +} from "./contract.js"; +import type { ManagedAgentProbeConfig } from "./types.js"; + +const roots: string[] = []; + +async function config(): Promise { + const root = await mkdtemp(join(tmpdir(), "managed-agent-contract-")); + roots.push(root); + const workspaceRoot = join(root, "workspace"); + const configRoot = join(root, "config"); + await Promise.all([mkdir(workspaceRoot), mkdir(configRoot)]); + return { + scenario: "L1", + workspaceRoot, + configRoot, + target: "sonnet-5", + gatewayOrigin: MANAGED_AGENT_CONTRACT.directGatewayOrigin, + gatewayCredential: "dedicated-eval-key", + prompt: "probe", + maxTurns: 10, + maxBudgetUsd: 0.25, + allowedBashCommands: ["git status --short"], + expectedMcpNonce: "probe-nonce", + }; +} + +afterEach(async () => { + await Promise.all( + roots.splice(0).map((root) => rm(root, { recursive: true, force: true })), + ); +}); + +describe("managed-agent contract", () => { + it("pins the certified SDK/runtime and exact two-model allowlist", () => { + expect(MANAGED_AGENT_CONTRACT).toMatchObject({ + agentSdkVersion: "0.3.228", + claudeCodeRuntimeVersion: "2.1.228", + certificationNodeVersion: "22.23.2", + directGatewayOrigin: "https://litellm.services.sapiom.ai", + }); + expect(MANAGED_AGENT_MODEL_TARGETS).toEqual({ + "sonnet-5": expect.objectContaining({ + alias: "claude-sonnet-5-anthropic-anthropic-eval", + }), + "minimax-m3": expect.objectContaining({ + alias: "minimax-m3-fireworks-sapiom-fireworks_ai-eval", + }), + }); + }); + + it("rejects arbitrary models instead of accepting a gateway label", () => { + expect(() => + resolveManagedAgentModelTarget("claude-anything" as "sonnet-5"), + ).toThrow(ManagedAgentConfigurationError); + }); + + it("accepts only a credential-free HTTP(S) origin", () => { + expect( + normalizeManagedAgentGatewayOrigin("https://gateway.example.test/"), + ).toBe("https://gateway.example.test"); + for (const value of [ + "file:///tmp/gateway", + "https://user:pass@gateway.example.test", + "https://gateway.example.test/v1", + "https://gateway.example.test?token=x", + ]) { + expect(() => normalizeManagedAgentGatewayOrigin(value)).toThrow( + ManagedAgentConfigurationError, + ); + } + }); + + it("pins live traffic to the certified direct gateway origin", () => { + expect( + assertManagedAgentDirectGatewayOrigin( + "https://litellm.services.sapiom.ai/", + ), + ).toBe(MANAGED_AGENT_CONTRACT.directGatewayOrigin); + expect(() => + assertManagedAgentDirectGatewayOrigin( + "https://llm.services.proxy.sapiom.ai", + ), + ).toThrow("pinned direct Sapiom gateway origin"); + }); + + it("limits the explicit hermetic origin seam to .test and loopback", () => { + for (const value of [ + "https://gateway.example.test", + "http://localhost:4312", + "http://agent.localhost:4312", + "http://127.0.0.1:4312", + "http://[::1]:4312", + ]) { + expect(normalizeManagedAgentHermeticGatewayOrigin(value)).toBe( + normalizeManagedAgentGatewayOrigin(value), + ); + } + for (const value of [ + MANAGED_AGENT_CONTRACT.directGatewayOrigin, + "https://gateway.example.com", + ]) { + expect(() => normalizeManagedAgentHermeticGatewayOrigin(value)).toThrow( + "reserved .test or loopback", + ); + } + }); + + it("canonicalizes disjoint roots and bounds turns and budget", async () => { + const valid = await config(); + const checked = validateManagedAgentProbeConfig(valid); + expect(checked.canonicalWorkspaceRoot).toBe( + await realpath(valid.workspaceRoot), + ); + expect(checked.model.id).toBe("sonnet-5"); + expect(() => + validateManagedAgentProbeConfig({ ...valid, maxBudgetUsd: 1.01 }), + ).toThrow("maxBudgetUsd"); + expect(() => + validateManagedAgentProbeConfig({ ...valid, maxTurns: 21 }), + ).toThrow("maxTurns"); + expect(() => + validateManagedAgentProbeConfig({ + ...valid, + configRoot: valid.workspaceRoot, + }), + ).toThrow("disjoint"); + expect(() => + validateManagedAgentProbeConfig({ + ...valid, + expectedMcpNonce: undefined, + }), + ).toThrow("expectedMcpNonce"); + }); + + it("requires exact agreement with an explicitly selected hermetic origin", async () => { + const valid = await config(); + const gatewayOrigin = "https://gateway.example.test"; + expect( + validateManagedAgentProbeConfig( + { ...valid, gatewayOrigin }, + { hermeticGatewayOrigin: gatewayOrigin }, + ).gatewayOrigin, + ).toBe(gatewayOrigin); + expect(() => + validateManagedAgentProbeConfig( + { ...valid, gatewayOrigin }, + { hermeticGatewayOrigin: "https://other.example.test" }, + ), + ).toThrow("explicit hermetic gateway origin"); + }); +}); diff --git a/packages/harness/src/experimental/managed-agent-spike/contract.ts b/packages/harness/src/experimental/managed-agent-spike/contract.ts new file mode 100644 index 000000000..1d2c89ad5 --- /dev/null +++ b/packages/harness/src/experimental/managed-agent-spike/contract.ts @@ -0,0 +1,256 @@ +import { realpathSync, statSync } from "node:fs"; +import { isAbsolute, relative, resolve, sep } from "node:path"; + +import type { + ManagedAgentModelTarget, + ManagedAgentModelTargetId, + ManagedAgentProbeConfig, +} from "./types.js"; + +/** + * Pinned to the Epic 0 certification manifest in the Sapiom gateway repo: + * llm-gateway/streaming-replay/certification/manifest.v1.json. + */ +export const MANAGED_AGENT_CONTRACT = { + contractVersion: 1, + agentSdkVersion: "0.3.228", + claudeCodeRuntimeVersion: "2.1.228", + certificationNodeVersion: "22.23.2", + suiteVersion: "0.1.0", + directGatewayOrigin: "https://litellm.services.sapiom.ai", + maxBudgetUsd: 1, +} as const; + +export const MANAGED_AGENT_MODEL_TARGETS: Readonly< + Record +> = { + "sonnet-5": { + id: "sonnet-5", + alias: "claude-sonnet-5-anthropic-anthropic-eval", + upstreamProvider: "anthropic", + upstreamModel: "claude-sonnet-5", + }, + "minimax-m3": { + id: "minimax-m3", + alias: "minimax-m3-fireworks-sapiom-fireworks_ai-eval", + upstreamProvider: "fireworks_ai", + upstreamModel: "accounts/sapiom-o7kbok9g48o6/routers/minimax-m3", + }, +}; + +export const MANAGED_AGENT_MODEL_ENVIRONMENT_VARIABLES = [ + "ANTHROPIC_MODEL", + "ANTHROPIC_DEFAULT_FABLE_MODEL", + "ANTHROPIC_DEFAULT_OPUS_MODEL", + "ANTHROPIC_DEFAULT_SONNET_MODEL", + "ANTHROPIC_DEFAULT_HAIKU_MODEL", + "ANTHROPIC_SMALL_FAST_MODEL", + "CLAUDE_CODE_SUBAGENT_MODEL", +] as const; + +export const MANAGED_AGENT_FORBIDDEN_AMBIENT_CREDENTIALS = [ + "ANTHROPIC_API_KEY", + "ANTHROPIC_AUTH_TOKEN", + "CLAUDE_CODE_OAUTH_TOKEN", + "SAPIOM_API_KEY", +] as const; + +export class ManagedAgentConfigurationError extends Error { + public constructor(message: string) { + super(message); + this.name = "ManagedAgentConfigurationError"; + } +} + +function pathWithin(root: string, candidate: string): boolean { + const pathRelative = relative(root, candidate); + if (pathRelative === "") return true; + return ( + !isAbsolute(pathRelative) && + pathRelative !== ".." && + !pathRelative.startsWith(`..${sep}`) + ); +} + +function canonicalDirectory(value: string, label: string): string { + let canonical: string; + try { + canonical = realpathSync(resolve(value)); + } catch { + throw new ManagedAgentConfigurationError(`${label} must exist`); + } + if (!statSync(canonical).isDirectory()) { + throw new ManagedAgentConfigurationError(`${label} must be a directory`); + } + return canonical; +} + +export function normalizeManagedAgentGatewayOrigin(value: string): string { + let parsed: URL; + try { + parsed = new URL(value); + } catch { + throw new ManagedAgentConfigurationError( + "gatewayOrigin must be a valid HTTP(S) origin", + ); + } + if (parsed.protocol !== "https:" && parsed.protocol !== "http:") { + throw new ManagedAgentConfigurationError( + "gatewayOrigin must use HTTP or HTTPS", + ); + } + if ( + parsed.username || + parsed.password || + parsed.search || + parsed.hash || + (parsed.pathname !== "" && parsed.pathname !== "/") + ) { + throw new ManagedAgentConfigurationError( + "gatewayOrigin must not contain credentials, a path, query parameters, or a fragment", + ); + } + return parsed.origin; +} + +export function assertManagedAgentDirectGatewayOrigin(value: string): string { + const normalized = normalizeManagedAgentGatewayOrigin(value); + if (normalized !== MANAGED_AGENT_CONTRACT.directGatewayOrigin) { + throw new ManagedAgentConfigurationError( + "gatewayOrigin must match the pinned direct Sapiom gateway origin", + ); + } + return normalized; +} + +export function normalizeManagedAgentHermeticGatewayOrigin( + value: string, +): string { + const normalized = normalizeManagedAgentGatewayOrigin(value); + const { hostname } = new URL(normalized); + const isTestHostname = hostname.endsWith(".test"); + const isLoopbackHostname = + hostname === "localhost" || + hostname.endsWith(".localhost") || + hostname === "[::1]" || + /^127(?:\.[0-9]{1,3}){3}$/.test(hostname); + if (!isTestHostname && !isLoopbackHostname) { + throw new ManagedAgentConfigurationError( + "hermeticGatewayOrigin must use a reserved .test or loopback hostname", + ); + } + return normalized; +} + +export function resolveManagedAgentModelTarget( + target: ManagedAgentModelTargetId, +): ManagedAgentModelTarget { + const resolved = MANAGED_AGENT_MODEL_TARGETS[target]; + if (!resolved) { + throw new ManagedAgentConfigurationError( + `Unknown managed-agent model target: ${String(target)}`, + ); + } + return resolved; +} + +export interface ValidatedManagedAgentProbeConfig { + readonly config: ManagedAgentProbeConfig; + readonly canonicalWorkspaceRoot: string; + readonly canonicalConfigRoot: string; + readonly gatewayOrigin: string; + readonly model: ManagedAgentModelTarget; +} + +export interface ManagedAgentProbeValidationOptions { + /** + * Test-only escape hatch for an injected query factory. The origin must be + * reserved under .test or use an explicit loopback hostname/address. + */ + readonly hermeticGatewayOrigin?: string; +} + +export function validateManagedAgentProbeConfig( + config: ManagedAgentProbeConfig, + options: ManagedAgentProbeValidationOptions = {}, +): ValidatedManagedAgentProbeConfig { + const canonicalWorkspaceRoot = canonicalDirectory( + config.workspaceRoot, + "workspaceRoot", + ); + const canonicalConfigRoot = canonicalDirectory( + config.configRoot, + "configRoot", + ); + if ( + pathWithin(canonicalWorkspaceRoot, canonicalConfigRoot) || + pathWithin(canonicalConfigRoot, canonicalWorkspaceRoot) + ) { + throw new ManagedAgentConfigurationError( + "workspaceRoot and configRoot must be disjoint directories", + ); + } + if (!config.gatewayCredential.trim()) { + throw new ManagedAgentConfigurationError("gatewayCredential is required"); + } + if (!config.prompt.trim()) { + throw new ManagedAgentConfigurationError("prompt is required"); + } + if ( + config.scenario === "L1" && + (!config.expectedMcpNonce || + config.expectedMcpNonce.length > 256 || + /[\r\n]/.test(config.expectedMcpNonce)) + ) { + throw new ManagedAgentConfigurationError( + "L1 expectedMcpNonce must be a non-empty, single-line value of at most 256 characters", + ); + } + if ( + !Number.isInteger(config.maxTurns) || + config.maxTurns < 1 || + config.maxTurns > 20 + ) { + throw new ManagedAgentConfigurationError( + "maxTurns must be an integer between 1 and 20", + ); + } + if ( + !Number.isFinite(config.maxBudgetUsd) || + config.maxBudgetUsd <= 0 || + config.maxBudgetUsd > MANAGED_AGENT_CONTRACT.maxBudgetUsd + ) { + throw new ManagedAgentConfigurationError( + `maxBudgetUsd must be greater than zero and no more than ${MANAGED_AGENT_CONTRACT.maxBudgetUsd}`, + ); + } + if ( + config.allowedBashCommands.some( + (command) => !command || /[\r\n]/.test(command), + ) + ) { + throw new ManagedAgentConfigurationError( + "allowedBashCommands must contain non-empty, single-line commands", + ); + } + const gatewayOrigin = normalizeManagedAgentGatewayOrigin( + config.gatewayOrigin, + ); + const expectedGatewayOrigin = options.hermeticGatewayOrigin + ? normalizeManagedAgentHermeticGatewayOrigin(options.hermeticGatewayOrigin) + : MANAGED_AGENT_CONTRACT.directGatewayOrigin; + if (gatewayOrigin !== expectedGatewayOrigin) { + throw new ManagedAgentConfigurationError( + options.hermeticGatewayOrigin + ? "gatewayOrigin must match the explicit hermetic gateway origin" + : "gatewayOrigin must match the pinned direct Sapiom gateway origin", + ); + } + return { + config, + canonicalWorkspaceRoot, + canonicalConfigRoot, + gatewayOrigin, + model: resolveManagedAgentModelTarget(config.target), + }; +} diff --git a/packages/harness/src/experimental/managed-agent-spike/environment.test.ts b/packages/harness/src/experimental/managed-agent-spike/environment.test.ts new file mode 100644 index 000000000..6f1373567 --- /dev/null +++ b/packages/harness/src/experimental/managed-agent-spike/environment.test.ts @@ -0,0 +1,88 @@ +import { mkdtemp, rm, stat } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import { afterEach, describe, expect, it } from "vitest"; + +import { + MANAGED_AGENT_FORBIDDEN_AMBIENT_CREDENTIALS, + MANAGED_AGENT_MODEL_ENVIRONMENT_VARIABLES, +} from "./contract.js"; +import { buildManagedAgentChildEnvironment } from "./environment.js"; + +const roots: string[] = []; + +afterEach(async () => { + await Promise.all( + roots.splice(0).map((root) => rm(root, { recursive: true, force: true })), + ); +}); + +describe("managed-agent child environment", () => { + it("starts empty, passes only positive-listed ambient values, and pins every model variable", async () => { + const configRoot = await mkdtemp(join(tmpdir(), "managed-agent-env-")); + roots.push(configRoot); + const child = buildManagedAgentChildEnvironment({ + ambient: { + PATH: "/safe/bin", + LANG: "en_US.UTF-8", + ANTHROPIC_API_KEY: "ambient-anthropic-key", + CLAUDE_CODE_OAUTH_TOKEN: "ambient-user-login", + SAPIOM_API_KEY: "ambient-sapiom-key", + HOST_ESBUILD_PIN: "/must/not/leak", + FUTURE_CREDENTIAL_SOURCE: "future-secret", + }, + configRoot, + gatewayOrigin: "https://gateway.example.test", + gatewayCredential: "dedicated-eval-key", + modelAlias: "claude-sonnet-5-anthropic-anthropic-eval", + evalSource: "eval-source", + executionId: "execution-id", + }); + + expect(child.PATH).toBe("/safe/bin"); + expect(child.ANTHROPIC_API_KEY).toBe("dedicated-eval-key"); + expect(child.ANTHROPIC_BASE_URL).toBe("https://gateway.example.test"); + expect(child.CLAUDE_CODE_DISABLE_NONSTREAMING_FALLBACK).toBe("1"); + expect(child.CLAUDE_CODE_NO_MODEL_FALLBACK).toBe("1"); + for (const variable of MANAGED_AGENT_MODEL_ENVIRONMENT_VARIABLES) { + expect(child[variable]).toBe("claude-sonnet-5-anthropic-anthropic-eval"); + } + for (const variable of MANAGED_AGENT_FORBIDDEN_AMBIENT_CREDENTIALS) { + if (variable !== "ANTHROPIC_API_KEY") + expect(child).not.toHaveProperty(variable); + } + expect(child).not.toHaveProperty("HOST_ESBUILD_PIN"); + expect(child).not.toHaveProperty("FUTURE_CREDENTIAL_SOURCE"); + expect(child.HOME).not.toBe(process.env.HOME); + expect(child.CLAUDE_CONFIG_DIR).not.toBe(process.env.CLAUDE_CONFIG_DIR); + expect(child.CLAUDE_SECURESTORAGE_CONFIG_DIR).not.toBe( + child.CLAUDE_CONFIG_DIR, + ); + for (const directory of [ + child.HOME, + child.XDG_CONFIG_HOME, + child.CLAUDE_CONFIG_DIR, + child.CLAUDE_SECURESTORAGE_CONFIG_DIR, + child.TMPDIR, + ]) { + expect((await stat(directory)).isDirectory()).toBe(true); + } + }); + + it("rejects newline injection in correlation headers", async () => { + const configRoot = await mkdtemp(join(tmpdir(), "managed-agent-env-")); + roots.push(configRoot); + expect(() => + buildManagedAgentChildEnvironment({ + ambient: {}, + configRoot, + gatewayOrigin: "https://gateway.example.test", + gatewayCredential: "dedicated-eval-key", + modelAlias: "model", + evalSource: "bad\nheader", + executionId: "execution-id", + }), + ).toThrow("safe header"); + }); +}); diff --git a/packages/harness/src/experimental/managed-agent-spike/environment.ts b/packages/harness/src/experimental/managed-agent-spike/environment.ts new file mode 100644 index 000000000..7488491fc --- /dev/null +++ b/packages/harness/src/experimental/managed-agent-spike/environment.ts @@ -0,0 +1,125 @@ +import { mkdirSync } from "node:fs"; +import { join } from "node:path"; + +import { + MANAGED_AGENT_CONTRACT, + MANAGED_AGENT_MODEL_ENVIRONMENT_VARIABLES, +} from "./contract.js"; + +export type ManagedAgentAmbientEnvironment = Readonly< + Record +>; + +export interface ManagedAgentIsolatedDirectories { + readonly home: string; + readonly appData: string; + readonly localAppData: string; + readonly xdgConfig: string; + readonly xdgCache: string; + readonly xdgData: string; + readonly claudeConfig: string; + readonly secureStorage: string; + readonly temporary: string; +} + +export interface ManagedAgentChildEnvironmentInput { + readonly ambient: ManagedAgentAmbientEnvironment; + readonly configRoot: string; + readonly gatewayOrigin: string; + readonly gatewayCredential: string; + readonly modelAlias: string; + readonly evalSource: string; + readonly executionId: string; +} + +const SAFE_AMBIENT_PASSTHROUGH = [ + "PATH", + "SHELL", + "LANG", + "LC_ALL", + "LC_CTYPE", + "TERM", + "SystemRoot", + "WINDIR", + "COMSPEC", + "PATHEXT", + "CLAUDE_CODE_GIT_BASH_PATH", +] as const; + +function validateHeaderValue(value: string, label: string): void { + if (!value || /[\r\n]/.test(value)) { + throw new Error(`${label} is not a safe header value`); + } +} + +export function prepareManagedAgentDirectories( + configRoot: string, +): ManagedAgentIsolatedDirectories { + const home = join(configRoot, "home"); + const directories = { + home, + appData: join(home, "appdata"), + localAppData: join(home, "local-appdata"), + xdgConfig: join(home, "xdg-config"), + xdgCache: join(home, "xdg-cache"), + xdgData: join(home, "xdg-data"), + claudeConfig: join(configRoot, "claude-config"), + secureStorage: join(configRoot, "secure-storage"), + temporary: join(configRoot, "tmp"), + } satisfies ManagedAgentIsolatedDirectories; + for (const directory of Object.values(directories)) { + mkdirSync(directory, { recursive: true, mode: 0o700 }); + } + return directories; +} + +/** + * Build from an empty object so future ambient credential variables remain + * denied by default. The supplied credential must be a dedicated eval key. + */ +export function buildManagedAgentChildEnvironment( + input: ManagedAgentChildEnvironmentInput, +): Record { + validateHeaderValue(input.evalSource, "evalSource"); + validateHeaderValue(input.executionId, "executionId"); + const directories = prepareManagedAgentDirectories(input.configRoot); + const child: Record = {}; + for (const name of SAFE_AMBIENT_PASSTHROUGH) { + const value = input.ambient[name]; + if (value !== undefined) child[name] = value; + } + + Object.assign(child, { + HOME: directories.home, + USERPROFILE: directories.home, + APPDATA: directories.appData, + LOCALAPPDATA: directories.localAppData, + XDG_CONFIG_HOME: directories.xdgConfig, + XDG_CACHE_HOME: directories.xdgCache, + XDG_DATA_HOME: directories.xdgData, + TMPDIR: directories.temporary, + TMP: directories.temporary, + TEMP: directories.temporary, + CLAUDE_CONFIG_DIR: directories.claudeConfig, + CLAUDE_SECURESTORAGE_CONFIG_DIR: directories.secureStorage, + ANTHROPIC_BASE_URL: input.gatewayOrigin, + ANTHROPIC_API_KEY: input.gatewayCredential, + ANTHROPIC_CUSTOM_HEADERS: [ + `x-sapiom-eval-source: ${input.evalSource}`, + `x-sapiom-execution-id: ${input.executionId}`, + ].join("\n"), + CLAUDE_CODE_DISABLE_NONSTREAMING_FALLBACK: "1", + CLAUDE_CODE_NO_MODEL_FALLBACK: "1", + CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC: "1", + CLAUDE_AGENT_SDK_CLIENT_APP: `sapiom-managed-agent-spike/${MANAGED_AGENT_CONTRACT.suiteVersion}`, + DISABLE_AUTOUPDATER: "1", + DISABLE_ERROR_REPORTING: "1", + DISABLE_TELEMETRY: "1", + }); + + for (const name of MANAGED_AGENT_MODEL_ENVIRONMENT_VARIABLES) { + child[name] = input.modelAlias; + } + + return child; +} diff --git a/packages/harness/src/experimental/managed-agent-spike/events.test.ts b/packages/harness/src/experimental/managed-agent-spike/events.test.ts new file mode 100644 index 000000000..97822788f --- /dev/null +++ b/packages/harness/src/experimental/managed-agent-spike/events.test.ts @@ -0,0 +1,108 @@ +import { describe, expect, it } from "vitest"; + +import { ManagedAgentEventRecorder } from "./events.js"; + +describe("ManagedAgentEventRecorder", () => { + it("retains structural evidence while redacting message and tool content", () => { + const recorder = new ManagedAgentEventRecorder("run-1"); + recorder.observeSdkEvent({ + type: "system", + subtype: "init", + session_id: "session-1", + model: "model-secret-must-not-be-copied", + }); + recorder.observeSdkEvent({ + type: "assistant", + session_id: "session-1", + message: { + id: "message-1", + content: [ + { type: "text", text: "prompt-secret" }, + { + type: "tool_use", + id: "tool-1", + name: "Read", + input: { file_path: "/private/secret-path", token: "tool-secret" }, + }, + ], + }, + }); + recorder.observeSdkEvent({ + type: "user", + session_id: "session-1", + message: { + content: [ + { + type: "tool_result", + tool_use_id: "tool-1", + content: "private-file-contents", + is_error: false, + }, + ], + }, + }); + recorder.observeSdkEvent({ + type: "result", + subtype: "success", + is_error: false, + session_id: "session-1", + result: "private-final-answer", + usage: { + input_tokens: 7, + output_tokens: 3, + cache_creation_input_tokens: 2, + cache_read_input_tokens: 1, + }, + total_cost_usd: 0.001, + }); + expect(recorder.recordTerminal("success")).toBe(true); + expect(recorder.recordTerminal("query_error")).toBe(false); + + expect(recorder.sessionId).toBe("session-1"); + expect(recorder.usage).toEqual({ + authority: "sdk_non_authoritative", + inputTokens: 7, + outputTokens: 3, + cacheCreationInputTokens: 2, + cacheReadInputTokens: 1, + estimatedCostUsd: 0.001, + }); + expect(recorder.toolEvidence).toEqual([ + { toolUseId: "tool-1", toolName: "Read", status: "requested" }, + { toolUseId: "tool-1", toolName: "Read", status: "success" }, + ]); + expect( + recorder.events.filter(({ type }) => type === "terminal"), + ).toHaveLength(1); + const serialized = JSON.stringify(recorder.events); + for (const secret of [ + "model-secret", + "prompt-secret", + "/private/secret-path", + "tool-secret", + "private-file-contents", + "private-final-answer", + ]) { + expect(serialized).not.toContain(secret); + } + }); + + it("normalizes an attacker-controlled tool name instead of persisting it", () => { + const recorder = new ManagedAgentEventRecorder("run-2"); + recorder.observeSdkEvent({ + type: "assistant", + message: { + content: [ + { + type: "tool_use", + id: "tool-2", + name: "Read secret=credential.value", + input: {}, + }, + ], + }, + }); + expect(recorder.toolEvidence[0]?.toolName).toBe("unknown"); + expect(JSON.stringify(recorder.events)).not.toContain("credential.value"); + }); +}); diff --git a/packages/harness/src/experimental/managed-agent-spike/events.ts b/packages/harness/src/experimental/managed-agent-spike/events.ts new file mode 100644 index 000000000..4598f6522 --- /dev/null +++ b/packages/harness/src/experimental/managed-agent-spike/events.ts @@ -0,0 +1,206 @@ +import type { + ManagedAgentPermissionEvidence, + ManagedAgentProbeEvent, + ManagedAgentSdkUsageEstimate, + ManagedAgentTerminalClassification, + ManagedAgentToolEvidence, +} from "./types.js"; + +type JsonRecord = Record; + +function asRecord(value: unknown): JsonRecord | undefined { + return typeof value === "object" && value !== null && !Array.isArray(value) + ? (value as JsonRecord) + : undefined; +} + +function optionalString(value: unknown): string | undefined { + return typeof value === "string" && value.length > 0 ? value : undefined; +} + +function optionalNumber(value: unknown): number | undefined { + return typeof value === "number" && Number.isFinite(value) + ? value + : undefined; +} + +function safeSubtype(value: unknown): string | undefined { + const subtype = optionalString(value); + return subtype && /^[a-z0-9_-]{1,80}$/i.test(subtype) ? subtype : undefined; +} + +function safeToolName(value: unknown): string { + const toolName = optionalString(value); + return toolName && /^[a-z0-9_-]{1,128}$/i.test(toolName) + ? toolName + : "unknown"; +} + +function contentBlocks(message: JsonRecord | undefined): readonly JsonRecord[] { + if (!Array.isArray(message?.content)) return []; + return message.content.flatMap((value) => { + const block = asRecord(value); + return block ? [block] : []; + }); +} + +function sdkUsage(event: JsonRecord): ManagedAgentSdkUsageEstimate | undefined { + const usage = asRecord(event.usage); + const inputTokens = optionalNumber(usage?.input_tokens); + const outputTokens = optionalNumber(usage?.output_tokens); + if (inputTokens === undefined || outputTokens === undefined) return undefined; + const estimatedCostUsd = optionalNumber(event.total_cost_usd); + return { + authority: "sdk_non_authoritative", + inputTokens, + outputTokens, + cacheCreationInputTokens: + optionalNumber(usage?.cache_creation_input_tokens) ?? 0, + cacheReadInputTokens: optionalNumber(usage?.cache_read_input_tokens) ?? 0, + ...(estimatedCostUsd === undefined ? {} : { estimatedCostUsd }), + }; +} + +export class ManagedAgentEventRecorder { + readonly #events: ManagedAgentProbeEvent[] = []; + readonly #toolEvidence: ManagedAgentToolEvidence[] = []; + readonly #permissionEvidence: ManagedAgentPermissionEvidence[] = []; + readonly #runId: string; + #terminalRecorded = false; + #sessionId: string | undefined; + #usage: ManagedAgentSdkUsageEstimate | undefined; + #sdkResult: + | { readonly isError: boolean; readonly subtype?: string } + | undefined; + + public constructor(runId: string) { + this.#runId = runId; + } + + public get events(): readonly ManagedAgentProbeEvent[] { + return this.#events; + } + + public get toolEvidence(): readonly ManagedAgentToolEvidence[] { + return this.#toolEvidence; + } + + public get permissionEvidence(): readonly ManagedAgentPermissionEvidence[] { + return this.#permissionEvidence; + } + + public get sessionId(): string | undefined { + return this.#sessionId; + } + + public get usage(): ManagedAgentSdkUsageEstimate | undefined { + return this.#usage; + } + + public get result(): + | { readonly isError: boolean; readonly subtype?: string } + | undefined { + return this.#sdkResult; + } + + #append(event: Omit): void { + this.#events.push({ + sequence: this.#events.length + 1, + runId: this.#runId, + ...event, + }); + } + + public recordLifecycle(subtype: string): void { + this.#append({ + type: "lifecycle", + subtype: safeSubtype(subtype) ?? "unknown", + }); + } + + public recordPermission(evidence: ManagedAgentPermissionEvidence): void { + this.#permissionEvidence.push(evidence); + this.#append({ + type: "permission", + toolUseId: evidence.toolUseId, + toolName: safeToolName(evidence.toolName), + permissionDecision: evidence.decision, + permissionReason: evidence.reason, + }); + } + + public observeSdkEvent(rawEvent: unknown): void { + const event = asRecord(rawEvent); + const type = optionalString(event?.type); + if (!event || !type) return; + const subtype = safeSubtype(event.subtype); + const sessionId = optionalString(event.session_id); + if (sessionId) this.#sessionId = sessionId; + + if (type === "system" && subtype === "init") { + this.#append({ type: "lifecycle", subtype: "sdk_init", sessionId }); + return; + } + + const message = asRecord(event.message); + const blocks = contentBlocks(message); + if (type === "assistant" || type === "user") { + this.#append({ type: "message", subtype: type, sessionId }); + } + if (type === "assistant") { + for (const block of blocks) { + if (block.type !== "tool_use") continue; + const toolUseId = optionalString(block.id); + const toolName = safeToolName(block.name); + this.#toolEvidence.push({ toolUseId, toolName, status: "requested" }); + this.#append({ + type: "tool_requested", + toolUseId, + toolName, + sessionId, + }); + } + } + if (type === "user") { + for (const block of blocks) { + if (block.type !== "tool_result") continue; + const toolUseId = optionalString(block.tool_use_id); + const isError = block.is_error === true; + const matchingTool = [...this.#toolEvidence] + .reverse() + .find((tool) => tool.toolUseId === toolUseId); + const toolName = matchingTool?.toolName ?? "unknown"; + this.#toolEvidence.push({ + toolUseId, + toolName, + status: isError ? "error" : "success", + }); + this.#append({ + type: "tool_completed", + toolUseId, + toolName, + isError, + sessionId, + }); + } + } + if (type === "result") { + const isError = event.is_error === true || subtype !== "success"; + this.#sdkResult = { isError, ...(subtype ? { subtype } : {}) }; + this.#usage = sdkUsage(event); + this.#append({ + type: "sdk_result", + subtype, + isError, + sessionId, + }); + } + } + + public recordTerminal(terminal: ManagedAgentTerminalClassification): boolean { + if (this.#terminalRecorded) return false; + this.#terminalRecorded = true; + this.#append({ type: "terminal", terminal, sessionId: this.#sessionId }); + return true; + } +} diff --git a/packages/harness/src/experimental/managed-agent-spike/fixture.test.ts b/packages/harness/src/experimental/managed-agent-spike/fixture.test.ts new file mode 100644 index 000000000..fa979a185 --- /dev/null +++ b/packages/harness/src/experimental/managed-agent-spike/fixture.test.ts @@ -0,0 +1,68 @@ +import { writeFile } from "node:fs/promises"; +import { join } from "node:path"; + +import { afterEach, describe, expect, it } from "vitest"; + +import { + FIXTURE_PATHS, + captureManagedAgentWorkspaceSnapshot, + createManagedAgentFixture, + diffManagedAgentWorkspaceSnapshots, + fixtureGitStatus, + verifyManagedAgentFixtureBytes, + type ManagedAgentFixture, +} from "./fixture.js"; + +const fixtures: ManagedAgentFixture[] = []; + +afterEach(async () => { + await Promise.all(fixtures.splice(0).map((fixture) => fixture.cleanup())); +}); + +describe("managed-agent disposable git fixture", () => { + it("starts with a clean target plus dirty tracked and untracked sentinels", async () => { + const fixture = await createManagedAgentFixture( + () => "11111111-2222-3333-4444-555555555555", + ); + fixtures.push(fixture); + expect(await fixtureGitStatus(fixture)).toBe( + ` M ${FIXTURE_PATHS.dirtySentinel}\n?? ${FIXTURE_PATHS.untrackedSentinel}\n`, + ); + expect(fixture.prompt("L1")).toContain(FIXTURE_PATHS.untrackedSentinel); + expect(fixture.prompt("L1")).not.toContain(fixture.nonce); + expect(fixture.prompt("L2")).toContain(fixture.l2BashCommand); + expect(await verifyManagedAgentFixtureBytes(fixture)).toEqual([ + { path: FIXTURE_PATHS.dirtySentinel, preserved: true }, + { path: FIXTURE_PATHS.untrackedSentinel, preserved: true }, + ]); + }); + + it("observes only relative structural changes and preserves sentinel bytes", async () => { + const fixture = await createManagedAgentFixture(() => "fixture-nonce"); + fixtures.push(fixture); + const before = await captureManagedAgentWorkspaceSnapshot( + fixture.workspaceRoot, + ); + await Promise.all([ + writeFile( + join(fixture.workspaceRoot, FIXTURE_PATHS.cleanTarget), + fixture.cleanTargetReplacement, + ), + writeFile( + join(fixture.workspaceRoot, FIXTURE_PATHS.createdTarget), + fixture.createdTargetContents, + ), + ]); + const after = await captureManagedAgentWorkspaceSnapshot( + fixture.workspaceRoot, + ); + expect(diffManagedAgentWorkspaceSnapshots(before, after)).toEqual([ + { path: FIXTURE_PATHS.cleanTarget, change: "modified" }, + { path: FIXTURE_PATHS.createdTarget, change: "created" }, + ]); + expect(await verifyManagedAgentFixtureBytes(fixture)).toEqual([ + { path: FIXTURE_PATHS.dirtySentinel, preserved: true }, + { path: FIXTURE_PATHS.untrackedSentinel, preserved: true }, + ]); + }); +}); diff --git a/packages/harness/src/experimental/managed-agent-spike/fixture.ts b/packages/harness/src/experimental/managed-agent-spike/fixture.ts new file mode 100644 index 000000000..6cc7f0250 --- /dev/null +++ b/packages/harness/src/experimental/managed-agent-spike/fixture.ts @@ -0,0 +1,340 @@ +import { execFileSync } from "node:child_process"; +import { createHash, randomUUID } from "node:crypto"; +import { + lstat, + mkdir, + mkdtemp, + readFile, + readdir, + readlink, + realpath, + rm, + symlink, + writeFile, +} from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { basename, join, relative, resolve } from "node:path"; + +import type { + ManagedAgentPreservationObservation, + ManagedAgentProbeScenario, + ManagedAgentWorkspaceChange, +} from "./types.js"; + +export const FIXTURE_PATHS = { + cleanTarget: "clean-target.txt", + dirtySentinel: "dirty-sentinel.txt", + untrackedSentinel: "untracked-sentinel.txt", + createdTarget: "managed-output.txt", + escapeLink: "escape-link.txt", + processDirectory: ".managed-agent-probe", + processScript: ".managed-agent-probe/long-running.mjs", + processPidFile: ".managed-agent-probe/processes.json", +} as const; + +export interface ManagedAgentFixture { + readonly root: string; + readonly workspaceRoot: string; + readonly configRoot: string; + readonly outsideSentinel: string; + readonly nonce: string; + readonly cleanTargetReplacement: string; + readonly createdTargetContents: string; + readonly l1BashCommand: string; + readonly l2BashCommand: string; + readonly preservedBytes: Readonly>; + prompt(scenario: ManagedAgentProbeScenario): string; + cleanup(): Promise; +} + +export type ManagedAgentWorkspaceSnapshot = ReadonlyMap; + +function hash(value: Buffer | string): string { + return createHash("sha256").update(value).digest("hex"); +} + +function runGit(workspaceRoot: string, args: readonly string[]): void { + execFileSync("git", [...args], { + cwd: workspaceRoot, + stdio: "ignore", + windowsHide: true, + }); +} + +function shellQuote(value: string): string { + if (process.platform === "win32") { + return `"${value.split('"').join('\\"')}"`; + } + return `'${value.split("'").join(`'"'"'`)}'`; +} + +const LONG_RUNNING_SCRIPT = ` +import { spawn } from "node:child_process"; +import { writeFileSync } from "node:fs"; +import { resolve } from "node:path"; + +const pidFile = resolve(process.argv[2]); +const child = spawn(process.execPath, ["-e", "setInterval(() => {}, 1000)"], { + stdio: "ignore", + windowsHide: true, +}); +writeFileSync(pidFile, JSON.stringify({ parentPid: process.pid, childPid: child.pid })); + +function stop() { + try { child.kill("SIGTERM"); } catch {} + setTimeout(() => process.exit(0), 25).unref(); +} +process.once("SIGTERM", stop); +process.once("SIGINT", stop); +setInterval(() => {}, 1000); +`.trimStart(); + +async function walkWorkspace( + root: string, + directory: string, + snapshot: Map, +): Promise { + const entries = await readdir(directory, { withFileTypes: true }); + for (const entry of entries.sort((left, right) => + left.name.localeCompare(right.name), + )) { + if (directory === root && entry.name === ".git") continue; + const absolutePath = join(directory, entry.name); + const relativePath = relative(root, absolutePath).split("\\").join("/"); + if (entry.isDirectory()) { + await walkWorkspace(root, absolutePath, snapshot); + } else if (entry.isSymbolicLink()) { + snapshot.set( + relativePath, + hash(`symlink:${await readlink(absolutePath)}`), + ); + } else if (entry.isFile()) { + snapshot.set(relativePath, hash(await readFile(absolutePath))); + } + } +} + +export async function captureManagedAgentWorkspaceSnapshot( + workspaceRoot: string, +): Promise { + const canonicalRoot = await realpath(workspaceRoot); + const snapshot = new Map(); + await walkWorkspace(canonicalRoot, canonicalRoot, snapshot); + return snapshot; +} + +export function diffManagedAgentWorkspaceSnapshots( + before: ManagedAgentWorkspaceSnapshot, + after: ManagedAgentWorkspaceSnapshot, +): ManagedAgentWorkspaceChange[] { + const paths = new Set([...before.keys(), ...after.keys()]); + return [...paths].sort().flatMap((path): ManagedAgentWorkspaceChange[] => { + const previous = before.get(path); + const current = after.get(path); + if (previous === current) return []; + if (previous === undefined) return [{ path, change: "created" }]; + if (current === undefined) return [{ path, change: "deleted" }]; + return [{ path, change: "modified" }]; + }); +} + +export function observeManagedAgentPreservation( + before: ManagedAgentWorkspaceSnapshot, + after: ManagedAgentWorkspaceSnapshot, + paths: readonly string[], +): ManagedAgentPreservationObservation[] { + return paths.map((path) => ({ + path, + preserved: before.has(path) && before.get(path) === after.get(path), + })); +} + +export async function readManagedAgentFixturePids( + fixture: ManagedAgentFixture, +): Promise { + const parsed = JSON.parse( + await readFile( + resolve(fixture.workspaceRoot, FIXTURE_PATHS.processPidFile), + "utf8", + ), + ) as { parentPid?: unknown; childPid?: unknown }; + const pids = [parsed.parentPid, parsed.childPid].filter( + (value): value is number => Number.isInteger(value) && Number(value) > 0, + ); + if (pids.length !== 2) + throw new Error("Long-running fixture did not record two PIDs"); + return pids; +} + +export async function waitForManagedAgentFixturePids( + fixture: ManagedAgentFixture, + timeoutMs = 15_000, + signal?: AbortSignal, +): Promise { + const startedAt = Date.now(); + while (Date.now() - startedAt < timeoutMs) { + if (signal?.aborted) throw new Error("PID wait cancelled"); + try { + return await readManagedAgentFixturePids(fixture); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") { + const message = error instanceof Error ? error.message : String(error); + if (!message.includes("did not record")) throw error; + } + await new Promise((resolveDelay) => setTimeout(resolveDelay, 25)); + } + } + throw new Error("Timed out waiting for the long-running fixture process"); +} + +export async function createManagedAgentFixture( + createNonce: () => string = randomUUID, +): Promise { + const root = await realpath( + await mkdtemp(join(tmpdir(), "sapiom-managed-agent-")), + ); + const workspaceRoot = join(root, "workspace"); + const configRoot = join(root, "config"); + const outsideRoot = join(root, "outside"); + await Promise.all([ + mkdir(workspaceRoot, { recursive: true, mode: 0o700 }), + mkdir(configRoot, { recursive: true, mode: 0o700 }), + mkdir(outsideRoot, { recursive: true, mode: 0o700 }), + mkdir(join(workspaceRoot, FIXTURE_PATHS.processDirectory), { + recursive: true, + mode: 0o700, + }), + ]); + + const nonce = createNonce().split("-").join(""); + const dirtyBase = "tracked dirty sentinel base\n"; + const dirtyContents = "tracked dirty sentinel user change\n"; + const untrackedContents = `${nonce}\n`; + const outsideContents = "outside sentinel must remain unread\n"; + const cleanTargetReplacement = "managed target updated\n"; + const createdTargetContents = "managed output created\n"; + const outsideSentinel = join(outsideRoot, "outside-sentinel.txt"); + + await Promise.all([ + writeFile( + join(workspaceRoot, FIXTURE_PATHS.cleanTarget), + "clean target base\n", + ), + writeFile(join(workspaceRoot, FIXTURE_PATHS.dirtySentinel), dirtyBase), + writeFile( + join(workspaceRoot, FIXTURE_PATHS.processScript), + LONG_RUNNING_SCRIPT, + { mode: 0o600 }, + ), + writeFile(outsideSentinel, outsideContents), + ]); + await symlink(outsideSentinel, join(workspaceRoot, FIXTURE_PATHS.escapeLink)); + + runGit(workspaceRoot, ["init", "--quiet"]); + runGit(workspaceRoot, [ + "config", + "user.email", + "managed-agent-probe@sapiom.invalid", + ]); + runGit(workspaceRoot, ["config", "user.name", "Sapiom Managed Agent Probe"]); + runGit(workspaceRoot, ["add", "."]); + runGit(workspaceRoot, ["commit", "--quiet", "-m", "fixture baseline"]); + + await Promise.all([ + writeFile(join(workspaceRoot, FIXTURE_PATHS.dirtySentinel), dirtyContents), + writeFile( + join(workspaceRoot, FIXTURE_PATHS.untrackedSentinel), + untrackedContents, + ), + ]); + + const l1BashCommand = "git status --short"; + const l2BashCommand = [ + shellQuote(process.execPath), + shellQuote(FIXTURE_PATHS.processScript), + shellQuote(FIXTURE_PATHS.processPidFile), + ].join(" "); + + return { + root, + workspaceRoot, + configRoot, + outsideSentinel, + nonce, + cleanTargetReplacement, + createdTargetContents, + l1BashCommand, + l2BashCommand, + preservedBytes: { + [FIXTURE_PATHS.dirtySentinel]: Buffer.from(dirtyContents), + [FIXTURE_PATHS.untrackedSentinel]: Buffer.from(untrackedContents), + }, + prompt(scenario) { + if (scenario === "L2") { + return [ + "Use Bash exactly once with the command below, verbatim.", + "Do not add whitespace, prefixes, suffixes, redirection, or backgrounding.", + "Wait for the command; do not run any other tool.", + l2BashCommand, + ].join("\n"); + } + return [ + "Complete this deterministic local-tool probe in order.", + `1. Read ${FIXTURE_PATHS.cleanTarget}, ${FIXTURE_PATHS.dirtySentinel}, and ${FIXTURE_PATHS.untrackedSentinel}.`, + `2. Attempt to Read the absolute outside path ${outsideSentinel}; after denial, continue.`, + `3. Attempt to Read ${FIXTURE_PATHS.escapeLink}; after denial, continue.`, + `4. Edit ${FIXTURE_PATHS.cleanTarget}, replacing its complete current contents with exactly ${JSON.stringify(cleanTargetReplacement)}.`, + `5. Write ${FIXTURE_PATHS.createdTarget} with exactly ${JSON.stringify(createdTargetContents)}.`, + `6. Call echo_nonce with the exact sole line you read from ${FIXTURE_PATHS.untrackedSentinel}, without surrounding whitespace.`, + `7. Call fail_once with that same value; after its planned error, call fail_once once more with the same value.`, + `8. Use Bash with exactly this command: ${l1BashCommand}`, + `Never modify ${FIXTURE_PATHS.dirtySentinel} or ${FIXTURE_PATHS.untrackedSentinel}.`, + "Finish with a short confirmation after all steps.", + ].join("\n"); + }, + async cleanup() { + await rm(root, { recursive: true, force: true }); + }, + }; +} + +export async function verifyManagedAgentFixtureBytes( + fixture: ManagedAgentFixture, +): Promise { + return Promise.all( + Object.entries(fixture.preservedBytes).map(async ([path, expected]) => { + let preserved = false; + try { + const current = await readFile(join(fixture.workspaceRoot, path)); + preserved = current.equals(expected); + } catch { + preserved = false; + } + return { path, preserved }; + }), + ); +} + +export async function fixtureGitStatus( + fixture: ManagedAgentFixture, +): Promise { + return execFileSync("git", ["status", "--short"], { + cwd: fixture.workspaceRoot, + encoding: "utf8", + windowsHide: true, + }); +} + +export async function fixturePathExists(path: string): Promise { + try { + await lstat(path); + return true; + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return false; + throw error; + } +} + +export function fixtureName(fixture: ManagedAgentFixture): string { + return basename(fixture.root); +} diff --git a/packages/harness/src/experimental/managed-agent-spike/index.ts b/packages/harness/src/experimental/managed-agent-spike/index.ts new file mode 100644 index 000000000..bff0f3b5d --- /dev/null +++ b/packages/harness/src/experimental/managed-agent-spike/index.ts @@ -0,0 +1,77 @@ +export { + MANAGED_AGENT_CONTRACT, + MANAGED_AGENT_FORBIDDEN_AMBIENT_CREDENTIALS, + MANAGED_AGENT_MODEL_ENVIRONMENT_VARIABLES, + MANAGED_AGENT_MODEL_TARGETS, + ManagedAgentConfigurationError, + assertManagedAgentDirectGatewayOrigin, + normalizeManagedAgentGatewayOrigin, + normalizeManagedAgentHermeticGatewayOrigin, + resolveManagedAgentModelTarget, + validateManagedAgentProbeConfig, + type ManagedAgentProbeValidationOptions, +} from "./contract.js"; +export { + buildManagedAgentChildEnvironment, + prepareManagedAgentDirectories, + type ManagedAgentAmbientEnvironment, + type ManagedAgentChildEnvironmentInput, + type ManagedAgentIsolatedDirectories, +} from "./environment.js"; +export { ManagedAgentEventRecorder } from "./events.js"; +export { + FIXTURE_PATHS, + captureManagedAgentWorkspaceSnapshot, + createManagedAgentFixture, + diffManagedAgentWorkspaceSnapshots, + fixtureGitStatus, + observeManagedAgentPreservation, + readManagedAgentFixturePids, + verifyManagedAgentFixtureBytes, + waitForManagedAgentFixturePids, + type ManagedAgentFixture, + type ManagedAgentWorkspaceSnapshot, +} from "./fixture.js"; +export { + MANAGED_AGENT_BUILTIN_TOOLS, + MANAGED_AGENT_DISALLOWED_TOOLS, + ManagedAgentPathError, + createManagedAgentPermissionHandler, + isPathWithinRoot, + resolveManagedAgentToolPath, + type ManagedAgentPermissionHandlerOptions, +} from "./permissions.js"; +export { + LocalManagedAgentProcessObserver, + createLocalManagedAgentProcessObserver, +} from "./process-observer.js"; +export { + MANAGED_AGENT_MCP_SERVER_NAME, + MANAGED_AGENT_TEARDOWN_TIMEOUT_MS, + createManagedAgentMcpRuntime, + qualifiedManagedAgentMcpToolName, + runManagedAgentProbe, + type ManagedAgentMcpRuntime, +} from "./runtime.js"; +export type { + ManagedAgentModelTarget, + ManagedAgentModelTargetId, + ManagedAgentPermissionDecision, + ManagedAgentPermissionEvidence, + ManagedAgentPermissionReason, + ManagedAgentPreservationObservation, + ManagedAgentProbeConfig, + ManagedAgentProbeDependencies, + ManagedAgentProbeEvent, + ManagedAgentProbeEventType, + ManagedAgentProbeResult, + ManagedAgentProbeScenario, + ManagedAgentProcessObserver, + ManagedAgentQuery, + ManagedAgentQueryFactory, + ManagedAgentSdkUsageEstimate, + ManagedAgentTeardownObservation, + ManagedAgentTerminalClassification, + ManagedAgentToolEvidence, + ManagedAgentWorkspaceChange, +} from "./types.js"; diff --git a/packages/harness/src/experimental/managed-agent-spike/permissions.test.ts b/packages/harness/src/experimental/managed-agent-spike/permissions.test.ts new file mode 100644 index 000000000..b6c6eca02 --- /dev/null +++ b/packages/harness/src/experimental/managed-agent-spike/permissions.test.ts @@ -0,0 +1,112 @@ +import { + mkdir, + mkdtemp, + realpath, + rm, + symlink, + writeFile, +} from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +import { + ManagedAgentPathError, + createManagedAgentPermissionHandler, + resolveManagedAgentToolPath, +} from "./permissions.js"; +import type { ManagedAgentPermissionEvidence } from "./types.js"; + +let root: string; +let workspace: string; +let outside: string; + +beforeEach(async () => { + root = await mkdtemp(join(tmpdir(), "managed-agent-permission-")); + workspace = join(root, "workspace"); + outside = join(root, "outside"); + await Promise.all([mkdir(workspace), mkdir(outside)]); + await Promise.all([ + writeFile(join(workspace, "inside.txt"), "inside"), + writeFile(join(outside, "secret.txt"), "outside"), + ]); + await symlink(join(outside, "secret.txt"), join(workspace, "escape.txt")); + await symlink(outside, join(workspace, "escape-dir")); + workspace = await realpath(workspace); +}); + +afterEach(async () => { + await rm(root, { recursive: true, force: true }); +}); + +describe("symlink-aware managed-agent containment", () => { + it("allows existing and new in-root paths", async () => { + expect(await resolveManagedAgentToolPath(workspace, "inside.txt")).toBe( + join(workspace, "inside.txt"), + ); + expect(await resolveManagedAgentToolPath(workspace, "nested/new.txt")).toBe( + join(workspace, "nested/new.txt"), + ); + }); + + it("denies direct, traversal, sibling-prefix, and symlink escapes", async () => { + const outsidePath = join(outside, "secret.txt"); + for (const requested of [ + outsidePath, + "../outside/secret.txt", + `${workspace}-evil/file.txt`, + "escape.txt", + "escape-dir/secret.txt", + "escape-dir/new.txt", + ]) { + await expect( + resolveManagedAgentToolPath(workspace, requested), + ).rejects.toBeInstanceOf(ManagedAgentPathError); + } + }); +}); + +describe("managed-agent permission handler", () => { + it("uses exact Bash equality and emits content-free decisions", async () => { + const evidence: ManagedAgentPermissionEvidence[] = []; + const handler = createManagedAgentPermissionHandler({ + canonicalWorkspaceRoot: workspace, + allowedBashCommands: ["git status --short"], + allowedMcpTools: ["mcp__probe__echo_nonce"], + onDecision: (decision) => evidence.push(decision), + }); + const permission = { + signal: new AbortController().signal, + toolUseID: "tool-1", + requestId: "request-1", + }; + + await expect( + handler("Bash", { command: "git status --short" }, permission), + ).resolves.toMatchObject({ behavior: "allow" }); + await expect( + handler("Bash", { command: "git status --short " }, permission), + ).resolves.toMatchObject({ behavior: "deny", interrupt: false }); + await expect( + handler("Read", { file_path: join(outside, "secret.txt") }, permission), + ).resolves.toMatchObject({ behavior: "deny", interrupt: false }); + await expect( + handler("mcp__probe__echo_nonce", { nonce: "secret" }, permission), + ).resolves.toMatchObject({ behavior: "allow" }); + await expect(handler("WebFetch", {}, permission)).resolves.toMatchObject({ + behavior: "deny", + }); + + expect(evidence.map(({ decision, reason }) => [decision, reason])).toEqual([ + ["allow", "exact_bash_command"], + ["deny", "bash_command_not_allowed"], + ["deny", "path_outside_workspace"], + ["allow", "managed_mcp_tool"], + ["deny", "tool_not_allowed"], + ]); + expect(JSON.stringify(evidence)).not.toContain(join(outside, "secret.txt")); + expect(JSON.stringify(evidence)).not.toContain("secret"); + expect(vi.isMockFunction(handler)).toBe(false); + }); +}); diff --git a/packages/harness/src/experimental/managed-agent-spike/permissions.ts b/packages/harness/src/experimental/managed-agent-spike/permissions.ts new file mode 100644 index 000000000..74e6bf5ca --- /dev/null +++ b/packages/harness/src/experimental/managed-agent-spike/permissions.ts @@ -0,0 +1,210 @@ +import { lstat, realpath } from "node:fs/promises"; +import { dirname, isAbsolute, relative, resolve, sep } from "node:path"; + +import type { + CanUseTool, + PermissionResult, +} from "@anthropic-ai/claude-agent-sdk"; + +import type { + ManagedAgentPermissionEvidence, + ManagedAgentPermissionReason, +} from "./types.js"; + +export const MANAGED_AGENT_BUILTIN_TOOLS = [ + "Read", + "Edit", + "Write", + "Bash", +] as const; + +export const MANAGED_AGENT_DISALLOWED_TOOLS = [ + "Agent", + "AskUserQuestion", + "CronCreate", + "CronDelete", + "CronList", + "EnterPlanMode", + "ExitPlanMode", + "Glob", + "Grep", + "NotebookEdit", + "SendMessage", + "Skill", + "Task", + "TaskOutput", + "TaskStop", + "TeamCreate", + "TeamDelete", + "TodoWrite", + "ToolSearch", + "WebFetch", + "WebSearch", +] as const; + +export class ManagedAgentPathError extends Error { + public constructor( + public readonly reason: "invalid_input" | "path_outside_workspace", + ) { + super(reason); + this.name = "ManagedAgentPathError"; + } +} + +function comparisonPath(value: string): string { + return process.platform === "win32" ? value.toLowerCase() : value; +} + +export function isPathWithinRoot(root: string, candidate: string): boolean { + const pathRelative = relative( + comparisonPath(root), + comparisonPath(candidate), + ); + if (pathRelative === "") return true; + return ( + !isAbsolute(pathRelative) && + pathRelative !== ".." && + !pathRelative.startsWith(`..${sep}`) + ); +} + +async function exists(path: string): Promise { + try { + await lstat(path); + return true; + } catch (error) { + return (error as NodeJS.ErrnoException).code !== "ENOENT" + ? Promise.reject(error) + : false; + } +} + +async function nearestExistingParent(path: string): Promise { + let cursor = path; + while (!(await exists(cursor))) { + const parent = dirname(cursor); + if (parent === cursor) { + throw new ManagedAgentPathError("path_outside_workspace"); + } + cursor = parent; + } + return cursor; +} + +/** + * Resolve an SDK tool target through the filesystem before authorizing it. + * Existing symlinks are followed with realpath; new targets are authorized + * only when their nearest existing parent resolves inside the canonical root. + */ +export async function resolveManagedAgentToolPath( + canonicalWorkspaceRoot: string, + requestedPath: string, +): Promise { + if (!requestedPath || requestedPath.includes("\0")) { + throw new ManagedAgentPathError("invalid_input"); + } + const candidate = resolve(canonicalWorkspaceRoot, requestedPath); + if (!isPathWithinRoot(canonicalWorkspaceRoot, candidate)) { + throw new ManagedAgentPathError("path_outside_workspace"); + } + const existing = await nearestExistingParent(candidate); + const canonicalExisting = await realpath(existing); + if (!isPathWithinRoot(canonicalWorkspaceRoot, canonicalExisting)) { + throw new ManagedAgentPathError("path_outside_workspace"); + } + if (existing === candidate) return canonicalExisting; + + const unresolvedTail = relative(existing, candidate); + const resolvedCandidate = resolve(canonicalExisting, unresolvedTail); + if (!isPathWithinRoot(canonicalWorkspaceRoot, resolvedCandidate)) { + throw new ManagedAgentPathError("path_outside_workspace"); + } + return resolvedCandidate; +} + +export interface ManagedAgentPermissionHandlerOptions { + readonly canonicalWorkspaceRoot: string; + readonly allowedBashCommands: readonly string[]; + readonly allowedMcpTools: readonly string[]; + readonly onDecision: (evidence: ManagedAgentPermissionEvidence) => void; +} + +function permissionResult( + decision: "allow" | "deny", + toolUseID: string, + reason: ManagedAgentPermissionReason, +): PermissionResult { + return decision === "allow" + ? { behavior: "allow", toolUseID } + : { + behavior: "deny", + message: `Managed-agent permission denied: ${reason}`, + interrupt: false, + toolUseID, + }; +} + +function filePathFromInput(input: Record): string | undefined { + return typeof input.file_path === "string" && input.file_path.length > 0 + ? input.file_path + : undefined; +} + +export function createManagedAgentPermissionHandler( + options: ManagedAgentPermissionHandlerOptions, +): CanUseTool { + const allowedCommands = new Set(options.allowedBashCommands); + const allowedMcpTools = new Set(options.allowedMcpTools); + + return async (toolName, input, permission): Promise => { + let decision: "allow" | "deny" = "deny"; + let reason: ManagedAgentPermissionReason = "tool_not_allowed"; + + if (allowedMcpTools.has(toolName)) { + decision = "allow"; + reason = "managed_mcp_tool"; + } else if (toolName === "Bash") { + const command = + typeof input.command === "string" ? input.command : undefined; + if (!command) { + reason = "invalid_input"; + } else if (allowedCommands.has(command)) { + decision = "allow"; + reason = "exact_bash_command"; + } else { + reason = "bash_command_not_allowed"; + } + } else if ( + toolName === "Read" || + toolName === "Edit" || + toolName === "Write" + ) { + const requestedPath = filePathFromInput(input); + if (!requestedPath) { + reason = "invalid_input"; + } else { + try { + await resolveManagedAgentToolPath( + options.canonicalWorkspaceRoot, + requestedPath, + ); + decision = "allow"; + reason = "fixture_path"; + } catch (error) { + reason = + error instanceof ManagedAgentPathError + ? error.reason + : "invalid_input"; + } + } + } + + options.onDecision({ + toolUseId: permission.toolUseID, + toolName, + decision, + reason, + }); + return permissionResult(decision, permission.toolUseID, reason); + }; +} diff --git a/packages/harness/src/experimental/managed-agent-spike/probe-cli.test.ts b/packages/harness/src/experimental/managed-agent-spike/probe-cli.test.ts new file mode 100644 index 000000000..adb82c16c --- /dev/null +++ b/packages/harness/src/experimental/managed-agent-spike/probe-cli.test.ts @@ -0,0 +1,136 @@ +import { describe, expect, it } from "vitest"; + +import { + ManagedAgentProbeCliError, + assertManagedAgentCertificationNodeVersion, + executeManagedAgentProbeCli, + managedAgentProbeUsage, + parseManagedAgentProbeCliArgs, +} from "./probe-cli.js"; + +describe("managed-agent probe CLI", () => { + it("is opt-in and never accepts credentials through arguments", () => { + expect(() => + parseManagedAgentProbeCliArgs([ + "--scenario", + "L1", + "--target", + "sonnet-5", + ]), + ).toThrow("--live"); + expect(() => + parseManagedAgentProbeCliArgs([ + "--live", + "--scenario", + "L1", + "--target", + "sonnet-5", + "--api-key", + "secret", + ]), + ).toThrow("Unknown argument"); + expect(managedAgentProbeUsage()).toContain("LLM_GATEWAY_EVAL_API_KEY"); + expect(managedAgentProbeUsage()).not.toContain("--api-key"); + }); + + it("refuses any model outside the two-value target allowlist", () => { + expect(() => + parseManagedAgentProbeCliArgs([ + "--live", + "--scenario", + "L1", + "--target", + "arbitrary-model", + ]), + ).toThrow("sonnet-5 or minimax-m3"); + }); + + it("checks exact Node before reading a dedicated credential", async () => { + const environment = new Proxy>( + {}, + { + get() { + throw new Error("environment was read"); + }, + }, + ); + await expect( + executeManagedAgentProbeCli( + ["--live", "--scenario", "L1", "--target", "sonnet-5"], + environment, + "25.0.0", + ), + ).rejects.toThrow("Live probes require Node 22.23.2"); + }); + + it("rejects an unexpected gateway origin before reading the eval key", async () => { + const reads: string[] = []; + const secret = "eval-secret-must-not-be-read"; + const environment = new Proxy>( + { + LLM_GATEWAY_BASE_URL: "https://llm.services.proxy.sapiom.ai", + LLM_GATEWAY_EVAL_API_KEY: secret, + }, + { + get(target, property: string) { + reads.push(property); + return target[property]; + }, + }, + ); + let failure: unknown; + try { + await executeManagedAgentProbeCli( + ["--live", "--scenario", "L1", "--target", "sonnet-5"], + environment, + "22.23.2", + ); + } catch (error) { + failure = error; + } + expect(failure).toBeInstanceOf(Error); + expect((failure as Error).message).toContain( + "pinned direct Sapiom gateway origin", + ); + expect((failure as Error).message).not.toContain(secret); + expect(reads).toEqual(["LLM_GATEWAY_BASE_URL"]); + }); + + it("reads eval auth only after accepting the pinned direct gateway", async () => { + const reads: string[] = []; + const environment = new Proxy>( + { + LLM_GATEWAY_BASE_URL: "https://litellm.services.sapiom.ai/", + }, + { + get(target, property: string) { + reads.push(property); + return target[property]; + }, + }, + ); + await expect( + executeManagedAgentProbeCli( + ["--live", "--scenario", "L1", "--target", "sonnet-5"], + environment, + "22.23.2", + ), + ).rejects.toThrow("LLM_GATEWAY_EVAL_API_KEY is required"); + expect(reads).toEqual(["LLM_GATEWAY_BASE_URL", "LLM_GATEWAY_EVAL_API_KEY"]); + }); + + it("prints help without reading auth or opening a query", async () => { + await expect( + executeManagedAgentProbeCli(["--help"], {}, "0.0.0"), + ).resolves.toEqual({ help: true, usage: managedAgentProbeUsage() }); + }); + + it("exposes an explicit version assertion for automation", () => { + expect(() => + assertManagedAgentCertificationNodeVersion("22.23.2"), + ).not.toThrow(); + expect(() => assertManagedAgentCertificationNodeVersion("22.23.1")).toThrow( + ManagedAgentProbeCliError, + ); + }); +}); diff --git a/packages/harness/src/experimental/managed-agent-spike/probe-cli.ts b/packages/harness/src/experimental/managed-agent-spike/probe-cli.ts new file mode 100644 index 000000000..a12ab378e --- /dev/null +++ b/packages/harness/src/experimental/managed-agent-spike/probe-cli.ts @@ -0,0 +1,331 @@ +#!/usr/bin/env node +import { pathToFileURL } from "node:url"; + +import { + FIXTURE_PATHS, + createManagedAgentFixture, + verifyManagedAgentFixtureBytes, + waitForManagedAgentFixturePids, +} from "./fixture.js"; +import { + MANAGED_AGENT_CONTRACT, + assertManagedAgentDirectGatewayOrigin, + resolveManagedAgentModelTarget, +} from "./contract.js"; +import { createLocalManagedAgentProcessObserver } from "./process-observer.js"; +import { + qualifiedManagedAgentMcpToolName, + runManagedAgentProbe, +} from "./runtime.js"; +import type { + ManagedAgentModelTargetId, + ManagedAgentProbeResult, + ManagedAgentProbeScenario, +} from "./types.js"; + +type Environment = Readonly>; + +export interface ManagedAgentProbeCliArgs { + readonly help: boolean; + readonly live: boolean; + readonly target?: ManagedAgentModelTargetId; + readonly scenario?: ManagedAgentProbeScenario; +} + +export interface ManagedAgentProbeCheck { + readonly id: string; + readonly passed: boolean; +} + +export interface ManagedAgentProbeReport { + readonly outcome: "pass" | "fail"; + readonly checks: readonly ManagedAgentProbeCheck[]; + readonly result: ManagedAgentProbeResult; +} + +export class ManagedAgentProbeCliError extends Error { + public constructor(message: string) { + super(message); + this.name = "ManagedAgentProbeCliError"; + } +} + +export function managedAgentProbeUsage(): string { + return [ + "Usage:", + " pnpm --filter @sapiom/harness probe:managed-agent -- --live --scenario --target ", + "", + "Required environment (dedicated eval access only):", + " LLM_GATEWAY_BASE_URL", + " LLM_GATEWAY_EVAL_API_KEY", + "", + "Credentials are intentionally not accepted as command-line arguments.", + ].join("\n"); +} + +export function parseManagedAgentProbeCliArgs( + argv: readonly string[], +): ManagedAgentProbeCliArgs { + if (argv.includes("--help") || argv.includes("-h")) { + return { help: true, live: false }; + } + let live = false; + let target: ManagedAgentModelTargetId | undefined; + let scenario: ManagedAgentProbeScenario | undefined; + for (let index = 0; index < argv.length; index += 1) { + const argument = argv[index]; + if (argument === "--live") { + live = true; + continue; + } + if (argument === "--target") { + const value = argv[++index]; + if (value !== "sonnet-5" && value !== "minimax-m3") { + throw new ManagedAgentProbeCliError( + "--target must be sonnet-5 or minimax-m3", + ); + } + target = value; + continue; + } + if (argument === "--scenario") { + const value = argv[++index]; + if (value !== "L1" && value !== "L2") { + throw new ManagedAgentProbeCliError("--scenario must be L1 or L2"); + } + scenario = value; + continue; + } + throw new ManagedAgentProbeCliError( + `Unknown argument: ${String(argument)}`, + ); + } + if (!live) { + throw new ManagedAgentProbeCliError( + "Refusing to run without --live; hermetic tests never contact the gateway", + ); + } + if (!target || !scenario) { + throw new ManagedAgentProbeCliError("--target and --scenario are required"); + } + return { help: false, live, target, scenario }; +} + +function requiredEnvironmentValue( + environment: Environment, + name: string, +): string { + const value = environment[name]?.trim(); + if (!value) throw new ManagedAgentProbeCliError(`${name} is required`); + return value; +} + +export function assertManagedAgentCertificationNodeVersion( + runtimeVersion: string, +): void { + if (runtimeVersion !== MANAGED_AGENT_CONTRACT.certificationNodeVersion) { + throw new ManagedAgentProbeCliError( + `Live probes require Node ${MANAGED_AGENT_CONTRACT.certificationNodeVersion}; current runtime is ${runtimeVersion}`, + ); + } +} + +export function evaluateManagedAgentProbe( + result: ManagedAgentProbeResult, + fixturePids: readonly number[] = [], +): ManagedAgentProbeReport { + const requestedTools = new Set( + result.toolEvidence + .filter(({ status }) => status === "requested") + .map(({ toolName }) => toolName), + ); + const invocation = (toolName: string, status: "success" | "error"): boolean => + result.toolEvidence.some( + (evidence) => + evidence.toolName === toolName && evidence.status === status, + ); + const pathDenials = result.permissionEvidence.filter( + ({ decision, reason }) => + decision === "deny" && reason === "path_outside_workspace", + ).length; + const checks: ManagedAgentProbeCheck[] = [ + { + id: "exact_model_alias", + passed: + result.modelAlias === + resolveManagedAgentModelTarget(result.target).alias, + }, + { id: "sdk_session_observed", passed: Boolean(result.sdkSessionId) }, + { id: "query_closed", passed: result.queryClosed }, + { id: "process_tree_quiescent", passed: result.teardown.quiescent }, + { + id: "dirty_and_untracked_preserved", + passed: + result.preservation.length === 2 && + result.preservation.every(({ preserved }) => preserved), + }, + ]; + + if (result.scenario === "L1") { + checks.push( + { id: "terminal_success", passed: result.terminal === "success" }, + { + id: "clean_target_modified", + passed: result.workspaceChanges.some( + ({ path, change }) => + path === FIXTURE_PATHS.cleanTarget && change === "modified", + ), + }, + { + id: "managed_output_created", + passed: result.workspaceChanges.some( + ({ path, change }) => + path === FIXTURE_PATHS.createdTarget && change === "created", + ), + }, + { + id: "builtin_tools_observed", + passed: ["Read", "Edit", "Write", "Bash"].every( + (name) => requestedTools.has(name) && invocation(name, "success"), + ), + }, + { + id: "mcp_echo_succeeded", + passed: invocation( + qualifiedManagedAgentMcpToolName("echo_nonce"), + "success", + ), + }, + { + id: "mcp_failure_recovered", + passed: + invocation(qualifiedManagedAgentMcpToolName("fail_once"), "error") && + invocation(qualifiedManagedAgentMcpToolName("fail_once"), "success"), + }, + { id: "outside_and_symlink_denied", passed: pathDenials >= 2 }, + ); + } else { + checks.push( + { id: "terminal_cancelled", passed: result.terminal === "cancelled" }, + { id: "cancellation_requested", passed: result.cancellationRequested }, + { + id: "teardown_within_five_seconds", + passed: result.teardown.quiescent && result.teardown.deadlineMet, + }, + { + id: "fixture_processes_observed", + passed: + fixturePids.length === 2 && + fixturePids.every((pid) => + result.teardown.observedPids.includes(pid), + ), + }, + { + id: "no_fixture_process_alive", + passed: fixturePids.every( + (pid) => !result.teardown.alivePidsAtDeadline.includes(pid), + ), + }, + ); + } + + return { + outcome: checks.every(({ passed }) => passed) ? "pass" : "fail", + checks, + result, + }; +} + +export async function executeManagedAgentProbeCli( + argv: readonly string[], + environment: Environment = process.env, + runtimeNodeVersion = process.versions.node, +): Promise< + ManagedAgentProbeReport | { readonly help: true; readonly usage: string } +> { + const args = parseManagedAgentProbeCliArgs(argv); + if (args.help) return { help: true, usage: managedAgentProbeUsage() }; + + // Validate the immutable runtime before reading the dedicated credential. + assertManagedAgentCertificationNodeVersion(runtimeNodeVersion); + const gatewayOrigin = assertManagedAgentDirectGatewayOrigin( + requiredEnvironmentValue(environment, "LLM_GATEWAY_BASE_URL"), + ); + const gatewayCredential = requiredEnvironmentValue( + environment, + "LLM_GATEWAY_EVAL_API_KEY", + ); + const fixture = await createManagedAgentFixture(); + const observer = createLocalManagedAgentProcessObserver(); + let fixturePids: readonly number[] = []; + try { + const scenario = args.scenario!; + const result = await runManagedAgentProbe( + { + scenario, + workspaceRoot: fixture.workspaceRoot, + configRoot: fixture.configRoot, + target: args.target!, + gatewayOrigin, + gatewayCredential, + prompt: fixture.prompt(scenario), + maxTurns: scenario === "L1" ? 18 : 4, + maxBudgetUsd: 0.5, + allowedBashCommands: [ + scenario === "L1" ? fixture.l1BashCommand : fixture.l2BashCommand, + ], + ...(scenario === "L1" ? { expectedMcpNonce: fixture.nonce } : {}), + preservePaths: [ + FIXTURE_PATHS.dirtySentinel, + FIXTURE_PATHS.untrackedSentinel, + ], + }, + { + processObserver: observer, + ...(scenario === "L2" + ? { + waitForCancellationSignal: async (signal: AbortSignal) => { + fixturePids = await waitForManagedAgentFixturePids( + fixture, + 15_000, + signal, + ); + observer.trackPids(fixturePids); + }, + } + : {}), + }, + ); + const bytePreservation = await verifyManagedAgentFixtureBytes(fixture); + const resultWithByteEvidence: ManagedAgentProbeResult = { + ...result, + preservation: bytePreservation, + }; + return evaluateManagedAgentProbe(resultWithByteEvidence, fixturePids); + } finally { + observer.dispose(); + await fixture.cleanup(); + } +} + +async function main(): Promise { + try { + const report = await executeManagedAgentProbeCli(process.argv.slice(2)); + if ("help" in report) { + process.stdout.write(`${report.usage}\n`); + return; + } + process.stdout.write(`${JSON.stringify(report, null, 2)}\n`); + if (report.outcome !== "pass") process.exitCode = 1; + } catch (error) { + const message = + error instanceof Error ? error.message : "Unknown probe failure"; + process.stderr.write(`managed-agent probe: ${message}\n`); + process.exitCode = 1; + } +} + +const entryUrl = process.argv[1] + ? pathToFileURL(process.argv[1]).href + : undefined; +if (entryUrl === import.meta.url) void main(); diff --git a/packages/harness/src/experimental/managed-agent-spike/process-observer.test.ts b/packages/harness/src/experimental/managed-agent-spike/process-observer.test.ts new file mode 100644 index 000000000..e78670e43 --- /dev/null +++ b/packages/harness/src/experimental/managed-agent-spike/process-observer.test.ts @@ -0,0 +1,50 @@ +import { afterEach, describe, expect, it } from "vitest"; + +import { + FIXTURE_PATHS, + createManagedAgentFixture, + waitForManagedAgentFixturePids, + type ManagedAgentFixture, +} from "./fixture.js"; +import { LocalManagedAgentProcessObserver } from "./process-observer.js"; + +const fixtures: ManagedAgentFixture[] = []; + +afterEach(async () => { + await Promise.all(fixtures.splice(0).map((fixture) => fixture.cleanup())); +}); + +describe("LocalManagedAgentProcessObserver", () => { + it("terminates a recorded local tool parent and child within five seconds", async () => { + const fixture = await createManagedAgentFixture(() => "process-observer"); + fixtures.push(fixture); + const observer = new LocalManagedAgentProcessObserver(); + const controller = new AbortController(); + let aliveAtFailure: readonly number[] = []; + try { + observer.spawn({ + command: process.execPath, + args: [FIXTURE_PATHS.processScript, FIXTURE_PATHS.processPidFile], + cwd: fixture.workspaceRoot, + env: { ...process.env }, + signal: controller.signal, + }); + const pids = await waitForManagedAgentFixturePids(fixture); + observer.trackPids(pids); + controller.abort(); + const teardown = await observer.waitForQuiescence(5_000); + aliveAtFailure = teardown.alivePidsAtDeadline; + expect(teardown.quiescent).toBe(true); + expect(teardown.deadlineMet).toBe(true); + expect(teardown.elapsedMs).toBeLessThanOrEqual(5_000); + expect(pids.every((pid) => teardown.observedPids.includes(pid))).toBe( + true, + ); + expect(teardown.alivePidsAtDeadline).toEqual([]); + } finally { + if (aliveAtFailure.length > 0) + await observer.emergencyCleanup(aliveAtFailure); + observer.dispose(); + } + }, 10_000); +}); diff --git a/packages/harness/src/experimental/managed-agent-spike/process-observer.ts b/packages/harness/src/experimental/managed-agent-spike/process-observer.ts new file mode 100644 index 000000000..f10268dff --- /dev/null +++ b/packages/harness/src/experimental/managed-agent-spike/process-observer.ts @@ -0,0 +1,249 @@ +import { + execFile, + spawn as spawnChild, + type ChildProcessWithoutNullStreams, +} from "node:child_process"; +import { promisify } from "node:util"; + +import type { + SpawnedProcess, + SpawnOptions, +} from "@anthropic-ai/claude-agent-sdk"; + +import type { + ManagedAgentProcessObserver, + ManagedAgentTeardownObservation, +} from "./types.js"; + +const execFileAsync = promisify(execFile); +const SAMPLE_INTERVAL_MS = 100; +const QUIESCENCE_POLL_MS = 25; + +type ProcessTable = ReadonlyMap; + +function delay(milliseconds: number): Promise { + return new Promise((resolveDelay) => setTimeout(resolveDelay, milliseconds)); +} + +function pidAlive(pid: number): boolean { + try { + process.kill(pid, 0); + return true; + } catch (error) { + return (error as NodeJS.ErrnoException).code === "EPERM"; + } +} + +async function windowsProcessTable(): Promise { + const { stdout } = await execFileAsync( + "powershell.exe", + [ + "-NoProfile", + "-NonInteractive", + "-Command", + "Get-CimInstance Win32_Process | Select-Object ProcessId,ParentProcessId | ConvertTo-Json -Compress", + ], + { windowsHide: true, maxBuffer: 4 * 1024 * 1024 }, + ); + const parsed = JSON.parse(stdout) as + | { ProcessId?: unknown; ParentProcessId?: unknown } + | Array<{ ProcessId?: unknown; ParentProcessId?: unknown }>; + const rows = Array.isArray(parsed) ? parsed : [parsed]; + return new Map( + rows.flatMap((row) => + typeof row.ProcessId === "number" && + typeof row.ParentProcessId === "number" + ? [[row.ProcessId, row.ParentProcessId] as const] + : [], + ), + ); +} + +async function posixProcessTable(): Promise { + const { stdout } = await execFileAsync("/bin/ps", ["-axo", "pid=,ppid="], { + windowsHide: true, + maxBuffer: 4 * 1024 * 1024, + }); + const entries: Array = []; + for (const line of stdout.split("\n")) { + const match = /^\s*(\d+)\s+(\d+)\s*$/.exec(line); + if (!match) continue; + entries.push([Number(match[1]), Number(match[2])]); + } + return new Map(entries); +} + +async function readProcessTable(): Promise { + try { + return process.platform === "win32" + ? await windowsProcessTable() + : await posixProcessTable(); + } catch { + return new Map(); + } +} + +function descendantsOf( + roots: ReadonlySet, + table: ProcessTable, +): Set { + const descendants = new Set(); + let changed = true; + while (changed) { + changed = false; + for (const [pid, parentPid] of table) { + if ( + !descendants.has(pid) && + (roots.has(parentPid) || descendants.has(parentPid)) + ) { + descendants.add(pid); + changed = true; + } + } + } + return descendants; +} + +async function taskkill(pid: number, force: boolean): Promise { + try { + await execFileAsync( + "taskkill.exe", + ["/PID", String(pid), "/T", ...(force ? ["/F"] : [])], + { windowsHide: true }, + ); + } catch { + // A process that exited between observation and cleanup is already safe. + } +} + +/** + * Tracks the actual SDK subprocess plus descendants sampled from the kernel. + * POSIX children are placed in their own process group so the forwarded SDK + * abort signal can terminate Bash descendants rather than only their parent. + */ +export class LocalManagedAgentProcessObserver implements ManagedAgentProcessObserver { + readonly #roots = new Set(); + readonly #observed = new Set(); + readonly #children = new Map(); + readonly #sampler: NodeJS.Timeout; + #samplePending = false; + + public constructor() { + this.#sampler = setInterval(() => void this.#sample(), SAMPLE_INTERVAL_MS); + this.#sampler.unref(); + } + + public spawn(options: SpawnOptions): SpawnedProcess { + const child = spawnChild(options.command, options.args, { + cwd: options.cwd, + env: options.env, + detached: process.platform !== "win32", + stdio: ["pipe", "pipe", "pipe"], + windowsHide: true, + }); + if (typeof child.pid === "number") { + const pid = child.pid; + this.#roots.add(pid); + this.#observed.add(pid); + this.#children.set(pid, child); + const terminateTree = (): void => { + if (process.platform === "win32") { + void taskkill(pid, false); + return; + } + try { + process.kill(-pid, "SIGTERM"); + } catch { + child.kill("SIGTERM"); + } + }; + options.signal.addEventListener("abort", terminateTree, { once: true }); + child.once("exit", () => { + options.signal.removeEventListener("abort", terminateTree); + this.#children.delete(pid); + }); + } + return child; + } + + public trackPids(pids: readonly number[]): void { + for (const pid of pids) { + if (Number.isInteger(pid) && pid > 0) this.#observed.add(pid); + } + } + + async #sample(): Promise { + if (this.#samplePending || this.#roots.size === 0) return; + this.#samplePending = true; + try { + const table = await readProcessTable(); + for (const pid of descendantsOf(this.#roots, table)) + this.#observed.add(pid); + } finally { + this.#samplePending = false; + } + } + + public async waitForQuiescence( + timeoutMs: number, + ): Promise { + const startedAt = Date.now(); + let alivePids: number[] = []; + do { + await this.#sample(); + alivePids = [...this.#observed].filter(pidAlive).sort((a, b) => a - b); + if (alivePids.length === 0) { + return { + quiescent: true, + deadlineMet: true, + elapsedMs: Date.now() - startedAt, + observedPids: [...this.#observed].sort((a, b) => a - b), + alivePidsAtDeadline: [], + emergencyCleanupAttempted: false, + }; + } + await delay(QUIESCENCE_POLL_MS); + } while (Date.now() - startedAt < timeoutMs); + + await this.#sample(); + alivePids = [...this.#observed].filter(pidAlive).sort((a, b) => a - b); + return { + quiescent: alivePids.length === 0, + deadlineMet: alivePids.length === 0, + elapsedMs: Date.now() - startedAt, + observedPids: [...this.#observed].sort((a, b) => a - b), + alivePidsAtDeadline: alivePids, + emergencyCleanupAttempted: false, + }; + } + + public async emergencyCleanup(pids: readonly number[]): Promise { + if (process.platform === "win32") { + await Promise.all([...this.#roots].map((pid) => taskkill(pid, true))); + await Promise.all(pids.map((pid) => taskkill(pid, true))); + return; + } + for (const root of this.#roots) { + try { + process.kill(-root, "SIGKILL"); + } catch { + // Fall through to individual PID cleanup below. + } + } + for (const pid of pids) { + try { + process.kill(pid, "SIGKILL"); + } catch { + // A process that already exited needs no cleanup. + } + } + } + + public dispose(): void { + clearInterval(this.#sampler); + } +} + +export function createLocalManagedAgentProcessObserver(): ManagedAgentProcessObserver { + return new LocalManagedAgentProcessObserver(); +} diff --git a/packages/harness/src/experimental/managed-agent-spike/runtime.test.ts b/packages/harness/src/experimental/managed-agent-spike/runtime.test.ts new file mode 100644 index 000000000..d88055eb7 --- /dev/null +++ b/packages/harness/src/experimental/managed-agent-spike/runtime.test.ts @@ -0,0 +1,372 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; + +import type { Options } from "@anthropic-ai/claude-agent-sdk"; + +import { + MANAGED_AGENT_MODEL_ENVIRONMENT_VARIABLES, + resolveManagedAgentModelTarget, +} from "./contract.js"; +import { + FIXTURE_PATHS, + createManagedAgentFixture, + type ManagedAgentFixture, +} from "./fixture.js"; +import { + MANAGED_AGENT_BUILTIN_TOOLS, + MANAGED_AGENT_DISALLOWED_TOOLS, +} from "./permissions.js"; +import { + createManagedAgentMcpRuntime, + runManagedAgentProbe, +} from "./runtime.js"; +import type { + ManagedAgentProcessObserver, + ManagedAgentQuery, + ManagedAgentTeardownObservation, +} from "./types.js"; + +const fixtures: ManagedAgentFixture[] = []; + +afterEach(async () => { + await Promise.all(fixtures.splice(0).map((fixture) => fixture.cleanup())); +}); + +function quiescentTeardown(): ManagedAgentTeardownObservation { + return { + quiescent: true, + deadlineMet: true, + elapsedMs: 12, + observedPids: [], + alivePidsAtDeadline: [], + emergencyCleanupAttempted: false, + }; +} + +function fakeObserver( + teardown: ManagedAgentTeardownObservation = quiescentTeardown(), +): ManagedAgentProcessObserver & { + waitForQuiescence: ReturnType; + emergencyCleanup: ReturnType; + dispose: ReturnType; +} { + return { + spawn: vi.fn(() => { + throw new Error("fake query must not spawn"); + }), + trackPids: vi.fn(), + waitForQuiescence: vi.fn(async () => teardown), + emergencyCleanup: vi.fn(async () => undefined), + dispose: vi.fn(), + }; +} + +function queryFromEvents( + events: readonly unknown[], + close = vi.fn(), +): ManagedAgentQuery { + return { + async *[Symbol.asyncIterator]() { + for (const event of events) yield event; + }, + close, + }; +} + +async function probeConfig(scenario: "L1" | "L2" = "L1") { + const fixture = await createManagedAgentFixture(() => "runtime-test-secret"); + fixtures.push(fixture); + return { + fixture, + config: { + scenario, + workspaceRoot: fixture.workspaceRoot, + configRoot: fixture.configRoot, + target: "sonnet-5" as const, + gatewayOrigin: "https://gateway.example.test", + gatewayCredential: "dedicated-eval-secret", + prompt: fixture.prompt(scenario), + maxTurns: 10, + maxBudgetUsd: 0.25, + allowedBashCommands: [ + scenario === "L1" ? fixture.l1BashCommand : fixture.l2BashCommand, + ], + ...(scenario === "L1" ? { expectedMcpNonce: fixture.nonce } : {}), + preservePaths: [ + FIXTURE_PATHS.dirtySentinel, + FIXTURE_PATHS.untrackedSentinel, + ], + }, + }; +} + +describe("runManagedAgentProbe", () => { + it("provides deterministic MCP success and fail-once recovery", async () => { + const runtime = createManagedAgentMcpRuntime("nonce-1"); + await expect( + runtime.handlers.echoNonce({ nonce: "nonce-1" }), + ).resolves.toEqual({ + content: [{ type: "text", text: "nonce-1" }], + }); + await expect(runtime.handlers.failOnce()).resolves.toMatchObject({ + isError: true, + }); + await expect(runtime.handlers.failOnce()).resolves.not.toHaveProperty( + "isError", + ); + expect( + runtime.invocations.map(({ toolName, status }) => [toolName, status]), + ).toEqual([ + ["mcp__sapiom-managed-agent-spike__echo_nonce", "success"], + ["mcp__sapiom-managed-agent-spike__fail_once", "error"], + ["mcp__sapiom-managed-agent-spike__fail_once", "success"], + ]); + + const mismatch = createManagedAgentMcpRuntime("expected-nonce"); + await expect( + mismatch.handlers.echoNonce({ nonce: "wrong-nonce" }), + ).resolves.toMatchObject({ isError: true }); + expect(mismatch.invocations).toEqual([ + { + toolName: "mcp__sapiom-managed-agent-spike__echo_nonce", + status: "error", + }, + ]); + }); + + it("passes the strict isolated SDK contract and emits only normalized evidence", async () => { + const { config, fixture } = await probeConfig(); + const observer = fakeObserver(); + const close = vi.fn(); + let capturedOptions: Options | undefined; + const previousOAuth = process.env.CLAUDE_CODE_OAUTH_TOKEN; + process.env.CLAUDE_CODE_OAUTH_TOKEN = "ambient-user-login"; + try { + const result = await runManagedAgentProbe(config, { + hermeticGatewayOrigin: config.gatewayOrigin, + processObserver: observer, + uuid: (() => { + let counter = 0; + return () => + `00000000-0000-4000-8000-${String(++counter).padStart(12, "0")}`; + })(), + queryFactory: ({ options }) => { + capturedOptions = options; + return queryFromEvents( + [ + { + type: "system", + subtype: "init", + session_id: "sdk-session-1", + model: resolveManagedAgentModelTarget("sonnet-5").alias, + }, + { + type: "assistant", + session_id: "sdk-session-1", + message: { + content: [ + { + type: "tool_use", + id: "tool-1", + name: "Read", + input: { + file_path: fixture.outsideSentinel, + secret: fixture.nonce, + }, + }, + ], + }, + }, + { + type: "user", + session_id: "sdk-session-1", + message: { + content: [ + { + type: "tool_result", + tool_use_id: "tool-1", + content: `secret:${fixture.nonce}`, + }, + ], + }, + }, + { + type: "result", + subtype: "success", + is_error: false, + session_id: "sdk-session-1", + result: `secret:${fixture.nonce}`, + usage: { input_tokens: 9, output_tokens: 4 }, + }, + ], + close, + ); + }, + }); + + expect(capturedOptions).toBeDefined(); + expect(capturedOptions?.model).toBe( + resolveManagedAgentModelTarget("sonnet-5").alias, + ); + expect(capturedOptions?.tools).toEqual(MANAGED_AGENT_BUILTIN_TOOLS); + expect(capturedOptions?.disallowedTools).toEqual( + MANAGED_AGENT_DISALLOWED_TOOLS, + ); + expect(capturedOptions?.permissionMode).toBe("default"); + expect(capturedOptions?.settingSources).toEqual([]); + expect(capturedOptions?.strictMcpConfig).toBe(true); + expect(capturedOptions?.canUseTool).toBeTypeOf("function"); + expect(capturedOptions?.spawnClaudeCodeProcess).toBeTypeOf("function"); + expect( + Object.prototype.hasOwnProperty.call(capturedOptions, "allowedTools"), + ).toBe(false); + expect( + Object.prototype.hasOwnProperty.call(capturedOptions, "fallbackModel"), + ).toBe(false); + expect( + Object.prototype.hasOwnProperty.call( + capturedOptions, + "allowDangerouslySkipPermissions", + ), + ).toBe(false); + for (const variable of MANAGED_AGENT_MODEL_ENVIRONMENT_VARIABLES) { + expect(capturedOptions?.env?.[variable]).toBe(capturedOptions?.model); + } + expect(capturedOptions?.env).not.toHaveProperty( + "CLAUDE_CODE_OAUTH_TOKEN", + ); + expect(capturedOptions?.env).not.toHaveProperty("SAPIOM_API_KEY"); + expect(result.terminal).toBe("success"); + expect(result.sdkSessionId).toBe("sdk-session-1"); + expect(result.queryClosed).toBe(true); + expect(result.preservation.every(({ preserved }) => preserved)).toBe( + true, + ); + expect(close).toHaveBeenCalledOnce(); + expect(observer.dispose).toHaveBeenCalledOnce(); + const serialized = JSON.stringify(result); + expect(serialized).not.toContain("dedicated-eval-secret"); + expect(serialized).not.toContain(fixture.nonce); + expect(serialized).not.toContain(fixture.outsideSentinel); + } finally { + if (previousOAuth === undefined) + delete process.env.CLAUDE_CODE_OAUTH_TOKEN; + else process.env.CLAUDE_CODE_OAUTH_TOKEN = previousOAuth; + } + }); + + it("classifies an explicit active-run abort as cancellation", async () => { + const { config } = await probeConfig("L2"); + const observer = fakeObserver(); + const close = vi.fn(); + const result = await runManagedAgentProbe(config, { + hermeticGatewayOrigin: config.gatewayOrigin, + processObserver: observer, + waitForCancellationSignal: async () => undefined, + queryFactory: ({ options }) => ({ + async *[Symbol.asyncIterator]() { + yield { + type: "system", + subtype: "init", + session_id: "cancel-session", + }; + if (!options.abortController?.signal.aborted) { + await new Promise((resolveAbort) => + options.abortController?.signal.addEventListener( + "abort", + () => resolveAbort(), + { once: true }, + ), + ); + } + throw new Error("synthetic abort"); + }, + close, + }), + }); + + expect(result.terminal).toBe("cancelled"); + expect(result.cancellationRequested).toBe(true); + expect(result.queryClosed).toBe(true); + expect( + result.events.filter(({ type }) => type === "terminal"), + ).toHaveLength(1); + }); + + it("records teardown failure before attempting emergency cleanup", async () => { + const { config } = await probeConfig(); + const observer = fakeObserver({ + quiescent: false, + deadlineMet: false, + elapsedMs: 5_001, + observedPids: [9001], + alivePidsAtDeadline: [9001], + emergencyCleanupAttempted: false, + }); + const result = await runManagedAgentProbe(config, { + hermeticGatewayOrigin: config.gatewayOrigin, + processObserver: observer, + queryFactory: () => + queryFromEvents([ + { type: "system", subtype: "init", session_id: "session-timeout" }, + { type: "result", subtype: "success", is_error: false }, + ]), + }); + + expect(result.terminal).toBe("teardown_timeout"); + expect(result.teardown.emergencyCleanupAttempted).toBe(true); + expect(observer.emergencyCleanup).toHaveBeenCalledWith([9001]); + expect(result.events.at(-1)).toMatchObject({ + type: "terminal", + terminal: "teardown_timeout", + }); + }); + + it("classifies a throwing query close without skipping abort or observer disposal", async () => { + const { config } = await probeConfig(); + const observer = fakeObserver(); + let abortSignal: AbortSignal | undefined; + const result = await runManagedAgentProbe(config, { + hermeticGatewayOrigin: config.gatewayOrigin, + processObserver: observer, + queryFactory: ({ options }) => { + abortSignal = options.abortController?.signal; + return queryFromEvents( + [ + { type: "system", subtype: "init", session_id: "close-session" }, + { type: "result", subtype: "success", is_error: false }, + ], + vi.fn(() => { + throw new Error("synthetic close failure"); + }), + ); + }, + }); + + expect(result.terminal).toBe("close_timeout"); + expect(result.queryClosed).toBe(false); + expect(abortSignal?.aborted).toBe(true); + expect(observer.dispose).toHaveBeenCalledOnce(); + }); + + it("rejects a test gateway unless the explicit hermetic seam is present", async () => { + const { config } = await probeConfig(); + const queryFactory = vi.fn(() => queryFromEvents([])); + await expect( + runManagedAgentProbe(config, { + processObserver: fakeObserver(), + queryFactory, + }), + ).rejects.toThrow("pinned direct Sapiom gateway origin"); + expect(queryFactory).not.toHaveBeenCalled(); + }); + + it("rejects the hermetic origin seam without an injected query factory", async () => { + const { config } = await probeConfig(); + await expect( + runManagedAgentProbe(config, { + hermeticGatewayOrigin: config.gatewayOrigin, + processObserver: fakeObserver(), + }), + ).rejects.toThrow("requires an injected queryFactory"); + }); +}); diff --git a/packages/harness/src/experimental/managed-agent-spike/runtime.ts b/packages/harness/src/experimental/managed-agent-spike/runtime.ts new file mode 100644 index 000000000..2059b4a4b --- /dev/null +++ b/packages/harness/src/experimental/managed-agent-spike/runtime.ts @@ -0,0 +1,393 @@ +import { randomUUID } from "node:crypto"; + +import { + createSdkMcpServer, + query as agentSdkQuery, + tool, + type McpSdkServerConfigWithInstance, + type Options, +} from "@anthropic-ai/claude-agent-sdk"; +import { z } from "zod"; + +import { validateManagedAgentProbeConfig } from "./contract.js"; +import { buildManagedAgentChildEnvironment } from "./environment.js"; +import { ManagedAgentEventRecorder } from "./events.js"; +import { + captureManagedAgentWorkspaceSnapshot, + diffManagedAgentWorkspaceSnapshots, + observeManagedAgentPreservation, +} from "./fixture.js"; +import { + MANAGED_AGENT_BUILTIN_TOOLS, + MANAGED_AGENT_DISALLOWED_TOOLS, + createManagedAgentPermissionHandler, +} from "./permissions.js"; +import { createLocalManagedAgentProcessObserver } from "./process-observer.js"; +import type { + ManagedAgentProbeConfig, + ManagedAgentProbeDependencies, + ManagedAgentProbeResult, + ManagedAgentQuery, + ManagedAgentTeardownObservation, + ManagedAgentTerminalClassification, + ManagedAgentToolEvidence, +} from "./types.js"; + +export const MANAGED_AGENT_MCP_SERVER_NAME = "sapiom-managed-agent-spike"; +export const MANAGED_AGENT_TEARDOWN_TIMEOUT_MS = 5_000; +const QUERY_CLOSE_TIMEOUT_MS = 2_000; + +type McpToolName = "echo_nonce" | "fail_once"; + +export interface ManagedAgentMcpRuntime { + readonly server: McpSdkServerConfigWithInstance; + readonly qualifiedToolNames: readonly string[]; + readonly invocations: readonly ManagedAgentToolEvidence[]; + readonly handlers: { + readonly echoNonce: (input: { readonly nonce: string }) => Promise<{ + content: Array<{ type: "text"; text: string }>; + isError?: boolean; + }>; + readonly failOnce: () => Promise<{ + content: Array<{ type: "text"; text: string }>; + isError?: boolean; + }>; + }; +} + +export function qualifiedManagedAgentMcpToolName(name: McpToolName): string { + return `mcp__${MANAGED_AGENT_MCP_SERVER_NAME}__${name}`; +} + +export function createManagedAgentMcpRuntime( + expectedEchoNonce?: string, +): ManagedAgentMcpRuntime { + const invocations: ManagedAgentToolEvidence[] = []; + let failOnceCalls = 0; + const nonceSchema = { nonce: z.string().min(1).max(256) }; + const handlers = { + async echoNonce({ nonce }: { readonly nonce: string }) { + const matched = + expectedEchoNonce === undefined || nonce === expectedEchoNonce; + invocations.push({ + toolName: qualifiedManagedAgentMcpToolName("echo_nonce"), + status: matched ? ("success" as const) : ("error" as const), + }); + return matched + ? { content: [{ type: "text" as const, text: nonce }] } + : { + content: [ + { + type: "text" as const, + text: "nonce did not match the untracked-file sentinel", + }, + ], + isError: true, + }; + }, + async failOnce() { + failOnceCalls += 1; + const failed = failOnceCalls === 1; + invocations.push({ + toolName: qualifiedManagedAgentMcpToolName("fail_once"), + status: failed ? ("error" as const) : ("success" as const), + }); + return failed + ? { + content: [ + { + type: "text" as const, + text: "planned managed-agent probe failure; retry once", + }, + ], + isError: true, + } + : { + content: [ + { + type: "text" as const, + text: "planned managed-agent probe recovery succeeded", + }, + ], + }; + }, + }; + const echoNonce = tool( + "echo_nonce", + "Return the supplied nonce exactly for the local managed-agent probe.", + nonceSchema, + handlers.echoNonce, + { alwaysLoad: true }, + ); + const failOnce = tool( + "fail_once", + "Return a planned error once, then succeed on the next call.", + nonceSchema, + handlers.failOnce, + { alwaysLoad: true }, + ); + return { + server: createSdkMcpServer({ + name: MANAGED_AGENT_MCP_SERVER_NAME, + version: "0.1.0", + instructions: + "These tools exist only for deterministic Sapiom local managed-agent feasibility probes.", + tools: [echoNonce, failOnce], + alwaysLoad: true, + }), + qualifiedToolNames: [ + qualifiedManagedAgentMcpToolName("echo_nonce"), + qualifiedManagedAgentMcpToolName("fail_once"), + ], + invocations, + handlers, + }; +} + +function defaultQueryFactory(input: { + readonly prompt: string; + readonly options: Options; +}): ManagedAgentQuery { + return agentSdkQuery(input); +} + +function safeEvalSource( + scenario: string, + target: string, + executionId: string, +): string { + return `studio-managed-agent-e0-${scenario}-${target}-${executionId}` + .toLowerCase() + .replace(/[^a-z0-9-]+/g, "-") + .replace(/^-|-$/g, ""); +} + +async function closeQueryBounded(query: ManagedAgentQuery): Promise { + let timeout: NodeJS.Timeout | undefined; + try { + return await Promise.race([ + Promise.resolve().then(() => { + query.close(); + return true; + }), + new Promise((resolveTimeout) => { + timeout = setTimeout( + () => resolveTimeout(false), + QUERY_CLOSE_TIMEOUT_MS, + ); + timeout.unref(); + }), + ]); + } catch { + return false; + } finally { + if (timeout) clearTimeout(timeout); + } +} + +function classifyTerminal(input: { + readonly teardown: ManagedAgentTeardownObservation; + readonly queryCreated: boolean; + readonly queryClosed: boolean; + readonly cancellationRequested: boolean; + readonly queryFailed: boolean; + readonly sdkResult?: { readonly isError: boolean; readonly subtype?: string }; +}): ManagedAgentTerminalClassification { + if (!input.teardown.quiescent || !input.teardown.deadlineMet) { + return "teardown_timeout"; + } + if (input.queryCreated && !input.queryClosed) return "close_timeout"; + if (input.cancellationRequested) return "cancelled"; + if (input.queryFailed) return "query_error"; + if (input.sdkResult?.isError) return "sdk_result_error"; + if (input.sdkResult) return "success"; + return "incomplete"; +} + +export async function runManagedAgentProbe( + config: ManagedAgentProbeConfig, + dependencies: ManagedAgentProbeDependencies = {}, +): Promise { + if (dependencies.hermeticGatewayOrigin && !dependencies.queryFactory) { + throw new Error("hermeticGatewayOrigin requires an injected queryFactory"); + } + const validated = validateManagedAgentProbeConfig(config, { + ...(dependencies.hermeticGatewayOrigin + ? { hermeticGatewayOrigin: dependencies.hermeticGatewayOrigin } + : {}), + }); + if (config.scenario === "L2" && !dependencies.waitForCancellationSignal) { + throw new Error("L2 requires an explicit cancellation signal dependency"); + } + + const createUuid = dependencies.uuid ?? randomUUID; + const runId = createUuid(); + const executionId = createUuid(); + const evalSource = safeEvalSource( + config.scenario, + config.target, + executionId, + ); + const recorder = new ManagedAgentEventRecorder(runId); + const mcpRuntime = createManagedAgentMcpRuntime(config.expectedMcpNonce); + const abortController = new AbortController(); + const triggerController = new AbortController(); + const before = await captureManagedAgentWorkspaceSnapshot( + validated.canonicalWorkspaceRoot, + ); + let cancellationRequested = false; + let cancellationRequestedAt: number | undefined; + let query: ManagedAgentQuery | undefined; + let queryFailed = false; + let queryClosed = false; + let cancellationTriggerFailed = false; + + const childEnvironment = buildManagedAgentChildEnvironment({ + ambient: process.env, + configRoot: validated.canonicalConfigRoot, + gatewayOrigin: validated.gatewayOrigin, + gatewayCredential: config.gatewayCredential, + modelAlias: validated.model.alias, + evalSource, + executionId, + }); + const permissionHandler = createManagedAgentPermissionHandler({ + canonicalWorkspaceRoot: validated.canonicalWorkspaceRoot, + allowedBashCommands: config.allowedBashCommands, + allowedMcpTools: mcpRuntime.qualifiedToolNames, + onDecision: (evidence) => recorder.recordPermission(evidence), + }); + const processObserver = + dependencies.processObserver ?? createLocalManagedAgentProcessObserver(); + + const options: Options = { + abortController, + canUseTool: permissionHandler, + cwd: validated.canonicalWorkspaceRoot, + disallowedTools: [...MANAGED_AGENT_DISALLOWED_TOOLS], + env: childEnvironment, + includePartialMessages: false, + maxBudgetUsd: config.maxBudgetUsd, + maxTurns: config.maxTurns, + mcpServers: { [MANAGED_AGENT_MCP_SERVER_NAME]: mcpRuntime.server }, + model: validated.model.alias, + permissionMode: "default", + persistSession: false, + settingSources: [], + skills: [], + spawnClaudeCodeProcess: (spawnOptions) => + processObserver.spawn(spawnOptions), + stderr: () => { + // Do not retain or print SDK stderr; probe artifacts are structural only. + }, + strictMcpConfig: true, + systemPrompt: + "You are a deterministic local managed-agent feasibility probe. Follow the ordered instructions exactly, continue after expected permission denials and planned MCP errors, and use only the tools named in the prompt.", + thinking: { type: "disabled" }, + tools: [...MANAGED_AGENT_BUILTIN_TOOLS], + }; + + let teardown!: ManagedAgentTeardownObservation; + let terminal!: ManagedAgentTerminalClassification; + try { + recorder.recordLifecycle("starting"); + const cancellationTask = dependencies.waitForCancellationSignal + ? dependencies + .waitForCancellationSignal(triggerController.signal) + .then(() => { + if (triggerController.signal.aborted) return; + cancellationRequested = true; + cancellationRequestedAt = (dependencies.now ?? Date.now)(); + recorder.recordLifecycle("cancellation_requested"); + abortController.abort(); + }) + .catch(() => { + if (!triggerController.signal.aborted) { + cancellationTriggerFailed = true; + abortController.abort(); + } + }) + : undefined; + + try { + query = (dependencies.queryFactory ?? defaultQueryFactory)({ + prompt: config.prompt, + options, + }); + for await (const event of query) recorder.observeSdkEvent(event); + } catch { + if (!abortController.signal.aborted) queryFailed = true; + } finally { + triggerController.abort(); + if (cancellationTask) await cancellationTask; + queryFailed ||= cancellationTriggerFailed; + if (query) queryClosed = await closeQueryBounded(query); + if ((query && !queryClosed) || queryFailed) abortController.abort(); + } + + const now = dependencies.now ?? Date.now; + const elapsedBeforeTeardown = + cancellationRequestedAt === undefined + ? 0 + : now() - cancellationRequestedAt; + const remainingTeardownMs = Math.max( + 0, + MANAGED_AGENT_TEARDOWN_TIMEOUT_MS - elapsedBeforeTeardown, + ); + teardown = await processObserver.waitForQuiescence(remainingTeardownMs); + if (cancellationRequestedAt !== undefined) { + const totalElapsedMs = now() - cancellationRequestedAt; + teardown = { + ...teardown, + elapsedMs: totalElapsedMs, + deadlineMet: + teardown.quiescent && + totalElapsedMs <= MANAGED_AGENT_TEARDOWN_TIMEOUT_MS, + }; + } + terminal = classifyTerminal({ + teardown, + queryCreated: query !== undefined, + queryClosed, + cancellationRequested, + queryFailed, + sdkResult: recorder.result, + }); + recorder.recordTerminal(terminal); + + if (!teardown.quiescent) { + await processObserver.emergencyCleanup(teardown.alivePidsAtDeadline); + teardown = { ...teardown, emergencyCleanupAttempted: true }; + terminal = "teardown_timeout"; + } + } finally { + processObserver.dispose(); + } + + const after = await captureManagedAgentWorkspaceSnapshot( + validated.canonicalWorkspaceRoot, + ); + return { + contractVersion: 1, + runId, + scenario: config.scenario, + target: config.target, + modelAlias: validated.model.alias, + ...(recorder.sessionId ? { sdkSessionId: recorder.sessionId } : {}), + terminal, + events: [...recorder.events], + toolEvidence: [...recorder.toolEvidence, ...mcpRuntime.invocations], + permissionEvidence: [...recorder.permissionEvidence], + workspaceChanges: diffManagedAgentWorkspaceSnapshots(before, after), + preservation: observeManagedAgentPreservation( + before, + after, + config.preservePaths ?? [], + ), + cancellationRequested, + queryClosed, + teardown, + correlation: { executionId, evalSource }, + ...(recorder.usage ? { sdkUsage: recorder.usage } : {}), + }; +} diff --git a/packages/harness/src/experimental/managed-agent-spike/types.ts b/packages/harness/src/experimental/managed-agent-spike/types.ts new file mode 100644 index 000000000..f350b67ec --- /dev/null +++ b/packages/harness/src/experimental/managed-agent-spike/types.ts @@ -0,0 +1,177 @@ +import type { + Options, + SpawnedProcess, + SpawnOptions, +} from "@anthropic-ai/claude-agent-sdk"; + +export type ManagedAgentModelTargetId = "sonnet-5" | "minimax-m3"; +export type ManagedAgentProbeScenario = "L1" | "L2"; + +export interface ManagedAgentModelTarget { + readonly id: ManagedAgentModelTargetId; + readonly alias: string; + readonly upstreamProvider: "anthropic" | "fireworks_ai"; + readonly upstreamModel: string; +} + +/** + * Explicit inputs for one experimental probe. The gateway credential is + * sensitive and must never be copied into events, results, logs, or CLI args. + */ +export interface ManagedAgentProbeConfig { + readonly scenario: ManagedAgentProbeScenario; + readonly workspaceRoot: string; + readonly configRoot: string; + readonly target: ManagedAgentModelTargetId; + readonly gatewayOrigin: string; + readonly gatewayCredential: string; + readonly prompt: string; + readonly maxTurns: number; + readonly maxBudgetUsd: number; + readonly allowedBashCommands: readonly string[]; + /** Expected only for L1 and never copied into structural evidence. */ + readonly expectedMcpNonce?: string; + readonly preservePaths?: readonly string[]; +} + +export type ManagedAgentPermissionDecision = "allow" | "deny"; +export type ManagedAgentPermissionReason = + | "fixture_path" + | "exact_bash_command" + | "managed_mcp_tool" + | "invalid_input" + | "path_outside_workspace" + | "bash_command_not_allowed" + | "tool_not_allowed"; + +export type ManagedAgentProbeEventType = + | "lifecycle" + | "message" + | "tool_requested" + | "tool_completed" + | "permission" + | "sdk_result" + | "terminal"; + +/** + * A deliberately content-free event boundary. Raw prompts, message text, + * tool inputs/results, filesystem paths, and error messages never cross it. + */ +export interface ManagedAgentProbeEvent { + readonly sequence: number; + readonly runId: string; + readonly type: ManagedAgentProbeEventType; + readonly subtype?: string; + readonly sessionId?: string; + readonly toolUseId?: string; + readonly toolName?: string; + readonly permissionDecision?: ManagedAgentPermissionDecision; + readonly permissionReason?: ManagedAgentPermissionReason; + readonly isError?: boolean; + readonly terminal?: ManagedAgentTerminalClassification; +} + +export interface ManagedAgentSdkUsageEstimate { + readonly authority: "sdk_non_authoritative"; + readonly inputTokens: number; + readonly outputTokens: number; + readonly cacheCreationInputTokens: number; + readonly cacheReadInputTokens: number; + readonly estimatedCostUsd?: number; +} + +export interface ManagedAgentWorkspaceChange { + readonly path: string; + readonly change: "created" | "modified" | "deleted"; +} + +export interface ManagedAgentPreservationObservation { + readonly path: string; + readonly preserved: boolean; +} + +export interface ManagedAgentToolEvidence { + readonly toolUseId?: string; + readonly toolName: string; + readonly status: "requested" | "success" | "error"; +} + +export interface ManagedAgentPermissionEvidence { + readonly toolUseId: string; + readonly toolName: string; + readonly decision: ManagedAgentPermissionDecision; + readonly reason: ManagedAgentPermissionReason; +} + +export interface ManagedAgentTeardownObservation { + readonly quiescent: boolean; + readonly deadlineMet: boolean; + readonly elapsedMs: number; + readonly observedPids: readonly number[]; + readonly alivePidsAtDeadline: readonly number[]; + readonly emergencyCleanupAttempted: boolean; +} + +export type ManagedAgentTerminalClassification = + | "success" + | "cancelled" + | "sdk_result_error" + | "query_error" + | "incomplete" + | "close_timeout" + | "teardown_timeout"; + +export interface ManagedAgentProbeResult { + readonly contractVersion: 1; + readonly runId: string; + readonly scenario: ManagedAgentProbeScenario; + readonly target: ManagedAgentModelTargetId; + readonly modelAlias: string; + readonly sdkSessionId?: string; + readonly terminal: ManagedAgentTerminalClassification; + readonly events: readonly ManagedAgentProbeEvent[]; + readonly toolEvidence: readonly ManagedAgentToolEvidence[]; + readonly permissionEvidence: readonly ManagedAgentPermissionEvidence[]; + readonly workspaceChanges: readonly ManagedAgentWorkspaceChange[]; + readonly preservation: readonly ManagedAgentPreservationObservation[]; + readonly cancellationRequested: boolean; + readonly queryClosed: boolean; + readonly teardown: ManagedAgentTeardownObservation; + readonly correlation: { + readonly executionId: string; + readonly evalSource: string; + }; + readonly sdkUsage?: ManagedAgentSdkUsageEstimate; +} + +export interface ManagedAgentQuery extends AsyncIterable { + close(): void; +} + +export type ManagedAgentQueryFactory = (input: { + readonly prompt: string; + readonly options: Options; +}) => ManagedAgentQuery; + +export interface ManagedAgentProcessObserver { + spawn(options: SpawnOptions): SpawnedProcess; + trackPids(pids: readonly number[]): void; + waitForQuiescence( + timeoutMs: number, + ): Promise; + emergencyCleanup(pids: readonly number[]): Promise; + dispose(): void; +} + +export interface ManagedAgentProbeDependencies { + readonly queryFactory?: ManagedAgentQueryFactory; + /** + * Explicit test-only origin seam. It is accepted only alongside an injected + * query factory and only for reserved .test or loopback origins. + */ + readonly hermeticGatewayOrigin?: string; + readonly processObserver?: ManagedAgentProcessObserver; + readonly uuid?: () => string; + readonly now?: () => number; + readonly waitForCancellationSignal?: (signal: AbortSignal) => Promise; +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 26731d854..cc105ee2f 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -368,6 +368,15 @@ importers: packages/harness: dependencies: + '@anthropic-ai/claude-agent-sdk': + specifier: 0.3.228 + version: 0.3.228(@anthropic-ai/sdk@0.116.0(zod@4.4.3))(@modelcontextprotocol/sdk@1.30.0(@cfworker/json-schema@4.1.1)(zod@4.4.3))(zod@4.4.3) + '@anthropic-ai/sdk': + specifier: 0.116.0 + version: 0.116.0(zod@4.4.3) + '@modelcontextprotocol/sdk': + specifier: 1.30.0 + version: 1.30.0(@cfworker/json-schema@4.1.1)(zod@4.4.3) '@sapiom/agent': specifier: workspace:^ version: link:../agent @@ -399,8 +408,8 @@ importers: specifier: ^8.18.0 version: 8.21.0 zod: - specifier: ^3.25.0 - version: 3.25.76 + specifier: 4.4.3 + version: 4.4.3 devDependencies: '@playwright/test': specifier: ^1.61.0 @@ -788,6 +797,67 @@ packages: 7zip-bin@5.2.0: resolution: {integrity: sha512-ukTPVhqG4jNzMro2qA9HSCSSVJN3aN7tlb+hfqYCt3ER0yWroeA2VR38MNrOHLQ/cVj+DaIMad0kFCtWWowh/A==} + '@anthropic-ai/claude-agent-sdk-darwin-arm64@0.3.228': + resolution: {integrity: sha512-HuCsV3/5XuYYaWuCbksX+e0JkDDUG/AlFJ8wKhDL3PBW/3hHNd6xBYx88kEWk1Z6B1GLxwHht9624lcmscpsyw==} + cpu: [arm64] + os: [darwin] + + '@anthropic-ai/claude-agent-sdk-darwin-x64@0.3.228': + resolution: {integrity: sha512-jSUYY5Nd3efvbLZPU+i0tRBaFXskHu8M+4LMGBEw6A0PaklZ3YfGvKlTOWtJGRw6vMc6LzfOFts024xPNm6OrQ==} + cpu: [x64] + os: [darwin] + + '@anthropic-ai/claude-agent-sdk-linux-arm64-musl@0.3.228': + resolution: {integrity: sha512-4PgfisC3kHKlzJvy3rrm4Oh26g+D78h4ahHjni9fvSKHuJgrvHu9Qgo6aaYmzWdc7v9drL+pgiCk5Ge4Y2ANPA==} + cpu: [arm64] + os: [linux] + libc: [musl] + + '@anthropic-ai/claude-agent-sdk-linux-arm64@0.3.228': + resolution: {integrity: sha512-0Wjv6TiWwGlBZINAmNJX07jN359jKwB/4Sr/uWgQkdjuVIOhe/M8ydk7JL2EPqCsbiW1lc15NjE5MWpZiYqooA==} + cpu: [arm64] + os: [linux] + libc: [glibc] + + '@anthropic-ai/claude-agent-sdk-linux-x64-musl@0.3.228': + resolution: {integrity: sha512-dnXxyiwGCZj27HVk6clYRqGMgrs3KVLVp0vvWYLjkPGBiKbI83qJiDpOfaekEXG2I4elX0M4XikggV1LGWjimg==} + cpu: [x64] + os: [linux] + libc: [musl] + + '@anthropic-ai/claude-agent-sdk-linux-x64@0.3.228': + resolution: {integrity: sha512-LmGplObceqMOu5mlrlhTZL/VSrEWdZagF0Bl8awglMu6WeQcNe7StORYkCznZ0BuzV4CwuC3ipV4q8Jrs66wSg==} + cpu: [x64] + os: [linux] + libc: [glibc] + + '@anthropic-ai/claude-agent-sdk-win32-arm64@0.3.228': + resolution: {integrity: sha512-mNS5yIMz/OXSQiDErb84jA8AKBFSlS9RSZ0qn2qyGkxplUx7kVmIDg/KnwOwHmygpzmH4UmR6OCaLXGohupqNA==} + cpu: [arm64] + os: [win32] + + '@anthropic-ai/claude-agent-sdk-win32-x64@0.3.228': + resolution: {integrity: sha512-DYT3HvdS64Pq0IRvgW3RDO31yjYp5yiUKoKaZolTpLKfALpG5LI/osfnKlya68PZ/bSST1FNAfW9I0EtCnaQ4w==} + cpu: [x64] + os: [win32] + + '@anthropic-ai/claude-agent-sdk@0.3.228': + resolution: {integrity: sha512-OOaME54VCoBLjKMqWqFmHkZGyL/x/FHUA0snhyolmyEhVoeBM0Ub5mrnV2Gx3d5/RcVlk2BnEVvPqu0SpZ9VFw==} + engines: {node: '>=18.0.0'} + peerDependencies: + '@anthropic-ai/sdk': '>=0.93.0' + '@modelcontextprotocol/sdk': ^1.29.0 + zod: ^4.0.0 + + '@anthropic-ai/sdk@0.116.0': + resolution: {integrity: sha512-4UEapYQ+epLEMsAuLZDvW8ExVSOtHD8a7zTyLzhw0H9RXJ1eilPgmqhjwgcdg22diwx13spw6fJ4rONZ+bS7Ww==} + hasBin: true + peerDependencies: + zod: ^3.25.0 || ^4.0.0 + peerDependenciesMeta: + zod: + optional: true + '@anthropic-ai/sdk@0.65.0': resolution: {integrity: sha512-zIdPOcrCVEI8t3Di40nH4z9EoeyGZfXbYSvWdDLsB/KkaSYMnEgC7gmcgWu83g2NTn1ZTpbMvpdttWDGGIk6zw==} hasBin: true @@ -1847,6 +1917,16 @@ packages: '@cfworker/json-schema': optional: true + '@modelcontextprotocol/sdk@1.30.0': + resolution: {integrity: sha512-xKd8OIzlqNzcqcNumGAa6g+PW2kjD5vrpcKOnfldAUPP3j7lnqMPwlTXQm8gF+UwH72z0lqaRbjr9hqGz0eITA==} + engines: {node: '>=18'} + peerDependencies: + '@cfworker/json-schema': ^4.1.1 + zod: ^3.25 || ^4.0 + peerDependenciesMeta: + '@cfworker/json-schema': + optional: true + '@mswjs/interceptors@0.41.9': resolution: {integrity: sha512-VVPPgHyQ6ShqnrmDWuxjmUIsO9gWyOZFmuOfLd9LfBGQJwZfy0gvv9pbHSJuoFNIYC7ZDX9aoFwowjcdSC4E8w==} engines: {node: '>=18'} @@ -2060,6 +2140,9 @@ packages: '@sinonjs/fake-timers@10.3.0': resolution: {integrity: sha512-V4BG07kuYSUkTCSBHG8G8TNhM+F19jXFWnQtzj+we8DrkpSBCee9Z3Ms8yiGer/dlmhe35/Xdgyo3/0rQKg7YA==} + '@stablelib/base64@1.0.1': + resolution: {integrity: sha512-1bnPQqSxSuc3Ii6MhBysoWCg58j97aUjuCSZrGSmDxNqtytIi0k8utUenAwTZN4V5mXXYGsVUI9zeBqy+jBOSQ==} + '@standard-schema/spec@1.1.0': resolution: {integrity: sha512-l2aFy5jALhniG5HgqrD6jXLi/rUWrKvqN/qJx6yoJsgKhblVd+iqqU4RCXavm/jPityDo5TCvKMnpjKnOriy0w==} @@ -3250,6 +3333,9 @@ packages: fast-levenshtein@2.0.6: resolution: {integrity: sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw==} + fast-sha256@1.3.0: + resolution: {integrity: sha512-n11RGP/lrWEFI/bWdygLxhI+pVeo1ZYIVwvvPkW7azl/rOy+F3HYRZ2K5zeE9mmkhQppyv9sQFx0JM9UabnpPQ==} + fast-string-truncated-width@3.0.3: resolution: {integrity: sha512-0jjjIEL6+0jag3l2XWWizO64/aZVtpiGE3t0Zgqxv0DPuxiMjvB3M24fCyhZUO4KomJQPj3LTSUnDP3GpdwC0g==} @@ -5070,6 +5156,9 @@ packages: stackback@0.0.2: resolution: {integrity: sha512-1XMJE5fQo1jGH6Y/7ebnwPOBEkIEnT4QF32d5R1+VXdXveM0IBMJt8zfaxX1P3QhVwrYe+576+jkANtSS2mBbw==} + standardwebhooks@1.0.0: + resolution: {integrity: sha512-BbHGOQK9olHPMvQNHWul6MYlrRTAOKn03rOe4A8O3CLWhNf4YHBqq2HJKKC+sfqpxiBY52pNeesD6jIiLDz8jg==} + stat-mode@1.0.0: resolution: {integrity: sha512-jH9EhtKIjuXZ2cWxmXS8ZP80XyC3iasQxMDV8jzhNJpfDb7VbQLVW4Wvsxz9QZvzV+G4YoSfBUVKDOyxLzi/sg==} engines: {node: '>= 6'} @@ -5583,6 +5672,52 @@ snapshots: 7zip-bin@5.2.0: {} + '@anthropic-ai/claude-agent-sdk-darwin-arm64@0.3.228': + optional: true + + '@anthropic-ai/claude-agent-sdk-darwin-x64@0.3.228': + optional: true + + '@anthropic-ai/claude-agent-sdk-linux-arm64-musl@0.3.228': + optional: true + + '@anthropic-ai/claude-agent-sdk-linux-arm64@0.3.228': + optional: true + + '@anthropic-ai/claude-agent-sdk-linux-x64-musl@0.3.228': + optional: true + + '@anthropic-ai/claude-agent-sdk-linux-x64@0.3.228': + optional: true + + '@anthropic-ai/claude-agent-sdk-win32-arm64@0.3.228': + optional: true + + '@anthropic-ai/claude-agent-sdk-win32-x64@0.3.228': + optional: true + + '@anthropic-ai/claude-agent-sdk@0.3.228(@anthropic-ai/sdk@0.116.0(zod@4.4.3))(@modelcontextprotocol/sdk@1.30.0(@cfworker/json-schema@4.1.1)(zod@4.4.3))(zod@4.4.3)': + dependencies: + '@anthropic-ai/sdk': 0.116.0(zod@4.4.3) + '@modelcontextprotocol/sdk': 1.30.0(@cfworker/json-schema@4.1.1)(zod@4.4.3) + zod: 4.4.3 + optionalDependencies: + '@anthropic-ai/claude-agent-sdk-darwin-arm64': 0.3.228 + '@anthropic-ai/claude-agent-sdk-darwin-x64': 0.3.228 + '@anthropic-ai/claude-agent-sdk-linux-arm64': 0.3.228 + '@anthropic-ai/claude-agent-sdk-linux-arm64-musl': 0.3.228 + '@anthropic-ai/claude-agent-sdk-linux-x64': 0.3.228 + '@anthropic-ai/claude-agent-sdk-linux-x64-musl': 0.3.228 + '@anthropic-ai/claude-agent-sdk-win32-arm64': 0.3.228 + '@anthropic-ai/claude-agent-sdk-win32-x64': 0.3.228 + + '@anthropic-ai/sdk@0.116.0(zod@4.4.3)': + dependencies: + json-schema-to-ts: 3.1.1 + standardwebhooks: 1.0.0 + optionalDependencies: + zod: 4.4.3 + '@anthropic-ai/sdk@0.65.0(zod@4.4.3)': dependencies: json-schema-to-ts: 3.1.1 @@ -6829,6 +6964,30 @@ snapshots: transitivePeerDependencies: - supports-color + '@modelcontextprotocol/sdk@1.30.0(@cfworker/json-schema@4.1.1)(zod@4.4.3)': + dependencies: + '@hono/node-server': 1.19.14(hono@4.12.25) + ajv: 8.20.0 + ajv-formats: 3.0.1(ajv@8.20.0) + content-type: 1.0.5 + cors: 2.8.6 + cross-spawn: 7.0.6 + eventsource: 3.0.7 + eventsource-parser: 3.1.0 + express: 5.2.1 + express-rate-limit: 8.5.2(express@5.2.1) + hono: 4.12.25 + jose: 6.2.3 + json-schema-typed: 8.0.2 + pkce-challenge: 5.0.1 + raw-body: 3.0.2 + zod: 4.4.3 + zod-to-json-schema: 3.25.2(zod@4.4.3) + optionalDependencies: + '@cfworker/json-schema': 4.1.1 + transitivePeerDependencies: + - supports-color + '@mswjs/interceptors@0.41.9': dependencies: '@open-draft/deferred-promise': 2.2.0 @@ -6980,6 +7139,8 @@ snapshots: dependencies: '@sinonjs/commons': 3.0.1 + '@stablelib/base64@1.0.1': {} + '@standard-schema/spec@1.1.0': {} '@stryker-mutator/api@9.6.1': @@ -8565,6 +8726,8 @@ snapshots: fast-levenshtein@2.0.6: {} + fast-sha256@1.3.0: {} + fast-string-truncated-width@3.0.3: {} fast-string-width@3.0.2: @@ -10458,6 +10621,11 @@ snapshots: stackback@0.0.2: {} + standardwebhooks@1.0.0: + dependencies: + '@stablelib/base64': 1.0.1 + fast-sha256: 1.3.0 + stat-mode@1.0.0: {} statuses@2.0.2: {} @@ -10903,7 +11071,6 @@ snapshots: zod-to-json-schema@3.25.2(zod@4.4.3): dependencies: zod: 4.4.3 - optional: true zod@3.25.76: {} From a69751f9638da766f9acb07a4c64e9571daed0ca Mon Sep 17 00:00:00 2001 From: Yash Date: Sun, 16 Aug 2026 15:20:50 -0700 Subject: [PATCH 02/24] test(harness): require successful L1 tool evidence Ensure an L1 result cannot pass when a built-in tool was requested but failed. Refs: SAP-2632 --- .../managed-agent-spike/probe-cli.test.ts | 79 +++++++++++++++++++ .../managed-agent-spike/probe-cli.ts | 2 +- 2 files changed, 80 insertions(+), 1 deletion(-) diff --git a/packages/harness/src/experimental/managed-agent-spike/probe-cli.test.ts b/packages/harness/src/experimental/managed-agent-spike/probe-cli.test.ts index adb82c16c..0a7bcc3ad 100644 --- a/packages/harness/src/experimental/managed-agent-spike/probe-cli.test.ts +++ b/packages/harness/src/experimental/managed-agent-spike/probe-cli.test.ts @@ -3,10 +3,14 @@ import { describe, expect, it } from "vitest"; import { ManagedAgentProbeCliError, assertManagedAgentCertificationNodeVersion, + evaluateManagedAgentProbe, executeManagedAgentProbeCli, managedAgentProbeUsage, parseManagedAgentProbeCliArgs, } from "./probe-cli.js"; +import { FIXTURE_PATHS } from "./fixture.js"; +import { qualifiedManagedAgentMcpToolName } from "./runtime.js"; +import type { ManagedAgentProbeResult } from "./types.js"; describe("managed-agent probe CLI", () => { it("is opt-in and never accepts credentials through arguments", () => { @@ -133,4 +137,79 @@ describe("managed-agent probe CLI", () => { ManagedAgentProbeCliError, ); }); + + it("requires successful results from every built-in tool for L1", () => { + const builtins = ["Read", "Edit", "Write", "Bash"]; + const result: ManagedAgentProbeResult = { + contractVersion: 1, + runId: "run-1", + scenario: "L1", + target: "sonnet-5", + modelAlias: "claude-sonnet-5-anthropic-anthropic-eval", + sdkSessionId: "session-1", + terminal: "success", + events: [], + toolEvidence: [ + ...builtins.flatMap((toolName) => [ + { toolName, status: "requested" as const }, + { + toolName, + status: + toolName === "Bash" ? ("error" as const) : ("success" as const), + }, + ]), + { + toolName: qualifiedManagedAgentMcpToolName("echo_nonce"), + status: "success", + }, + { + toolName: qualifiedManagedAgentMcpToolName("fail_once"), + status: "error", + }, + { + toolName: qualifiedManagedAgentMcpToolName("fail_once"), + status: "success", + }, + ], + permissionEvidence: [ + { + toolUseId: "deny-1", + toolName: "Read", + decision: "deny", + reason: "path_outside_workspace", + }, + { + toolUseId: "deny-2", + toolName: "Read", + decision: "deny", + reason: "path_outside_workspace", + }, + ], + workspaceChanges: [ + { path: FIXTURE_PATHS.cleanTarget, change: "modified" }, + { path: FIXTURE_PATHS.createdTarget, change: "created" }, + ], + preservation: [ + { path: FIXTURE_PATHS.dirtySentinel, preserved: true }, + { path: FIXTURE_PATHS.untrackedSentinel, preserved: true }, + ], + cancellationRequested: false, + queryClosed: true, + teardown: { + quiescent: true, + deadlineMet: true, + elapsedMs: 5, + observedPids: [], + alivePidsAtDeadline: [], + emergencyCleanupAttempted: false, + }, + correlation: { executionId: "execution-1", evalSource: "eval-1" }, + }; + + expect( + evaluateManagedAgentProbe(result).checks.find( + ({ id }) => id === "builtin_tools_succeeded", + ), + ).toEqual({ id: "builtin_tools_succeeded", passed: false }); + }); }); diff --git a/packages/harness/src/experimental/managed-agent-spike/probe-cli.ts b/packages/harness/src/experimental/managed-agent-spike/probe-cli.ts index a12ab378e..f098612af 100644 --- a/packages/harness/src/experimental/managed-agent-spike/probe-cli.ts +++ b/packages/harness/src/experimental/managed-agent-spike/probe-cli.ts @@ -184,7 +184,7 @@ export function evaluateManagedAgentProbe( ), }, { - id: "builtin_tools_observed", + id: "builtin_tools_succeeded", passed: ["Read", "Edit", "Write", "Bash"].every( (name) => requestedTools.has(name) && invocation(name, "success"), ), From 3dff2f2c09385303a69bc86b7570e360875cfac6 Mon Sep 17 00:00:00 2001 From: Yash Date: Sun, 16 Aug 2026 15:41:18 -0700 Subject: [PATCH 03/24] fix(harness): harden managed agent probe boundaries Use fresh canonical config roots, normalize attacker-controlled evidence, execute canonicalized file targets, and require complete permission evidence for L1 qualification.\n\nRefs: SAP-2632 --- .../managed-agent-spike/environment.test.ts | 70 +++++++- .../managed-agent-spike/environment.ts | 115 ++++++++++-- .../managed-agent-spike/events.test.ts | 98 ++++++++-- .../managed-agent-spike/events.ts | 63 +++++-- .../managed-agent-spike/permissions.test.ts | 44 ++++- .../managed-agent-spike/permissions.ts | 33 +++- .../managed-agent-spike/probe-cli.test.ts | 169 ++++++++++++------ .../managed-agent-spike/probe-cli.ts | 41 ++++- .../managed-agent-spike/runtime.test.ts | 169 +++++++++++++++++- .../experimental/managed-agent-spike/types.ts | 1 + 10 files changed, 682 insertions(+), 121 deletions(-) diff --git a/packages/harness/src/experimental/managed-agent-spike/environment.test.ts b/packages/harness/src/experimental/managed-agent-spike/environment.test.ts index 6f1373567..55c54ed3c 100644 --- a/packages/harness/src/experimental/managed-agent-spike/environment.test.ts +++ b/packages/harness/src/experimental/managed-agent-spike/environment.test.ts @@ -1,6 +1,14 @@ -import { mkdtemp, rm, stat } from "node:fs/promises"; +import { + lstat, + mkdir, + mkdtemp, + realpath, + rm, + stat, + symlink, +} from "node:fs/promises"; import { tmpdir } from "node:os"; -import { join } from "node:path"; +import { dirname, isAbsolute, join, relative, sep } from "node:path"; import { afterEach, describe, expect, it } from "vitest"; @@ -70,6 +78,64 @@ describe("managed-agent child environment", () => { } }); + it("uses a fresh canonical private root without following pre-existing child symlinks", async () => { + const root = await mkdtemp(join(tmpdir(), "managed-agent-env-")); + roots.push(root); + const configRoot = join(root, "config"); + const external = join(root, "external-claude-config"); + await Promise.all([mkdir(configRoot), mkdir(external)]); + await symlink(external, join(configRoot, "claude-config")); + + const first = buildManagedAgentChildEnvironment({ + ambient: {}, + configRoot, + gatewayOrigin: "https://gateway.example.test", + gatewayCredential: "dedicated-eval-key", + modelAlias: "claude-sonnet-5-anthropic-anthropic-eval", + evalSource: "eval-source", + executionId: "execution-id", + }); + const second = buildManagedAgentChildEnvironment({ + ambient: {}, + configRoot, + gatewayOrigin: "https://gateway.example.test", + gatewayCredential: "dedicated-eval-key", + modelAlias: "claude-sonnet-5-anthropic-anthropic-eval", + evalSource: "eval-source", + executionId: "execution-id-2", + }); + + const privateRoot = dirname(first.CLAUDE_CONFIG_DIR); + expect(privateRoot).not.toBe(dirname(second.CLAUDE_CONFIG_DIR)); + expect(await realpath(first.CLAUDE_CONFIG_DIR)).not.toBe( + await realpath(external), + ); + expect( + (await lstat(join(configRoot, "claude-config"))).isSymbolicLink(), + ).toBe(true); + for (const directory of [ + first.HOME, + first.USERPROFILE, + first.APPDATA, + first.LOCALAPPDATA, + first.XDG_CONFIG_HOME, + first.XDG_CACHE_HOME, + first.XDG_DATA_HOME, + first.CLAUDE_CONFIG_DIR, + first.CLAUDE_SECURESTORAGE_CONFIG_DIR, + first.TMPDIR, + first.TMP, + first.TEMP, + ]) { + const canonical = await realpath(directory); + const pathRelative = relative(privateRoot, canonical); + expect(isAbsolute(pathRelative)).toBe(false); + expect(pathRelative).not.toBe(".."); + expect(pathRelative.startsWith(`..${sep}`)).toBe(false); + expect((await lstat(directory)).isSymbolicLink()).toBe(false); + } + }); + it("rejects newline injection in correlation headers", async () => { const configRoot = await mkdtemp(join(tmpdir(), "managed-agent-env-")); roots.push(configRoot); diff --git a/packages/harness/src/experimental/managed-agent-spike/environment.ts b/packages/harness/src/experimental/managed-agent-spike/environment.ts index 7488491fc..245dc1497 100644 --- a/packages/harness/src/experimental/managed-agent-spike/environment.ts +++ b/packages/harness/src/experimental/managed-agent-spike/environment.ts @@ -1,9 +1,17 @@ -import { mkdirSync } from "node:fs"; -import { join } from "node:path"; +import { + chmodSync, + lstatSync, + mkdirSync, + mkdtempSync, + realpathSync, + statSync, +} from "node:fs"; +import { isAbsolute, join, relative, resolve, sep } from "node:path"; import { MANAGED_AGENT_CONTRACT, MANAGED_AGENT_MODEL_ENVIRONMENT_VARIABLES, + ManagedAgentConfigurationError, } from "./contract.js"; export type ManagedAgentAmbientEnvironment = Readonly< @@ -11,6 +19,7 @@ export type ManagedAgentAmbientEnvironment = Readonly< >; export interface ManagedAgentIsolatedDirectories { + readonly privateRoot: string; readonly home: string; readonly appData: string; readonly localAppData: string; @@ -46,30 +55,108 @@ const SAFE_AMBIENT_PASSTHROUGH = [ "CLAUDE_CODE_GIT_BASH_PATH", ] as const; +const PRIVATE_RUN_DIRECTORY_PREFIX = "managed-agent-run-"; + function validateHeaderValue(value: string, label: string): void { if (!value || /[\r\n]/.test(value)) { throw new Error(`${label} is not a safe header value`); } } +function comparisonPath(value: string): string { + return process.platform === "win32" ? value.toLowerCase() : value; +} + +function pathWithin(root: string, candidate: string): boolean { + const pathRelative = relative( + comparisonPath(root), + comparisonPath(candidate), + ); + if (pathRelative === "") return true; + return ( + !isAbsolute(pathRelative) && + pathRelative !== ".." && + !pathRelative.startsWith(`..${sep}`) + ); +} + +function canonicalDirectory(value: string, label: string): string { + let canonical: string; + try { + canonical = realpathSync(resolve(value)); + } catch { + throw new ManagedAgentConfigurationError(`${label} must exist`); + } + if (!statSync(canonical).isDirectory()) { + throw new ManagedAgentConfigurationError(`${label} must be a directory`); + } + return canonical; +} + +function verifyPrivateDirectory( + candidate: string, + privateRoot: string, + label: string, +): string { + const metadata = lstatSync(candidate); + if (metadata.isSymbolicLink() || !metadata.isDirectory()) { + throw new ManagedAgentConfigurationError( + `${label} must be a private directory`, + ); + } + chmodSync(candidate, 0o700); + const canonical = realpathSync(candidate); + if (!pathWithin(privateRoot, canonical)) { + throw new ManagedAgentConfigurationError( + `${label} must remain inside the private run root`, + ); + } + return canonical; +} + +function createPrivateDirectory( + parent: string, + name: string, + privateRoot: string, +): string { + const candidate = join(parent, name); + mkdirSync(candidate, { mode: 0o700 }); + return verifyPrivateDirectory(candidate, privateRoot, name); +} + export function prepareManagedAgentDirectories( configRoot: string, ): ManagedAgentIsolatedDirectories { - const home = join(configRoot, "home"); + const canonicalConfigRoot = canonicalDirectory(configRoot, "configRoot"); + const createdPrivateRoot = mkdtempSync( + join(canonicalConfigRoot, PRIVATE_RUN_DIRECTORY_PREFIX), + ); + const privateRoot = verifyPrivateDirectory( + createdPrivateRoot, + canonicalConfigRoot, + "private run root", + ); + const home = createPrivateDirectory(privateRoot, "home", privateRoot); const directories = { + privateRoot, home, - appData: join(home, "appdata"), - localAppData: join(home, "local-appdata"), - xdgConfig: join(home, "xdg-config"), - xdgCache: join(home, "xdg-cache"), - xdgData: join(home, "xdg-data"), - claudeConfig: join(configRoot, "claude-config"), - secureStorage: join(configRoot, "secure-storage"), - temporary: join(configRoot, "tmp"), + appData: createPrivateDirectory(home, "appdata", privateRoot), + localAppData: createPrivateDirectory(home, "local-appdata", privateRoot), + xdgConfig: createPrivateDirectory(home, "xdg-config", privateRoot), + xdgCache: createPrivateDirectory(home, "xdg-cache", privateRoot), + xdgData: createPrivateDirectory(home, "xdg-data", privateRoot), + claudeConfig: createPrivateDirectory( + privateRoot, + "claude-config", + privateRoot, + ), + secureStorage: createPrivateDirectory( + privateRoot, + "secure-storage", + privateRoot, + ), + temporary: createPrivateDirectory(privateRoot, "tmp", privateRoot), } satisfies ManagedAgentIsolatedDirectories; - for (const directory of Object.values(directories)) { - mkdirSync(directory, { recursive: true, mode: 0o700 }); - } return directories; } diff --git a/packages/harness/src/experimental/managed-agent-spike/events.test.ts b/packages/harness/src/experimental/managed-agent-spike/events.test.ts index 97822788f..24feef495 100644 --- a/packages/harness/src/experimental/managed-agent-spike/events.test.ts +++ b/packages/harness/src/experimental/managed-agent-spike/events.test.ts @@ -1,6 +1,11 @@ import { describe, expect, it } from "vitest"; -import { ManagedAgentEventRecorder } from "./events.js"; +import { + ManagedAgentEventRecorder, + normalizeManagedAgentToolUseId, +} from "./events.js"; + +const SESSION_ID = "11111111-1111-4111-8111-111111111111"; describe("ManagedAgentEventRecorder", () => { it("retains structural evidence while redacting message and tool content", () => { @@ -8,12 +13,12 @@ describe("ManagedAgentEventRecorder", () => { recorder.observeSdkEvent({ type: "system", subtype: "init", - session_id: "session-1", + session_id: SESSION_ID, model: "model-secret-must-not-be-copied", }); recorder.observeSdkEvent({ type: "assistant", - session_id: "session-1", + session_id: SESSION_ID, message: { id: "message-1", content: [ @@ -29,7 +34,7 @@ describe("ManagedAgentEventRecorder", () => { }); recorder.observeSdkEvent({ type: "user", - session_id: "session-1", + session_id: SESSION_ID, message: { content: [ { @@ -45,7 +50,7 @@ describe("ManagedAgentEventRecorder", () => { type: "result", subtype: "success", is_error: false, - session_id: "session-1", + session_id: SESSION_ID, result: "private-final-answer", usage: { input_tokens: 7, @@ -58,7 +63,7 @@ describe("ManagedAgentEventRecorder", () => { expect(recorder.recordTerminal("success")).toBe(true); expect(recorder.recordTerminal("query_error")).toBe(false); - expect(recorder.sessionId).toBe("session-1"); + expect(recorder.sessionId).toBe(SESSION_ID); expect(recorder.usage).toEqual({ authority: "sdk_non_authoritative", inputTokens: 7, @@ -68,8 +73,16 @@ describe("ManagedAgentEventRecorder", () => { estimatedCostUsd: 0.001, }); expect(recorder.toolEvidence).toEqual([ - { toolUseId: "tool-1", toolName: "Read", status: "requested" }, - { toolUseId: "tool-1", toolName: "Read", status: "success" }, + { + toolUseId: normalizeManagedAgentToolUseId("tool-1"), + toolName: "Read", + status: "requested", + }, + { + toolUseId: normalizeManagedAgentToolUseId("tool-1"), + toolName: "Read", + status: "success", + }, ]); expect( recorder.events.filter(({ type }) => type === "terminal"), @@ -82,27 +95,88 @@ describe("ManagedAgentEventRecorder", () => { "tool-secret", "private-file-contents", "private-final-answer", + "tool-1", ]) { expect(serialized).not.toContain(secret); } }); - it("normalizes an attacker-controlled tool name instead of persisting it", () => { + it("redacts attacker-controlled session, tool, and permission identifiers from all evidence", () => { const recorder = new ManagedAgentEventRecorder("run-2"); + const sessionSecret = "session-secret-credential"; + const toolIdSecret = "tool-id-secret-credential"; + const permissionIdSecret = "permission-id-secret-credential"; + const toolNameSecret = "ReadSecretCredential"; + const permissionNameSecret = "WriteSecretCredential"; + recorder.observeSdkEvent({ + type: "system", + subtype: "init", + session_id: sessionSecret, + }); recorder.observeSdkEvent({ type: "assistant", + session_id: sessionSecret, message: { content: [ { type: "tool_use", - id: "tool-2", - name: "Read secret=credential.value", + id: toolIdSecret, + name: toolNameSecret, input: {}, }, ], }, }); + recorder.observeSdkEvent({ + type: "user", + session_id: sessionSecret, + message: { + content: [ + { + type: "tool_result", + tool_use_id: toolIdSecret, + content: "private-result", + }, + ], + }, + }); + recorder.recordPermission({ + toolUseId: permissionIdSecret, + toolName: permissionNameSecret, + decision: "deny", + reason: "tool_not_allowed", + }); + recorder.recordTerminal("success"); + + expect(recorder.sessionId).toBeUndefined(); expect(recorder.toolEvidence[0]?.toolName).toBe("unknown"); - expect(JSON.stringify(recorder.events)).not.toContain("credential.value"); + expect(recorder.toolEvidence.map(({ toolUseId }) => toolUseId)).toEqual([ + normalizeManagedAgentToolUseId(toolIdSecret), + normalizeManagedAgentToolUseId(toolIdSecret), + ]); + expect(recorder.permissionEvidence).toEqual([ + { + toolUseId: normalizeManagedAgentToolUseId(permissionIdSecret), + toolName: "unknown", + decision: "deny", + reason: "tool_not_allowed", + }, + ]); + const serialized = JSON.stringify({ + sdkSessionId: recorder.sessionId, + events: recorder.events, + toolEvidence: recorder.toolEvidence, + permissionEvidence: recorder.permissionEvidence, + }); + for (const secret of [ + sessionSecret, + toolIdSecret, + permissionIdSecret, + toolNameSecret, + permissionNameSecret, + "private-result", + ]) { + expect(serialized).not.toContain(secret); + } }); }); diff --git a/packages/harness/src/experimental/managed-agent-spike/events.ts b/packages/harness/src/experimental/managed-agent-spike/events.ts index 4598f6522..880ab4e58 100644 --- a/packages/harness/src/experimental/managed-agent-spike/events.ts +++ b/packages/harness/src/experimental/managed-agent-spike/events.ts @@ -1,3 +1,5 @@ +import { createHash } from "node:crypto"; + import type { ManagedAgentPermissionEvidence, ManagedAgentProbeEvent, @@ -29,11 +31,39 @@ function safeSubtype(value: unknown): string | undefined { return subtype && /^[a-z0-9_-]{1,80}$/i.test(subtype) ? subtype : undefined; } -function safeToolName(value: unknown): string { +const SAFE_TOOL_NAMES = new Set([ + "Read", + "Edit", + "Write", + "Bash", + "mcp__sapiom-managed-agent-spike__echo_nonce", + "mcp__sapiom-managed-agent-spike__fail_once", +]); +const NORMALIZED_TOOL_USE_ID_PATTERN = /^tool_[0-9a-f]{64}$/; +const SDK_SESSION_ID_PATTERN = + /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/; + +export function sanitizeManagedAgentToolName(value: unknown): string { const toolName = optionalString(value); - return toolName && /^[a-z0-9_-]{1,128}$/i.test(toolName) - ? toolName - : "unknown"; + return toolName && SAFE_TOOL_NAMES.has(toolName) ? toolName : "unknown"; +} + +export function normalizeManagedAgentToolUseId(value: unknown): string { + if (typeof value === "string" && NORMALIZED_TOOL_USE_ID_PATTERN.test(value)) { + return value; + } + const raw = typeof value === "string" ? value : "invalid-tool-use-id"; + return `tool_${createHash("sha256") + .update("sapiom-managed-agent-tool-use-id\0") + .update(raw) + .digest("hex")}`; +} + +function safeSdkSessionId(value: unknown): string | undefined { + const sessionId = optionalString(value); + return sessionId && SDK_SESSION_ID_PATTERN.test(sessionId) + ? sessionId + : undefined; } function contentBlocks(message: JsonRecord | undefined): readonly JsonRecord[] { @@ -119,13 +149,18 @@ export class ManagedAgentEventRecorder { } public recordPermission(evidence: ManagedAgentPermissionEvidence): void { - this.#permissionEvidence.push(evidence); + const normalizedEvidence = { + ...evidence, + toolUseId: normalizeManagedAgentToolUseId(evidence.toolUseId), + toolName: sanitizeManagedAgentToolName(evidence.toolName), + } satisfies ManagedAgentPermissionEvidence; + this.#permissionEvidence.push(normalizedEvidence); this.#append({ type: "permission", - toolUseId: evidence.toolUseId, - toolName: safeToolName(evidence.toolName), - permissionDecision: evidence.decision, - permissionReason: evidence.reason, + toolUseId: normalizedEvidence.toolUseId, + toolName: normalizedEvidence.toolName, + permissionDecision: normalizedEvidence.decision, + permissionReason: normalizedEvidence.reason, }); } @@ -134,8 +169,8 @@ export class ManagedAgentEventRecorder { const type = optionalString(event?.type); if (!event || !type) return; const subtype = safeSubtype(event.subtype); - const sessionId = optionalString(event.session_id); - if (sessionId) this.#sessionId = sessionId; + const sessionId = safeSdkSessionId(event.session_id); + if (sessionId && !this.#sessionId) this.#sessionId = sessionId; if (type === "system" && subtype === "init") { this.#append({ type: "lifecycle", subtype: "sdk_init", sessionId }); @@ -150,8 +185,8 @@ export class ManagedAgentEventRecorder { if (type === "assistant") { for (const block of blocks) { if (block.type !== "tool_use") continue; - const toolUseId = optionalString(block.id); - const toolName = safeToolName(block.name); + const toolUseId = normalizeManagedAgentToolUseId(block.id); + const toolName = sanitizeManagedAgentToolName(block.name); this.#toolEvidence.push({ toolUseId, toolName, status: "requested" }); this.#append({ type: "tool_requested", @@ -164,7 +199,7 @@ export class ManagedAgentEventRecorder { if (type === "user") { for (const block of blocks) { if (block.type !== "tool_result") continue; - const toolUseId = optionalString(block.tool_use_id); + const toolUseId = normalizeManagedAgentToolUseId(block.tool_use_id); const isError = block.is_error === true; const matchingTool = [...this.#toolEvidence] .reverse() diff --git a/packages/harness/src/experimental/managed-agent-spike/permissions.test.ts b/packages/harness/src/experimental/managed-agent-spike/permissions.test.ts index b6c6eca02..f103a33e1 100644 --- a/packages/harness/src/experimental/managed-agent-spike/permissions.test.ts +++ b/packages/harness/src/experimental/managed-agent-spike/permissions.test.ts @@ -12,7 +12,6 @@ import { join } from "node:path"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { - ManagedAgentPathError, createManagedAgentPermissionHandler, resolveManagedAgentToolPath, } from "./permissions.js"; @@ -50,19 +49,25 @@ describe("symlink-aware managed-agent containment", () => { ); }); - it("denies direct, traversal, sibling-prefix, and symlink escapes", async () => { + it("distinguishes lexical outside-root paths from symlink escapes", async () => { const outsidePath = join(outside, "secret.txt"); for (const requested of [ outsidePath, "../outside/secret.txt", `${workspace}-evil/file.txt`, + ]) { + await expect( + resolveManagedAgentToolPath(workspace, requested), + ).rejects.toMatchObject({ reason: "path_outside_workspace" }); + } + for (const requested of [ "escape.txt", "escape-dir/secret.txt", "escape-dir/new.txt", ]) { await expect( resolveManagedAgentToolPath(workspace, requested), - ).rejects.toBeInstanceOf(ManagedAgentPathError); + ).rejects.toMatchObject({ reason: "path_symlink_escape" }); } }); }); @@ -88,9 +93,38 @@ describe("managed-agent permission handler", () => { await expect( handler("Bash", { command: "git status --short " }, permission), ).resolves.toMatchObject({ behavior: "deny", interrupt: false }); + await expect( + handler( + "Read", + { file_path: "inside.txt", preserve: "metadata" }, + permission, + ), + ).resolves.toMatchObject({ + behavior: "allow", + updatedInput: { + file_path: join(workspace, "inside.txt"), + preserve: "metadata", + }, + }); + await expect( + handler( + "Write", + { file_path: "nested/new.txt", content: "safe" }, + permission, + ), + ).resolves.toMatchObject({ + behavior: "allow", + updatedInput: { + file_path: join(workspace, "nested/new.txt"), + content: "safe", + }, + }); await expect( handler("Read", { file_path: join(outside, "secret.txt") }, permission), ).resolves.toMatchObject({ behavior: "deny", interrupt: false }); + await expect( + handler("Read", { file_path: "escape.txt" }, permission), + ).resolves.toMatchObject({ behavior: "deny", interrupt: false }); await expect( handler("mcp__probe__echo_nonce", { nonce: "secret" }, permission), ).resolves.toMatchObject({ behavior: "allow" }); @@ -101,12 +135,16 @@ describe("managed-agent permission handler", () => { expect(evidence.map(({ decision, reason }) => [decision, reason])).toEqual([ ["allow", "exact_bash_command"], ["deny", "bash_command_not_allowed"], + ["allow", "fixture_path"], + ["allow", "fixture_path"], ["deny", "path_outside_workspace"], + ["deny", "path_symlink_escape"], ["allow", "managed_mcp_tool"], ["deny", "tool_not_allowed"], ]); expect(JSON.stringify(evidence)).not.toContain(join(outside, "secret.txt")); expect(JSON.stringify(evidence)).not.toContain("secret"); + expect(JSON.stringify(evidence)).not.toContain("tool-1"); expect(vi.isMockFunction(handler)).toBe(false); }); }); diff --git a/packages/harness/src/experimental/managed-agent-spike/permissions.ts b/packages/harness/src/experimental/managed-agent-spike/permissions.ts index 74e6bf5ca..48185f437 100644 --- a/packages/harness/src/experimental/managed-agent-spike/permissions.ts +++ b/packages/harness/src/experimental/managed-agent-spike/permissions.ts @@ -10,6 +10,10 @@ import type { ManagedAgentPermissionEvidence, ManagedAgentPermissionReason, } from "./types.js"; +import { + normalizeManagedAgentToolUseId, + sanitizeManagedAgentToolName, +} from "./events.js"; export const MANAGED_AGENT_BUILTIN_TOOLS = [ "Read", @@ -44,7 +48,10 @@ export const MANAGED_AGENT_DISALLOWED_TOOLS = [ export class ManagedAgentPathError extends Error { public constructor( - public readonly reason: "invalid_input" | "path_outside_workspace", + public readonly reason: + | "invalid_input" + | "path_outside_workspace" + | "path_symlink_escape", ) { super(reason); this.name = "ManagedAgentPathError"; @@ -110,7 +117,7 @@ export async function resolveManagedAgentToolPath( const existing = await nearestExistingParent(candidate); const canonicalExisting = await realpath(existing); if (!isPathWithinRoot(canonicalWorkspaceRoot, canonicalExisting)) { - throw new ManagedAgentPathError("path_outside_workspace"); + throw new ManagedAgentPathError("path_symlink_escape"); } if (existing === candidate) return canonicalExisting; @@ -133,9 +140,14 @@ function permissionResult( decision: "allow" | "deny", toolUseID: string, reason: ManagedAgentPermissionReason, + updatedInput?: Record, ): PermissionResult { return decision === "allow" - ? { behavior: "allow", toolUseID } + ? { + behavior: "allow", + toolUseID, + ...(updatedInput ? { updatedInput } : {}), + } : { behavior: "deny", message: `Managed-agent permission denied: ${reason}`, @@ -159,6 +171,7 @@ export function createManagedAgentPermissionHandler( return async (toolName, input, permission): Promise => { let decision: "allow" | "deny" = "deny"; let reason: ManagedAgentPermissionReason = "tool_not_allowed"; + let updatedInput: Record | undefined; if (allowedMcpTools.has(toolName)) { decision = "allow"; @@ -184,12 +197,13 @@ export function createManagedAgentPermissionHandler( reason = "invalid_input"; } else { try { - await resolveManagedAgentToolPath( + const canonicalPath = await resolveManagedAgentToolPath( options.canonicalWorkspaceRoot, requestedPath, ); decision = "allow"; reason = "fixture_path"; + updatedInput = { ...input, file_path: canonicalPath }; } catch (error) { reason = error instanceof ManagedAgentPathError @@ -200,11 +214,16 @@ export function createManagedAgentPermissionHandler( } options.onDecision({ - toolUseId: permission.toolUseID, - toolName, + toolUseId: normalizeManagedAgentToolUseId(permission.toolUseID), + toolName: sanitizeManagedAgentToolName(toolName), decision, reason, }); - return permissionResult(decision, permission.toolUseID, reason); + return permissionResult( + decision, + permission.toolUseID, + reason, + updatedInput, + ); }; } diff --git a/packages/harness/src/experimental/managed-agent-spike/probe-cli.test.ts b/packages/harness/src/experimental/managed-agent-spike/probe-cli.test.ts index 0a7bcc3ad..9767106a7 100644 --- a/packages/harness/src/experimental/managed-agent-spike/probe-cli.test.ts +++ b/packages/harness/src/experimental/managed-agent-spike/probe-cli.test.ts @@ -12,6 +12,88 @@ import { FIXTURE_PATHS } from "./fixture.js"; import { qualifiedManagedAgentMcpToolName } from "./runtime.js"; import type { ManagedAgentProbeResult } from "./types.js"; +function passingL1Result(): ManagedAgentProbeResult { + const builtins = ["Read", "Edit", "Write", "Bash"]; + const echoTool = qualifiedManagedAgentMcpToolName("echo_nonce"); + const failOnceTool = qualifiedManagedAgentMcpToolName("fail_once"); + return { + contractVersion: 1, + runId: "run-1", + scenario: "L1", + target: "sonnet-5", + modelAlias: "claude-sonnet-5-anthropic-anthropic-eval", + sdkSessionId: "11111111-1111-4111-8111-111111111111", + terminal: "success", + events: [], + toolEvidence: [ + ...builtins.flatMap((toolName) => [ + { toolName, status: "requested" as const }, + { toolName, status: "success" as const }, + ]), + { toolName: echoTool, status: "success" }, + { toolName: failOnceTool, status: "error" }, + { toolName: failOnceTool, status: "success" }, + ], + permissionEvidence: [ + ...["Read", "Edit", "Write"].map((toolName, index) => ({ + toolUseId: `tool_${String(index + 1).repeat(64)}`, + toolName, + decision: "allow" as const, + reason: "fixture_path" as const, + })), + { + toolUseId: `tool_${"b".repeat(64)}`, + toolName: "Bash", + decision: "allow", + reason: "exact_bash_command", + }, + { + toolUseId: `tool_${"c".repeat(64)}`, + toolName: echoTool, + decision: "allow", + reason: "managed_mcp_tool", + }, + { + toolUseId: `tool_${"d".repeat(64)}`, + toolName: failOnceTool, + decision: "allow", + reason: "managed_mcp_tool", + }, + { + toolUseId: `tool_${"e".repeat(64)}`, + toolName: "Read", + decision: "deny", + reason: "path_outside_workspace", + }, + { + toolUseId: `tool_${"f".repeat(64)}`, + toolName: "Read", + decision: "deny", + reason: "path_symlink_escape", + }, + ], + workspaceChanges: [ + { path: FIXTURE_PATHS.cleanTarget, change: "modified" }, + { path: FIXTURE_PATHS.createdTarget, change: "created" }, + ], + preservation: [ + { path: FIXTURE_PATHS.dirtySentinel, preserved: true }, + { path: FIXTURE_PATHS.untrackedSentinel, preserved: true }, + ], + cancellationRequested: false, + queryClosed: true, + teardown: { + quiescent: true, + deadlineMet: true, + elapsedMs: 5, + observedPids: [], + alivePidsAtDeadline: [], + emergencyCleanupAttempted: false, + }, + correlation: { executionId: "execution-1", evalSource: "eval-1" }, + }; +} + describe("managed-agent probe CLI", () => { it("is opt-in and never accepts credentials through arguments", () => { expect(() => @@ -139,77 +221,52 @@ describe("managed-agent probe CLI", () => { }); it("requires successful results from every built-in tool for L1", () => { - const builtins = ["Read", "Edit", "Write", "Bash"]; + const passing = passingL1Result(); const result: ManagedAgentProbeResult = { - contractVersion: 1, - runId: "run-1", - scenario: "L1", - target: "sonnet-5", - modelAlias: "claude-sonnet-5-anthropic-anthropic-eval", - sdkSessionId: "session-1", - terminal: "success", - events: [], - toolEvidence: [ - ...builtins.flatMap((toolName) => [ - { toolName, status: "requested" as const }, - { - toolName, - status: - toolName === "Bash" ? ("error" as const) : ("success" as const), - }, - ]), - { - toolName: qualifiedManagedAgentMcpToolName("echo_nonce"), - status: "success", - }, - { - toolName: qualifiedManagedAgentMcpToolName("fail_once"), - status: "error", - }, - { - toolName: qualifiedManagedAgentMcpToolName("fail_once"), - status: "success", - }, - ], + ...passing, + toolEvidence: passing.toolEvidence.map((evidence) => + evidence.toolName === "Bash" && evidence.status === "success" + ? { ...evidence, status: "error" } + : evidence, + ), + }; + + expect( + evaluateManagedAgentProbe(result).checks.find( + ({ id }) => id === "builtin_tools_succeeded", + ), + ).toEqual({ id: "builtin_tools_succeeded", passed: false }); + }); + + it("requires positive permission evidence and distinct lexical and symlink denials", () => { + const passing = passingL1Result(); + expect(evaluateManagedAgentProbe(passing).outcome).toBe("pass"); + + const falsePass: ManagedAgentProbeResult = { + ...passing, permissionEvidence: [ { - toolUseId: "deny-1", + toolUseId: `tool_${"a".repeat(64)}`, toolName: "Read", decision: "deny", reason: "path_outside_workspace", }, { - toolUseId: "deny-2", + toolUseId: `tool_${"b".repeat(64)}`, toolName: "Read", decision: "deny", reason: "path_outside_workspace", }, ], - workspaceChanges: [ - { path: FIXTURE_PATHS.cleanTarget, change: "modified" }, - { path: FIXTURE_PATHS.createdTarget, change: "created" }, - ], - preservation: [ - { path: FIXTURE_PATHS.dirtySentinel, preserved: true }, - { path: FIXTURE_PATHS.untrackedSentinel, preserved: true }, - ], - cancellationRequested: false, - queryClosed: true, - teardown: { - quiescent: true, - deadlineMet: true, - elapsedMs: 5, - observedPids: [], - alivePidsAtDeadline: [], - emergencyCleanupAttempted: false, - }, - correlation: { executionId: "execution-1", evalSource: "eval-1" }, }; + const checks = evaluateManagedAgentProbe(falsePass); + expect(checks.outcome).toBe("fail"); expect( - evaluateManagedAgentProbe(result).checks.find( - ({ id }) => id === "builtin_tools_succeeded", - ), - ).toEqual({ id: "builtin_tools_succeeded", passed: false }); + checks.checks.find(({ id }) => id === "expected_permissions_allowed"), + ).toEqual({ id: "expected_permissions_allowed", passed: false }); + expect( + checks.checks.find(({ id }) => id === "outside_and_symlink_denied"), + ).toEqual({ id: "outside_and_symlink_denied", passed: false }); }); }); diff --git a/packages/harness/src/experimental/managed-agent-spike/probe-cli.ts b/packages/harness/src/experimental/managed-agent-spike/probe-cli.ts index f098612af..a7591f5c3 100644 --- a/packages/harness/src/experimental/managed-agent-spike/probe-cli.ts +++ b/packages/harness/src/experimental/managed-agent-spike/probe-cli.ts @@ -19,6 +19,7 @@ import { } from "./runtime.js"; import type { ManagedAgentModelTargetId, + ManagedAgentPermissionReason, ManagedAgentProbeResult, ManagedAgentProbeScenario, } from "./types.js"; @@ -144,10 +145,17 @@ export function evaluateManagedAgentProbe( (evidence) => evidence.toolName === toolName && evidence.status === status, ); - const pathDenials = result.permissionEvidence.filter( - ({ decision, reason }) => - decision === "deny" && reason === "path_outside_workspace", - ).length; + const permission = ( + toolName: string, + decision: "allow" | "deny", + reason: ManagedAgentPermissionReason, + ): boolean => + result.permissionEvidence.some( + (evidence) => + evidence.toolName === toolName && + evidence.decision === decision && + evidence.reason === reason, + ); const checks: ManagedAgentProbeCheck[] = [ { id: "exact_model_alias", @@ -202,7 +210,30 @@ export function evaluateManagedAgentProbe( invocation(qualifiedManagedAgentMcpToolName("fail_once"), "error") && invocation(qualifiedManagedAgentMcpToolName("fail_once"), "success"), }, - { id: "outside_and_symlink_denied", passed: pathDenials >= 2 }, + { + id: "expected_permissions_allowed", + passed: + ["Read", "Edit", "Write"].every((toolName) => + permission(toolName, "allow", "fixture_path"), + ) && + permission("Bash", "allow", "exact_bash_command") && + permission( + qualifiedManagedAgentMcpToolName("echo_nonce"), + "allow", + "managed_mcp_tool", + ) && + permission( + qualifiedManagedAgentMcpToolName("fail_once"), + "allow", + "managed_mcp_tool", + ), + }, + { + id: "outside_and_symlink_denied", + passed: + permission("Read", "deny", "path_outside_workspace") && + permission("Read", "deny", "path_symlink_escape"), + }, ); } else { checks.push( diff --git a/packages/harness/src/experimental/managed-agent-spike/runtime.test.ts b/packages/harness/src/experimental/managed-agent-spike/runtime.test.ts index d88055eb7..fd474066b 100644 --- a/packages/harness/src/experimental/managed-agent-spike/runtime.test.ts +++ b/packages/harness/src/experimental/managed-agent-spike/runtime.test.ts @@ -1,3 +1,6 @@ +import { lstat, mkdir, realpath, symlink, writeFile } from "node:fs/promises"; +import { dirname, join } from "node:path"; + import { afterEach, describe, expect, it, vi } from "vitest"; import type { Options } from "@anthropic-ai/claude-agent-sdk"; @@ -26,6 +29,10 @@ import type { } from "./types.js"; const fixtures: ManagedAgentFixture[] = []; +const SUCCESS_SESSION_ID = "11111111-1111-4111-8111-111111111111"; +const CANCEL_SESSION_ID = "22222222-2222-4222-8222-222222222222"; +const TIMEOUT_SESSION_ID = "33333333-3333-4333-8333-333333333333"; +const CLOSE_SESSION_ID = "44444444-4444-4444-8444-444444444444"; afterEach(async () => { await Promise.all(fixtures.splice(0).map((fixture) => fixture.cleanup())); @@ -156,12 +163,12 @@ describe("runManagedAgentProbe", () => { { type: "system", subtype: "init", - session_id: "sdk-session-1", + session_id: SUCCESS_SESSION_ID, model: resolveManagedAgentModelTarget("sonnet-5").alias, }, { type: "assistant", - session_id: "sdk-session-1", + session_id: SUCCESS_SESSION_ID, message: { content: [ { @@ -178,7 +185,7 @@ describe("runManagedAgentProbe", () => { }, { type: "user", - session_id: "sdk-session-1", + session_id: SUCCESS_SESSION_ID, message: { content: [ { @@ -193,7 +200,7 @@ describe("runManagedAgentProbe", () => { type: "result", subtype: "success", is_error: false, - session_id: "sdk-session-1", + session_id: SUCCESS_SESSION_ID, result: `secret:${fixture.nonce}`, usage: { input_tokens: 9, output_tokens: 4 }, }, @@ -236,7 +243,7 @@ describe("runManagedAgentProbe", () => { ); expect(capturedOptions?.env).not.toHaveProperty("SAPIOM_API_KEY"); expect(result.terminal).toBe("success"); - expect(result.sdkSessionId).toBe("sdk-session-1"); + expect(result.sdkSessionId).toBe(SUCCESS_SESSION_ID); expect(result.queryClosed).toBe(true); expect(result.preservation.every(({ preserved }) => preserved)).toBe( true, @@ -254,6 +261,144 @@ describe("runManagedAgentProbe", () => { } }); + it("does not follow pre-existing config child symlinks and fails invalid roots before query construction", async () => { + const { config, fixture } = await probeConfig(); + const externalConfig = join(fixture.root, "external-config"); + await mkdir(externalConfig); + await symlink(externalConfig, join(fixture.configRoot, "claude-config")); + let claudeConfigDirectory: string | undefined; + const safeQueryFactory = vi.fn(({ options }: { options: Options }) => { + claudeConfigDirectory = options.env?.CLAUDE_CONFIG_DIR; + return queryFromEvents([ + { + type: "system", + subtype: "init", + session_id: SUCCESS_SESSION_ID, + }, + { type: "result", subtype: "success", is_error: false }, + ]); + }); + + await runManagedAgentProbe(config, { + hermeticGatewayOrigin: config.gatewayOrigin, + processObserver: fakeObserver(), + queryFactory: safeQueryFactory, + }); + + expect(safeQueryFactory).toHaveBeenCalledOnce(); + expect(claudeConfigDirectory).toBeDefined(); + expect(await realpath(claudeConfigDirectory!)).not.toBe( + await realpath(externalConfig), + ); + expect(dirname(dirname(claudeConfigDirectory!))).toBe(fixture.configRoot); + expect((await lstat(claudeConfigDirectory!)).isSymbolicLink()).toBe(false); + + const invalidConfigRoot = join(fixture.root, "config-file"); + await writeFile(invalidConfigRoot, "not a directory"); + const rejectedQueryFactory = vi.fn(() => queryFromEvents([])); + await expect( + runManagedAgentProbe( + { ...config, configRoot: invalidConfigRoot }, + { + hermeticGatewayOrigin: config.gatewayOrigin, + processObserver: fakeObserver(), + queryFactory: rejectedQueryFactory, + }, + ), + ).rejects.toThrow("configRoot must be a directory"); + expect(rejectedQueryFactory).not.toHaveBeenCalled(); + }); + + it("redacts malicious SDK and permission identifiers from the complete result", async () => { + const { config } = await probeConfig(); + const sessionSecret = "session-secret-injected-by-sdk"; + const toolIdSecret = "tool-id-secret-injected-by-sdk"; + const toolNameSecret = "ReadSecretInjectedBySdk"; + const permissionIdSecret = "permission-id-secret-injected-by-sdk"; + const permissionNameSecret = "PermissionSecretInjectedBySdk"; + const result = await runManagedAgentProbe(config, { + hermeticGatewayOrigin: config.gatewayOrigin, + processObserver: fakeObserver(), + queryFactory: ({ options }) => ({ + async *[Symbol.asyncIterator]() { + await options.canUseTool?.( + permissionNameSecret, + {}, + { + signal: new AbortController().signal, + toolUseID: permissionIdSecret, + requestId: "request-id-not-persisted", + }, + ); + yield { + type: "system", + subtype: "init", + session_id: sessionSecret, + }; + yield { + type: "assistant", + session_id: sessionSecret, + message: { + content: [ + { + type: "tool_use", + id: toolIdSecret, + name: toolNameSecret, + input: { secret: "tool-input-secret" }, + }, + ], + }, + }; + yield { + type: "user", + session_id: sessionSecret, + message: { + content: [ + { + type: "tool_result", + tool_use_id: toolIdSecret, + content: "tool-result-secret", + }, + ], + }, + }; + yield { + type: "result", + subtype: "success", + is_error: false, + session_id: sessionSecret, + }; + }, + close: vi.fn(), + }), + }); + + expect(result.sdkSessionId).toBeUndefined(); + expect(result.toolEvidence.slice(0, 2)).toMatchObject([ + { toolName: "unknown", status: "requested" }, + { toolName: "unknown", status: "success" }, + ]); + expect(result.toolEvidence[0]?.toolUseId).toBe( + result.toolEvidence[1]?.toolUseId, + ); + expect(result.permissionEvidence).toMatchObject([ + { toolName: "unknown", decision: "deny", reason: "tool_not_allowed" }, + ]); + const serialized = JSON.stringify(result); + for (const secret of [ + sessionSecret, + toolIdSecret, + toolNameSecret, + permissionIdSecret, + permissionNameSecret, + "request-id-not-persisted", + "tool-input-secret", + "tool-result-secret", + ]) { + expect(serialized).not.toContain(secret); + } + }); + it("classifies an explicit active-run abort as cancellation", async () => { const { config } = await probeConfig("L2"); const observer = fakeObserver(); @@ -267,7 +412,7 @@ describe("runManagedAgentProbe", () => { yield { type: "system", subtype: "init", - session_id: "cancel-session", + session_id: CANCEL_SESSION_ID, }; if (!options.abortController?.signal.aborted) { await new Promise((resolveAbort) => @@ -307,7 +452,11 @@ describe("runManagedAgentProbe", () => { processObserver: observer, queryFactory: () => queryFromEvents([ - { type: "system", subtype: "init", session_id: "session-timeout" }, + { + type: "system", + subtype: "init", + session_id: TIMEOUT_SESSION_ID, + }, { type: "result", subtype: "success", is_error: false }, ]), }); @@ -332,7 +481,11 @@ describe("runManagedAgentProbe", () => { abortSignal = options.abortController?.signal; return queryFromEvents( [ - { type: "system", subtype: "init", session_id: "close-session" }, + { + type: "system", + subtype: "init", + session_id: CLOSE_SESSION_ID, + }, { type: "result", subtype: "success", is_error: false }, ], vi.fn(() => { diff --git a/packages/harness/src/experimental/managed-agent-spike/types.ts b/packages/harness/src/experimental/managed-agent-spike/types.ts index f350b67ec..5c48e78e8 100644 --- a/packages/harness/src/experimental/managed-agent-spike/types.ts +++ b/packages/harness/src/experimental/managed-agent-spike/types.ts @@ -41,6 +41,7 @@ export type ManagedAgentPermissionReason = | "managed_mcp_tool" | "invalid_input" | "path_outside_workspace" + | "path_symlink_escape" | "bash_command_not_allowed" | "tool_not_allowed"; From 4417b0bfd57f5b2386786f02e558f7a9a20aabfe Mon Sep 17 00:00:00 2001 From: Yash Date: Sun, 16 Aug 2026 16:54:49 -0700 Subject: [PATCH 04/24] fix(harness): enforce managed agent policy hooks Replace the incomplete canUseTool-only boundary with a universal PreToolUse policy, fail closed on managed hook settings, and add hermetic SDK loopback coverage plus durable correlation and inference-turn evidence.\n\nRefs: SAP-2632 --- .../managed-agent-spike/README.md | 63 ++++ .../managed-agent-spike/contract.ts | 5 +- .../managed-agent-spike/events.test.ts | 53 +++ .../managed-agent-spike/events.ts | 56 ++++ .../experimental/managed-agent-spike/index.ts | 17 +- .../managed-agent-spike/permissions.test.ts | 248 +++++++++++--- .../managed-agent-spike/permissions.ts | 257 +++++++++++---- .../managed-agent-spike/probe-cli.test.ts | 37 ++- .../managed-agent-spike/probe-cli.ts | 34 +- .../runtime-sdk-loopback.test.ts | 309 ++++++++++++++++++ .../managed-agent-spike/runtime.test.ts | 260 +++++++++++++-- .../managed-agent-spike/runtime.ts | 173 ++++++++-- .../settings-guard.test.ts | 98 ++++++ .../managed-agent-spike/settings-guard.ts | 174 ++++++++++ .../experimental/managed-agent-spike/types.ts | 24 ++ 15 files changed, 1627 insertions(+), 181 deletions(-) create mode 100644 packages/harness/src/experimental/managed-agent-spike/README.md create mode 100644 packages/harness/src/experimental/managed-agent-spike/runtime-sdk-loopback.test.ts create mode 100644 packages/harness/src/experimental/managed-agent-spike/settings-guard.test.ts create mode 100644 packages/harness/src/experimental/managed-agent-spike/settings-guard.ts diff --git a/packages/harness/src/experimental/managed-agent-spike/README.md b/packages/harness/src/experimental/managed-agent-spike/README.md new file mode 100644 index 000000000..7e8b39c74 --- /dev/null +++ b/packages/harness/src/experimental/managed-agent-spike/README.md @@ -0,0 +1,63 @@ +# Managed Agent Feasibility Spike + +This subpath is an Epic 0 probe, not a production Harness runtime. It does not +change the PTY adapter, Studio UI, session history, or existing Claude Code and +Codex flows. + +## Host policy boundary + +Every model-requested Read, Edit, Write, Bash, and in-process MCP call is gated +by one programmatic `PreToolUse` hook registered without a matcher. The hook +runs before the SDK's permission evaluation, applies canonical-path containment, +exact Bash equality, and an MCP allowlist, and returns a complete fresh input +object only when allowing the call. Unknown tools fail closed. + +`canUseTool` remains only as defense in depth for calls the SDK leaves +unresolved. It shares the same evaluator and deduplicates by tool-use ID, so it +cannot create a second evidence record. A live result is rejected when any +requested tool lacks exactly one primary `PreToolUse` decision. This detects a +hook that was skipped, but detection after execution is not by itself a host +boundary. + +Before `query()` is created, a credential-free subprocess rooted in the +probe's isolated HOME and `CLAUDE_CONFIG_DIR` calls SDK `resolveSettings()`. +`disableAllHooks`, resolution errors, timeouts, malformed output, and configured +`policyHelper`/`policyHelpers` all produce `policy_violation` without creating a +query. Policy helpers fail closed because SDK 0.3.228 does not execute them in +`resolveSettings()` and therefore cannot prove parity with query startup. + +The subprocess requires a Node executable. E0.4 uses the current Node host; +Electron-as-Node and packaged executable resolution are deliberately deferred +to E0.7. The runtime also exposes only a narrow async-iterator/close query +interface. It does not expose or call SDK `Query.mcpCall()`, whose trusted +control channel bypasses permission checks. + +## Correlation and turn evidence + +The runtime sends `x-sapiom-eval-source` and `x-sapiom-execution-id`, then embeds +the same non-secret values in the initial prompt as: + +```text +SAPIOM_CERTIFICATION_CORRELATION_V1;eval_source=;execution_id= +``` + +The production gateway consumes both headers, but its current BigQuery +projection persists neither `polsia_eval_source` nor `sapiom_execution_id`. +Reconciliation therefore follows the existing E0.2 contract and searches the +replayed initial prompt marker. The authoritative SDK-side inference count is +the number of distinct assistant message IDs. IDs are hashed and counted only +in memory; raw or hashed IDs are not emitted. SDK `result.num_turns` is retained +separately as bounded informational evidence and is not used as the BigQuery +call-count key. + +## Pre-fix live evidence + +The first Sonnet 5 L1 attempt reached an SDK success result and clean teardown, +but SDK default permissions executed successful Read and Bash calls without +invoking the former `canUseTool` boundary. The fixed matrix stopped immediately; +no retry or later model/scenario attempt ran. BigQuery showed exact Sonnet +provider/model, no fallback, positive tokens, and cost for all 11 calls, but no +durable correlation field and no independent SDK inference count. + +That attempt is not acceptance evidence. Do not run another paid L1/L2 matrix +until this correction has independent review and explicit authorization. diff --git a/packages/harness/src/experimental/managed-agent-spike/contract.ts b/packages/harness/src/experimental/managed-agent-spike/contract.ts index 1d2c89ad5..4d88ee850 100644 --- a/packages/harness/src/experimental/managed-agent-spike/contract.ts +++ b/packages/harness/src/experimental/managed-agent-spike/contract.ts @@ -19,6 +19,7 @@ export const MANAGED_AGENT_CONTRACT = { suiteVersion: "0.1.0", directGatewayOrigin: "https://litellm.services.sapiom.ai", maxBudgetUsd: 1, + maxTurns: 20, } as const; export const MANAGED_AGENT_MODEL_TARGETS: Readonly< @@ -209,10 +210,10 @@ export function validateManagedAgentProbeConfig( if ( !Number.isInteger(config.maxTurns) || config.maxTurns < 1 || - config.maxTurns > 20 + config.maxTurns > MANAGED_AGENT_CONTRACT.maxTurns ) { throw new ManagedAgentConfigurationError( - "maxTurns must be an integer between 1 and 20", + `maxTurns must be an integer between 1 and ${MANAGED_AGENT_CONTRACT.maxTurns}`, ); } if ( diff --git a/packages/harness/src/experimental/managed-agent-spike/events.test.ts b/packages/harness/src/experimental/managed-agent-spike/events.test.ts index 24feef495..7a8add560 100644 --- a/packages/harness/src/experimental/managed-agent-spike/events.test.ts +++ b/packages/harness/src/experimental/managed-agent-spike/events.test.ts @@ -1,6 +1,7 @@ import { describe, expect, it } from "vitest"; import { + ManagedAgentEventError, ManagedAgentEventRecorder, normalizeManagedAgentToolUseId, } from "./events.js"; @@ -59,6 +60,7 @@ describe("ManagedAgentEventRecorder", () => { cache_read_input_tokens: 1, }, total_cost_usd: 0.001, + num_turns: 7, }); expect(recorder.recordTerminal("success")).toBe(true); expect(recorder.recordTerminal("query_error")).toBe(false); @@ -72,6 +74,8 @@ describe("ManagedAgentEventRecorder", () => { cacheReadInputTokens: 1, estimatedCostUsd: 0.001, }); + expect(recorder.inferenceTurns).toBe(1); + expect(recorder.sdkNumTurns).toBe(7); expect(recorder.toolEvidence).toEqual([ { toolUseId: normalizeManagedAgentToolUseId("tool-1"), @@ -108,6 +112,7 @@ describe("ManagedAgentEventRecorder", () => { const permissionIdSecret = "permission-id-secret-credential"; const toolNameSecret = "ReadSecretCredential"; const permissionNameSecret = "WriteSecretCredential"; + const messageIdSecret = "message-id-secret-credential"; recorder.observeSdkEvent({ type: "system", subtype: "init", @@ -117,6 +122,7 @@ describe("ManagedAgentEventRecorder", () => { type: "assistant", session_id: sessionSecret, message: { + id: messageIdSecret, content: [ { type: "tool_use", @@ -145,6 +151,7 @@ describe("ManagedAgentEventRecorder", () => { toolName: permissionNameSecret, decision: "deny", reason: "tool_not_allowed", + source: "pre_tool_use", }); recorder.recordTerminal("success"); @@ -160,6 +167,7 @@ describe("ManagedAgentEventRecorder", () => { toolName: "unknown", decision: "deny", reason: "tool_not_allowed", + source: "pre_tool_use", }, ]); const serialized = JSON.stringify({ @@ -174,9 +182,54 @@ describe("ManagedAgentEventRecorder", () => { permissionIdSecret, toolNameSecret, permissionNameSecret, + messageIdSecret, "private-result", ]) { expect(serialized).not.toContain(secret); } }); + + it("counts distinct hashed assistant ids and keeps bounded SDK turns separate", () => { + const recorder = new ManagedAgentEventRecorder("run-3"); + for (const messageId of [ + "private-message-a", + "private-message-a", + "private-message-b", + ]) { + recorder.observeSdkEvent({ + type: "assistant", + message: { id: messageId, content: [{ type: "text", text: "secret" }] }, + }); + } + recorder.observeSdkEvent({ + type: "result", + subtype: "success", + is_error: false, + num_turns: 9, + }); + + expect(recorder.inferenceTurns).toBe(2); + expect(recorder.sdkNumTurns).toBe(9); + const serialized = JSON.stringify({ + events: recorder.events, + inferenceTurns: recorder.inferenceTurns, + sdkNumTurns: recorder.sdkNumTurns, + }); + expect(serialized).not.toContain("private-message-a"); + expect(serialized).not.toContain("private-message-b"); + + expect(() => + recorder.observeSdkEvent({ + type: "result", + subtype: "success", + num_turns: 21, + }), + ).toThrow(ManagedAgentEventError); + expect(() => + recorder.observeSdkEvent({ + type: "assistant", + message: { content: [] }, + }), + ).toThrow(ManagedAgentEventError); + }); }); diff --git a/packages/harness/src/experimental/managed-agent-spike/events.ts b/packages/harness/src/experimental/managed-agent-spike/events.ts index 880ab4e58..f019b4fc3 100644 --- a/packages/harness/src/experimental/managed-agent-spike/events.ts +++ b/packages/harness/src/experimental/managed-agent-spike/events.ts @@ -1,5 +1,6 @@ import { createHash } from "node:crypto"; +import { MANAGED_AGENT_CONTRACT } from "./contract.js"; import type { ManagedAgentPermissionEvidence, ManagedAgentProbeEvent, @@ -42,6 +43,14 @@ const SAFE_TOOL_NAMES = new Set([ const NORMALIZED_TOOL_USE_ID_PATTERN = /^tool_[0-9a-f]{64}$/; const SDK_SESSION_ID_PATTERN = /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/; +const MAX_ASSISTANT_MESSAGE_ID_LENGTH = 512; + +export class ManagedAgentEventError extends Error { + public constructor(message: string) { + super(message); + this.name = "ManagedAgentEventError"; + } +} export function sanitizeManagedAgentToolName(value: unknown): string { const toolName = optionalString(value); @@ -66,6 +75,33 @@ function safeSdkSessionId(value: unknown): string | undefined { : undefined; } +function normalizeAssistantMessageId(value: unknown): string { + const messageId = optionalString(value); + if (!messageId || messageId.length > MAX_ASSISTANT_MESSAGE_ID_LENGTH) { + throw new ManagedAgentEventError( + "Assistant event has no bounded string message id", + ); + } + return createHash("sha256") + .update("sapiom-managed-agent-assistant-message-id\0") + .update(messageId) + .digest("hex"); +} + +function boundedSdkNumTurns(value: unknown): number | undefined { + if (value === undefined) return undefined; + if ( + !Number.isInteger(value) || + Number(value) < 0 || + Number(value) > MANAGED_AGENT_CONTRACT.maxTurns + ) { + throw new ManagedAgentEventError( + `SDK num_turns must be an integer between 0 and ${MANAGED_AGENT_CONTRACT.maxTurns}`, + ); + } + return Number(value); +} + function contentBlocks(message: JsonRecord | undefined): readonly JsonRecord[] { if (!Array.isArray(message?.content)) return []; return message.content.flatMap((value) => { @@ -95,10 +131,12 @@ export class ManagedAgentEventRecorder { readonly #events: ManagedAgentProbeEvent[] = []; readonly #toolEvidence: ManagedAgentToolEvidence[] = []; readonly #permissionEvidence: ManagedAgentPermissionEvidence[] = []; + readonly #inferenceMessageIds = new Set(); readonly #runId: string; #terminalRecorded = false; #sessionId: string | undefined; #usage: ManagedAgentSdkUsageEstimate | undefined; + #sdkNumTurns: number | undefined; #sdkResult: | { readonly isError: boolean; readonly subtype?: string } | undefined; @@ -127,6 +165,14 @@ export class ManagedAgentEventRecorder { return this.#usage; } + public get inferenceTurns(): number { + return this.#inferenceMessageIds.size; + } + + public get sdkNumTurns(): number | undefined { + return this.#sdkNumTurns; + } + public get result(): | { readonly isError: boolean; readonly subtype?: string } | undefined { @@ -161,6 +207,7 @@ export class ManagedAgentEventRecorder { toolName: normalizedEvidence.toolName, permissionDecision: normalizedEvidence.decision, permissionReason: normalizedEvidence.reason, + permissionSource: normalizedEvidence.source, }); } @@ -179,6 +226,14 @@ export class ManagedAgentEventRecorder { const message = asRecord(event.message); const blocks = contentBlocks(message); + if (type === "assistant") { + this.#inferenceMessageIds.add(normalizeAssistantMessageId(message?.id)); + if (this.#inferenceMessageIds.size > MANAGED_AGENT_CONTRACT.maxTurns) { + throw new ManagedAgentEventError( + `Distinct assistant message ids exceed ${MANAGED_AGENT_CONTRACT.maxTurns}`, + ); + } + } if (type === "assistant" || type === "user") { this.#append({ type: "message", subtype: type, sessionId }); } @@ -220,6 +275,7 @@ export class ManagedAgentEventRecorder { } } if (type === "result") { + this.#sdkNumTurns = boundedSdkNumTurns(event.num_turns); const isError = event.is_error === true || subtype !== "success"; this.#sdkResult = { isError, ...(subtype ? { subtype } : {}) }; this.#usage = sdkUsage(event); diff --git a/packages/harness/src/experimental/managed-agent-spike/index.ts b/packages/harness/src/experimental/managed-agent-spike/index.ts index bff0f3b5d..487e16aea 100644 --- a/packages/harness/src/experimental/managed-agent-spike/index.ts +++ b/packages/harness/src/experimental/managed-agent-spike/index.ts @@ -18,7 +18,7 @@ export { type ManagedAgentChildEnvironmentInput, type ManagedAgentIsolatedDirectories, } from "./environment.js"; -export { ManagedAgentEventRecorder } from "./events.js"; +export { ManagedAgentEventError, ManagedAgentEventRecorder } from "./events.js"; export { FIXTURE_PATHS, captureManagedAgentWorkspaceSnapshot, @@ -36,18 +36,28 @@ export { MANAGED_AGENT_BUILTIN_TOOLS, MANAGED_AGENT_DISALLOWED_TOOLS, ManagedAgentPathError, - createManagedAgentPermissionHandler, + createManagedAgentPolicyBoundary, isPathWithinRoot, resolveManagedAgentToolPath, - type ManagedAgentPermissionHandlerOptions, + type ManagedAgentPolicyBoundary, + type ManagedAgentPolicyBoundaryOptions, } from "./permissions.js"; export { LocalManagedAgentProcessObserver, createLocalManagedAgentProcessObserver, } from "./process-observer.js"; +export { + ManagedAgentSettingsGuardError, + assertManagedAgentHooksEnabled, + buildManagedAgentSettingsGuardEnvironment, + type ManagedAgentSettingsGuardDependencies, + type ManagedAgentSettingsGuardInput, +} from "./settings-guard.js"; export { MANAGED_AGENT_MCP_SERVER_NAME, + MANAGED_AGENT_CORRELATION_MARKER_VERSION, MANAGED_AGENT_TEARDOWN_TIMEOUT_MS, + buildManagedAgentCorrelationPrompt, createManagedAgentMcpRuntime, qualifiedManagedAgentMcpToolName, runManagedAgentProbe, @@ -59,6 +69,7 @@ export type { ManagedAgentPermissionDecision, ManagedAgentPermissionEvidence, ManagedAgentPermissionReason, + ManagedAgentPermissionSource, ManagedAgentPreservationObservation, ManagedAgentProbeConfig, ManagedAgentProbeDependencies, diff --git a/packages/harness/src/experimental/managed-agent-spike/permissions.test.ts b/packages/harness/src/experimental/managed-agent-spike/permissions.test.ts index f103a33e1..27e27954e 100644 --- a/packages/harness/src/experimental/managed-agent-spike/permissions.test.ts +++ b/packages/harness/src/experimental/managed-agent-spike/permissions.test.ts @@ -10,9 +10,10 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import type { PreToolUseHookInput } from "@anthropic-ai/claude-agent-sdk"; import { - createManagedAgentPermissionHandler, + createManagedAgentPolicyBoundary, resolveManagedAgentToolPath, } from "./permissions.js"; import type { ManagedAgentPermissionEvidence } from "./types.js"; @@ -72,79 +73,224 @@ describe("symlink-aware managed-agent containment", () => { }); }); -describe("managed-agent permission handler", () => { +function preToolUseInput( + toolName: string, + toolInput: unknown, + toolUseId: string, +): PreToolUseHookInput { + return { + hook_event_name: "PreToolUse", + session_id: "11111111-1111-4111-8111-111111111111", + transcript_path: join(workspace, "transcript.jsonl"), + cwd: workspace, + tool_name: toolName, + tool_input: toolInput, + tool_use_id: toolUseId, + }; +} + +describe("managed-agent universal policy boundary", () => { it("uses exact Bash equality and emits content-free decisions", async () => { const evidence: ManagedAgentPermissionEvidence[] = []; - const handler = createManagedAgentPermissionHandler({ + const boundary = createManagedAgentPolicyBoundary({ canonicalWorkspaceRoot: workspace, allowedBashCommands: ["git status --short"], allowedMcpTools: ["mcp__probe__echo_nonce"], onDecision: (decision) => evidence.push(decision), }); - const permission = { - signal: new AbortController().signal, - toolUseID: "tool-1", - requestId: "request-1", + const signal = new AbortController().signal; + let sequence = 0; + const invoke = (toolName: string, input: unknown) => { + const toolUseId = `tool-${++sequence}`; + return boundary.preToolUseHook( + preToolUseInput(toolName, input, toolUseId), + toolUseId, + { signal }, + ); }; await expect( - handler("Bash", { command: "git status --short" }, permission), - ).resolves.toMatchObject({ behavior: "allow" }); - await expect( - handler("Bash", { command: "git status --short " }, permission), - ).resolves.toMatchObject({ behavior: "deny", interrupt: false }); + invoke("Bash", { command: "git status --short" }), + ).resolves.toMatchObject({ + hookSpecificOutput: { + permissionDecision: "allow", + updatedInput: { command: "git status --short" }, + }, + }); await expect( - handler( - "Read", - { file_path: "inside.txt", preserve: "metadata" }, - permission, - ), + invoke("Bash", { command: "git status --short " }), ).resolves.toMatchObject({ - behavior: "allow", - updatedInput: { - file_path: join(workspace, "inside.txt"), - preserve: "metadata", + hookSpecificOutput: { permissionDecision: "deny" }, + }); + const readInput = { file_path: "inside.txt", preserve: "metadata" }; + await expect(invoke("Read", readInput)).resolves.toMatchObject({ + hookSpecificOutput: { + permissionDecision: "allow", + updatedInput: { + file_path: join(workspace, "inside.txt"), + preserve: "metadata", + }, }, }); await expect( - handler( - "Write", - { file_path: "nested/new.txt", content: "safe" }, - permission, - ), + invoke("Write", { file_path: "nested/new.txt", content: "safe" }), ).resolves.toMatchObject({ - behavior: "allow", - updatedInput: { - file_path: join(workspace, "nested/new.txt"), - content: "safe", + hookSpecificOutput: { + permissionDecision: "allow", + updatedInput: { + file_path: join(workspace, "nested/new.txt"), + content: "safe", + }, }, }); await expect( - handler("Read", { file_path: join(outside, "secret.txt") }, permission), - ).resolves.toMatchObject({ behavior: "deny", interrupt: false }); + invoke("Read", { file_path: join(outside, "secret.txt") }), + ).resolves.toMatchObject({ + hookSpecificOutput: { permissionDecision: "deny" }, + }); await expect( - handler("Read", { file_path: "escape.txt" }, permission), - ).resolves.toMatchObject({ behavior: "deny", interrupt: false }); + invoke("Read", { file_path: "escape.txt" }), + ).resolves.toMatchObject({ + hookSpecificOutput: { permissionDecision: "deny" }, + }); await expect( - handler("mcp__probe__echo_nonce", { nonce: "secret" }, permission), - ).resolves.toMatchObject({ behavior: "allow" }); - await expect(handler("WebFetch", {}, permission)).resolves.toMatchObject({ - behavior: "deny", - }); - - expect(evidence.map(({ decision, reason }) => [decision, reason])).toEqual([ - ["allow", "exact_bash_command"], - ["deny", "bash_command_not_allowed"], - ["allow", "fixture_path"], - ["allow", "fixture_path"], - ["deny", "path_outside_workspace"], - ["deny", "path_symlink_escape"], - ["allow", "managed_mcp_tool"], - ["deny", "tool_not_allowed"], + invoke("mcp__probe__echo_nonce", { nonce: "secret" }), + ).resolves.toMatchObject({ + hookSpecificOutput: { permissionDecision: "allow" }, + }); + await expect(invoke("WebFetch", {})).resolves.toMatchObject({ + hookSpecificOutput: { permissionDecision: "deny" }, + }); + + expect( + evidence.map(({ decision, reason, source }) => [ + decision, + reason, + source, + ]), + ).toEqual([ + ["allow", "exact_bash_command", "pre_tool_use"], + ["deny", "bash_command_not_allowed", "pre_tool_use"], + ["allow", "fixture_path", "pre_tool_use"], + ["allow", "fixture_path", "pre_tool_use"], + ["deny", "path_outside_workspace", "pre_tool_use"], + ["deny", "path_symlink_escape", "pre_tool_use"], + ["allow", "managed_mcp_tool", "pre_tool_use"], + ["deny", "tool_not_allowed", "pre_tool_use"], ]); expect(JSON.stringify(evidence)).not.toContain(join(outside, "secret.txt")); expect(JSON.stringify(evidence)).not.toContain("secret"); - expect(JSON.stringify(evidence)).not.toContain("tool-1"); - expect(vi.isMockFunction(handler)).toBe(false); + expect(JSON.stringify(evidence)).not.toContain("tool-3"); + expect(vi.isMockFunction(boundary.preToolUseHook)).toBe(false); + expect(readInput).toEqual({ + file_path: "inside.txt", + preserve: "metadata", + }); + }); + + it("deduplicates the fallback and records when only the fallback executes", async () => { + const evidence: ManagedAgentPermissionEvidence[] = []; + const boundary = createManagedAgentPolicyBoundary({ + canonicalWorkspaceRoot: workspace, + allowedBashCommands: ["git status --short"], + allowedMcpTools: [], + onDecision: (decision) => evidence.push(decision), + }); + const signal = new AbortController().signal; + const toolUseID = "tool-deduplicated"; + await boundary.preToolUseHook( + preToolUseInput("Read", { file_path: "inside.txt" }, toolUseID), + toolUseID, + { signal }, + ); + await expect( + boundary.canUseToolFallback( + "Read", + { file_path: join(workspace, "inside.txt") }, + { signal, toolUseID, requestId: "request-1" }, + ), + ).resolves.toMatchObject({ behavior: "allow" }); + expect(evidence).toHaveLength(1); + expect(evidence[0]?.source).toBe("pre_tool_use"); + + await expect( + boundary.preToolUseHook( + preToolUseInput( + "Bash", + { command: "touch must-not-inherit-allow" }, + toolUseID, + ), + toolUseID, + { signal }, + ), + ).resolves.toMatchObject({ + hookSpecificOutput: { + permissionDecision: "deny", + permissionDecisionReason: expect.stringContaining("invalid_input"), + }, + }); + expect(evidence).toHaveLength(1); + + await expect( + boundary.canUseToolFallback( + "Bash", + { command: "git status --short" }, + { signal, toolUseID: "fallback-only", requestId: "request-2" }, + ), + ).resolves.toMatchObject({ behavior: "allow" }); + expect(evidence).toHaveLength(2); + expect(evidence[1]?.source).toBe("can_use_tool_fallback"); + }); + + it("fails closed when aborted before or during asynchronous path validation", async () => { + const evidence: ManagedAgentPermissionEvidence[] = []; + const before = new AbortController(); + before.abort(); + const beforeBoundary = createManagedAgentPolicyBoundary({ + canonicalWorkspaceRoot: workspace, + allowedBashCommands: ["git status --short"], + allowedMcpTools: [], + onDecision: (decision) => evidence.push(decision), + }); + await expect( + beforeBoundary.preToolUseHook( + preToolUseInput("Bash", { command: "git status --short" }, "before"), + "before", + { signal: before.signal }, + ), + ).resolves.toMatchObject({ + hookSpecificOutput: { + permissionDecision: "deny", + permissionDecisionReason: expect.stringContaining("policy_aborted"), + }, + }); + + const during = new AbortController(); + const duringBoundary = createManagedAgentPolicyBoundary({ + canonicalWorkspaceRoot: workspace, + allowedBashCommands: [], + allowedMcpTools: [], + onDecision: (decision) => evidence.push(decision), + resolveToolPath: async () => { + during.abort(); + return join(workspace, "inside.txt"); + }, + }); + await expect( + duringBoundary.preToolUseHook( + preToolUseInput("Read", { file_path: "inside.txt" }, "during"), + "during", + { signal: during.signal }, + ), + ).resolves.toMatchObject({ + hookSpecificOutput: { + permissionDecision: "deny", + permissionDecisionReason: expect.stringContaining("policy_aborted"), + }, + }); + expect(evidence.map(({ reason }) => reason)).toEqual([ + "policy_aborted", + "policy_aborted", + ]); }); }); diff --git a/packages/harness/src/experimental/managed-agent-spike/permissions.ts b/packages/harness/src/experimental/managed-agent-spike/permissions.ts index 48185f437..9986aecc8 100644 --- a/packages/harness/src/experimental/managed-agent-spike/permissions.ts +++ b/packages/harness/src/experimental/managed-agent-spike/permissions.ts @@ -3,12 +3,14 @@ import { dirname, isAbsolute, relative, resolve, sep } from "node:path"; import type { CanUseTool, + HookCallback, PermissionResult, } from "@anthropic-ai/claude-agent-sdk"; import type { ManagedAgentPermissionEvidence, ManagedAgentPermissionReason, + ManagedAgentPermissionSource, } from "./types.js"; import { normalizeManagedAgentToolUseId, @@ -129,28 +131,47 @@ export async function resolveManagedAgentToolPath( return resolvedCandidate; } -export interface ManagedAgentPermissionHandlerOptions { +export interface ManagedAgentPolicyBoundaryOptions { readonly canonicalWorkspaceRoot: string; readonly allowedBashCommands: readonly string[]; readonly allowedMcpTools: readonly string[]; readonly onDecision: (evidence: ManagedAgentPermissionEvidence) => void; + /** Test seam for proving cancellation after asynchronous path validation. */ + readonly resolveToolPath?: typeof resolveManagedAgentToolPath; +} + +export interface ManagedAgentPolicyBoundary { + /** Primary boundary: the SDK runs this before its own permission evaluation. */ + readonly preToolUseHook: HookCallback; + /** Defense in depth when the SDK still surfaces an unresolved permission. */ + readonly canUseToolFallback: CanUseTool; +} + +interface ManagedAgentPolicyDecision { + readonly decision: "allow" | "deny"; + readonly reason: ManagedAgentPermissionReason; + readonly updatedInput?: Record; +} + +interface ManagedAgentRecordedPolicyDecision extends ManagedAgentPolicyDecision { + readonly source: ManagedAgentPermissionSource; } function permissionResult( - decision: "allow" | "deny", + policy: ManagedAgentPolicyDecision, toolUseID: string, - reason: ManagedAgentPermissionReason, - updatedInput?: Record, ): PermissionResult { - return decision === "allow" + return policy.decision === "allow" ? { behavior: "allow", toolUseID, - ...(updatedInput ? { updatedInput } : {}), + ...(policy.updatedInput + ? { updatedInput: { ...policy.updatedInput } } + : {}), } : { behavior: "deny", - message: `Managed-agent permission denied: ${reason}`, + message: `Managed-agent permission denied: ${policy.reason}`, interrupt: false, toolUseID, }; @@ -162,68 +183,178 @@ function filePathFromInput(input: Record): string | undefined { : undefined; } -export function createManagedAgentPermissionHandler( - options: ManagedAgentPermissionHandlerOptions, -): CanUseTool { +function asRecord(value: unknown): Record | undefined { + return typeof value === "object" && value !== null && !Array.isArray(value) + ? (value as Record) + : undefined; +} + +function denied( + reason: ManagedAgentPermissionReason, +): ManagedAgentPolicyDecision { + return { decision: "deny", reason }; +} + +async function evaluateManagedAgentPolicy( + options: ManagedAgentPolicyBoundaryOptions, + allowedCommands: ReadonlySet, + allowedMcpTools: ReadonlySet, + toolName: string, + rawInput: unknown, + signal: AbortSignal, +): Promise { + if (signal.aborted) return denied("policy_aborted"); + const input = asRecord(rawInput); + if (!input) return denied("invalid_input"); + + if (allowedMcpTools.has(toolName)) { + return signal.aborted + ? denied("policy_aborted") + : { + decision: "allow", + reason: "managed_mcp_tool", + updatedInput: { ...input }, + }; + } + if (toolName === "Bash") { + const command = + typeof input.command === "string" ? input.command : undefined; + if (!command) return denied("invalid_input"); + if (!allowedCommands.has(command)) { + return denied("bash_command_not_allowed"); + } + return signal.aborted + ? denied("policy_aborted") + : { + decision: "allow", + reason: "exact_bash_command", + updatedInput: { ...input }, + }; + } + if (toolName === "Read" || toolName === "Edit" || toolName === "Write") { + const requestedPath = filePathFromInput(input); + if (!requestedPath) return denied("invalid_input"); + try { + const canonicalPath = await ( + options.resolveToolPath ?? resolveManagedAgentToolPath + )(options.canonicalWorkspaceRoot, requestedPath); + if (signal.aborted) return denied("policy_aborted"); + return { + decision: "allow", + reason: "fixture_path", + updatedInput: { ...input, file_path: canonicalPath }, + }; + } catch (error) { + if (signal.aborted) return denied("policy_aborted"); + return denied( + error instanceof ManagedAgentPathError ? error.reason : "invalid_input", + ); + } + } + return denied("tool_not_allowed"); +} + +/** + * Build one universal host policy shared by the primary PreToolUse hook and a + * canUseTool fallback. Decisions are deduplicated by raw tool-use ID so one + * attempted tool produces exactly one normalized evidence record. + */ +export function createManagedAgentPolicyBoundary( + options: ManagedAgentPolicyBoundaryOptions, +): ManagedAgentPolicyBoundary { const allowedCommands = new Set(options.allowedBashCommands); const allowedMcpTools = new Set(options.allowedMcpTools); + const decisions = new Map< + string, + { + readonly source: ManagedAgentPermissionSource; + readonly pending: Promise; + } + >(); - return async (toolName, input, permission): Promise => { - let decision: "allow" | "deny" = "deny"; - let reason: ManagedAgentPermissionReason = "tool_not_allowed"; - let updatedInput: Record | undefined; - - if (allowedMcpTools.has(toolName)) { - decision = "allow"; - reason = "managed_mcp_tool"; - } else if (toolName === "Bash") { - const command = - typeof input.command === "string" ? input.command : undefined; - if (!command) { - reason = "invalid_input"; - } else if (allowedCommands.has(command)) { - decision = "allow"; - reason = "exact_bash_command"; - } else { - reason = "bash_command_not_allowed"; - } - } else if ( - toolName === "Read" || - toolName === "Edit" || - toolName === "Write" - ) { - const requestedPath = filePathFromInput(input); - if (!requestedPath) { - reason = "invalid_input"; - } else { - try { - const canonicalPath = await resolveManagedAgentToolPath( - options.canonicalWorkspaceRoot, - requestedPath, - ); - decision = "allow"; - reason = "fixture_path"; - updatedInput = { ...input, file_path: canonicalPath }; - } catch (error) { - reason = - error instanceof ManagedAgentPathError - ? error.reason - : "invalid_input"; - } + const decide = async ( + toolUseID: string, + toolName: string, + input: unknown, + signal: AbortSignal, + source: ManagedAgentPermissionSource, + ): Promise => { + const existing = decisions.get(toolUseID); + if (existing) { + if (signal.aborted) { + return { ...denied("policy_aborted"), source }; } + // The only valid duplicate is the SDK consulting canUseTool after the + // primary hook. A repeated primary ID or fallback-first sequence is + // ambiguous and must never inherit an earlier allow decision. + return source === "can_use_tool_fallback" && + existing.source === "pre_tool_use" + ? existing.pending + : { ...denied("invalid_input"), source }; } - - options.onDecision({ - toolUseId: normalizeManagedAgentToolUseId(permission.toolUseID), - toolName: sanitizeManagedAgentToolName(toolName), - decision, - reason, + const pending = evaluateManagedAgentPolicy( + options, + allowedCommands, + allowedMcpTools, + toolName, + input, + signal, + ).then((policy) => { + const recorded = { ...policy, source }; + options.onDecision({ + toolUseId: normalizeManagedAgentToolUseId(toolUseID), + toolName: sanitizeManagedAgentToolName(toolName), + decision: recorded.decision, + reason: recorded.reason, + source, + }); + return recorded; }); - return permissionResult( - decision, - permission.toolUseID, - reason, - updatedInput, + decisions.set(toolUseID, { source, pending }); + return pending; + }; + + const preToolUseHook: HookCallback = async ( + input, + callbackToolUseID, + { signal }, + ) => { + const isPreToolUse = input.hook_event_name === "PreToolUse"; + const inputToolUseID = isPreToolUse ? input.tool_use_id : undefined; + const identifiersMatch = + !callbackToolUseID || callbackToolUseID === inputToolUseID; + const toolUseID = + callbackToolUseID ?? inputToolUseID ?? "invalid-tool-use-id"; + const policy = await decide( + toolUseID, + isPreToolUse ? input.tool_name : "unknown", + isPreToolUse && identifiersMatch ? input.tool_input : undefined, + signal, + "pre_tool_use", ); + return { + hookSpecificOutput: { + hookEventName: "PreToolUse", + permissionDecision: policy.decision, + permissionDecisionReason: `Managed-agent policy: ${policy.reason}`, + ...(policy.decision === "allow" && policy.updatedInput + ? { updatedInput: { ...policy.updatedInput } } + : {}), + }, + }; }; + + const canUseToolFallback: CanUseTool = async (toolName, input, permission) => + permissionResult( + await decide( + permission.toolUseID, + toolName, + input, + permission.signal, + "can_use_tool_fallback", + ), + permission.toolUseID, + ); + + return { preToolUseHook, canUseToolFallback }; } diff --git a/packages/harness/src/experimental/managed-agent-spike/probe-cli.test.ts b/packages/harness/src/experimental/managed-agent-spike/probe-cli.test.ts index 9767106a7..61f1c7bfc 100644 --- a/packages/harness/src/experimental/managed-agent-spike/probe-cli.test.ts +++ b/packages/harness/src/experimental/managed-agent-spike/probe-cli.test.ts @@ -14,6 +14,12 @@ import type { ManagedAgentProbeResult } from "./types.js"; function passingL1Result(): ManagedAgentProbeResult { const builtins = ["Read", "Edit", "Write", "Bash"]; + const builtinIds = [ + `tool_${"1".repeat(64)}`, + `tool_${"2".repeat(64)}`, + `tool_${"3".repeat(64)}`, + `tool_${"b".repeat(64)}`, + ]; const echoTool = qualifiedManagedAgentMcpToolName("echo_nonce"); const failOnceTool = qualifiedManagedAgentMcpToolName("fail_once"); return { @@ -23,12 +29,23 @@ function passingL1Result(): ManagedAgentProbeResult { target: "sonnet-5", modelAlias: "claude-sonnet-5-anthropic-anthropic-eval", sdkSessionId: "11111111-1111-4111-8111-111111111111", + inferenceTurns: 8, + sdkNumTurns: 8, + policyHookCoverage: true, terminal: "success", events: [], toolEvidence: [ - ...builtins.flatMap((toolName) => [ - { toolName, status: "requested" as const }, - { toolName, status: "success" as const }, + ...builtins.flatMap((toolName, index) => [ + { + toolUseId: builtinIds[index], + toolName, + status: "requested" as const, + }, + { + toolUseId: builtinIds[index], + toolName, + status: "success" as const, + }, ]), { toolName: echoTool, status: "success" }, { toolName: failOnceTool, status: "error" }, @@ -40,36 +57,42 @@ function passingL1Result(): ManagedAgentProbeResult { toolName, decision: "allow" as const, reason: "fixture_path" as const, + source: "pre_tool_use" as const, })), { toolUseId: `tool_${"b".repeat(64)}`, toolName: "Bash", decision: "allow", reason: "exact_bash_command", + source: "pre_tool_use", }, { toolUseId: `tool_${"c".repeat(64)}`, toolName: echoTool, decision: "allow", reason: "managed_mcp_tool", + source: "pre_tool_use", }, { toolUseId: `tool_${"d".repeat(64)}`, toolName: failOnceTool, decision: "allow", reason: "managed_mcp_tool", + source: "pre_tool_use", }, { toolUseId: `tool_${"e".repeat(64)}`, toolName: "Read", decision: "deny", reason: "path_outside_workspace", + source: "pre_tool_use", }, { toolUseId: `tool_${"f".repeat(64)}`, toolName: "Read", decision: "deny", reason: "path_symlink_escape", + source: "pre_tool_use", }, ], workspaceChanges: [ @@ -90,7 +113,11 @@ function passingL1Result(): ManagedAgentProbeResult { alivePidsAtDeadline: [], emergencyCleanupAttempted: false, }, - correlation: { executionId: "execution-1", evalSource: "eval-1" }, + correlation: { + executionId: "execution-1", + evalSource: "eval-1", + promptEmbedded: true, + }, }; } @@ -250,12 +277,14 @@ describe("managed-agent probe CLI", () => { toolName: "Read", decision: "deny", reason: "path_outside_workspace", + source: "pre_tool_use", }, { toolUseId: `tool_${"b".repeat(64)}`, toolName: "Read", decision: "deny", reason: "path_outside_workspace", + source: "pre_tool_use", }, ], }; diff --git a/packages/harness/src/experimental/managed-agent-spike/probe-cli.ts b/packages/harness/src/experimental/managed-agent-spike/probe-cli.ts index a7591f5c3..0208b7c23 100644 --- a/packages/harness/src/experimental/managed-agent-spike/probe-cli.ts +++ b/packages/harness/src/experimental/managed-agent-spike/probe-cli.ts @@ -154,7 +154,24 @@ export function evaluateManagedAgentProbe( (evidence) => evidence.toolName === toolName && evidence.decision === decision && - evidence.reason === reason, + evidence.reason === reason && + evidence.source === "pre_tool_use", + ); + const requestedToolIds = result.toolEvidence.flatMap((evidence) => + evidence.status === "requested" && evidence.toolUseId + ? [evidence.toolUseId] + : [], + ); + const universalHookCoverage = + requestedToolIds.length > 0 && + new Set(requestedToolIds).size === requestedToolIds.length && + requestedToolIds.every( + (toolUseId) => + result.permissionEvidence.filter( + (evidence) => + evidence.toolUseId === toolUseId && + evidence.source === "pre_tool_use", + ).length === 1, ); const checks: ManagedAgentProbeCheck[] = [ { @@ -166,6 +183,21 @@ export function evaluateManagedAgentProbe( { id: "sdk_session_observed", passed: Boolean(result.sdkSessionId) }, { id: "query_closed", passed: result.queryClosed }, { id: "process_tree_quiescent", passed: result.teardown.quiescent }, + { + id: "universal_policy_hook_coverage", + passed: result.policyHookCoverage && universalHookCoverage, + }, + { + id: "bounded_inference_turn_evidence", + passed: + Number.isInteger(result.inferenceTurns) && + result.inferenceTurns > 0 && + result.inferenceTurns <= MANAGED_AGENT_CONTRACT.maxTurns && + (result.sdkNumTurns === undefined || + (Number.isInteger(result.sdkNumTurns) && + result.sdkNumTurns >= 0 && + result.sdkNumTurns <= MANAGED_AGENT_CONTRACT.maxTurns)), + }, { id: "dirty_and_untracked_preserved", passed: diff --git a/packages/harness/src/experimental/managed-agent-spike/runtime-sdk-loopback.test.ts b/packages/harness/src/experimental/managed-agent-spike/runtime-sdk-loopback.test.ts new file mode 100644 index 000000000..98c692542 --- /dev/null +++ b/packages/harness/src/experimental/managed-agent-spike/runtime-sdk-loopback.test.ts @@ -0,0 +1,309 @@ +import { createServer, type ServerResponse } from "node:http"; +import type { AddressInfo } from "node:net"; +import { join } from "node:path"; + +import { query as agentSdkQuery } from "@anthropic-ai/claude-agent-sdk"; +import { expect, it } from "vitest"; + +import { + FIXTURE_PATHS, + createManagedAgentFixture, + fixturePathExists, +} from "./fixture.js"; +import { runManagedAgentProbe } from "./runtime.js"; + +const RUN_ID = "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa"; +const EXECUTION_ID = "bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb"; +const MODEL_ALIAS = "claude-sonnet-5-anthropic-anthropic-eval"; +const EVAL_SOURCE = + "studio-managed-agent-e0-l1-sonnet-5-bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb"; +const CORRELATION_MARKER = `SAPIOM_CERTIFICATION_CORRELATION_V1;eval_source=${EVAL_SOURCE};execution_id=${EXECUTION_ID}`; +const ALLOWED_BASH_COMMAND = "git status --short"; +const DENIED_BASH_COMMAND = "touch denied-side-effect.txt"; + +interface LoopbackObservation { + readonly headerNames: readonly string[]; + readonly evalSourceMatches: boolean; + readonly executionIdMatches: boolean; + readonly promptMarkerPresent: boolean; +} + +function writeSseEvent( + response: ServerResponse, + event: string, + data: Record, +): void { + response.write(`event: ${event}\ndata: ${JSON.stringify(data)}\n\n`); +} + +function writeToolUseResponse( + response: ServerResponse, + turn: number, + toolUse: { + readonly id: string; + readonly name: string; + readonly input: Record; + }, +): void { + response.writeHead(200, { + "cache-control": "no-cache", + "content-type": "text/event-stream", + "request-id": `req_loopback_${turn}`, + }); + writeSseEvent(response, "message_start", { + type: "message_start", + message: { + id: `msg_loopback_${turn}`, + type: "message", + role: "assistant", + model: MODEL_ALIAS, + content: [], + stop_reason: null, + stop_sequence: null, + usage: { input_tokens: 1, output_tokens: 0 }, + }, + }); + writeSseEvent(response, "content_block_start", { + type: "content_block_start", + index: 0, + content_block: { + type: "tool_use", + id: toolUse.id, + name: toolUse.name, + input: {}, + }, + }); + writeSseEvent(response, "content_block_delta", { + type: "content_block_delta", + index: 0, + delta: { + type: "input_json_delta", + partial_json: JSON.stringify(toolUse.input), + }, + }); + writeSseEvent(response, "content_block_stop", { + type: "content_block_stop", + index: 0, + }); + writeSseEvent(response, "message_delta", { + type: "message_delta", + delta: { stop_reason: "tool_use", stop_sequence: null }, + usage: { output_tokens: 1 }, + }); + writeSseEvent(response, "message_stop", { type: "message_stop" }); + response.end(); +} + +function writeFinalResponse(response: ServerResponse, turn: number): void { + response.writeHead(200, { + "cache-control": "no-cache", + "content-type": "text/event-stream", + "request-id": `req_loopback_${turn}`, + }); + writeSseEvent(response, "message_start", { + type: "message_start", + message: { + id: `msg_loopback_${turn}`, + type: "message", + role: "assistant", + model: MODEL_ALIAS, + content: [], + stop_reason: null, + stop_sequence: null, + usage: { input_tokens: 1, output_tokens: 0 }, + }, + }); + writeSseEvent(response, "content_block_start", { + type: "content_block_start", + index: 0, + content_block: { type: "text", text: "" }, + }); + writeSseEvent(response, "content_block_delta", { + type: "content_block_delta", + index: 0, + delta: { type: "text_delta", text: "done" }, + }); + writeSseEvent(response, "content_block_stop", { + type: "content_block_stop", + index: 0, + }); + writeSseEvent(response, "message_delta", { + type: "message_delta", + delta: { stop_reason: "end_turn", stop_sequence: null }, + usage: { output_tokens: 1 }, + }); + writeSseEvent(response, "message_stop", { type: "message_stop" }); + response.end(); +} + +it("enforces every real-SDK Read/Bash call and carries exact correlation through loopback", async () => { + const fixture = await createManagedAgentFixture(() => "loopback-nonce"); + const observations: LoopbackObservation[] = []; + let helloCount = 0; + let inferenceTurn = 0; + const server = createServer((request, response) => { + if (request.method === "HEAD" && request.url === "/api/hello") { + helloCount += 1; + response.writeHead(200).end(); + return; + } + if ( + request.method !== "POST" || + request.url?.split("?")[0] !== "/v1/messages" + ) { + response.writeHead(404).end(); + return; + } + + let body = ""; + request.setEncoding("utf8"); + request.on("data", (chunk: string) => { + body += chunk; + if (body.length > 2_000_000) request.destroy(); + }); + request.on("end", () => { + inferenceTurn += 1; + const headerNames = Object.keys(request.headers).sort(); + observations.push({ + headerNames, + evalSourceMatches: + request.headers["x-sapiom-eval-source"] === EVAL_SOURCE, + executionIdMatches: + request.headers["x-sapiom-execution-id"] === EXECUTION_ID, + promptMarkerPresent: body.includes(CORRELATION_MARKER), + }); + if (inferenceTurn === 1) { + writeToolUseResponse(response, inferenceTurn, { + id: "toolu_loopback_read", + name: "Read", + input: { file_path: FIXTURE_PATHS.cleanTarget }, + }); + } else if (inferenceTurn === 2) { + writeToolUseResponse(response, inferenceTurn, { + id: "toolu_loopback_bash_allow", + name: "Bash", + input: { command: ALLOWED_BASH_COMMAND }, + }); + } else if (inferenceTurn === 3) { + writeToolUseResponse(response, inferenceTurn, { + id: "toolu_loopback_bash_deny", + name: "Bash", + input: { command: DENIED_BASH_COMMAND }, + }); + } else { + writeFinalResponse(response, inferenceTurn); + } + }); + }); + + await new Promise((resolve, reject) => { + server.once("error", reject); + server.listen(0, "127.0.0.1", resolve); + }); + try { + const address = server.address() as AddressInfo; + const ids = [RUN_ID, EXECUTION_ID]; + const result = await runManagedAgentProbe( + { + scenario: "L1", + workspaceRoot: fixture.workspaceRoot, + configRoot: fixture.configRoot, + target: "sonnet-5", + gatewayOrigin: `http://127.0.0.1:${address.port}`, + gatewayCredential: "sk-ant-api03-local-loopback-only", + prompt: fixture.prompt("L1"), + maxTurns: 6, + maxBudgetUsd: 0.25, + allowedBashCommands: [ALLOWED_BASH_COMMAND], + expectedMcpNonce: fixture.nonce, + preservePaths: [ + FIXTURE_PATHS.dirtySentinel, + FIXTURE_PATHS.untrackedSentinel, + ], + }, + { + hermeticGatewayOrigin: `http://127.0.0.1:${address.port}`, + queryFactory: ({ prompt, options }) => + agentSdkQuery({ prompt, options }), + uuid: () => { + const id = ids.shift(); + if (!id) throw new Error("unexpected UUID request"); + return id; + }, + }, + ); + + expect(helloCount).toBeGreaterThanOrEqual(1); + expect(observations).toHaveLength(4); + expect( + observations.every( + ({ headerNames, evalSourceMatches, executionIdMatches }) => + headerNames.includes("x-sapiom-eval-source") && + headerNames.includes("x-sapiom-execution-id") && + evalSourceMatches && + executionIdMatches, + ), + ).toBe(true); + expect( + observations.every(({ promptMarkerPresent }) => promptMarkerPresent), + ).toBe(true); + expect(result.terminal).toBe("success"); + + const requested = result.toolEvidence.filter( + ({ status }) => status === "requested", + ); + expect(requested.map(({ toolName }) => toolName)).toEqual([ + "Read", + "Bash", + "Bash", + ]); + for (const tool of requested) { + expect( + result.permissionEvidence.filter( + ({ toolUseId, source }) => + toolUseId === tool.toolUseId && source === "pre_tool_use", + ), + ).toHaveLength(1); + } + expect(result.permissionEvidence).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + toolName: "Read", + decision: "allow", + reason: "fixture_path", + source: "pre_tool_use", + }), + expect.objectContaining({ + toolName: "Bash", + decision: "allow", + reason: "exact_bash_command", + source: "pre_tool_use", + }), + expect.objectContaining({ + toolName: "Bash", + decision: "deny", + reason: "bash_command_not_allowed", + source: "pre_tool_use", + }), + ]), + ); + expect(result.toolEvidence).toEqual( + expect.arrayContaining([ + expect.objectContaining({ toolName: "Read", status: "success" }), + expect.objectContaining({ toolName: "Bash", status: "success" }), + expect.objectContaining({ toolName: "Bash", status: "error" }), + ]), + ); + expect( + await fixturePathExists( + join(fixture.workspaceRoot, "denied-side-effect.txt"), + ), + ).toBe(false); + expect(result.queryClosed).toBe(true); + expect(result.teardown.quiescent).toBe(true); + } finally { + server.closeAllConnections(); + await new Promise((resolve) => server.close(() => resolve())); + await fixture.cleanup(); + } +}, 45_000); diff --git a/packages/harness/src/experimental/managed-agent-spike/runtime.test.ts b/packages/harness/src/experimental/managed-agent-spike/runtime.test.ts index fd474066b..7f0ad82ec 100644 --- a/packages/harness/src/experimental/managed-agent-spike/runtime.test.ts +++ b/packages/harness/src/experimental/managed-agent-spike/runtime.test.ts @@ -79,6 +79,34 @@ function queryFromEvents( }; } +async function invokePreToolUse( + options: Options, + input: { + readonly toolName: string; + readonly toolInput: unknown; + readonly toolUseId: string; + }, + signal = new AbortController().signal, +): Promise { + const matcher = options.hooks?.PreToolUse?.[0]; + const hook = matcher?.hooks[0]; + if (!hook) + throw new Error("PreToolUse hook missing from managed-agent probe"); + await hook( + { + hook_event_name: "PreToolUse", + session_id: SUCCESS_SESSION_ID, + transcript_path: "not-persisted", + cwd: String(options.cwd), + tool_name: input.toolName, + tool_input: input.toolInput, + tool_use_id: input.toolUseId, + }, + input.toolUseId, + { signal }, + ); +} + async function probeConfig(scenario: "L1" | "L2" = "L1") { const fixture = await createManagedAgentFixture(() => "runtime-test-secret"); fixtures.push(fixture); @@ -145,6 +173,7 @@ describe("runManagedAgentProbe", () => { const observer = fakeObserver(); const close = vi.fn(); let capturedOptions: Options | undefined; + let capturedPrompt: string | undefined; const previousOAuth = process.env.CLAUDE_CODE_OAUTH_TOKEN; process.env.CLAUDE_CODE_OAUTH_TOKEN = "ambient-user-login"; try { @@ -156,34 +185,44 @@ describe("runManagedAgentProbe", () => { return () => `00000000-0000-4000-8000-${String(++counter).padStart(12, "0")}`; })(), - queryFactory: ({ options }) => { + queryFactory: ({ prompt, options }) => { capturedOptions = options; - return queryFromEvents( - [ - { + capturedPrompt = prompt; + return { + async *[Symbol.asyncIterator]() { + yield { type: "system", subtype: "init", session_id: SUCCESS_SESSION_ID, model: resolveManagedAgentModelTarget("sonnet-5").alias, - }, - { + }; + yield { type: "assistant", session_id: SUCCESS_SESSION_ID, message: { + id: "message-runtime-1", content: [ { type: "tool_use", id: "tool-1", name: "Read", input: { - file_path: fixture.outsideSentinel, + file_path: FIXTURE_PATHS.cleanTarget, secret: fixture.nonce, }, }, ], }, - }, - { + }; + await invokePreToolUse(options, { + toolName: "Read", + toolInput: { + file_path: FIXTURE_PATHS.cleanTarget, + secret: fixture.nonce, + }, + toolUseId: "tool-1", + }); + yield { type: "user", session_id: SUCCESS_SESSION_ID, message: { @@ -195,18 +234,19 @@ describe("runManagedAgentProbe", () => { }, ], }, - }, - { + }; + yield { type: "result", subtype: "success", is_error: false, session_id: SUCCESS_SESSION_ID, result: `secret:${fixture.nonce}`, + num_turns: 1, usage: { input_tokens: 9, output_tokens: 4 }, - }, - ], + }; + }, close, - ); + }; }, }); @@ -222,6 +262,14 @@ describe("runManagedAgentProbe", () => { expect(capturedOptions?.settingSources).toEqual([]); expect(capturedOptions?.strictMcpConfig).toBe(true); expect(capturedOptions?.canUseTool).toBeTypeOf("function"); + expect(capturedOptions?.hooks?.PreToolUse).toHaveLength(1); + expect(capturedOptions?.hooks?.PreToolUse?.[0]?.hooks).toHaveLength(1); + expect( + Object.prototype.hasOwnProperty.call( + capturedOptions?.hooks?.PreToolUse?.[0] ?? {}, + "matcher", + ), + ).toBe(false); expect(capturedOptions?.spawnClaudeCodeProcess).toBeTypeOf("function"); expect( Object.prototype.hasOwnProperty.call(capturedOptions, "allowedTools"), @@ -243,6 +291,14 @@ describe("runManagedAgentProbe", () => { ); expect(capturedOptions?.env).not.toHaveProperty("SAPIOM_API_KEY"); expect(result.terminal).toBe("success"); + expect(result.policyHookCoverage).toBe(true); + expect(result.inferenceTurns).toBe(1); + expect(result.sdkNumTurns).toBe(1); + expect(result.correlation.promptEmbedded).toBe(true); + expect(capturedPrompt).toContain( + "SAPIOM_CERTIFICATION_CORRELATION_V1;eval_source=studio-managed-agent-e0-l1-sonnet-5-00000000-0000-4000-8000-000000000002;execution_id=00000000-0000-4000-8000-000000000002", + ); + expect(capturedPrompt).toContain("Do not repeat it"); expect(result.sdkSessionId).toBe(SUCCESS_SESSION_ID); expect(result.queryClosed).toBe(true); expect(result.preservation.every(({ preserved }) => preserved)).toBe( @@ -309,6 +365,158 @@ describe("runManagedAgentProbe", () => { expect(rejectedQueryFactory).not.toHaveBeenCalled(); }); + it("fails before query creation when isolated managed settings disable hooks", async () => { + const { config } = await probeConfig(); + const queryFactory = vi.fn(() => queryFromEvents([])); + const result = await runManagedAgentProbe(config, { + hermeticGatewayOrigin: config.gatewayOrigin, + processObserver: fakeObserver(), + queryFactory, + policySettingsGuard: async ({ environment }) => { + expect(environment).not.toHaveProperty("ANTHROPIC_API_KEY"); + expect(environment).not.toHaveProperty("ANTHROPIC_BASE_URL"); + expect(environment).not.toHaveProperty("ANTHROPIC_CUSTOM_HEADERS"); + expect(environment).toHaveProperty("CLAUDE_CONFIG_DIR"); + throw new Error("disableAllHooks"); + }, + }); + + expect(queryFactory).not.toHaveBeenCalled(); + expect(result.terminal).toBe("policy_violation"); + expect(result.policyHookCoverage).toBe(false); + expect(result.queryClosed).toBe(false); + expect(result.workspaceChanges).toEqual([]); + expect( + result.events.filter(({ type }) => type === "terminal"), + ).toHaveLength(1); + }); + + it("preserves teardown failure priority when policy preflight fails", async () => { + const { config } = await probeConfig(); + const observer = fakeObserver({ + quiescent: false, + deadlineMet: false, + elapsedMs: 5_001, + observedPids: [8001], + alivePidsAtDeadline: [8001], + emergencyCleanupAttempted: false, + }); + const result = await runManagedAgentProbe(config, { + hermeticGatewayOrigin: config.gatewayOrigin, + processObserver: observer, + queryFactory: vi.fn(() => queryFromEvents([])), + policySettingsGuard: async () => { + throw new Error("disableAllHooks"); + }, + }); + + expect(result.terminal).toBe("teardown_timeout"); + expect(result.events.at(-1)).toMatchObject({ + type: "terminal", + terminal: "teardown_timeout", + }); + expect(observer.emergencyCleanup).toHaveBeenCalledWith([8001]); + }); + + it("rejects a successful stream when a requested tool has no primary hook decision", async () => { + const { config } = await probeConfig(); + const result = await runManagedAgentProbe(config, { + hermeticGatewayOrigin: config.gatewayOrigin, + processObserver: fakeObserver(), + policySettingsGuard: async () => undefined, + queryFactory: () => + queryFromEvents([ + { + type: "assistant", + message: { + id: "message-with-disabled-hook", + content: [ + { + type: "tool_use", + id: "tool-with-disabled-hook", + name: "Read", + input: { file_path: FIXTURE_PATHS.cleanTarget }, + }, + ], + }, + }, + { + type: "result", + subtype: "success", + is_error: false, + num_turns: 1, + }, + ]), + }); + + expect(result.policyHookCoverage).toBe(false); + expect(result.terminal).toBe("policy_violation"); + expect(result.permissionEvidence).toEqual([]); + }); + + it("rejects duplicate requested tool ids instead of reusing one policy decision", async () => { + const { config } = await probeConfig(); + const duplicateToolUseId = "duplicate-tool-use-id"; + const result = await runManagedAgentProbe(config, { + hermeticGatewayOrigin: config.gatewayOrigin, + processObserver: fakeObserver(), + policySettingsGuard: async () => undefined, + queryFactory: ({ options }) => ({ + async *[Symbol.asyncIterator]() { + yield { + type: "assistant", + message: { + id: "duplicate-tool-message-1", + content: [ + { + type: "tool_use", + id: duplicateToolUseId, + name: "Read", + input: { file_path: FIXTURE_PATHS.cleanTarget }, + }, + ], + }, + }; + await invokePreToolUse(options, { + toolName: "Read", + toolInput: { file_path: FIXTURE_PATHS.cleanTarget }, + toolUseId: duplicateToolUseId, + }); + yield { + type: "assistant", + message: { + id: "duplicate-tool-message-2", + content: [ + { + type: "tool_use", + id: duplicateToolUseId, + name: "Bash", + input: { command: "touch must-not-inherit-allow" }, + }, + ], + }, + }; + await invokePreToolUse(options, { + toolName: "Bash", + toolInput: { command: "touch must-not-inherit-allow" }, + toolUseId: duplicateToolUseId, + }); + yield { + type: "result", + subtype: "success", + is_error: false, + num_turns: 2, + }; + }, + close: vi.fn(), + }), + }); + + expect(result.permissionEvidence).toHaveLength(1); + expect(result.policyHookCoverage).toBe(false); + expect(result.terminal).toBe("policy_violation"); + }); + it("redacts malicious SDK and permission identifiers from the complete result", async () => { const { config } = await probeConfig(); const sessionSecret = "session-secret-injected-by-sdk"; @@ -321,15 +529,6 @@ describe("runManagedAgentProbe", () => { processObserver: fakeObserver(), queryFactory: ({ options }) => ({ async *[Symbol.asyncIterator]() { - await options.canUseTool?.( - permissionNameSecret, - {}, - { - signal: new AbortController().signal, - toolUseID: permissionIdSecret, - requestId: "request-id-not-persisted", - }, - ); yield { type: "system", subtype: "init", @@ -339,6 +538,7 @@ describe("runManagedAgentProbe", () => { type: "assistant", session_id: sessionSecret, message: { + id: permissionIdSecret, content: [ { type: "tool_use", @@ -349,6 +549,11 @@ describe("runManagedAgentProbe", () => { ], }, }; + await invokePreToolUse(options, { + toolName: permissionNameSecret, + toolInput: { secret: "tool-input-secret" }, + toolUseId: toolIdSecret, + }); yield { type: "user", session_id: sessionSecret, @@ -367,6 +572,7 @@ describe("runManagedAgentProbe", () => { subtype: "success", is_error: false, session_id: sessionSecret, + num_turns: 1, }; }, close: vi.fn(), @@ -382,7 +588,12 @@ describe("runManagedAgentProbe", () => { result.toolEvidence[1]?.toolUseId, ); expect(result.permissionEvidence).toMatchObject([ - { toolName: "unknown", decision: "deny", reason: "tool_not_allowed" }, + { + toolName: "unknown", + decision: "deny", + reason: "tool_not_allowed", + source: "pre_tool_use", + }, ]); const serialized = JSON.stringify(result); for (const secret of [ @@ -391,7 +602,6 @@ describe("runManagedAgentProbe", () => { toolNameSecret, permissionIdSecret, permissionNameSecret, - "request-id-not-persisted", "tool-input-secret", "tool-result-secret", ]) { diff --git a/packages/harness/src/experimental/managed-agent-spike/runtime.ts b/packages/harness/src/experimental/managed-agent-spike/runtime.ts index 2059b4a4b..81091dc87 100644 --- a/packages/harness/src/experimental/managed-agent-spike/runtime.ts +++ b/packages/harness/src/experimental/managed-agent-spike/runtime.ts @@ -20,9 +20,13 @@ import { import { MANAGED_AGENT_BUILTIN_TOOLS, MANAGED_AGENT_DISALLOWED_TOOLS, - createManagedAgentPermissionHandler, + createManagedAgentPolicyBoundary, } from "./permissions.js"; import { createLocalManagedAgentProcessObserver } from "./process-observer.js"; +import { + assertManagedAgentHooksEnabled, + buildManagedAgentSettingsGuardEnvironment, +} from "./settings-guard.js"; import type { ManagedAgentProbeConfig, ManagedAgentProbeDependencies, @@ -35,6 +39,8 @@ import type { export const MANAGED_AGENT_MCP_SERVER_NAME = "sapiom-managed-agent-spike"; export const MANAGED_AGENT_TEARDOWN_TIMEOUT_MS = 5_000; +export const MANAGED_AGENT_CORRELATION_MARKER_VERSION = + "SAPIOM_CERTIFICATION_CORRELATION_V1"; const QUERY_CLOSE_TIMEOUT_MS = 2_000; type McpToolName = "echo_nonce" | "fail_once"; @@ -148,6 +154,8 @@ function defaultQueryFactory(input: { readonly prompt: string; readonly options: Options; }): ManagedAgentQuery { + // The narrow return type intentionally withholds control-channel methods, + // especially Query.mcpCall(), because those calls bypass permission checks. return agentSdkQuery(input); } @@ -162,6 +170,48 @@ function safeEvalSource( .replace(/^-|-$/g, ""); } +export function buildManagedAgentCorrelationPrompt(input: { + readonly prompt: string; + readonly evalSource: string; + readonly executionId: string; +}): string { + const marker = [ + MANAGED_AGENT_CORRELATION_MARKER_VERSION, + `eval_source=${input.evalSource}`, + `execution_id=${input.executionId}`, + ].join(";"); + return [ + marker, + "This is a non-secret certification marker. Do not repeat it.", + input.prompt, + ].join("\n"); +} + +function hasUniversalPolicyHookCoverage( + toolEvidence: readonly ManagedAgentToolEvidence[], + permissionEvidence: ManagedAgentProbeResult["permissionEvidence"], +): boolean { + const requested = toolEvidence.filter(({ status }) => status === "requested"); + const requestedIds = requested.flatMap(({ toolUseId }) => + toolUseId ? [toolUseId] : [], + ); + if ( + requestedIds.length !== requested.length || + new Set(requestedIds).size !== requestedIds.length + ) { + return false; + } + return requestedIds.every((toolUseId) => { + return ( + permissionEvidence.filter( + (evidence) => + evidence.toolUseId === toolUseId && + evidence.source === "pre_tool_use", + ).length === 1 + ); + }); +} + async function closeQueryBounded(query: ManagedAgentQuery): Promise { let timeout: NodeJS.Timeout | undefined; try { @@ -241,6 +291,7 @@ export async function runManagedAgentProbe( let queryFailed = false; let queryClosed = false; let cancellationTriggerFailed = false; + let policyPreflightFailed = false; const childEnvironment = buildManagedAgentChildEnvironment({ ambient: process.env, @@ -251,7 +302,7 @@ export async function runManagedAgentProbe( evalSource, executionId, }); - const permissionHandler = createManagedAgentPermissionHandler({ + const policyBoundary = createManagedAgentPolicyBoundary({ canonicalWorkspaceRoot: validated.canonicalWorkspaceRoot, allowedBashCommands: config.allowedBashCommands, allowedMcpTools: mcpRuntime.qualifiedToolNames, @@ -262,11 +313,22 @@ export async function runManagedAgentProbe( const options: Options = { abortController, - canUseTool: permissionHandler, + // PreToolUse is the universal boundary. canUseTool only handles an + // unresolved SDK permission as defense in depth; the shared evaluator + // deduplicates its evidence by tool-use ID. + canUseTool: policyBoundary.canUseToolFallback, cwd: validated.canonicalWorkspaceRoot, disallowedTools: [...MANAGED_AGENT_DISALLOWED_TOOLS], env: childEnvironment, includePartialMessages: false, + hooks: { + PreToolUse: [ + { + hooks: [policyBoundary.preToolUseHook], + timeout: 5, + }, + ], + }, maxBudgetUsd: config.maxBudgetUsd, maxTurns: config.maxTurns, mcpServers: { [MANAGED_AGENT_MCP_SERVER_NAME]: mcpRuntime.server }, @@ -289,40 +351,62 @@ export async function runManagedAgentProbe( let teardown!: ManagedAgentTeardownObservation; let terminal!: ManagedAgentTerminalClassification; + let policyHookCoverage = false; try { recorder.recordLifecycle("starting"); - const cancellationTask = dependencies.waitForCancellationSignal - ? dependencies - .waitForCancellationSignal(triggerController.signal) - .then(() => { - if (triggerController.signal.aborted) return; - cancellationRequested = true; - cancellationRequestedAt = (dependencies.now ?? Date.now)(); - recorder.recordLifecycle("cancellation_requested"); - abortController.abort(); - }) - .catch(() => { - if (!triggerController.signal.aborted) { - cancellationTriggerFailed = true; - abortController.abort(); - } - }) - : undefined; - try { - query = (dependencies.queryFactory ?? defaultQueryFactory)({ - prompt: config.prompt, - options, + await ( + dependencies.policySettingsGuard ?? assertManagedAgentHooksEnabled + )({ + cwd: validated.canonicalWorkspaceRoot, + environment: + buildManagedAgentSettingsGuardEnvironment(childEnvironment), }); - for await (const event of query) recorder.observeSdkEvent(event); } catch { - if (!abortController.signal.aborted) queryFailed = true; - } finally { + policyPreflightFailed = true; + recorder.recordLifecycle("policy_preflight_failed"); triggerController.abort(); - if (cancellationTask) await cancellationTask; - queryFailed ||= cancellationTriggerFailed; - if (query) queryClosed = await closeQueryBounded(query); - if ((query && !queryClosed) || queryFailed) abortController.abort(); + abortController.abort(); + } + const cancellationTask = + !policyPreflightFailed && dependencies.waitForCancellationSignal + ? dependencies + .waitForCancellationSignal(triggerController.signal) + .then(() => { + if (triggerController.signal.aborted) return; + cancellationRequested = true; + cancellationRequestedAt = (dependencies.now ?? Date.now)(); + recorder.recordLifecycle("cancellation_requested"); + abortController.abort(); + }) + .catch(() => { + if (!triggerController.signal.aborted) { + cancellationTriggerFailed = true; + abortController.abort(); + } + }) + : undefined; + + if (!policyPreflightFailed) { + try { + query = (dependencies.queryFactory ?? defaultQueryFactory)({ + prompt: buildManagedAgentCorrelationPrompt({ + prompt: config.prompt, + evalSource, + executionId, + }), + options, + }); + for await (const event of query) recorder.observeSdkEvent(event); + } catch { + if (!abortController.signal.aborted) queryFailed = true; + } finally { + triggerController.abort(); + if (cancellationTask) await cancellationTask; + queryFailed ||= cancellationTriggerFailed; + if (query) queryClosed = await closeQueryBounded(query); + if ((query && !queryClosed) || queryFailed) abortController.abort(); + } } const now = dependencies.now ?? Date.now; @@ -353,6 +437,26 @@ export async function runManagedAgentProbe( queryFailed, sdkResult: recorder.result, }); + if ( + policyPreflightFailed && + terminal !== "teardown_timeout" && + terminal !== "close_timeout" + ) { + terminal = "policy_violation"; + } + policyHookCoverage = + !policyPreflightFailed && + hasUniversalPolicyHookCoverage( + recorder.toolEvidence, + recorder.permissionEvidence, + ); + if ( + !policyHookCoverage && + terminal !== "teardown_timeout" && + terminal !== "close_timeout" + ) { + terminal = "policy_violation"; + } recorder.recordTerminal(terminal); if (!teardown.quiescent) { @@ -374,6 +478,11 @@ export async function runManagedAgentProbe( target: config.target, modelAlias: validated.model.alias, ...(recorder.sessionId ? { sdkSessionId: recorder.sessionId } : {}), + inferenceTurns: recorder.inferenceTurns, + ...(recorder.sdkNumTurns === undefined + ? {} + : { sdkNumTurns: recorder.sdkNumTurns }), + policyHookCoverage, terminal, events: [...recorder.events], toolEvidence: [...recorder.toolEvidence, ...mcpRuntime.invocations], @@ -387,7 +496,7 @@ export async function runManagedAgentProbe( cancellationRequested, queryClosed, teardown, - correlation: { executionId, evalSource }, + correlation: { executionId, evalSource, promptEmbedded: true }, ...(recorder.usage ? { sdkUsage: recorder.usage } : {}), }; } diff --git a/packages/harness/src/experimental/managed-agent-spike/settings-guard.test.ts b/packages/harness/src/experimental/managed-agent-spike/settings-guard.test.ts new file mode 100644 index 000000000..8a278c20d --- /dev/null +++ b/packages/harness/src/experimental/managed-agent-spike/settings-guard.test.ts @@ -0,0 +1,98 @@ +import { describe, expect, it } from "vitest"; + +import { + ManagedAgentSettingsGuardError, + assertManagedAgentHooksEnabled, + buildManagedAgentSettingsGuardEnvironment, +} from "./settings-guard.js"; + +describe("managed-agent settings guard", () => { + it("removes gateway and credential inputs before resolution", () => { + expect( + buildManagedAgentSettingsGuardEnvironment({ + PATH: "/safe/bin", + HOME: "/isolated/home", + CLAUDE_CONFIG_DIR: "/isolated/claude", + ANTHROPIC_API_KEY: "credential", + ANTHROPIC_AUTH_TOKEN: "auth-token", + ANTHROPIC_BASE_URL: "https://gateway.invalid", + ANTHROPIC_CUSTOM_HEADERS: "x-secret: value", + }), + ).toEqual({ + PATH: "/safe/bin", + HOME: "/isolated/home", + CLAUDE_CONFIG_DIR: "/isolated/claude", + }); + }); + + it("accepts only the exact enabled contract", async () => { + const input = { cwd: process.cwd(), environment: {} }; + await expect( + assertManagedAgentHooksEnabled(input, { + run: async () => ({ + stdout: JSON.stringify({ + contractVersion: 1, + disableAllHooks: false, + policyHelperConfigured: false, + }), + }), + }), + ).resolves.toBeUndefined(); + + for (const stdout of [ + "not-json", + "{}", + JSON.stringify({ + contractVersion: 1, + disableAllHooks: "false", + policyHelperConfigured: false, + }), + JSON.stringify({ + contractVersion: 1, + disableAllHooks: false, + policyHelperConfigured: false, + unexpected: true, + }), + ]) { + await expect( + assertManagedAgentHooksEnabled(input, { + run: async () => ({ stdout }), + }), + ).rejects.toBeInstanceOf(ManagedAgentSettingsGuardError); + } + }); + + it("fails closed on disabled hooks or resolution errors", async () => { + const input = { cwd: process.cwd(), environment: {} }; + await expect( + assertManagedAgentHooksEnabled(input, { + run: async () => ({ + stdout: JSON.stringify({ + contractVersion: 1, + disableAllHooks: true, + policyHelperConfigured: false, + }), + }), + }), + ).rejects.toThrow("disabled by managed settings"); + await expect( + assertManagedAgentHooksEnabled(input, { + run: async () => { + throw new Error("private resolver error"); + }, + }), + ).rejects.toThrow("could not be resolved"); + + await expect( + assertManagedAgentHooksEnabled(input, { + run: async () => ({ + stdout: JSON.stringify({ + contractVersion: 1, + disableAllHooks: false, + policyHelperConfigured: true, + }), + }), + }), + ).rejects.toThrow("unresolved policy helper"); + }); +}); diff --git a/packages/harness/src/experimental/managed-agent-spike/settings-guard.ts b/packages/harness/src/experimental/managed-agent-spike/settings-guard.ts new file mode 100644 index 000000000..982cc5d63 --- /dev/null +++ b/packages/harness/src/experimental/managed-agent-spike/settings-guard.ts @@ -0,0 +1,174 @@ +import { execFile } from "node:child_process"; +import { createRequire } from "node:module"; +import { promisify } from "node:util"; +import { pathToFileURL } from "node:url"; + +const execFileAsync = promisify(execFile); +const SETTINGS_GUARD_TIMEOUT_MS = 5_000; +const SETTINGS_GUARD_MODULE_ENV = "SAPIOM_MANAGED_AGENT_SETTINGS_SDK_URL"; +const SETTINGS_GUARD_SCRIPT = ` +const moduleUrl = process.env.${SETTINGS_GUARD_MODULE_ENV}; +if (!moduleUrl) throw new Error("missing sdk module url"); +const { resolveSettings } = await import(moduleUrl); +const resolved = await resolveSettings({ cwd: process.cwd(), settingSources: [] }); +const isRecord = (candidate) => + typeof candidate === "object" && candidate !== null && !Array.isArray(candidate); +if ( + !isRecord(resolved) || + !isRecord(resolved.effective) || + !Array.isArray(resolved.sources) || + !resolved.sources.every((source) => isRecord(source) && isRecord(source.settings)) +) { + throw new Error("malformed resolved settings"); +} +const value = resolved?.effective?.disableAllHooks; +const settings = [resolved.effective, ...resolved.sources.map((source) => source.settings)]; +const policyHelperConfigured = settings.some((candidate) => + candidate && (candidate.policyHelper !== undefined || candidate.policyHelpers !== undefined) +); +if (value !== undefined && typeof value !== "boolean") { + throw new Error("malformed disableAllHooks setting"); +} +process.stdout.write(JSON.stringify({ + contractVersion: 1, + disableAllHooks: value === true, + policyHelperConfigured, +})); +`; + +const CREDENTIAL_AND_GATEWAY_ENVIRONMENT = new Set([ + "ANTHROPIC_API_KEY", + "ANTHROPIC_AUTH_TOKEN", + "ANTHROPIC_CUSTOM_HEADERS", + "ANTHROPIC_BASE_URL", +]); + +export class ManagedAgentSettingsGuardError extends Error { + public constructor(message: string) { + super(message); + this.name = "ManagedAgentSettingsGuardError"; + } +} + +export interface ManagedAgentSettingsGuardInput { + readonly cwd: string; + readonly environment: Readonly>; + /** + * Explicit Node executable seam for future packaged hosts. E0.4 intentionally + * does not certify Electron-as-Node; E0.7 owns that packaging proof. + */ + readonly nodeExecutable?: string; +} + +export interface ManagedAgentSettingsGuardDependencies { + readonly run?: (input: { + readonly cwd: string; + readonly environment: Readonly>; + readonly nodeExecutable?: string; + }) => Promise<{ readonly stdout: string }>; +} + +export function buildManagedAgentSettingsGuardEnvironment( + childEnvironment: Readonly>, +): Record { + return Object.fromEntries( + Object.entries(childEnvironment).filter( + ([name]) => !CREDENTIAL_AND_GATEWAY_ENVIRONMENT.has(name), + ), + ); +} + +async function runSettingsResolver(input: { + readonly cwd: string; + readonly environment: Readonly>; + readonly nodeExecutable?: string; +}): Promise<{ readonly stdout: string }> { + const sdkModulePath = createRequire(import.meta.url).resolve( + "@anthropic-ai/claude-agent-sdk", + ); + const environment = { + ...input.environment, + [SETTINGS_GUARD_MODULE_ENV]: pathToFileURL(sdkModulePath).href, + }; + try { + const result = await execFileAsync( + input.nodeExecutable ?? process.execPath, + ["--input-type=module", "--eval", SETTINGS_GUARD_SCRIPT], + { + cwd: input.cwd, + env: environment, + timeout: SETTINGS_GUARD_TIMEOUT_MS, + maxBuffer: 4_096, + windowsHide: true, + }, + ); + return { stdout: result.stdout }; + } catch { + throw new ManagedAgentSettingsGuardError( + "Managed-agent hook settings could not be resolved", + ); + } +} + +/** + * Resolve managed settings in an isolated, credential-free subprocess before + * query construction. Any uncertain result fails closed. + */ +export async function assertManagedAgentHooksEnabled( + input: ManagedAgentSettingsGuardInput, + dependencies: ManagedAgentSettingsGuardDependencies = {}, +): Promise { + if (process.versions.electron && !input.nodeExecutable) { + throw new ManagedAgentSettingsGuardError( + "Managed-agent settings guard requires an explicit Node executable in Electron", + ); + } + let stdout: string; + try { + ({ stdout } = await (dependencies.run ?? runSettingsResolver)(input)); + } catch (error) { + if (error instanceof ManagedAgentSettingsGuardError) throw error; + throw new ManagedAgentSettingsGuardError( + "Managed-agent hook settings could not be resolved", + ); + } + + let parsed: unknown; + try { + parsed = JSON.parse(stdout); + } catch { + throw new ManagedAgentSettingsGuardError( + "Managed-agent hook settings result was malformed", + ); + } + const result = + typeof parsed === "object" && parsed !== null && !Array.isArray(parsed) + ? (parsed as Record) + : undefined; + if ( + !result || + result.contractVersion !== 1 || + typeof result.disableAllHooks !== "boolean" || + typeof result.policyHelperConfigured !== "boolean" || + Object.keys(result).some( + (key) => + key !== "contractVersion" && + key !== "disableAllHooks" && + key !== "policyHelperConfigured", + ) + ) { + throw new ManagedAgentSettingsGuardError( + "Managed-agent hook settings result was malformed", + ); + } + if (result.disableAllHooks) { + throw new ManagedAgentSettingsGuardError( + "Managed-agent hooks are disabled by managed settings", + ); + } + if (result.policyHelperConfigured) { + throw new ManagedAgentSettingsGuardError( + "Managed-agent hook settings use an unresolved policy helper", + ); + } +} diff --git a/packages/harness/src/experimental/managed-agent-spike/types.ts b/packages/harness/src/experimental/managed-agent-spike/types.ts index 5c48e78e8..bf88d6cce 100644 --- a/packages/harness/src/experimental/managed-agent-spike/types.ts +++ b/packages/harness/src/experimental/managed-agent-spike/types.ts @@ -39,12 +39,17 @@ export type ManagedAgentPermissionReason = | "fixture_path" | "exact_bash_command" | "managed_mcp_tool" + | "policy_aborted" | "invalid_input" | "path_outside_workspace" | "path_symlink_escape" | "bash_command_not_allowed" | "tool_not_allowed"; +export type ManagedAgentPermissionSource = + | "pre_tool_use" + | "can_use_tool_fallback"; + export type ManagedAgentProbeEventType = | "lifecycle" | "message" @@ -68,6 +73,7 @@ export interface ManagedAgentProbeEvent { readonly toolName?: string; readonly permissionDecision?: ManagedAgentPermissionDecision; readonly permissionReason?: ManagedAgentPermissionReason; + readonly permissionSource?: ManagedAgentPermissionSource; readonly isError?: boolean; readonly terminal?: ManagedAgentTerminalClassification; } @@ -102,6 +108,7 @@ export interface ManagedAgentPermissionEvidence { readonly toolName: string; readonly decision: ManagedAgentPermissionDecision; readonly reason: ManagedAgentPermissionReason; + readonly source: ManagedAgentPermissionSource; } export interface ManagedAgentTeardownObservation { @@ -118,6 +125,7 @@ export type ManagedAgentTerminalClassification = | "cancelled" | "sdk_result_error" | "query_error" + | "policy_violation" | "incomplete" | "close_timeout" | "teardown_timeout"; @@ -129,6 +137,12 @@ export interface ManagedAgentProbeResult { readonly target: ManagedAgentModelTargetId; readonly modelAlias: string; readonly sdkSessionId?: string; + /** Distinct, hashed assistant message IDs; authoritative for BQ call count. */ + readonly inferenceTurns: number; + /** SDK result.num_turns; informational and not a gateway reconciliation key. */ + readonly sdkNumTurns?: number; + /** False if any requested tool lacked exactly one primary PreToolUse decision. */ + readonly policyHookCoverage: boolean; readonly terminal: ManagedAgentTerminalClassification; readonly events: readonly ManagedAgentProbeEvent[]; readonly toolEvidence: readonly ManagedAgentToolEvidence[]; @@ -141,10 +155,15 @@ export interface ManagedAgentProbeResult { readonly correlation: { readonly executionId: string; readonly evalSource: string; + readonly promptEmbedded: true; }; readonly sdkUsage?: ManagedAgentSdkUsageEstimate; } +/** + * Deliberately excludes Agent SDK control-channel methods. In particular, + * Query.mcpCall bypasses permission checks and is outside this host boundary. + */ export interface ManagedAgentQuery extends AsyncIterable { close(): void; } @@ -175,4 +194,9 @@ export interface ManagedAgentProbeDependencies { readonly uuid?: () => string; readonly now?: () => number; readonly waitForCancellationSignal?: (signal: AbortSignal) => Promise; + readonly policySettingsGuard?: (input: { + readonly cwd: string; + readonly environment: Readonly>; + readonly nodeExecutable?: string; + }) => Promise; } From 6af3cf68992bd49bdd6761d1044c4dc22f357f17 Mon Sep 17 00:00:00 2001 From: Yash Date: Sun, 16 Aug 2026 17:27:45 -0700 Subject: [PATCH 05/24] fix(harness): resolve managed agent review findings --- .../managed-agent-spike/README.md | 14 ++ .../managed-agent-spike/events.test.ts | 51 +++++++ .../managed-agent-spike/events.ts | 21 ++- .../experimental/managed-agent-spike/index.ts | 7 +- .../managed-agent-spike/permissions.test.ts | 126 ++++++++++++++++++ .../managed-agent-spike/permissions.ts | 44 ++++-- .../runtime-sdk-loopback.test.ts | 82 +++++++++++- .../managed-agent-spike/runtime.test.ts | 54 ++++++++ .../managed-agent-spike/runtime.ts | 28 ++-- .../experimental/managed-agent-spike/types.ts | 3 +- 10 files changed, 406 insertions(+), 24 deletions(-) diff --git a/packages/harness/src/experimental/managed-agent-spike/README.md b/packages/harness/src/experimental/managed-agent-spike/README.md index 7e8b39c74..1f9474cac 100644 --- a/packages/harness/src/experimental/managed-agent-spike/README.md +++ b/packages/harness/src/experimental/managed-agent-spike/README.md @@ -12,6 +12,11 @@ runs before the SDK's permission evaluation, applies canonical-path containment, exact Bash equality, and an MCP allowlist, and returns a complete fresh input object only when allowing the call. Unknown tools fail closed. +The hook also requires a non-empty, bounded `tool_use_id` from the event. When +the SDK supplies the optional callback ID, it must be independently bounded and +exactly match the event ID. Invalid identifiers are denied before policy +evaluation and never become normalized permission evidence. + `canUseTool` remains only as defense in depth for calls the SDK leaves unresolved. It shares the same evaluator and deduplicates by tool-use ID, so it cannot create a second evidence record. A live result is rejected when any @@ -41,6 +46,10 @@ the same non-secret values in the initial prompt as: SAPIOM_CERTIFICATION_CORRELATION_V1;eval_source=;execution_id= ``` +`correlation.promptEmbedded` records whether that marked prompt reached query +construction. It remains false when the settings preflight prevents query +creation. + The production gateway consumes both headers, but its current BigQuery projection persists neither `polsia_eval_source` nor `sapiom_execution_id`. Reconciliation therefore follows the existing E0.2 contract and searches the @@ -50,6 +59,11 @@ in memory; raw or hashed IDs are not emitted. SDK `result.num_turns` is retained separately as bounded informational evidence and is not used as the BigQuery call-count key. +The hermetic pinned-SDK loopback exercises Read, allowed and denied Bash, and a +real in-process `echo_nonce` MCP turn. It requires one primary `PreToolUse` +decision for each request and separately verifies the MCP handler invocation +and SDK tool-result event. + ## Pre-fix live evidence The first Sonnet 5 L1 attempt reached an SDK success result and clean teardown, diff --git a/packages/harness/src/experimental/managed-agent-spike/events.test.ts b/packages/harness/src/experimental/managed-agent-spike/events.test.ts index 7a8add560..136c546a2 100644 --- a/packages/harness/src/experimental/managed-agent-spike/events.test.ts +++ b/packages/harness/src/experimental/managed-agent-spike/events.test.ts @@ -1,6 +1,7 @@ import { describe, expect, it } from "vitest"; import { + MANAGED_AGENT_TOOL_USE_ID_MAX_LENGTH, ManagedAgentEventError, ManagedAgentEventRecorder, normalizeManagedAgentToolUseId, @@ -189,6 +190,56 @@ describe("ManagedAgentEventRecorder", () => { } }); + it("rejects missing, empty, and overlong tool-use identifiers instead of normalizing sentinels", () => { + const invalidIds = [ + undefined, + "", + " ", + "x".repeat(MANAGED_AGENT_TOOL_USE_ID_MAX_LENGTH + 1), + ]; + for (const invalidId of invalidIds) { + expect(() => normalizeManagedAgentToolUseId(invalidId)).toThrow( + ManagedAgentEventError, + ); + + const requested = new ManagedAgentEventRecorder("invalid-requested"); + expect(() => + requested.observeSdkEvent({ + type: "assistant", + message: { + id: "bounded-message-id", + content: [ + { + type: "tool_use", + id: invalidId, + name: "Read", + input: { file_path: "private-path" }, + }, + ], + }, + }), + ).toThrow(ManagedAgentEventError); + expect(requested.toolEvidence).toEqual([]); + + const completed = new ManagedAgentEventRecorder("invalid-completed"); + expect(() => + completed.observeSdkEvent({ + type: "user", + message: { + content: [ + { + type: "tool_result", + tool_use_id: invalidId, + content: "private-result", + }, + ], + }, + }), + ).toThrow(ManagedAgentEventError); + expect(completed.toolEvidence).toEqual([]); + } + }); + it("counts distinct hashed assistant ids and keeps bounded SDK turns separate", () => { const recorder = new ManagedAgentEventRecorder("run-3"); for (const messageId of [ diff --git a/packages/harness/src/experimental/managed-agent-spike/events.ts b/packages/harness/src/experimental/managed-agent-spike/events.ts index f019b4fc3..48fa4fd1e 100644 --- a/packages/harness/src/experimental/managed-agent-spike/events.ts +++ b/packages/harness/src/experimental/managed-agent-spike/events.ts @@ -44,6 +44,7 @@ const NORMALIZED_TOOL_USE_ID_PATTERN = /^tool_[0-9a-f]{64}$/; const SDK_SESSION_ID_PATTERN = /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/; const MAX_ASSISTANT_MESSAGE_ID_LENGTH = 512; +export const MANAGED_AGENT_TOOL_USE_ID_MAX_LENGTH = 512; export class ManagedAgentEventError extends Error { public constructor(message: string) { @@ -57,14 +58,28 @@ export function sanitizeManagedAgentToolName(value: unknown): string { return toolName && SAFE_TOOL_NAMES.has(toolName) ? toolName : "unknown"; } +export function isBoundedManagedAgentToolUseId( + value: unknown, +): value is string { + return ( + typeof value === "string" && + value.trim().length > 0 && + value.length <= MANAGED_AGENT_TOOL_USE_ID_MAX_LENGTH + ); +} + export function normalizeManagedAgentToolUseId(value: unknown): string { - if (typeof value === "string" && NORMALIZED_TOOL_USE_ID_PATTERN.test(value)) { + if (!isBoundedManagedAgentToolUseId(value)) { + throw new ManagedAgentEventError( + "Managed-agent event has no bounded string tool-use id", + ); + } + if (NORMALIZED_TOOL_USE_ID_PATTERN.test(value)) { return value; } - const raw = typeof value === "string" ? value : "invalid-tool-use-id"; return `tool_${createHash("sha256") .update("sapiom-managed-agent-tool-use-id\0") - .update(raw) + .update(value) .digest("hex")}`; } diff --git a/packages/harness/src/experimental/managed-agent-spike/index.ts b/packages/harness/src/experimental/managed-agent-spike/index.ts index 487e16aea..c672d1521 100644 --- a/packages/harness/src/experimental/managed-agent-spike/index.ts +++ b/packages/harness/src/experimental/managed-agent-spike/index.ts @@ -18,7 +18,12 @@ export { type ManagedAgentChildEnvironmentInput, type ManagedAgentIsolatedDirectories, } from "./environment.js"; -export { ManagedAgentEventError, ManagedAgentEventRecorder } from "./events.js"; +export { + MANAGED_AGENT_TOOL_USE_ID_MAX_LENGTH, + ManagedAgentEventError, + ManagedAgentEventRecorder, + isBoundedManagedAgentToolUseId, +} from "./events.js"; export { FIXTURE_PATHS, captureManagedAgentWorkspaceSnapshot, diff --git a/packages/harness/src/experimental/managed-agent-spike/permissions.test.ts b/packages/harness/src/experimental/managed-agent-spike/permissions.test.ts index 27e27954e..5a93f2b70 100644 --- a/packages/harness/src/experimental/managed-agent-spike/permissions.test.ts +++ b/packages/harness/src/experimental/managed-agent-spike/permissions.test.ts @@ -12,6 +12,7 @@ import { join } from "node:path"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import type { PreToolUseHookInput } from "@anthropic-ai/claude-agent-sdk"; +import { MANAGED_AGENT_TOOL_USE_ID_MAX_LENGTH } from "./events.js"; import { createManagedAgentPolicyBoundary, resolveManagedAgentToolPath, @@ -188,6 +189,131 @@ describe("managed-agent universal policy boundary", () => { }); }); + it("rejects malformed hook identifiers before policy evaluation", async () => { + const evidence: ManagedAgentPermissionEvidence[] = []; + const resolveToolPath = vi.fn(async () => join(workspace, "inside.txt")); + const boundary = createManagedAgentPolicyBoundary({ + canonicalWorkspaceRoot: workspace, + allowedBashCommands: [], + allowedMcpTools: [], + onDecision: (decision) => evidence.push(decision), + resolveToolPath, + }); + const signal = new AbortController().signal; + const overlong = "x".repeat(MANAGED_AGENT_TOOL_USE_ID_MAX_LENGTH + 1); + const invalidIdentifiers: ReadonlyArray< + readonly [inputToolUseId: unknown, callbackToolUseId: unknown] + > = [ + [undefined, undefined], + ["", undefined], + [" ", undefined], + [overlong, undefined], + ["valid-input-id", ""], + ["valid-input-id", " "], + ["valid-input-id", overlong], + ["valid-input-id", "mismatched-callback-id"], + ]; + + for (const [inputToolUseId, callbackToolUseId] of invalidIdentifiers) { + const malformedInput = { + ...preToolUseInput("Read", { file_path: "inside.txt" }, "placeholder"), + tool_use_id: inputToolUseId, + } as unknown as PreToolUseHookInput; + await expect( + boundary.preToolUseHook( + malformedInput, + callbackToolUseId as string | undefined, + { signal }, + ), + ).resolves.toMatchObject({ + hookSpecificOutput: { + permissionDecision: "deny", + permissionDecisionReason: expect.stringContaining("invalid_input"), + }, + }); + } + + await expect( + boundary.canUseToolFallback( + "Read", + { file_path: "inside.txt" }, + { signal, toolUseID: "", requestId: "invalid-fallback" }, + ), + ).resolves.toMatchObject({ + behavior: "deny", + message: expect.stringContaining("invalid_input"), + }); + expect(resolveToolPath).not.toHaveBeenCalled(); + expect(evidence).toEqual([]); + + await expect( + boundary.preToolUseHook( + preToolUseInput("Read", { file_path: "inside.txt" }, "valid-input-id"), + undefined, + { signal }, + ), + ).resolves.toMatchObject({ + hookSpecificOutput: { permissionDecision: "allow" }, + }); + expect(resolveToolPath).toHaveBeenCalledOnce(); + expect(evidence).toHaveLength(1); + }); + + it("denies a concurrent duplicate primary identifier without sharing its pending allow", async () => { + const evidence: ManagedAgentPermissionEvidence[] = []; + let releasePathResolution: (() => void) | undefined; + let reportPathResolutionStarted: (() => void) | undefined; + const pathResolutionStarted = new Promise((resolveStarted) => { + reportPathResolutionStarted = resolveStarted; + }); + const releasePath = new Promise((resolvePath) => { + releasePathResolution = resolvePath; + }); + const resolveToolPath = vi.fn(async () => { + reportPathResolutionStarted?.(); + await releasePath; + return join(workspace, "inside.txt"); + }); + const boundary = createManagedAgentPolicyBoundary({ + canonicalWorkspaceRoot: workspace, + allowedBashCommands: [], + allowedMcpTools: [], + onDecision: (decision) => evidence.push(decision), + resolveToolPath, + }); + const signal = new AbortController().signal; + const toolUseId = "concurrent-tool-use-id"; + const first = boundary.preToolUseHook( + preToolUseInput("Read", { file_path: "inside.txt" }, toolUseId), + toolUseId, + { signal }, + ); + await pathResolutionStarted; + const duplicate = boundary.preToolUseHook( + preToolUseInput("Read", { file_path: "inside.txt" }, toolUseId), + toolUseId, + { signal }, + ); + releasePathResolution?.(); + + await expect(first).resolves.toMatchObject({ + hookSpecificOutput: { permissionDecision: "allow" }, + }); + await expect(duplicate).resolves.toMatchObject({ + hookSpecificOutput: { + permissionDecision: "deny", + permissionDecisionReason: expect.stringContaining("invalid_input"), + }, + }); + expect(resolveToolPath).toHaveBeenCalledOnce(); + expect(evidence).toHaveLength(1); + expect(evidence[0]).toMatchObject({ + decision: "allow", + reason: "fixture_path", + source: "pre_tool_use", + }); + }); + it("deduplicates the fallback and records when only the fallback executes", async () => { const evidence: ManagedAgentPermissionEvidence[] = []; const boundary = createManagedAgentPolicyBoundary({ diff --git a/packages/harness/src/experimental/managed-agent-spike/permissions.ts b/packages/harness/src/experimental/managed-agent-spike/permissions.ts index 9986aecc8..3f8ac94be 100644 --- a/packages/harness/src/experimental/managed-agent-spike/permissions.ts +++ b/packages/harness/src/experimental/managed-agent-spike/permissions.ts @@ -13,6 +13,7 @@ import type { ManagedAgentPermissionSource, } from "./types.js"; import { + isBoundedManagedAgentToolUseId, normalizeManagedAgentToolUseId, sanitizeManagedAgentToolName, } from "./events.js"; @@ -321,14 +322,33 @@ export function createManagedAgentPolicyBoundary( ) => { const isPreToolUse = input.hook_event_name === "PreToolUse"; const inputToolUseID = isPreToolUse ? input.tool_use_id : undefined; - const identifiersMatch = - !callbackToolUseID || callbackToolUseID === inputToolUseID; - const toolUseID = - callbackToolUseID ?? inputToolUseID ?? "invalid-tool-use-id"; + if (!isPreToolUse || !isBoundedManagedAgentToolUseId(inputToolUseID)) { + return { + hookSpecificOutput: { + hookEventName: "PreToolUse", + permissionDecision: "deny", + permissionDecisionReason: "Managed-agent policy: invalid_input", + }, + }; + } + if ( + callbackToolUseID !== undefined && + (!isBoundedManagedAgentToolUseId(callbackToolUseID) || + callbackToolUseID !== inputToolUseID) + ) { + return { + hookSpecificOutput: { + hookEventName: "PreToolUse", + permissionDecision: "deny", + permissionDecisionReason: "Managed-agent policy: invalid_input", + }, + }; + } + const toolUseID = inputToolUseID; const policy = await decide( toolUseID, - isPreToolUse ? input.tool_name : "unknown", - isPreToolUse && identifiersMatch ? input.tool_input : undefined, + input.tool_name, + input.tool_input, signal, "pre_tool_use", ); @@ -344,8 +364,15 @@ export function createManagedAgentPolicyBoundary( }; }; - const canUseToolFallback: CanUseTool = async (toolName, input, permission) => - permissionResult( + const canUseToolFallback: CanUseTool = async ( + toolName, + input, + permission, + ) => { + if (!isBoundedManagedAgentToolUseId(permission.toolUseID)) { + return permissionResult(denied("invalid_input"), permission.toolUseID); + } + return permissionResult( await decide( permission.toolUseID, toolName, @@ -355,6 +382,7 @@ export function createManagedAgentPolicyBoundary( ), permission.toolUseID, ); + }; return { preToolUseHook, canUseToolFallback }; } diff --git a/packages/harness/src/experimental/managed-agent-spike/runtime-sdk-loopback.test.ts b/packages/harness/src/experimental/managed-agent-spike/runtime-sdk-loopback.test.ts index 98c692542..c5b4b96f6 100644 --- a/packages/harness/src/experimental/managed-agent-spike/runtime-sdk-loopback.test.ts +++ b/packages/harness/src/experimental/managed-agent-spike/runtime-sdk-loopback.test.ts @@ -10,7 +10,10 @@ import { createManagedAgentFixture, fixturePathExists, } from "./fixture.js"; -import { runManagedAgentProbe } from "./runtime.js"; +import { + qualifiedManagedAgentMcpToolName, + runManagedAgentProbe, +} from "./runtime.js"; const RUN_ID = "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa"; const EXECUTION_ID = "bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb"; @@ -20,12 +23,49 @@ const EVAL_SOURCE = const CORRELATION_MARKER = `SAPIOM_CERTIFICATION_CORRELATION_V1;eval_source=${EVAL_SOURCE};execution_id=${EXECUTION_ID}`; const ALLOWED_BASH_COMMAND = "git status --short"; const DENIED_BASH_COMMAND = "touch denied-side-effect.txt"; +const ECHO_NONCE_TOOL = qualifiedManagedAgentMcpToolName("echo_nonce"); interface LoopbackObservation { readonly headerNames: readonly string[]; readonly evalSourceMatches: boolean; readonly executionIdMatches: boolean; readonly promptMarkerPresent: boolean; + readonly mcpResultMatches: boolean; +} + +function containsExactText(value: unknown, expected: string): boolean { + if (value === expected) return true; + if (Array.isArray(value)) { + return value.some((entry) => containsExactText(entry, expected)); + } + if (typeof value !== "object" || value === null) return false; + return Object.values(value).some((entry) => + containsExactText(entry, expected), + ); +} + +function hasSuccessfulMcpResult(body: string, expectedNonce: string): boolean { + try { + const payload = JSON.parse(body) as { messages?: unknown }; + if (!Array.isArray(payload.messages)) return false; + return payload.messages.some((message) => { + if (typeof message !== "object" || message === null) return false; + const content = (message as { content?: unknown }).content; + if (!Array.isArray(content)) return false; + return content.some((block) => { + if (typeof block !== "object" || block === null) return false; + const result = block as Record; + return ( + result.type === "tool_result" && + result.tool_use_id === "toolu_loopback_mcp_echo" && + result.is_error !== true && + containsExactText(result.content, expectedNonce) + ); + }); + }); + } catch { + return false; + } } function writeSseEvent( @@ -136,7 +176,7 @@ function writeFinalResponse(response: ServerResponse, turn: number): void { response.end(); } -it("enforces every real-SDK Read/Bash call and carries exact correlation through loopback", async () => { +it("enforces real-SDK built-in and in-process MCP calls with exact loopback correlation", async () => { const fixture = await createManagedAgentFixture(() => "loopback-nonce"); const observations: LoopbackObservation[] = []; let helloCount = 0; @@ -171,6 +211,7 @@ it("enforces every real-SDK Read/Bash call and carries exact correlation through executionIdMatches: request.headers["x-sapiom-execution-id"] === EXECUTION_ID, promptMarkerPresent: body.includes(CORRELATION_MARKER), + mcpResultMatches: hasSuccessfulMcpResult(body, fixture.nonce), }); if (inferenceTurn === 1) { writeToolUseResponse(response, inferenceTurn, { @@ -190,6 +231,12 @@ it("enforces every real-SDK Read/Bash call and carries exact correlation through name: "Bash", input: { command: DENIED_BASH_COMMAND }, }); + } else if (inferenceTurn === 4) { + writeToolUseResponse(response, inferenceTurn, { + id: "toolu_loopback_mcp_echo", + name: ECHO_NONCE_TOOL, + input: { nonce: fixture.nonce }, + }); } else { writeFinalResponse(response, inferenceTurn); } @@ -234,7 +281,7 @@ it("enforces every real-SDK Read/Bash call and carries exact correlation through ); expect(helloCount).toBeGreaterThanOrEqual(1); - expect(observations).toHaveLength(4); + expect(observations).toHaveLength(5); expect( observations.every( ({ headerNames, evalSourceMatches, executionIdMatches }) => @@ -247,6 +294,9 @@ it("enforces every real-SDK Read/Bash call and carries exact correlation through expect( observations.every(({ promptMarkerPresent }) => promptMarkerPresent), ).toBe(true); + expect( + observations.map(({ mcpResultMatches }) => mcpResultMatches), + ).toEqual([false, false, false, false, true]); expect(result.terminal).toBe("success"); const requested = result.toolEvidence.filter( @@ -256,6 +306,7 @@ it("enforces every real-SDK Read/Bash call and carries exact correlation through "Read", "Bash", "Bash", + ECHO_NONCE_TOOL, ]); for (const tool of requested) { expect( @@ -285,6 +336,12 @@ it("enforces every real-SDK Read/Bash call and carries exact correlation through reason: "bash_command_not_allowed", source: "pre_tool_use", }), + expect.objectContaining({ + toolName: ECHO_NONCE_TOOL, + decision: "allow", + reason: "managed_mcp_tool", + source: "pre_tool_use", + }), ]), ); expect(result.toolEvidence).toEqual( @@ -294,6 +351,25 @@ it("enforces every real-SDK Read/Bash call and carries exact correlation through expect.objectContaining({ toolName: "Bash", status: "error" }), ]), ); + const requestedMcp = requested.find( + ({ toolName }) => toolName === ECHO_NONCE_TOOL, + ); + expect(requestedMcp?.toolUseId).toBeDefined(); + expect( + result.toolEvidence.filter( + ({ toolName, toolUseId, status }) => + toolName === ECHO_NONCE_TOOL && + toolUseId === requestedMcp?.toolUseId && + status === "success", + ), + ).toHaveLength(1); + expect( + result.toolEvidence.filter( + ({ toolName, toolUseId }) => + toolName === ECHO_NONCE_TOOL && toolUseId === undefined, + ), + ).toEqual([{ toolName: ECHO_NONCE_TOOL, status: "success" }]); + expect(result.policyHookCoverage).toBe(true); expect( await fixturePathExists( join(fixture.workspaceRoot, "denied-side-effect.txt"), diff --git a/packages/harness/src/experimental/managed-agent-spike/runtime.test.ts b/packages/harness/src/experimental/managed-agent-spike/runtime.test.ts index 7f0ad82ec..74759534e 100644 --- a/packages/harness/src/experimental/managed-agent-spike/runtime.test.ts +++ b/packages/harness/src/experimental/managed-agent-spike/runtime.test.ts @@ -385,12 +385,29 @@ describe("runManagedAgentProbe", () => { expect(result.terminal).toBe("policy_violation"); expect(result.policyHookCoverage).toBe(false); expect(result.queryClosed).toBe(false); + expect(result.correlation.promptEmbedded).toBe(false); expect(result.workspaceChanges).toEqual([]); expect( result.events.filter(({ type }) => type === "terminal"), ).toHaveLength(1); }); + it("does not claim prompt embedding when query construction fails", async () => { + const { config } = await probeConfig(); + const result = await runManagedAgentProbe(config, { + hermeticGatewayOrigin: config.gatewayOrigin, + processObserver: fakeObserver(), + policySettingsGuard: async () => undefined, + queryFactory: () => { + throw new Error("synthetic query construction failure"); + }, + }); + + expect(result.correlation.promptEmbedded).toBe(false); + expect(result.queryClosed).toBe(false); + expect(result.terminal).toBe("query_error"); + }); + it("preserves teardown failure priority when policy preflight fails", async () => { const { config } = await probeConfig(); const observer = fakeObserver({ @@ -454,6 +471,43 @@ describe("runManagedAgentProbe", () => { expect(result.permissionEvidence).toEqual([]); }); + it("cannot certify a malformed SDK tool-use identifier as policy-covered evidence", async () => { + const { config } = await probeConfig(); + const result = await runManagedAgentProbe(config, { + hermeticGatewayOrigin: config.gatewayOrigin, + processObserver: fakeObserver(), + policySettingsGuard: async () => undefined, + queryFactory: () => + queryFromEvents([ + { + type: "assistant", + message: { + id: "message-with-missing-tool-id", + content: [ + { + type: "tool_use", + name: "Read", + input: { file_path: FIXTURE_PATHS.cleanTarget }, + }, + ], + }, + }, + { + type: "result", + subtype: "success", + is_error: false, + num_turns: 1, + }, + ]), + }); + + expect(result.correlation.promptEmbedded).toBe(true); + expect(result.toolEvidence).toEqual([]); + expect(result.permissionEvidence).toEqual([]); + expect(result.policyHookCoverage).toBe(false); + expect(result.terminal).toBe("policy_violation"); + }); + it("rejects duplicate requested tool ids instead of reusing one policy decision", async () => { const { config } = await probeConfig(); const duplicateToolUseId = "duplicate-tool-use-id"; diff --git a/packages/harness/src/experimental/managed-agent-spike/runtime.ts b/packages/harness/src/experimental/managed-agent-spike/runtime.ts index 81091dc87..6a6b32031 100644 --- a/packages/harness/src/experimental/managed-agent-spike/runtime.ts +++ b/packages/harness/src/experimental/managed-agent-spike/runtime.ts @@ -11,7 +11,7 @@ import { z } from "zod"; import { validateManagedAgentProbeConfig } from "./contract.js"; import { buildManagedAgentChildEnvironment } from "./environment.js"; -import { ManagedAgentEventRecorder } from "./events.js"; +import { ManagedAgentEventError, ManagedAgentEventRecorder } from "./events.js"; import { captureManagedAgentWorkspaceSnapshot, diffManagedAgentWorkspaceSnapshots, @@ -292,6 +292,8 @@ export async function runManagedAgentProbe( let queryClosed = false; let cancellationTriggerFailed = false; let policyPreflightFailed = false; + let promptEmbedded = false; + let eventNormalizationFailed = false; const childEnvironment = buildManagedAgentChildEnvironment({ ambient: process.env, @@ -389,15 +391,24 @@ export async function runManagedAgentProbe( if (!policyPreflightFailed) { try { + const prompt = buildManagedAgentCorrelationPrompt({ + prompt: config.prompt, + evalSource, + executionId, + }); query = (dependencies.queryFactory ?? defaultQueryFactory)({ - prompt: buildManagedAgentCorrelationPrompt({ - prompt: config.prompt, - evalSource, - executionId, - }), + prompt, options, }); - for await (const event of query) recorder.observeSdkEvent(event); + promptEmbedded = true; + for await (const event of query) { + try { + recorder.observeSdkEvent(event); + } catch (error) { + eventNormalizationFailed = error instanceof ManagedAgentEventError; + throw error; + } + } } catch { if (!abortController.signal.aborted) queryFailed = true; } finally { @@ -446,6 +457,7 @@ export async function runManagedAgentProbe( } policyHookCoverage = !policyPreflightFailed && + !eventNormalizationFailed && hasUniversalPolicyHookCoverage( recorder.toolEvidence, recorder.permissionEvidence, @@ -496,7 +508,7 @@ export async function runManagedAgentProbe( cancellationRequested, queryClosed, teardown, - correlation: { executionId, evalSource, promptEmbedded: true }, + correlation: { executionId, evalSource, promptEmbedded }, ...(recorder.usage ? { sdkUsage: recorder.usage } : {}), }; } diff --git a/packages/harness/src/experimental/managed-agent-spike/types.ts b/packages/harness/src/experimental/managed-agent-spike/types.ts index bf88d6cce..982a244ad 100644 --- a/packages/harness/src/experimental/managed-agent-spike/types.ts +++ b/packages/harness/src/experimental/managed-agent-spike/types.ts @@ -155,7 +155,8 @@ export interface ManagedAgentProbeResult { readonly correlation: { readonly executionId: string; readonly evalSource: string; - readonly promptEmbedded: true; + /** True only after the query factory accepts the marked prompt. */ + readonly promptEmbedded: boolean; }; readonly sdkUsage?: ManagedAgentSdkUsageEstimate; } From 043b30e1889a25df9c9ed681cd812139db5ef9f8 Mon Sep 17 00:00:00 2001 From: Yash Date: Sun, 16 Aug 2026 17:36:51 -0700 Subject: [PATCH 06/24] fix(harness): record managed prompt handoff --- .../harness/src/experimental/managed-agent-spike/README.md | 6 +++--- .../src/experimental/managed-agent-spike/runtime.test.ts | 4 ++-- .../harness/src/experimental/managed-agent-spike/runtime.ts | 2 +- .../harness/src/experimental/managed-agent-spike/types.ts | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/packages/harness/src/experimental/managed-agent-spike/README.md b/packages/harness/src/experimental/managed-agent-spike/README.md index 1f9474cac..f7bdddeec 100644 --- a/packages/harness/src/experimental/managed-agent-spike/README.md +++ b/packages/harness/src/experimental/managed-agent-spike/README.md @@ -46,9 +46,9 @@ the same non-secret values in the initial prompt as: SAPIOM_CERTIFICATION_CORRELATION_V1;eval_source=;execution_id= ``` -`correlation.promptEmbedded` records whether that marked prompt reached query -construction. It remains false when the settings preflight prevents query -creation. +`correlation.promptEmbedded` records whether that marked prompt was handed to +the query factory. It remains false when the settings preflight prevents the +factory invocation. The production gateway consumes both headers, but its current BigQuery projection persists neither `polsia_eval_source` nor `sapiom_execution_id`. diff --git a/packages/harness/src/experimental/managed-agent-spike/runtime.test.ts b/packages/harness/src/experimental/managed-agent-spike/runtime.test.ts index 74759534e..3893879d0 100644 --- a/packages/harness/src/experimental/managed-agent-spike/runtime.test.ts +++ b/packages/harness/src/experimental/managed-agent-spike/runtime.test.ts @@ -392,7 +392,7 @@ describe("runManagedAgentProbe", () => { ).toHaveLength(1); }); - it("does not claim prompt embedding when query construction fails", async () => { + it("records prompt delivery when the query factory receives it and throws", async () => { const { config } = await probeConfig(); const result = await runManagedAgentProbe(config, { hermeticGatewayOrigin: config.gatewayOrigin, @@ -403,7 +403,7 @@ describe("runManagedAgentProbe", () => { }, }); - expect(result.correlation.promptEmbedded).toBe(false); + expect(result.correlation.promptEmbedded).toBe(true); expect(result.queryClosed).toBe(false); expect(result.terminal).toBe("query_error"); }); diff --git a/packages/harness/src/experimental/managed-agent-spike/runtime.ts b/packages/harness/src/experimental/managed-agent-spike/runtime.ts index 6a6b32031..608039c8f 100644 --- a/packages/harness/src/experimental/managed-agent-spike/runtime.ts +++ b/packages/harness/src/experimental/managed-agent-spike/runtime.ts @@ -396,11 +396,11 @@ export async function runManagedAgentProbe( evalSource, executionId, }); + promptEmbedded = true; query = (dependencies.queryFactory ?? defaultQueryFactory)({ prompt, options, }); - promptEmbedded = true; for await (const event of query) { try { recorder.observeSdkEvent(event); diff --git a/packages/harness/src/experimental/managed-agent-spike/types.ts b/packages/harness/src/experimental/managed-agent-spike/types.ts index 982a244ad..7c1187cdd 100644 --- a/packages/harness/src/experimental/managed-agent-spike/types.ts +++ b/packages/harness/src/experimental/managed-agent-spike/types.ts @@ -155,7 +155,7 @@ export interface ManagedAgentProbeResult { readonly correlation: { readonly executionId: string; readonly evalSource: string; - /** True only after the query factory accepts the marked prompt. */ + /** True only after the marked prompt is handed to the query factory. */ readonly promptEmbedded: boolean; }; readonly sdkUsage?: ManagedAgentSdkUsageEstimate; From b9d3e77d7f70bd3fb97861c1ff08125c1c21de47 Mon Sep 17 00:00:00 2001 From: Yash Date: Sun, 16 Aug 2026 18:30:15 -0700 Subject: [PATCH 07/24] fix(harness): diagnose managed agent policy failures --- .../managed-agent-spike/events.ts | 36 ++- .../managed-agent-spike/fixture.test.ts | 39 ++++ .../managed-agent-spike/fixture.ts | 27 ++- .../experimental/managed-agent-spike/index.ts | 5 + .../managed-agent-spike/permissions.test.ts | 58 +++++ .../managed-agent-spike/permissions.ts | 95 +++++++- .../managed-agent-spike/probe-cli.test.ts | 6 + .../runtime-sdk-loopback.test.ts | 214 +++++++++++++++++- .../managed-agent-spike/runtime.test.ts | 154 ++++++++++++- .../managed-agent-spike/runtime.ts | 110 ++++++++- .../experimental/managed-agent-spike/types.ts | 53 +++++ 11 files changed, 766 insertions(+), 31 deletions(-) diff --git a/packages/harness/src/experimental/managed-agent-spike/events.ts b/packages/harness/src/experimental/managed-agent-spike/events.ts index 48fa4fd1e..1d9b359e1 100644 --- a/packages/harness/src/experimental/managed-agent-spike/events.ts +++ b/packages/harness/src/experimental/managed-agent-spike/events.ts @@ -2,6 +2,7 @@ import { createHash } from "node:crypto"; import { MANAGED_AGENT_CONTRACT } from "./contract.js"; import type { + ManagedAgentEventNormalizationFailureReason, ManagedAgentPermissionEvidence, ManagedAgentProbeEvent, ManagedAgentSdkUsageEstimate, @@ -47,7 +48,10 @@ const MAX_ASSISTANT_MESSAGE_ID_LENGTH = 512; export const MANAGED_AGENT_TOOL_USE_ID_MAX_LENGTH = 512; export class ManagedAgentEventError extends Error { - public constructor(message: string) { + public constructor( + public readonly reason: ManagedAgentEventNormalizationFailureReason, + message: string, + ) { super(message); this.name = "ManagedAgentEventError"; } @@ -71,6 +75,7 @@ export function isBoundedManagedAgentToolUseId( export function normalizeManagedAgentToolUseId(value: unknown): string { if (!isBoundedManagedAgentToolUseId(value)) { throw new ManagedAgentEventError( + "tool_request_id_invalid", "Managed-agent event has no bounded string tool-use id", ); } @@ -94,6 +99,7 @@ function normalizeAssistantMessageId(value: unknown): string { const messageId = optionalString(value); if (!messageId || messageId.length > MAX_ASSISTANT_MESSAGE_ID_LENGTH) { throw new ManagedAgentEventError( + "assistant_message_id_invalid", "Assistant event has no bounded string message id", ); } @@ -111,6 +117,7 @@ function boundedSdkNumTurns(value: unknown): number | undefined { Number(value) > MANAGED_AGENT_CONTRACT.maxTurns ) { throw new ManagedAgentEventError( + "sdk_num_turns_invalid", `SDK num_turns must be an integer between 0 and ${MANAGED_AGENT_CONTRACT.maxTurns}`, ); } @@ -245,6 +252,7 @@ export class ManagedAgentEventRecorder { this.#inferenceMessageIds.add(normalizeAssistantMessageId(message?.id)); if (this.#inferenceMessageIds.size > MANAGED_AGENT_CONTRACT.maxTurns) { throw new ManagedAgentEventError( + "inference_turn_limit_exceeded", `Distinct assistant message ids exceed ${MANAGED_AGENT_CONTRACT.maxTurns}`, ); } @@ -255,7 +263,18 @@ export class ManagedAgentEventRecorder { if (type === "assistant") { for (const block of blocks) { if (block.type !== "tool_use") continue; - const toolUseId = normalizeManagedAgentToolUseId(block.id); + let toolUseId: string; + try { + toolUseId = normalizeManagedAgentToolUseId(block.id); + } catch (error) { + if (error instanceof ManagedAgentEventError) { + throw new ManagedAgentEventError( + "tool_request_id_invalid", + error.message, + ); + } + throw error; + } const toolName = sanitizeManagedAgentToolName(block.name); this.#toolEvidence.push({ toolUseId, toolName, status: "requested" }); this.#append({ @@ -269,7 +288,18 @@ export class ManagedAgentEventRecorder { if (type === "user") { for (const block of blocks) { if (block.type !== "tool_result") continue; - const toolUseId = normalizeManagedAgentToolUseId(block.tool_use_id); + let toolUseId: string; + try { + toolUseId = normalizeManagedAgentToolUseId(block.tool_use_id); + } catch (error) { + if (error instanceof ManagedAgentEventError) { + throw new ManagedAgentEventError( + "tool_result_id_invalid", + error.message, + ); + } + throw error; + } const isError = block.is_error === true; const matchingTool = [...this.#toolEvidence] .reverse() diff --git a/packages/harness/src/experimental/managed-agent-spike/fixture.test.ts b/packages/harness/src/experimental/managed-agent-spike/fixture.test.ts index fa979a185..47eded2f8 100644 --- a/packages/harness/src/experimental/managed-agent-spike/fixture.test.ts +++ b/packages/harness/src/experimental/managed-agent-spike/fixture.test.ts @@ -37,6 +37,45 @@ describe("managed-agent disposable git fixture", () => { ]); }); + it("renders L1 as eleven exact ordered calls without resolving the escape link", async () => { + const fixture = await createManagedAgentFixture(() => "prompt-contract"); + fixtures.push(fixture); + const prompt = fixture.prompt("L1"); + const numberedLines = prompt + .split("\n") + .filter((line) => /^\d+\./.test(line)); + + expect(numberedLines).toHaveLength(11); + expect(numberedLines.map((line) => Number.parseInt(line, 10))).toEqual([ + 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, + ]); + expect(numberedLines[4]).toContain( + JSON.stringify({ file_path: FIXTURE_PATHS.escapeLink }), + ); + expect(numberedLines[4]).toContain("exact relative path"); + expect(numberedLines[4]).not.toContain(fixture.outsideSentinel); + expect(numberedLines[5]).toContain( + JSON.stringify({ + file_path: FIXTURE_PATHS.cleanTarget, + old_string: "clean target base\n", + new_string: fixture.cleanTargetReplacement, + replace_all: false, + }), + ); + expect(numberedLines[8]).toContain("fail_once"); + expect(numberedLines[9]).toContain("fail_once"); + expect(numberedLines[10]).toContain( + JSON.stringify({ command: fixture.l1BashCommand }), + ); + expect(prompt.split(fixture.outsideSentinel)).toHaveLength(2); + expect(prompt.replace(fixture.outsideSentinel, "")).not.toContain( + fixture.root, + ); + expect(prompt).toContain( + "After call 11 completes, make no further tool calls", + ); + }); + it("observes only relative structural changes and preserves sentinel bytes", async () => { const fixture = await createManagedAgentFixture(() => "fixture-nonce"); fixtures.push(fixture); diff --git a/packages/harness/src/experimental/managed-agent-spike/fixture.ts b/packages/harness/src/experimental/managed-agent-spike/fixture.ts index 6cc7f0250..1cabf9b7b 100644 --- a/packages/harness/src/experimental/managed-agent-spike/fixture.ts +++ b/packages/harness/src/experimental/managed-agent-spike/fixture.ts @@ -207,6 +207,7 @@ export async function createManagedAgentFixture( ]); const nonce = createNonce().split("-").join(""); + const cleanTargetContents = "clean target base\n"; const dirtyBase = "tracked dirty sentinel base\n"; const dirtyContents = "tracked dirty sentinel user change\n"; const untrackedContents = `${nonce}\n`; @@ -218,7 +219,7 @@ export async function createManagedAgentFixture( await Promise.all([ writeFile( join(workspaceRoot, FIXTURE_PATHS.cleanTarget), - "clean target base\n", + cleanTargetContents, ), writeFile(join(workspaceRoot, FIXTURE_PATHS.dirtySentinel), dirtyBase), writeFile( @@ -279,17 +280,21 @@ export async function createManagedAgentFixture( ].join("\n"); } return [ - "Complete this deterministic local-tool probe in order.", - `1. Read ${FIXTURE_PATHS.cleanTarget}, ${FIXTURE_PATHS.dirtySentinel}, and ${FIXTURE_PATHS.untrackedSentinel}.`, - `2. Attempt to Read the absolute outside path ${outsideSentinel}; after denial, continue.`, - `3. Attempt to Read ${FIXTURE_PATHS.escapeLink}; after denial, continue.`, - `4. Edit ${FIXTURE_PATHS.cleanTarget}, replacing its complete current contents with exactly ${JSON.stringify(cleanTargetReplacement)}.`, - `5. Write ${FIXTURE_PATHS.createdTarget} with exactly ${JSON.stringify(createdTargetContents)}.`, - `6. Call echo_nonce with the exact sole line you read from ${FIXTURE_PATHS.untrackedSentinel}, without surrounding whitespace.`, - `7. Call fail_once with that same value; after its planned error, call fail_once once more with the same value.`, - `8. Use Bash with exactly this command: ${l1BashCommand}`, + "Perform exactly these 11 tool calls in numbered order. Make each numbered call exactly once, do not combine calls, and make no unlisted tool call.", + "Use every literal path, argument, and command exactly as written. Do not resolve, normalize, substitute, or retry a path or command. Continue after the two expected Read denials and the first expected fail_once error.", + `1. Call Read with exactly ${JSON.stringify({ file_path: FIXTURE_PATHS.cleanTarget })}.`, + `2. Call Read with exactly ${JSON.stringify({ file_path: FIXTURE_PATHS.dirtySentinel })}.`, + `3. Call Read with exactly ${JSON.stringify({ file_path: FIXTURE_PATHS.untrackedSentinel })}. Save its sole line without surrounding whitespace for calls 8-10.`, + `4. Call Read with exactly ${JSON.stringify({ file_path: outsideSentinel })}. A denial is expected; continue without retrying.`, + `5. Call Read with exactly ${JSON.stringify({ file_path: FIXTURE_PATHS.escapeLink })}. This must remain the exact relative path shown: do not replace it with an absolute path or its symlink target. A denial is expected; continue without retrying.`, + `6. Call Edit with exactly ${JSON.stringify({ file_path: FIXTURE_PATHS.cleanTarget, old_string: cleanTargetContents, new_string: cleanTargetReplacement, replace_all: false })}.`, + `7. Call Write with exactly ${JSON.stringify({ file_path: FIXTURE_PATHS.createdTarget, content: createdTargetContents })}.`, + `8. Call echo_nonce exactly once with the saved line as its nonce argument.`, + `9. Call fail_once with the saved line as its nonce argument. Its planned error is expected; continue.`, + `10. Call fail_once a second and final time with the same nonce argument.`, + `11. Call Bash with exactly ${JSON.stringify({ command: l1BashCommand })}.`, `Never modify ${FIXTURE_PATHS.dirtySentinel} or ${FIXTURE_PATHS.untrackedSentinel}.`, - "Finish with a short confirmation after all steps.", + "After call 11 completes, make no further tool calls and return one short final text confirmation.", ].join("\n"); }, async cleanup() { diff --git a/packages/harness/src/experimental/managed-agent-spike/index.ts b/packages/harness/src/experimental/managed-agent-spike/index.ts index c672d1521..e0a5db076 100644 --- a/packages/harness/src/experimental/managed-agent-spike/index.ts +++ b/packages/harness/src/experimental/managed-agent-spike/index.ts @@ -71,10 +71,13 @@ export { export type { ManagedAgentModelTarget, ManagedAgentModelTargetId, + ManagedAgentEventNormalizationFailureReason, ManagedAgentPermissionDecision, ManagedAgentPermissionEvidence, ManagedAgentPermissionReason, ManagedAgentPermissionSource, + ManagedAgentPolicyDiagnostic, + ManagedAgentPreToolUseGuardRejectionReason, ManagedAgentPreservationObservation, ManagedAgentProbeConfig, ManagedAgentProbeDependencies, @@ -84,10 +87,12 @@ export type { ManagedAgentProbeScenario, ManagedAgentProcessObserver, ManagedAgentQuery, + ManagedAgentQueryExecutionOutcome, ManagedAgentQueryFactory, ManagedAgentSdkUsageEstimate, ManagedAgentTeardownObservation, ManagedAgentTerminalClassification, + ManagedAgentTerminationEvidence, ManagedAgentToolEvidence, ManagedAgentWorkspaceChange, } from "./types.js"; diff --git a/packages/harness/src/experimental/managed-agent-spike/permissions.test.ts b/packages/harness/src/experimental/managed-agent-spike/permissions.test.ts index 5a93f2b70..b32d5058b 100644 --- a/packages/harness/src/experimental/managed-agent-spike/permissions.test.ts +++ b/packages/harness/src/experimental/managed-agent-spike/permissions.test.ts @@ -191,12 +191,18 @@ describe("managed-agent universal policy boundary", () => { it("rejects malformed hook identifiers before policy evaluation", async () => { const evidence: ManagedAgentPermissionEvidence[] = []; + const guardDiagnostics: Array<{ + reason: string; + toolName: string; + normalizedToolUseId?: string; + }> = []; const resolveToolPath = vi.fn(async () => join(workspace, "inside.txt")); const boundary = createManagedAgentPolicyBoundary({ canonicalWorkspaceRoot: workspace, allowedBashCommands: [], allowedMcpTools: [], onDecision: (decision) => evidence.push(decision), + onGuardRejection: (diagnostic) => guardDiagnostics.push(diagnostic), resolveToolPath, }); const signal = new AbortController().signal; @@ -245,6 +251,58 @@ describe("managed-agent universal policy boundary", () => { }); expect(resolveToolPath).not.toHaveBeenCalled(); expect(evidence).toEqual([]); + expect( + guardDiagnostics.map(({ reason, toolName, normalizedToolUseId }) => ({ + reason, + toolName, + correlated: normalizedToolUseId !== undefined, + })), + ).toEqual([ + { + reason: "input_tool_use_id_missing", + toolName: "Read", + correlated: false, + }, + { + reason: "input_tool_use_id_invalid", + toolName: "Read", + correlated: false, + }, + { + reason: "input_tool_use_id_invalid", + toolName: "Read", + correlated: false, + }, + { + reason: "input_tool_use_id_too_long", + toolName: "Read", + correlated: false, + }, + { + reason: "callback_tool_use_id_invalid", + toolName: "Read", + correlated: true, + }, + { + reason: "callback_tool_use_id_invalid", + toolName: "Read", + correlated: true, + }, + { + reason: "callback_tool_use_id_too_long", + toolName: "Read", + correlated: true, + }, + { + reason: "callback_tool_use_id_mismatch", + toolName: "Read", + correlated: true, + }, + ]); + const serializedDiagnostics = JSON.stringify(guardDiagnostics); + expect(serializedDiagnostics).not.toContain("valid-input-id"); + expect(serializedDiagnostics).not.toContain("mismatched-callback-id"); + expect(serializedDiagnostics).not.toContain(overlong); await expect( boundary.preToolUseHook( diff --git a/packages/harness/src/experimental/managed-agent-spike/permissions.ts b/packages/harness/src/experimental/managed-agent-spike/permissions.ts index 3f8ac94be..8d827e294 100644 --- a/packages/harness/src/experimental/managed-agent-spike/permissions.ts +++ b/packages/harness/src/experimental/managed-agent-spike/permissions.ts @@ -11,8 +11,10 @@ import type { ManagedAgentPermissionEvidence, ManagedAgentPermissionReason, ManagedAgentPermissionSource, + ManagedAgentPreToolUseGuardRejectionReason, } from "./types.js"; import { + MANAGED_AGENT_TOOL_USE_ID_MAX_LENGTH, isBoundedManagedAgentToolUseId, normalizeManagedAgentToolUseId, sanitizeManagedAgentToolName, @@ -137,10 +139,20 @@ export interface ManagedAgentPolicyBoundaryOptions { readonly allowedBashCommands: readonly string[]; readonly allowedMcpTools: readonly string[]; readonly onDecision: (evidence: ManagedAgentPermissionEvidence) => void; + readonly onGuardRejection?: ( + diagnostic: ManagedAgentPreToolUseGuardRejection, + ) => void; /** Test seam for proving cancellation after asynchronous path validation. */ readonly resolveToolPath?: typeof resolveManagedAgentToolPath; } +/** Internal correlation is normalized immediately and is removed from output. */ +export interface ManagedAgentPreToolUseGuardRejection { + readonly reason: ManagedAgentPreToolUseGuardRejectionReason; + readonly toolName: string; + readonly normalizedToolUseId?: string; +} + export interface ManagedAgentPolicyBoundary { /** Primary boundary: the SDK runs this before its own permission evaluation. */ readonly preToolUseHook: HookCallback; @@ -158,6 +170,26 @@ interface ManagedAgentRecordedPolicyDecision extends ManagedAgentPolicyDecision readonly source: ManagedAgentPermissionSource; } +function toolUseIdIssue( + value: unknown, + role: "input" | "callback", +): ManagedAgentPreToolUseGuardRejectionReason | undefined { + if (role === "input" && value === undefined) { + return "input_tool_use_id_missing"; + } + if (typeof value !== "string" || value.trim().length === 0) { + return role === "input" + ? "input_tool_use_id_invalid" + : "callback_tool_use_id_invalid"; + } + if (value.length > MANAGED_AGENT_TOOL_USE_ID_MAX_LENGTH) { + return role === "input" + ? "input_tool_use_id_too_long" + : "callback_tool_use_id_too_long"; + } + return undefined; +} + function permissionResult( policy: ManagedAgentPolicyDecision, toolUseID: string, @@ -273,6 +305,22 @@ export function createManagedAgentPolicyBoundary( } >(); + const recordGuardRejection = ( + reason: ManagedAgentPreToolUseGuardRejectionReason, + toolName: unknown, + inputToolUseID?: unknown, + ): void => { + options.onGuardRejection?.({ + reason, + toolName: sanitizeManagedAgentToolName(toolName), + ...(isBoundedManagedAgentToolUseId(inputToolUseID) + ? { + normalizedToolUseId: normalizeManagedAgentToolUseId(inputToolUseID), + } + : {}), + }); + }; + const decide = async ( toolUseID: string, toolName: string, @@ -321,8 +369,8 @@ export function createManagedAgentPolicyBoundary( { signal }, ) => { const isPreToolUse = input.hook_event_name === "PreToolUse"; - const inputToolUseID = isPreToolUse ? input.tool_use_id : undefined; - if (!isPreToolUse || !isBoundedManagedAgentToolUseId(inputToolUseID)) { + if (!isPreToolUse) { + recordGuardRejection("unexpected_hook_event", undefined); return { hookSpecificOutput: { hookEventName: "PreToolUse", @@ -331,11 +379,13 @@ export function createManagedAgentPolicyBoundary( }, }; } - if ( - callbackToolUseID !== undefined && - (!isBoundedManagedAgentToolUseId(callbackToolUseID) || - callbackToolUseID !== inputToolUseID) - ) { + const inputToolUseID = input.tool_use_id; + const inputIssue = toolUseIdIssue(inputToolUseID, "input"); + if (inputIssue || !isBoundedManagedAgentToolUseId(inputToolUseID)) { + recordGuardRejection( + inputIssue ?? "input_tool_use_id_invalid", + input.tool_name, + ); return { hookSpecificOutput: { hookEventName: "PreToolUse", @@ -344,6 +394,37 @@ export function createManagedAgentPolicyBoundary( }, }; } + if (callbackToolUseID !== undefined) { + const callbackIssue = toolUseIdIssue(callbackToolUseID, "callback"); + if (callbackIssue || !isBoundedManagedAgentToolUseId(callbackToolUseID)) { + recordGuardRejection( + callbackIssue ?? "callback_tool_use_id_invalid", + input.tool_name, + inputToolUseID, + ); + return { + hookSpecificOutput: { + hookEventName: "PreToolUse", + permissionDecision: "deny", + permissionDecisionReason: "Managed-agent policy: invalid_input", + }, + }; + } + if (callbackToolUseID !== inputToolUseID) { + recordGuardRejection( + "callback_tool_use_id_mismatch", + input.tool_name, + inputToolUseID, + ); + return { + hookSpecificOutput: { + hookEventName: "PreToolUse", + permissionDecision: "deny", + permissionDecisionReason: "Managed-agent policy: invalid_input", + }, + }; + } + } const toolUseID = inputToolUseID; const policy = await decide( toolUseID, diff --git a/packages/harness/src/experimental/managed-agent-spike/probe-cli.test.ts b/packages/harness/src/experimental/managed-agent-spike/probe-cli.test.ts index 61f1c7bfc..f24d02b82 100644 --- a/packages/harness/src/experimental/managed-agent-spike/probe-cli.test.ts +++ b/packages/harness/src/experimental/managed-agent-spike/probe-cli.test.ts @@ -33,6 +33,11 @@ function passingL1Result(): ManagedAgentProbeResult { sdkNumTurns: 8, policyHookCoverage: true, terminal: "success", + terminationEvidence: { + beforePolicyOverride: "success", + queryExecution: "iteration_completed", + sdkResult: "success", + }, events: [], toolEvidence: [ ...builtins.flatMap((toolName, index) => [ @@ -95,6 +100,7 @@ function passingL1Result(): ManagedAgentProbeResult { source: "pre_tool_use", }, ], + policyDiagnostics: [], workspaceChanges: [ { path: FIXTURE_PATHS.cleanTarget, change: "modified" }, { path: FIXTURE_PATHS.createdTarget, change: "created" }, diff --git a/packages/harness/src/experimental/managed-agent-spike/runtime-sdk-loopback.test.ts b/packages/harness/src/experimental/managed-agent-spike/runtime-sdk-loopback.test.ts index c5b4b96f6..d8363cfbd 100644 --- a/packages/harness/src/experimental/managed-agent-spike/runtime-sdk-loopback.test.ts +++ b/packages/harness/src/experimental/managed-agent-spike/runtime-sdk-loopback.test.ts @@ -1,6 +1,8 @@ +import { readFile } from "node:fs/promises"; import { createServer, type ServerResponse } from "node:http"; +import { createRequire } from "node:module"; import type { AddressInfo } from "node:net"; -import { join } from "node:path"; +import { dirname, join } from "node:path"; import { query as agentSdkQuery } from "@anthropic-ai/claude-agent-sdk"; import { expect, it } from "vitest"; @@ -10,6 +12,7 @@ import { createManagedAgentFixture, fixturePathExists, } from "./fixture.js"; +import { MANAGED_AGENT_CONTRACT } from "./contract.js"; import { qualifiedManagedAgentMcpToolName, runManagedAgentProbe, @@ -24,6 +27,7 @@ const CORRELATION_MARKER = `SAPIOM_CERTIFICATION_CORRELATION_V1;eval_source=${EV const ALLOWED_BASH_COMMAND = "git status --short"; const DENIED_BASH_COMMAND = "touch denied-side-effect.txt"; const ECHO_NONCE_TOOL = qualifiedManagedAgentMcpToolName("echo_nonce"); +const require = createRequire(import.meta.url); interface LoopbackObservation { readonly headerNames: readonly string[]; @@ -68,6 +72,35 @@ function hasSuccessfulMcpResult(body: string, expectedNonce: string): boolean { } } +function hasToolResult( + body: string, + toolUseId: string, + expectedError: boolean, +): boolean { + try { + const payload = JSON.parse(body) as { messages?: unknown }; + if (!Array.isArray(payload.messages)) return false; + return payload.messages.some((message) => { + if (typeof message !== "object" || message === null) return false; + const content = (message as { content?: unknown }).content; + return ( + Array.isArray(content) && + content.some( + (block) => + typeof block === "object" && + block !== null && + (block as Record).type === "tool_result" && + (block as Record).tool_use_id === toolUseId && + ((block as Record).is_error === true) === + expectedError, + ) + ); + }); + } catch { + return false; + } +} + function writeSseEvent( response: ServerResponse, event: string, @@ -383,3 +416,182 @@ it("enforces real-SDK built-in and in-process MCP calls with exact loopback corr await fixture.cleanup(); } }, 45_000); + +it.skipIf( + process.versions.node !== MANAGED_AGENT_CONTRACT.certificationNodeVersion, +)( + "keeps malformed real-SDK Edit requests outside strict primary-hook coverage", + async () => { + const fixture = await createManagedAgentFixture( + () => "loopback-malformed-edit", + ); + const malformedToolUseId = "toolu_loopback_malformed_edit"; + const validToolUseId = "toolu_loopback_valid_edit"; + const observedMalformedError: boolean[] = []; + const observedValidSuccess: boolean[] = []; + let inferenceTurn = 0; + const server = createServer((request, response) => { + if (request.method === "HEAD" && request.url === "/api/hello") { + response.writeHead(200).end(); + return; + } + if ( + request.method !== "POST" || + request.url?.split("?")[0] !== "/v1/messages" + ) { + response.writeHead(404).end(); + return; + } + + let body = ""; + request.setEncoding("utf8"); + request.on("data", (chunk: string) => { + body += chunk; + if (body.length > 2_000_000) request.destroy(); + }); + request.on("end", () => { + inferenceTurn += 1; + observedMalformedError.push( + hasToolResult(body, malformedToolUseId, true), + ); + observedValidSuccess.push(hasToolResult(body, validToolUseId, false)); + if (inferenceTurn === 1) { + writeToolUseResponse(response, inferenceTurn, { + id: malformedToolUseId, + name: "Edit", + input: { + file_path: FIXTURE_PATHS.cleanTarget, + new_string: fixture.cleanTargetReplacement, + }, + }); + } else if (inferenceTurn === 2) { + writeToolUseResponse(response, inferenceTurn, { + id: validToolUseId, + name: "Edit", + input: { + file_path: FIXTURE_PATHS.cleanTarget, + old_string: "clean target base\n", + new_string: fixture.cleanTargetReplacement, + replace_all: false, + }, + }); + } else { + writeFinalResponse(response, inferenceTurn); + } + }); + }); + + await new Promise((resolve, reject) => { + server.once("error", reject); + server.listen(0, "127.0.0.1", resolve); + }); + try { + const sdkPackage = JSON.parse( + await readFile( + join( + dirname(require.resolve("@anthropic-ai/claude-agent-sdk")), + "package.json", + ), + "utf8", + ), + ) as { version?: unknown }; + expect(sdkPackage.version).toBe(MANAGED_AGENT_CONTRACT.agentSdkVersion); + expect(process.versions.node).toBe( + MANAGED_AGENT_CONTRACT.certificationNodeVersion, + ); + + const address = server.address() as AddressInfo; + const ids = [RUN_ID, EXECUTION_ID]; + const result = await runManagedAgentProbe( + { + scenario: "L1", + workspaceRoot: fixture.workspaceRoot, + configRoot: fixture.configRoot, + target: "sonnet-5", + gatewayOrigin: `http://127.0.0.1:${address.port}`, + gatewayCredential: "sk-ant-api03-local-loopback-only", + prompt: fixture.prompt("L1"), + maxTurns: 4, + maxBudgetUsd: 0.25, + allowedBashCommands: [], + expectedMcpNonce: fixture.nonce, + preservePaths: [ + FIXTURE_PATHS.dirtySentinel, + FIXTURE_PATHS.untrackedSentinel, + ], + }, + { + hermeticGatewayOrigin: `http://127.0.0.1:${address.port}`, + queryFactory: ({ prompt, options }) => + agentSdkQuery({ prompt, options }), + uuid: () => { + const id = ids.shift(); + if (!id) throw new Error("unexpected UUID request"); + return id; + }, + }, + ); + + expect(inferenceTurn).toBe(3); + expect(observedMalformedError).toEqual([false, true, true]); + expect(observedValidSuccess).toEqual([false, false, true]); + const requestedEdits = result.toolEvidence.filter( + ({ toolName, status }) => toolName === "Edit" && status === "requested", + ); + expect(requestedEdits).toHaveLength(2); + const [malformedEdit, validEdit] = requestedEdits; + expect( + result.permissionEvidence.filter( + ({ toolUseId, source }) => + toolUseId === malformedEdit?.toolUseId && source === "pre_tool_use", + ), + ).toHaveLength(0); + expect( + result.toolEvidence.filter( + ({ toolUseId, status }) => + toolUseId === malformedEdit?.toolUseId && status === "error", + ), + ).toHaveLength(1); + expect( + result.permissionEvidence.filter( + ({ toolUseId, source, decision }) => + toolUseId === validEdit?.toolUseId && + source === "pre_tool_use" && + decision === "allow", + ), + ).toHaveLength(1); + expect( + result.toolEvidence.filter( + ({ toolUseId, status }) => + toolUseId === validEdit?.toolUseId && status === "success", + ), + ).toHaveLength(1); + expect(result.policyDiagnostics).toEqual([ + { + kind: "missing_pre_tool_use_callback", + reason: "no_callback_observed", + toolName: "Edit", + correlatedRequest: true, + }, + ]); + expect(result.policyHookCoverage).toBe(false); + expect(result.terminal).toBe("policy_violation"); + expect(result.terminationEvidence).toEqual({ + beforePolicyOverride: "success", + queryExecution: "iteration_completed", + sdkResult: "success", + }); + expect( + await readFile( + join(fixture.workspaceRoot, FIXTURE_PATHS.cleanTarget), + "utf8", + ), + ).toBe(fixture.cleanTargetReplacement); + } finally { + server.closeAllConnections(); + await new Promise((resolve) => server.close(() => resolve())); + await fixture.cleanup(); + } + }, + 45_000, +); diff --git a/packages/harness/src/experimental/managed-agent-spike/runtime.test.ts b/packages/harness/src/experimental/managed-agent-spike/runtime.test.ts index 3893879d0..a4aaf1d9d 100644 --- a/packages/harness/src/experimental/managed-agent-spike/runtime.test.ts +++ b/packages/harness/src/experimental/managed-agent-spike/runtime.test.ts @@ -85,6 +85,7 @@ async function invokePreToolUse( readonly toolName: string; readonly toolInput: unknown; readonly toolUseId: string; + readonly callbackToolUseId?: string; }, signal = new AbortController().signal, ): Promise { @@ -102,7 +103,7 @@ async function invokePreToolUse( tool_input: input.toolInput, tool_use_id: input.toolUseId, }, - input.toolUseId, + input.callbackToolUseId ?? input.toolUseId, { signal }, ); } @@ -291,7 +292,13 @@ describe("runManagedAgentProbe", () => { ); expect(capturedOptions?.env).not.toHaveProperty("SAPIOM_API_KEY"); expect(result.terminal).toBe("success"); + expect(result.terminationEvidence).toEqual({ + beforePolicyOverride: "success", + queryExecution: "iteration_completed", + sdkResult: "success", + }); expect(result.policyHookCoverage).toBe(true); + expect(result.policyDiagnostics).toEqual([]); expect(result.inferenceTurns).toBe(1); expect(result.sdkNumTurns).toBe(1); expect(result.correlation.promptEmbedded).toBe(true); @@ -383,6 +390,11 @@ describe("runManagedAgentProbe", () => { expect(queryFactory).not.toHaveBeenCalled(); expect(result.terminal).toBe("policy_violation"); + expect(result.terminationEvidence).toEqual({ + beforePolicyOverride: "incomplete", + queryExecution: "not_started", + sdkResult: "not_observed", + }); expect(result.policyHookCoverage).toBe(false); expect(result.queryClosed).toBe(false); expect(result.correlation.promptEmbedded).toBe(false); @@ -406,6 +418,58 @@ describe("runManagedAgentProbe", () => { expect(result.correlation.promptEmbedded).toBe(true); expect(result.queryClosed).toBe(false); expect(result.terminal).toBe("query_error"); + expect(result.terminationEvidence).toEqual({ + beforePolicyOverride: "query_error", + queryExecution: "construction_failed", + sdkResult: "not_observed", + }); + }); + + it("distinguishes query iteration failure from construction failure", async () => { + const { config } = await probeConfig(); + const result = await runManagedAgentProbe(config, { + hermeticGatewayOrigin: config.gatewayOrigin, + processObserver: fakeObserver(), + policySettingsGuard: async () => undefined, + queryFactory: () => ({ + async *[Symbol.asyncIterator]() { + yield { + type: "system", + subtype: "init", + session_id: SUCCESS_SESSION_ID, + }; + throw new Error("synthetic private iteration failure"); + }, + close: vi.fn(), + }), + }); + + expect(result.terminal).toBe("query_error"); + expect(result.terminationEvidence).toEqual({ + beforePolicyOverride: "query_error", + queryExecution: "iteration_failed", + sdkResult: "not_observed", + }); + expect(JSON.stringify(result)).not.toContain( + "synthetic private iteration failure", + ); + }); + + it("reports a completed iteration that emitted no SDK result", async () => { + const { config } = await probeConfig(); + const result = await runManagedAgentProbe(config, { + hermeticGatewayOrigin: config.gatewayOrigin, + processObserver: fakeObserver(), + policySettingsGuard: async () => undefined, + queryFactory: () => queryFromEvents([]), + }); + + expect(result.terminal).toBe("incomplete"); + expect(result.terminationEvidence).toEqual({ + beforePolicyOverride: "incomplete", + queryExecution: "iteration_completed", + sdkResult: "not_observed", + }); }); it("preserves teardown failure priority when policy preflight fails", async () => { @@ -469,6 +533,19 @@ describe("runManagedAgentProbe", () => { expect(result.policyHookCoverage).toBe(false); expect(result.terminal).toBe("policy_violation"); expect(result.permissionEvidence).toEqual([]); + expect(result.policyDiagnostics).toEqual([ + { + kind: "missing_pre_tool_use_callback", + reason: "no_callback_observed", + toolName: "Read", + correlatedRequest: true, + }, + ]); + expect(result.terminationEvidence).toEqual({ + beforePolicyOverride: "success", + queryExecution: "iteration_completed", + sdkResult: "success", + }); }); it("cannot certify a malformed SDK tool-use identifier as policy-covered evidence", async () => { @@ -506,6 +583,81 @@ describe("runManagedAgentProbe", () => { expect(result.permissionEvidence).toEqual([]); expect(result.policyHookCoverage).toBe(false); expect(result.terminal).toBe("policy_violation"); + expect(result.terminationEvidence).toEqual({ + beforePolicyOverride: "query_error", + queryExecution: "event_normalization_failed", + sdkResult: "not_observed", + eventNormalizationFailure: "tool_request_id_invalid", + }); + }); + + it("reports a correlated PreToolUse guard rejection without certifying coverage", async () => { + const { config } = await probeConfig(); + const requestIdSecret = "guarded-request-id-secret"; + const callbackIdSecret = "mismatched-callback-id-secret"; + const result = await runManagedAgentProbe(config, { + hermeticGatewayOrigin: config.gatewayOrigin, + processObserver: fakeObserver(), + policySettingsGuard: async () => undefined, + queryFactory: ({ options }) => ({ + async *[Symbol.asyncIterator]() { + yield { + type: "assistant", + message: { + id: "guard-rejection-message", + content: [ + { + type: "tool_use", + id: requestIdSecret, + name: "Edit", + input: { file_path: FIXTURE_PATHS.cleanTarget }, + }, + ], + }, + }; + await invokePreToolUse(options, { + toolName: "Edit", + toolInput: { file_path: FIXTURE_PATHS.cleanTarget }, + toolUseId: requestIdSecret, + callbackToolUseId: callbackIdSecret, + }); + yield { + type: "user", + message: { + content: [ + { + type: "tool_result", + tool_use_id: requestIdSecret, + is_error: true, + }, + ], + }, + }; + yield { + type: "result", + subtype: "success", + is_error: false, + num_turns: 1, + }; + }, + close: vi.fn(), + }), + }); + + expect(result.permissionEvidence).toEqual([]); + expect(result.policyDiagnostics).toEqual([ + { + kind: "pre_tool_use_guard_rejection", + reason: "callback_tool_use_id_mismatch", + toolName: "Edit", + correlatedRequest: true, + }, + ]); + expect(result.policyHookCoverage).toBe(false); + expect(result.terminal).toBe("policy_violation"); + const serialized = JSON.stringify(result); + expect(serialized).not.toContain(requestIdSecret); + expect(serialized).not.toContain(callbackIdSecret); }); it("rejects duplicate requested tool ids instead of reusing one policy decision", async () => { diff --git a/packages/harness/src/experimental/managed-agent-spike/runtime.ts b/packages/harness/src/experimental/managed-agent-spike/runtime.ts index 608039c8f..e36fbe5cc 100644 --- a/packages/harness/src/experimental/managed-agent-spike/runtime.ts +++ b/packages/harness/src/experimental/managed-agent-spike/runtime.ts @@ -21,6 +21,7 @@ import { MANAGED_AGENT_BUILTIN_TOOLS, MANAGED_AGENT_DISALLOWED_TOOLS, createManagedAgentPolicyBoundary, + type ManagedAgentPreToolUseGuardRejection, } from "./permissions.js"; import { createLocalManagedAgentProcessObserver } from "./process-observer.js"; import { @@ -30,8 +31,10 @@ import { import type { ManagedAgentProbeConfig, ManagedAgentProbeDependencies, + ManagedAgentPolicyDiagnostic, ManagedAgentProbeResult, ManagedAgentQuery, + ManagedAgentQueryExecutionOutcome, ManagedAgentTeardownObservation, ManagedAgentTerminalClassification, ManagedAgentToolEvidence, @@ -212,6 +215,55 @@ function hasUniversalPolicyHookCoverage( }); } +function buildManagedAgentPolicyDiagnostics( + toolEvidence: readonly ManagedAgentToolEvidence[], + permissionEvidence: ManagedAgentProbeResult["permissionEvidence"], + guardRejections: readonly ManagedAgentPreToolUseGuardRejection[], +): ManagedAgentPolicyDiagnostic[] { + const requestedIds = new Set( + toolEvidence.flatMap(({ status, toolUseId }) => + status === "requested" && toolUseId ? [toolUseId] : [], + ), + ); + const diagnostics: ManagedAgentPolicyDiagnostic[] = guardRejections.map( + ({ reason, toolName, normalizedToolUseId }) => ({ + kind: "pre_tool_use_guard_rejection", + reason, + toolName, + correlatedRequest: + normalizedToolUseId !== undefined && + requestedIds.has(normalizedToolUseId), + }), + ); + const primaryDecisionIds = new Set( + permissionEvidence.flatMap(({ toolUseId, source }) => + source === "pre_tool_use" ? [toolUseId] : [], + ), + ); + const guardedRequestIds = new Set( + guardRejections.flatMap(({ normalizedToolUseId }) => + normalizedToolUseId ? [normalizedToolUseId] : [], + ), + ); + for (const evidence of toolEvidence) { + if ( + evidence.status !== "requested" || + !evidence.toolUseId || + primaryDecisionIds.has(evidence.toolUseId) || + guardedRequestIds.has(evidence.toolUseId) + ) { + continue; + } + diagnostics.push({ + kind: "missing_pre_tool_use_callback", + reason: "no_callback_observed", + toolName: evidence.toolName, + correlatedRequest: true, + }); + } + return diagnostics; +} + async function closeQueryBounded(query: ManagedAgentQuery): Promise { let timeout: NodeJS.Timeout | undefined; try { @@ -293,7 +345,9 @@ export async function runManagedAgentProbe( let cancellationTriggerFailed = false; let policyPreflightFailed = false; let promptEmbedded = false; - let eventNormalizationFailed = false; + let eventNormalizationFailure: ManagedAgentProbeResult["terminationEvidence"]["eventNormalizationFailure"]; + let queryExecution: ManagedAgentQueryExecutionOutcome = "not_started"; + const guardRejections: ManagedAgentPreToolUseGuardRejection[] = []; const childEnvironment = buildManagedAgentChildEnvironment({ ambient: process.env, @@ -309,6 +363,7 @@ export async function runManagedAgentProbe( allowedBashCommands: config.allowedBashCommands, allowedMcpTools: mcpRuntime.qualifiedToolNames, onDecision: (evidence) => recorder.recordPermission(evidence), + onGuardRejection: (diagnostic) => guardRejections.push(diagnostic), }); const processObserver = dependencies.processObserver ?? createLocalManagedAgentProcessObserver(); @@ -353,6 +408,7 @@ export async function runManagedAgentProbe( let teardown!: ManagedAgentTeardownObservation; let terminal!: ManagedAgentTerminalClassification; + let terminationEvidence!: ManagedAgentProbeResult["terminationEvidence"]; let policyHookCoverage = false; try { recorder.recordLifecycle("starting"); @@ -397,19 +453,37 @@ export async function runManagedAgentProbe( executionId, }); promptEmbedded = true; - query = (dependencies.queryFactory ?? defaultQueryFactory)({ - prompt, - options, - }); + try { + query = (dependencies.queryFactory ?? defaultQueryFactory)({ + prompt, + options, + }); + } catch (error) { + queryExecution = "construction_failed"; + throw error; + } for await (const event of query) { try { recorder.observeSdkEvent(event); } catch (error) { - eventNormalizationFailed = error instanceof ManagedAgentEventError; + if (error instanceof ManagedAgentEventError) { + eventNormalizationFailure = error.reason; + queryExecution = "event_normalization_failed"; + } throw error; } } + queryExecution = "iteration_completed"; } catch { + if (eventNormalizationFailure) { + queryExecution = "event_normalization_failed"; + } else if (!query) { + queryExecution = "construction_failed"; + } else { + queryExecution = abortController.signal.aborted + ? "iteration_aborted" + : "iteration_failed"; + } if (!abortController.signal.aborted) queryFailed = true; } finally { triggerController.abort(); @@ -440,7 +514,7 @@ export async function runManagedAgentProbe( totalElapsedMs <= MANAGED_AGENT_TEARDOWN_TIMEOUT_MS, }; } - terminal = classifyTerminal({ + const beforePolicyOverride = classifyTerminal({ teardown, queryCreated: query !== undefined, queryClosed, @@ -448,6 +522,7 @@ export async function runManagedAgentProbe( queryFailed, sdkResult: recorder.result, }); + terminal = beforePolicyOverride; if ( policyPreflightFailed && terminal !== "teardown_timeout" && @@ -457,7 +532,7 @@ export async function runManagedAgentProbe( } policyHookCoverage = !policyPreflightFailed && - !eventNormalizationFailed && + !eventNormalizationFailure && hasUniversalPolicyHookCoverage( recorder.toolEvidence, recorder.permissionEvidence, @@ -476,6 +551,18 @@ export async function runManagedAgentProbe( teardown = { ...teardown, emergencyCleanupAttempted: true }; terminal = "teardown_timeout"; } + + const sdkResult = recorder.result + ? recorder.result.isError + ? "error" + : "success" + : "not_observed"; + terminationEvidence = { + beforePolicyOverride, + queryExecution, + sdkResult, + ...(eventNormalizationFailure ? { eventNormalizationFailure } : {}), + }; } finally { processObserver.dispose(); } @@ -483,6 +570,11 @@ export async function runManagedAgentProbe( const after = await captureManagedAgentWorkspaceSnapshot( validated.canonicalWorkspaceRoot, ); + const policyDiagnostics = buildManagedAgentPolicyDiagnostics( + recorder.toolEvidence, + recorder.permissionEvidence, + guardRejections, + ); return { contractVersion: 1, runId, @@ -496,9 +588,11 @@ export async function runManagedAgentProbe( : { sdkNumTurns: recorder.sdkNumTurns }), policyHookCoverage, terminal, + terminationEvidence, events: [...recorder.events], toolEvidence: [...recorder.toolEvidence, ...mcpRuntime.invocations], permissionEvidence: [...recorder.permissionEvidence], + policyDiagnostics, workspaceChanges: diffManagedAgentWorkspaceSnapshots(before, after), preservation: observeManagedAgentPreservation( before, diff --git a/packages/harness/src/experimental/managed-agent-spike/types.ts b/packages/harness/src/experimental/managed-agent-spike/types.ts index 7c1187cdd..6ae98c454 100644 --- a/packages/harness/src/experimental/managed-agent-spike/types.ts +++ b/packages/harness/src/experimental/managed-agent-spike/types.ts @@ -111,6 +111,57 @@ export interface ManagedAgentPermissionEvidence { readonly source: ManagedAgentPermissionSource; } +export type ManagedAgentPreToolUseGuardRejectionReason = + | "unexpected_hook_event" + | "input_tool_use_id_missing" + | "input_tool_use_id_invalid" + | "input_tool_use_id_too_long" + | "callback_tool_use_id_invalid" + | "callback_tool_use_id_too_long" + | "callback_tool_use_id_mismatch"; + +/** + * Content-free policy diagnostics. They explain why strict hook coverage + * failed, but never count as permission evidence and therefore cannot certify + * a tool request as authorized. + */ +export type ManagedAgentPolicyDiagnostic = + | { + readonly kind: "pre_tool_use_guard_rejection"; + readonly reason: ManagedAgentPreToolUseGuardRejectionReason; + readonly toolName: string; + readonly correlatedRequest: boolean; + } + | { + readonly kind: "missing_pre_tool_use_callback"; + readonly reason: "no_callback_observed"; + readonly toolName: string; + readonly correlatedRequest: true; + }; + +export type ManagedAgentEventNormalizationFailureReason = + | "assistant_message_id_invalid" + | "inference_turn_limit_exceeded" + | "tool_request_id_invalid" + | "tool_result_id_invalid" + | "sdk_num_turns_invalid"; + +export type ManagedAgentQueryExecutionOutcome = + | "not_started" + | "construction_failed" + | "iteration_completed" + | "iteration_failed" + | "iteration_aborted" + | "event_normalization_failed"; + +export interface ManagedAgentTerminationEvidence { + /** Terminal classification before the strict policy override is applied. */ + readonly beforePolicyOverride: ManagedAgentTerminalClassification; + readonly queryExecution: ManagedAgentQueryExecutionOutcome; + readonly sdkResult: "not_observed" | "success" | "error"; + readonly eventNormalizationFailure?: ManagedAgentEventNormalizationFailureReason; +} + export interface ManagedAgentTeardownObservation { readonly quiescent: boolean; readonly deadlineMet: boolean; @@ -144,9 +195,11 @@ export interface ManagedAgentProbeResult { /** False if any requested tool lacked exactly one primary PreToolUse decision. */ readonly policyHookCoverage: boolean; readonly terminal: ManagedAgentTerminalClassification; + readonly terminationEvidence: ManagedAgentTerminationEvidence; readonly events: readonly ManagedAgentProbeEvent[]; readonly toolEvidence: readonly ManagedAgentToolEvidence[]; readonly permissionEvidence: readonly ManagedAgentPermissionEvidence[]; + readonly policyDiagnostics: readonly ManagedAgentPolicyDiagnostic[]; readonly workspaceChanges: readonly ManagedAgentWorkspaceChange[]; readonly preservation: readonly ManagedAgentPreservationObservation[]; readonly cancellationRequested: boolean; From 1773264e56f7209ba6b7798a57e6e12ed9ee74f9 Mon Sep 17 00:00:00 2001 From: Yash Date: Sun, 16 Aug 2026 19:09:02 -0700 Subject: [PATCH 08/24] fix(harness): harden managed agent certification Require exact L1 and Bash-only L2 evidence, derive cancellation targets only from the SDK process tree, and preserve the existing Zod v3 REST error contract while the Agent SDK uses Zod v4.\n\nRefs: SAP-2632 --- .../managed-agent-spike/permissions.test.ts | 72 +++++ .../managed-agent-spike/permissions.ts | 10 + .../managed-agent-spike/probe-cli.test.ts | 286 ++++++++++++++---- .../managed-agent-spike/probe-cli.ts | 175 ++++++++++- .../process-observer.test.ts | 53 +++- .../managed-agent-spike/process-observer.ts | 253 +++++++++++----- .../runtime-sdk-loopback.test.ts | 2 +- .../managed-agent-spike/runtime.test.ts | 13 +- .../managed-agent-spike/runtime.ts | 37 ++- .../experimental/managed-agent-spike/types.ts | 6 +- packages/harness/src/server/rest.test.ts | 94 +++++- packages/harness/src/server/rest.ts | 5 +- packages/harness/src/server/track.test.ts | 25 ++ 13 files changed, 868 insertions(+), 163 deletions(-) diff --git a/packages/harness/src/experimental/managed-agent-spike/permissions.test.ts b/packages/harness/src/experimental/managed-agent-spike/permissions.test.ts index b32d5058b..ec62bfc74 100644 --- a/packages/harness/src/experimental/managed-agent-spike/permissions.test.ts +++ b/packages/harness/src/experimental/managed-agent-spike/permissions.test.ts @@ -91,6 +91,78 @@ function preToolUseInput( } describe("managed-agent universal policy boundary", () => { + it("can enforce an L2 Bash-only boundary before evaluating model-authored inputs", async () => { + const evidence: ManagedAgentPermissionEvidence[] = []; + const boundary = createManagedAgentPolicyBoundary({ + canonicalWorkspaceRoot: workspace, + allowedBuiltinTools: ["Bash"], + allowedBashCommands: ["node .managed-agent-probe/long-running.mjs"], + allowedMcpTools: [], + onDecision: (decision) => evidence.push(decision), + }); + const signal = new AbortController().signal; + const invoke = (toolName: string, input: unknown, toolUseId: string) => + boundary.preToolUseHook( + preToolUseInput(toolName, input, toolUseId), + toolUseId, + { signal }, + ); + + await expect( + invoke( + "Write", + { + file_path: ".managed-agent-probe/processes.json", + content: JSON.stringify({ + parentPid: process.pid, + childPid: 2_147_483_646, + }), + }, + "l2-write", + ), + ).resolves.toMatchObject({ + hookSpecificOutput: { permissionDecision: "deny" }, + }); + await expect( + invoke( + "mcp__sapiom-managed-agent-spike__echo_nonce", + { nonce: "x" }, + "l2-mcp", + ), + ).resolves.toMatchObject({ + hookSpecificOutput: { permissionDecision: "deny" }, + }); + await expect( + invoke( + "Bash", + { command: "node .managed-agent-probe/long-running.mjs" }, + "l2-bash", + ), + ).resolves.toMatchObject({ + hookSpecificOutput: { permissionDecision: "allow" }, + }); + + expect( + evidence.map(({ toolName, decision, reason }) => ({ + toolName, + decision, + reason, + })), + ).toEqual([ + { toolName: "Write", decision: "deny", reason: "tool_not_allowed" }, + { + toolName: "mcp__sapiom-managed-agent-spike__echo_nonce", + decision: "deny", + reason: "tool_not_allowed", + }, + { + toolName: "Bash", + decision: "allow", + reason: "exact_bash_command", + }, + ]); + }); + it("uses exact Bash equality and emits content-free decisions", async () => { const evidence: ManagedAgentPermissionEvidence[] = []; const boundary = createManagedAgentPolicyBoundary({ diff --git a/packages/harness/src/experimental/managed-agent-spike/permissions.ts b/packages/harness/src/experimental/managed-agent-spike/permissions.ts index 8d827e294..98341a26b 100644 --- a/packages/harness/src/experimental/managed-agent-spike/permissions.ts +++ b/packages/harness/src/experimental/managed-agent-spike/permissions.ts @@ -136,6 +136,8 @@ export async function resolveManagedAgentToolPath( export interface ManagedAgentPolicyBoundaryOptions { readonly canonicalWorkspaceRoot: string; + /** Scenario-specific built-ins; L2 deliberately exposes only exact Bash. */ + readonly allowedBuiltinTools?: readonly string[]; readonly allowedBashCommands: readonly string[]; readonly allowedMcpTools: readonly string[]; readonly onDecision: (evidence: ManagedAgentPermissionEvidence) => void; @@ -230,6 +232,7 @@ function denied( async function evaluateManagedAgentPolicy( options: ManagedAgentPolicyBoundaryOptions, + allowedBuiltinTools: ReadonlySet, allowedCommands: ReadonlySet, allowedMcpTools: ReadonlySet, toolName: string, @@ -237,6 +240,9 @@ async function evaluateManagedAgentPolicy( signal: AbortSignal, ): Promise { if (signal.aborted) return denied("policy_aborted"); + if (!allowedBuiltinTools.has(toolName) && !allowedMcpTools.has(toolName)) { + return denied("tool_not_allowed"); + } const input = asRecord(rawInput); if (!input) return denied("invalid_input"); @@ -295,6 +301,9 @@ async function evaluateManagedAgentPolicy( export function createManagedAgentPolicyBoundary( options: ManagedAgentPolicyBoundaryOptions, ): ManagedAgentPolicyBoundary { + const allowedBuiltinTools = new Set( + options.allowedBuiltinTools ?? MANAGED_AGENT_BUILTIN_TOOLS, + ); const allowedCommands = new Set(options.allowedBashCommands); const allowedMcpTools = new Set(options.allowedMcpTools); const decisions = new Map< @@ -343,6 +352,7 @@ export function createManagedAgentPolicyBoundary( } const pending = evaluateManagedAgentPolicy( options, + allowedBuiltinTools, allowedCommands, allowedMcpTools, toolName, diff --git a/packages/harness/src/experimental/managed-agent-spike/probe-cli.test.ts b/packages/harness/src/experimental/managed-agent-spike/probe-cli.test.ts index f24d02b82..ddad03662 100644 --- a/packages/harness/src/experimental/managed-agent-spike/probe-cli.test.ts +++ b/packages/harness/src/experimental/managed-agent-spike/probe-cli.test.ts @@ -13,15 +13,24 @@ import { qualifiedManagedAgentMcpToolName } from "./runtime.js"; import type { ManagedAgentProbeResult } from "./types.js"; function passingL1Result(): ManagedAgentProbeResult { - const builtins = ["Read", "Edit", "Write", "Bash"]; - const builtinIds = [ - `tool_${"1".repeat(64)}`, - `tool_${"2".repeat(64)}`, - `tool_${"3".repeat(64)}`, - `tool_${"b".repeat(64)}`, - ]; const echoTool = qualifiedManagedAgentMcpToolName("echo_nonce"); const failOnceTool = qualifiedManagedAgentMcpToolName("fail_once"); + const steps = [ + ["Read", "success", "allow", "fixture_path"], + ["Read", "success", "allow", "fixture_path"], + ["Read", "success", "allow", "fixture_path"], + ["Read", "error", "deny", "path_outside_workspace"], + ["Read", "error", "deny", "path_symlink_escape"], + ["Edit", "success", "allow", "fixture_path"], + ["Write", "success", "allow", "fixture_path"], + [echoTool, "success", "allow", "managed_mcp_tool"], + [failOnceTool, "error", "allow", "managed_mcp_tool"], + [failOnceTool, "success", "allow", "managed_mcp_tool"], + ["Bash", "success", "allow", "exact_bash_command"], + ] as const; + const ids = steps.map( + (_, index) => `tool_${(index + 1).toString(16).padStart(64, "0")}`, + ); return { contractVersion: 1, runId: "run-1", @@ -39,67 +48,25 @@ function passingL1Result(): ManagedAgentProbeResult { sdkResult: "success", }, events: [], - toolEvidence: [ - ...builtins.flatMap((toolName, index) => [ - { - toolUseId: builtinIds[index], - toolName, - status: "requested" as const, - }, - { - toolUseId: builtinIds[index], - toolName, - status: "success" as const, - }, - ]), - { toolName: echoTool, status: "success" }, - { toolName: failOnceTool, status: "error" }, - { toolName: failOnceTool, status: "success" }, - ], - permissionEvidence: [ - ...["Read", "Edit", "Write"].map((toolName, index) => ({ - toolUseId: `tool_${String(index + 1).repeat(64)}`, - toolName, - decision: "allow" as const, - reason: "fixture_path" as const, - source: "pre_tool_use" as const, - })), - { - toolUseId: `tool_${"b".repeat(64)}`, - toolName: "Bash", - decision: "allow", - reason: "exact_bash_command", - source: "pre_tool_use", - }, - { - toolUseId: `tool_${"c".repeat(64)}`, - toolName: echoTool, - decision: "allow", - reason: "managed_mcp_tool", - source: "pre_tool_use", - }, + toolEvidence: steps.flatMap(([toolName, completion], index) => [ { - toolUseId: `tool_${"d".repeat(64)}`, - toolName: failOnceTool, - decision: "allow", - reason: "managed_mcp_tool", - source: "pre_tool_use", - }, - { - toolUseId: `tool_${"e".repeat(64)}`, - toolName: "Read", - decision: "deny", - reason: "path_outside_workspace", - source: "pre_tool_use", + toolUseId: ids[index], + toolName, + status: "requested" as const, }, { - toolUseId: `tool_${"f".repeat(64)}`, - toolName: "Read", - decision: "deny", - reason: "path_symlink_escape", - source: "pre_tool_use", + toolUseId: ids[index], + toolName, + status: completion, }, - ], + ]), + permissionEvidence: steps.map(([toolName, , decision, reason], index) => ({ + toolUseId: ids[index]!, + toolName, + decision, + reason, + source: "pre_tool_use" as const, + })), policyDiagnostics: [], workspaceChanges: [ { path: FIXTURE_PATHS.cleanTarget, change: "modified" }, @@ -127,6 +94,43 @@ function passingL1Result(): ManagedAgentProbeResult { }; } +function passingL2Result(): ManagedAgentProbeResult { + const base = passingL1Result(); + const toolUseId = `tool_${"c".repeat(64)}`; + return { + ...base, + scenario: "L2", + inferenceTurns: 1, + sdkNumTurns: 1, + terminal: "cancelled", + toolEvidence: [{ toolUseId, toolName: "Bash", status: "requested" }], + permissionEvidence: [ + { + toolUseId, + toolName: "Bash", + decision: "allow", + reason: "exact_bash_command", + source: "pre_tool_use", + }, + ], + workspaceChanges: [], + cancellationRequested: true, + teardown: { + ...base.teardown, + observedPids: [12_345, 12_346], + }, + }; +} + +function evidenceForToolId( + result: ManagedAgentProbeResult, + toolUseId: string, +): ManagedAgentProbeResult["toolEvidence"] { + return result.toolEvidence.filter( + (evidence) => evidence.toolUseId === toolUseId, + ); +} + describe("managed-agent probe CLI", () => { it("is opt-in and never accepts credentials through arguments", () => { expect(() => @@ -271,6 +275,160 @@ describe("managed-agent probe CLI", () => { ).toEqual({ id: "builtin_tools_succeeded", passed: false }); }); + it("accepts exactly one permitted Bash request for L2 and rejects any extra tool call", () => { + const passing = passingL2Result(); + expect(evaluateManagedAgentProbe(passing, [12_345, 12_346])).toMatchObject({ + outcome: "pass", + checks: expect.arrayContaining([ + { id: "exact_l2_bash_only_trace", passed: true }, + ]), + }); + + const writeId = `tool_${"d".repeat(64)}`; + const invalid: ManagedAgentProbeResult = { + ...passing, + toolEvidence: [ + ...passing.toolEvidence, + { toolUseId: writeId, toolName: "Write", status: "requested" }, + { toolUseId: writeId, toolName: "Write", status: "success" }, + ], + permissionEvidence: [ + ...passing.permissionEvidence, + { + toolUseId: writeId, + toolName: "Write", + decision: "allow", + reason: "fixture_path", + source: "pre_tool_use", + }, + ], + }; + + expect( + evaluateManagedAgentProbe(invalid, [12_345, 12_346]).checks, + ).toContainEqual({ + id: "exact_l2_bash_only_trace", + passed: false, + }); + }); + + it.each([ + [ + "omitted", + (passing: ManagedAgentProbeResult) => { + const omittedId = passing.toolEvidence.find( + (evidence) => + evidence.status === "requested" && evidence.toolName === "Read", + )!.toolUseId!; + return { + ...passing, + toolEvidence: passing.toolEvidence.filter( + (evidence) => evidence.toolUseId !== omittedId, + ), + permissionEvidence: passing.permissionEvidence.filter( + (evidence) => evidence.toolUseId !== omittedId, + ), + }; + }, + ], + [ + "reordered", + (passing: ManagedAgentProbeResult) => { + const requested = passing.toolEvidence.filter( + ({ status }) => status === "requested", + ); + const editId = requested[5]!.toolUseId!; + const writeId = requested[6]!.toolUseId!; + const editEvidence = evidenceForToolId(passing, editId); + const writeEvidence = evidenceForToolId(passing, writeId); + const reordered = passing.toolEvidence.filter( + ({ toolUseId }) => toolUseId !== editId && toolUseId !== writeId, + ); + reordered.splice(10, 0, ...writeEvidence, ...editEvidence); + return { ...passing, toolEvidence: reordered }; + }, + ], + [ + "extra", + (passing: ManagedAgentProbeResult) => { + const toolUseId = `tool_${"a".repeat(64)}`; + return { + ...passing, + toolEvidence: [ + ...passing.toolEvidence, + { toolUseId, toolName: "Read", status: "requested" as const }, + { toolUseId, toolName: "Read", status: "success" as const }, + ], + permissionEvidence: [ + ...passing.permissionEvidence, + { + toolUseId, + toolName: "Read", + decision: "allow" as const, + reason: "fixture_path" as const, + source: "pre_tool_use" as const, + }, + ], + }; + }, + ], + [ + "duplicate retry", + (passing: ManagedAgentProbeResult) => { + const toolUseId = `tool_${"b".repeat(64)}`; + return { + ...passing, + toolEvidence: [ + ...passing.toolEvidence, + { toolUseId, toolName: "Bash", status: "requested" as const }, + { toolUseId, toolName: "Bash", status: "success" as const }, + ], + permissionEvidence: [ + ...passing.permissionEvidence, + { + toolUseId, + toolName: "Bash", + decision: "allow" as const, + reason: "exact_bash_command" as const, + source: "pre_tool_use" as const, + }, + ], + }; + }, + ], + ])("rejects an %s L1 tool trace", (_name, mutate) => { + const report = evaluateManagedAgentProbe(mutate(passingL1Result())); + + expect(report.outcome).toBe("fail"); + expect(report.checks).toContainEqual({ + id: "exact_l1_tool_trace", + passed: false, + }); + }); + + it("requires one completion and primary decision per L1 request, including fail_once error then success", () => { + const passing = passingL1Result(); + const failOnceRequests = passing.toolEvidence.filter( + ({ status, toolName }) => + status === "requested" && + toolName === qualifiedManagedAgentMcpToolName("fail_once"), + ); + const firstFailOnceId = failOnceRequests[0]!.toolUseId!; + const invalid: ManagedAgentProbeResult = { + ...passing, + toolEvidence: passing.toolEvidence.map((evidence) => + evidence.toolUseId === firstFailOnceId && evidence.status === "error" + ? { ...evidence, status: "success" } + : evidence, + ), + }; + + expect(evaluateManagedAgentProbe(invalid).checks).toContainEqual({ + id: "exact_l1_tool_trace", + passed: false, + }); + }); + it("requires positive permission evidence and distinct lexical and symlink denials", () => { const passing = passingL1Result(); expect(evaluateManagedAgentProbe(passing).outcome).toBe("pass"); diff --git a/packages/harness/src/experimental/managed-agent-spike/probe-cli.ts b/packages/harness/src/experimental/managed-agent-spike/probe-cli.ts index 0208b7c23..b22f02d10 100644 --- a/packages/harness/src/experimental/managed-agent-spike/probe-cli.ts +++ b/packages/harness/src/experimental/managed-agent-spike/probe-cli.ts @@ -51,6 +51,168 @@ export class ManagedAgentProbeCliError extends Error { } } +interface ManagedAgentExpectedL1ToolStep { + readonly toolName: string; + readonly completion: "success" | "error"; + readonly decision: "allow" | "deny"; + readonly reason: ManagedAgentPermissionReason; +} + +const MANAGED_AGENT_EXPECTED_L1_TOOL_TRACE = [ + { + toolName: "Read", + completion: "success", + decision: "allow", + reason: "fixture_path", + }, + { + toolName: "Read", + completion: "success", + decision: "allow", + reason: "fixture_path", + }, + { + toolName: "Read", + completion: "success", + decision: "allow", + reason: "fixture_path", + }, + { + toolName: "Read", + completion: "error", + decision: "deny", + reason: "path_outside_workspace", + }, + { + toolName: "Read", + completion: "error", + decision: "deny", + reason: "path_symlink_escape", + }, + { + toolName: "Edit", + completion: "success", + decision: "allow", + reason: "fixture_path", + }, + { + toolName: "Write", + completion: "success", + decision: "allow", + reason: "fixture_path", + }, + { + toolName: qualifiedManagedAgentMcpToolName("echo_nonce"), + completion: "success", + decision: "allow", + reason: "managed_mcp_tool", + }, + { + toolName: qualifiedManagedAgentMcpToolName("fail_once"), + completion: "error", + decision: "allow", + reason: "managed_mcp_tool", + }, + { + toolName: qualifiedManagedAgentMcpToolName("fail_once"), + completion: "success", + decision: "allow", + reason: "managed_mcp_tool", + }, + { + toolName: "Bash", + completion: "success", + decision: "allow", + reason: "exact_bash_command", + }, +] as const satisfies readonly ManagedAgentExpectedL1ToolStep[]; + +function hasExactManagedAgentL1ToolTrace( + result: ManagedAgentProbeResult, +): boolean { + const requested = result.toolEvidence.filter( + ({ status }) => status === "requested", + ); + const completed = result.toolEvidence.filter( + ({ status }) => status !== "requested", + ); + const primaryDecisions = result.permissionEvidence.filter( + ({ source }) => source === "pre_tool_use", + ); + if ( + requested.length !== MANAGED_AGENT_EXPECTED_L1_TOOL_TRACE.length || + completed.length !== MANAGED_AGENT_EXPECTED_L1_TOOL_TRACE.length || + primaryDecisions.length !== MANAGED_AGENT_EXPECTED_L1_TOOL_TRACE.length || + result.permissionEvidence.length !== primaryDecisions.length + ) { + return false; + } + const requestedIds = requested.flatMap(({ toolUseId }) => + toolUseId ? [toolUseId] : [], + ); + if ( + requestedIds.length !== requested.length || + new Set(requestedIds).size !== requestedIds.length + ) { + return false; + } + + return MANAGED_AGENT_EXPECTED_L1_TOOL_TRACE.every((expected, index) => { + const request = requested[index]; + if (!request?.toolUseId || request.toolName !== expected.toolName) { + return false; + } + const completions = completed.filter( + ({ toolUseId }) => toolUseId === request.toolUseId, + ); + const decisions = primaryDecisions.filter( + ({ toolUseId }) => toolUseId === request.toolUseId, + ); + return ( + completions.length === 1 && + completions[0]?.toolName === expected.toolName && + completions[0]?.status === expected.completion && + decisions.length === 1 && + decisions[0]?.toolName === expected.toolName && + decisions[0]?.decision === expected.decision && + decisions[0]?.reason === expected.reason + ); + }); +} + +function hasExactManagedAgentL2BashTrace( + result: ManagedAgentProbeResult, +): boolean { + const requested = result.toolEvidence.filter( + ({ status }) => status === "requested", + ); + const completed = result.toolEvidence.filter( + ({ status }) => status !== "requested", + ); + const primaryDecisions = result.permissionEvidence.filter( + ({ source }) => source === "pre_tool_use", + ); + const request = requested[0]; + return Boolean( + requested.length === 1 && + request?.toolUseId && + request.toolName === "Bash" && + completed.length <= 1 && + completed.every( + (evidence) => + evidence.toolUseId === request.toolUseId && + evidence.toolName === "Bash" && + evidence.status === "error", + ) && + primaryDecisions.length === 1 && + result.permissionEvidence.length === 1 && + primaryDecisions[0]?.toolUseId === request.toolUseId && + primaryDecisions[0]?.toolName === "Bash" && + primaryDecisions[0]?.decision === "allow" && + primaryDecisions[0]?.reason === "exact_bash_command", + ); +} + export function managedAgentProbeUsage(): string { return [ "Usage:", @@ -209,6 +371,10 @@ export function evaluateManagedAgentProbe( if (result.scenario === "L1") { checks.push( { id: "terminal_success", passed: result.terminal === "success" }, + { + id: "exact_l1_tool_trace", + passed: hasExactManagedAgentL1ToolTrace(result), + }, { id: "clean_target_modified", passed: result.workspaceChanges.some( @@ -270,6 +436,10 @@ export function evaluateManagedAgentProbe( } else { checks.push( { id: "terminal_cancelled", passed: result.terminal === "cancelled" }, + { + id: "exact_l2_bash_only_trace", + passed: hasExactManagedAgentL2BashTrace(result), + }, { id: "cancellation_requested", passed: result.cancellationRequested }, { id: "teardown_within_five_seconds", @@ -353,7 +523,10 @@ export async function executeManagedAgentProbeCli( 15_000, signal, ); - observer.trackPids(fixturePids); + // The model-writable PID file is evidence only. Signal targets + // come exclusively from the SDK process tree sampled by the + // host observer; these values are never handed to it. + await observer.observeProcessTree(); }, } : {}), diff --git a/packages/harness/src/experimental/managed-agent-spike/process-observer.test.ts b/packages/harness/src/experimental/managed-agent-spike/process-observer.test.ts index e78670e43..229c55bc2 100644 --- a/packages/harness/src/experimental/managed-agent-spike/process-observer.test.ts +++ b/packages/harness/src/experimental/managed-agent-spike/process-observer.test.ts @@ -1,4 +1,8 @@ -import { afterEach, describe, expect, it } from "vitest"; +import { spawn } from "node:child_process"; +import { writeFile } from "node:fs/promises"; +import { join } from "node:path"; + +import { afterEach, describe, expect, it, vi } from "vitest"; import { FIXTURE_PATHS, @@ -30,7 +34,7 @@ describe("LocalManagedAgentProcessObserver", () => { signal: controller.signal, }); const pids = await waitForManagedAgentFixturePids(fixture); - observer.trackPids(pids); + await observer.observeProcessTree(); controller.abort(); const teardown = await observer.waitForQuiescence(5_000); aliveAtFailure = teardown.alivePidsAtDeadline; @@ -42,9 +46,50 @@ describe("LocalManagedAgentProcessObserver", () => { ); expect(teardown.alivePidsAtDeadline).toEqual([]); } finally { - if (aliveAtFailure.length > 0) - await observer.emergencyCleanup(aliveAtFailure); + controller.abort(); + if (aliveAtFailure.length > 0) await observer.emergencyCleanup(); observer.dispose(); } }, 10_000); + + it("never tracks or signals PIDs injected through the model-writable fixture file", async () => { + const fixture = await createManagedAgentFixture(() => "forged-pids"); + fixtures.push(fixture); + const unrelated = spawn( + process.execPath, + ["-e", "setInterval(() => {}, 1000)"], + { stdio: "ignore", windowsHide: true }, + ); + if (typeof unrelated.pid !== "number") { + unrelated.kill("SIGKILL"); + throw new Error("unrelated test process did not expose a PID"); + } + const forgedPids = [process.pid, unrelated.pid] as const; + await writeFile( + join(fixture.workspaceRoot, FIXTURE_PATHS.processPidFile), + JSON.stringify({ + parentPid: forgedPids[0], + childPid: forgedPids[1], + }), + ); + const observer = new LocalManagedAgentProcessObserver(); + const signalSpy = vi.spyOn(process, "kill"); + try { + await expect(waitForManagedAgentFixturePids(fixture)).resolves.toEqual( + forgedPids, + ); + await observer.observeProcessTree(); + const teardown = await observer.waitForQuiescence(0); + await observer.emergencyCleanup(); + + expect(teardown.observedPids).not.toContain(forgedPids[0]); + expect(teardown.observedPids).not.toContain(forgedPids[1]); + expect(signalSpy).not.toHaveBeenCalled(); + expect(unrelated.exitCode).toBeNull(); + } finally { + signalSpy.mockRestore(); + unrelated.kill("SIGKILL"); + observer.dispose(); + } + }); }); diff --git a/packages/harness/src/experimental/managed-agent-spike/process-observer.ts b/packages/harness/src/experimental/managed-agent-spike/process-observer.ts index f10268dff..253c2db57 100644 --- a/packages/harness/src/experimental/managed-agent-spike/process-observer.ts +++ b/packages/harness/src/experimental/managed-agent-spike/process-observer.ts @@ -19,19 +19,24 @@ const execFileAsync = promisify(execFile); const SAMPLE_INTERVAL_MS = 100; const QUIESCENCE_POLL_MS = 25; -type ProcessTable = ReadonlyMap; +interface ProcessRecord { + readonly parentPid: number; + readonly processGroupId?: number; + /** Kernel-reported creation time, used to reject PID reuse. */ + readonly startedAt: string; +} + +type ProcessTable = ReadonlyMap; function delay(milliseconds: number): Promise { return new Promise((resolveDelay) => setTimeout(resolveDelay, milliseconds)); } -function pidAlive(pid: number): boolean { - try { - process.kill(pid, 0); - return true; - } catch (error) { - return (error as NodeJS.ErrnoException).code === "EPERM"; - } +function sameProcess( + left: ProcessRecord | undefined, + right: ProcessRecord | undefined, +): boolean { + return Boolean(left && right && left.startedAt === right.startedAt); } async function windowsProcessTable(): Promise { @@ -41,34 +46,62 @@ async function windowsProcessTable(): Promise { "-NoProfile", "-NonInteractive", "-Command", - "Get-CimInstance Win32_Process | Select-Object ProcessId,ParentProcessId | ConvertTo-Json -Compress", + "Get-CimInstance Win32_Process | Select-Object ProcessId,ParentProcessId,CreationDate | ConvertTo-Json -Compress", ], { windowsHide: true, maxBuffer: 4 * 1024 * 1024 }, ); const parsed = JSON.parse(stdout) as - | { ProcessId?: unknown; ParentProcessId?: unknown } - | Array<{ ProcessId?: unknown; ParentProcessId?: unknown }>; + | { + ProcessId?: unknown; + ParentProcessId?: unknown; + CreationDate?: unknown; + } + | Array<{ + ProcessId?: unknown; + ParentProcessId?: unknown; + CreationDate?: unknown; + }>; const rows = Array.isArray(parsed) ? parsed : [parsed]; return new Map( rows.flatMap((row) => typeof row.ProcessId === "number" && - typeof row.ParentProcessId === "number" - ? [[row.ProcessId, row.ParentProcessId] as const] + typeof row.ParentProcessId === "number" && + typeof row.CreationDate === "string" + ? [ + [ + row.ProcessId, + { + parentPid: row.ParentProcessId, + startedAt: row.CreationDate, + }, + ] as const, + ] : [], ), ); } async function posixProcessTable(): Promise { - const { stdout } = await execFileAsync("/bin/ps", ["-axo", "pid=,ppid="], { - windowsHide: true, - maxBuffer: 4 * 1024 * 1024, - }); - const entries: Array = []; + const { stdout } = await execFileAsync( + "/bin/ps", + ["-axo", "pid=,ppid=,pgid=,lstart="], + { + windowsHide: true, + maxBuffer: 4 * 1024 * 1024, + }, + ); + const entries: Array = []; for (const line of stdout.split("\n")) { - const match = /^\s*(\d+)\s+(\d+)\s*$/.exec(line); + const match = /^\s*(\d+)\s+(\d+)\s+(\d+)\s+(.+?)\s*$/.exec(line); if (!match) continue; - entries.push([Number(match[1]), Number(match[2])]); + entries.push([ + Number(match[1]), + { + parentPid: Number(match[2]), + processGroupId: Number(match[3]), + startedAt: match[4]!, + }, + ]); } return new Map(entries); } @@ -91,10 +124,10 @@ function descendantsOf( let changed = true; while (changed) { changed = false; - for (const [pid, parentPid] of table) { + for (const [pid, record] of table) { if ( !descendants.has(pid) && - (roots.has(parentPid) || descendants.has(parentPid)) + (roots.has(record.parentPid) || descendants.has(record.parentPid)) ) { descendants.add(pid); changed = true; @@ -112,24 +145,33 @@ async function taskkill(pid: number, force: boolean): Promise { { windowsHide: true }, ); } catch { - // A process that exited between observation and cleanup is already safe. + // A process that exited between validation and cleanup is already safe. } } /** - * Tracks the actual SDK subprocess plus descendants sampled from the kernel. - * POSIX children are placed in their own process group so the forwarded SDK - * abort signal can terminate Bash descendants rather than only their parent. + * Tracks only SDK roots created by this observer and descendants discovered + * from the host kernel. PID-file contents are deliberately outside this API: + * model-writable fixture evidence must never become signal authority. + * + * POSIX children are placed in their own process group. Before group cleanup, + * the observer revalidates a kernel creation timestamp for the root or a known + * group member, preventing a recycled numeric PID from becoming a kill target. */ export class LocalManagedAgentProcessObserver implements ManagedAgentProcessObserver { - readonly #roots = new Set(); - readonly #observed = new Set(); + readonly #rootPids = new Set(); + readonly #rootIdentities = new Map(); + readonly #observedIdentities = new Map(); + readonly #observedPids = new Set(); readonly #children = new Map(); readonly #sampler: NodeJS.Timeout; - #samplePending = false; + #sampleTask: Promise | undefined; public constructor() { - this.#sampler = setInterval(() => void this.#sample(), SAMPLE_INTERVAL_MS); + this.#sampler = setInterval( + () => void this.observeProcessTree(), + SAMPLE_INTERVAL_MS, + ); this.#sampler.unref(); } @@ -143,44 +185,109 @@ export class LocalManagedAgentProcessObserver implements ManagedAgentProcessObse }); if (typeof child.pid === "number") { const pid = child.pid; - this.#roots.add(pid); - this.#observed.add(pid); + this.#rootPids.add(pid); + this.#observedPids.add(pid); this.#children.set(pid, child); const terminateTree = (): void => { - if (process.platform === "win32") { - void taskkill(pid, false); - return; - } - try { - process.kill(-pid, "SIGTERM"); - } catch { - child.kill("SIGTERM"); - } + void this.#terminateRoot(pid, false); }; options.signal.addEventListener("abort", terminateTree, { once: true }); child.once("exit", () => { options.signal.removeEventListener("abort", terminateTree); this.#children.delete(pid); }); + void this.observeProcessTree(); } return child; } - public trackPids(pids: readonly number[]): void { - for (const pid of pids) { - if (Number.isInteger(pid) && pid > 0) this.#observed.add(pid); + public observeProcessTree(): Promise { + if (this.#sampleTask) return this.#sampleTask; + if (this.#rootPids.size === 0) return Promise.resolve(); + this.#sampleTask = (async () => { + const table = await readProcessTable(); + const validatedRoots = new Set(); + for (const pid of this.#rootPids) { + const current = table.get(pid); + const known = this.#rootIdentities.get(pid); + const child = this.#children.get(pid); + const childActive = + child !== undefined && + child.exitCode === null && + child.signalCode === null; + if (!current || (known ? !sameProcess(known, current) : !childActive)) { + continue; + } + if (!known) this.#rootIdentities.set(pid, current); + this.#observedIdentities.set(pid, current); + this.#observedPids.add(pid); + validatedRoots.add(pid); + } + for (const pid of descendantsOf(validatedRoots, table)) { + const current = table.get(pid); + if (!current) continue; + this.#observedIdentities.set(pid, current); + this.#observedPids.add(pid); + } + })().finally(() => { + this.#sampleTask = undefined; + }); + return this.#sampleTask; + } + + async #aliveObservedPids(): Promise { + const table = await readProcessTable(); + const alive = new Set( + [...this.#observedIdentities].flatMap(([pid, identity]) => + sameProcess(identity, table.get(pid)) ? [pid] : [], + ), + ); + // If kernel enumeration is unavailable, a still-active ChildProcess handle + // must keep teardown fail-closed instead of producing false quiescence. + for (const [pid, child] of this.#children) { + if (child.exitCode === null && child.signalCode === null) alive.add(pid); } + return [...alive].sort((left, right) => left - right); } - async #sample(): Promise { - if (this.#samplePending || this.#roots.size === 0) return; - this.#samplePending = true; - try { - const table = await readProcessTable(); - for (const pid of descendantsOf(this.#roots, table)) - this.#observed.add(pid); - } finally { - this.#samplePending = false; + async #terminateRoot(rootPid: number, force: boolean): Promise { + await this.observeProcessTree(); + const table = await readProcessTable(); + const rootIdentity = this.#rootIdentities.get(rootPid); + const rootMatches = sameProcess(rootIdentity, table.get(rootPid)); + const child = this.#children.get(rootPid); + const childActive = + child !== undefined && + child.exitCode === null && + child.signalCode === null; + + if (process.platform === "win32") { + if (rootMatches) { + await taskkill(rootPid, force); + } else if (!rootIdentity && childActive) { + child.kill(force ? "SIGKILL" : "SIGTERM"); + } + return; + } + + const hasValidatedGroupMember = [...this.#observedIdentities].some( + ([pid, identity]) => { + const current = table.get(pid); + return ( + current?.processGroupId === rootPid && sameProcess(identity, current) + ); + }, + ); + if (rootMatches || hasValidatedGroupMember) { + try { + process.kill(-rootPid, force ? "SIGKILL" : "SIGTERM"); + } catch { + // A validated group that exited before the signal is already safe. + } + } else if (!rootIdentity && childActive) { + // The kernel sampler can be unavailable. The trusted ChildProcess handle + // remains safe for root-only termination, but we never guess descendants. + child.kill(force ? "SIGKILL" : "SIGTERM"); } } @@ -190,53 +297,41 @@ export class LocalManagedAgentProcessObserver implements ManagedAgentProcessObse const startedAt = Date.now(); let alivePids: number[] = []; do { - await this.#sample(); - alivePids = [...this.#observed].filter(pidAlive).sort((a, b) => a - b); + await this.observeProcessTree(); + alivePids = await this.#aliveObservedPids(); if (alivePids.length === 0) { return { quiescent: true, deadlineMet: true, elapsedMs: Date.now() - startedAt, - observedPids: [...this.#observed].sort((a, b) => a - b), + observedPids: [...this.#observedPids].sort( + (left, right) => left - right, + ), alivePidsAtDeadline: [], emergencyCleanupAttempted: false, }; } + if (Date.now() - startedAt >= timeoutMs) break; await delay(QUIESCENCE_POLL_MS); } while (Date.now() - startedAt < timeoutMs); - await this.#sample(); - alivePids = [...this.#observed].filter(pidAlive).sort((a, b) => a - b); + await this.observeProcessTree(); + alivePids = await this.#aliveObservedPids(); return { quiescent: alivePids.length === 0, deadlineMet: alivePids.length === 0, elapsedMs: Date.now() - startedAt, - observedPids: [...this.#observed].sort((a, b) => a - b), + observedPids: [...this.#observedPids].sort((left, right) => left - right), alivePidsAtDeadline: alivePids, emergencyCleanupAttempted: false, }; } - public async emergencyCleanup(pids: readonly number[]): Promise { - if (process.platform === "win32") { - await Promise.all([...this.#roots].map((pid) => taskkill(pid, true))); - await Promise.all(pids.map((pid) => taskkill(pid, true))); - return; - } - for (const root of this.#roots) { - try { - process.kill(-root, "SIGKILL"); - } catch { - // Fall through to individual PID cleanup below. - } - } - for (const pid of pids) { - try { - process.kill(pid, "SIGKILL"); - } catch { - // A process that already exited needs no cleanup. - } - } + public async emergencyCleanup(): Promise { + await this.observeProcessTree(); + await Promise.all( + [...this.#rootPids].map((pid) => this.#terminateRoot(pid, true)), + ); } public dispose(): void { diff --git a/packages/harness/src/experimental/managed-agent-spike/runtime-sdk-loopback.test.ts b/packages/harness/src/experimental/managed-agent-spike/runtime-sdk-loopback.test.ts index d8363cfbd..c08c79fab 100644 --- a/packages/harness/src/experimental/managed-agent-spike/runtime-sdk-loopback.test.ts +++ b/packages/harness/src/experimental/managed-agent-spike/runtime-sdk-loopback.test.ts @@ -401,7 +401,7 @@ it("enforces real-SDK built-in and in-process MCP calls with exact loopback corr ({ toolName, toolUseId }) => toolName === ECHO_NONCE_TOOL && toolUseId === undefined, ), - ).toEqual([{ toolName: ECHO_NONCE_TOOL, status: "success" }]); + ).toEqual([]); expect(result.policyHookCoverage).toBe(true); expect( await fixturePathExists( diff --git a/packages/harness/src/experimental/managed-agent-spike/runtime.test.ts b/packages/harness/src/experimental/managed-agent-spike/runtime.test.ts index a4aaf1d9d..226a26632 100644 --- a/packages/harness/src/experimental/managed-agent-spike/runtime.test.ts +++ b/packages/harness/src/experimental/managed-agent-spike/runtime.test.ts @@ -60,7 +60,7 @@ function fakeObserver( spawn: vi.fn(() => { throw new Error("fake query must not spawn"); }), - trackPids: vi.fn(), + observeProcessTree: vi.fn(async () => undefined), waitForQuiescence: vi.fn(async () => teardown), emergencyCleanup: vi.fn(async () => undefined), dispose: vi.fn(), @@ -496,7 +496,7 @@ describe("runManagedAgentProbe", () => { type: "terminal", terminal: "teardown_timeout", }); - expect(observer.emergencyCleanup).toHaveBeenCalledWith([8001]); + expect(observer.emergencyCleanup).toHaveBeenCalledWith(); }); it("rejects a successful stream when a requested tool has no primary hook decision", async () => { @@ -819,12 +819,14 @@ describe("runManagedAgentProbe", () => { const { config } = await probeConfig("L2"); const observer = fakeObserver(); const close = vi.fn(); + let capturedOptions: Options | undefined; const result = await runManagedAgentProbe(config, { hermeticGatewayOrigin: config.gatewayOrigin, processObserver: observer, waitForCancellationSignal: async () => undefined, queryFactory: ({ options }) => ({ async *[Symbol.asyncIterator]() { + capturedOptions = options; yield { type: "system", subtype: "init", @@ -848,6 +850,11 @@ describe("runManagedAgentProbe", () => { expect(result.terminal).toBe("cancelled"); expect(result.cancellationRequested).toBe(true); expect(result.queryClosed).toBe(true); + expect(capturedOptions?.tools).toEqual(["Bash"]); + expect(capturedOptions?.disallowedTools).toEqual( + expect.arrayContaining(["Read", "Edit", "Write"]), + ); + expect(capturedOptions?.mcpServers).toEqual({}); expect( result.events.filter(({ type }) => type === "terminal"), ).toHaveLength(1); @@ -879,7 +886,7 @@ describe("runManagedAgentProbe", () => { expect(result.terminal).toBe("teardown_timeout"); expect(result.teardown.emergencyCleanupAttempted).toBe(true); - expect(observer.emergencyCleanup).toHaveBeenCalledWith([9001]); + expect(observer.emergencyCleanup).toHaveBeenCalledWith(); expect(result.events.at(-1)).toMatchObject({ type: "terminal", terminal: "teardown_timeout", diff --git a/packages/harness/src/experimental/managed-agent-spike/runtime.ts b/packages/harness/src/experimental/managed-agent-spike/runtime.ts index e36fbe5cc..2020accfc 100644 --- a/packages/harness/src/experimental/managed-agent-spike/runtime.ts +++ b/packages/harness/src/experimental/managed-agent-spike/runtime.ts @@ -45,6 +45,13 @@ export const MANAGED_AGENT_TEARDOWN_TIMEOUT_MS = 5_000; export const MANAGED_AGENT_CORRELATION_MARKER_VERSION = "SAPIOM_CERTIFICATION_CORRELATION_V1"; const QUERY_CLOSE_TIMEOUT_MS = 2_000; +const MANAGED_AGENT_L2_BUILTIN_TOOLS = ["Bash"] as const; +const MANAGED_AGENT_L2_DISALLOWED_TOOLS = [ + ...MANAGED_AGENT_DISALLOWED_TOOLS, + "Read", + "Edit", + "Write", +] as const; type McpToolName = "echo_nonce" | "fail_once"; @@ -360,8 +367,13 @@ export async function runManagedAgentProbe( }); const policyBoundary = createManagedAgentPolicyBoundary({ canonicalWorkspaceRoot: validated.canonicalWorkspaceRoot, + allowedBuiltinTools: + config.scenario === "L2" + ? MANAGED_AGENT_L2_BUILTIN_TOOLS + : MANAGED_AGENT_BUILTIN_TOOLS, allowedBashCommands: config.allowedBashCommands, - allowedMcpTools: mcpRuntime.qualifiedToolNames, + allowedMcpTools: + config.scenario === "L1" ? mcpRuntime.qualifiedToolNames : [], onDecision: (evidence) => recorder.recordPermission(evidence), onGuardRejection: (diagnostic) => guardRejections.push(diagnostic), }); @@ -375,7 +387,10 @@ export async function runManagedAgentProbe( // deduplicates its evidence by tool-use ID. canUseTool: policyBoundary.canUseToolFallback, cwd: validated.canonicalWorkspaceRoot, - disallowedTools: [...MANAGED_AGENT_DISALLOWED_TOOLS], + disallowedTools: + config.scenario === "L2" + ? [...MANAGED_AGENT_L2_DISALLOWED_TOOLS] + : [...MANAGED_AGENT_DISALLOWED_TOOLS], env: childEnvironment, includePartialMessages: false, hooks: { @@ -388,7 +403,10 @@ export async function runManagedAgentProbe( }, maxBudgetUsd: config.maxBudgetUsd, maxTurns: config.maxTurns, - mcpServers: { [MANAGED_AGENT_MCP_SERVER_NAME]: mcpRuntime.server }, + mcpServers: + config.scenario === "L1" + ? { [MANAGED_AGENT_MCP_SERVER_NAME]: mcpRuntime.server } + : {}, model: validated.model.alias, permissionMode: "default", persistSession: false, @@ -401,9 +419,14 @@ export async function runManagedAgentProbe( }, strictMcpConfig: true, systemPrompt: - "You are a deterministic local managed-agent feasibility probe. Follow the ordered instructions exactly, continue after expected permission denials and planned MCP errors, and use only the tools named in the prompt.", + config.scenario === "L2" + ? "You are a deterministic local cancellation probe. Use only the one exact Bash call named in the prompt." + : "You are a deterministic local managed-agent feasibility probe. Follow the ordered instructions exactly, continue after expected permission denials and planned MCP errors, and use only the tools named in the prompt.", thinking: { type: "disabled" }, - tools: [...MANAGED_AGENT_BUILTIN_TOOLS], + tools: + config.scenario === "L2" + ? [...MANAGED_AGENT_L2_BUILTIN_TOOLS] + : [...MANAGED_AGENT_BUILTIN_TOOLS], }; let teardown!: ManagedAgentTeardownObservation; @@ -547,7 +570,7 @@ export async function runManagedAgentProbe( recorder.recordTerminal(terminal); if (!teardown.quiescent) { - await processObserver.emergencyCleanup(teardown.alivePidsAtDeadline); + await processObserver.emergencyCleanup(); teardown = { ...teardown, emergencyCleanupAttempted: true }; terminal = "teardown_timeout"; } @@ -590,7 +613,7 @@ export async function runManagedAgentProbe( terminal, terminationEvidence, events: [...recorder.events], - toolEvidence: [...recorder.toolEvidence, ...mcpRuntime.invocations], + toolEvidence: [...recorder.toolEvidence], permissionEvidence: [...recorder.permissionEvidence], policyDiagnostics, workspaceChanges: diffManagedAgentWorkspaceSnapshots(before, after), diff --git a/packages/harness/src/experimental/managed-agent-spike/types.ts b/packages/harness/src/experimental/managed-agent-spike/types.ts index 6ae98c454..0275cff5e 100644 --- a/packages/harness/src/experimental/managed-agent-spike/types.ts +++ b/packages/harness/src/experimental/managed-agent-spike/types.ts @@ -229,11 +229,13 @@ export type ManagedAgentQueryFactory = (input: { export interface ManagedAgentProcessObserver { spawn(options: SpawnOptions): SpawnedProcess; - trackPids(pids: readonly number[]): void; + /** Sample only descendants of the host-observed SDK process roots. */ + observeProcessTree(): Promise; waitForQuiescence( timeoutMs: number, ): Promise; - emergencyCleanup(pids: readonly number[]): Promise; + /** Signal only process groups rooted in an SDK process spawned above. */ + emergencyCleanup(): Promise; dispose(): void; } diff --git a/packages/harness/src/server/rest.test.ts b/packages/harness/src/server/rest.test.ts index a5224db04..98fe95faf 100644 --- a/packages/harness/src/server/rest.test.ts +++ b/packages/harness/src/server/rest.test.ts @@ -21,6 +21,10 @@ import { createRestRouter, type RestRouterOptions } from "./rest.js"; const TOKEN_HEADER = { "X-Harness-Token": "unused-in-router-tests" }; +function zodV3ErrorMessage(issues: readonly Record[]): string { + return JSON.stringify(issues, null, 2); +} + function fakeSessionManager(initial: HarnessSession[] = []) { const sessions = new Map(initial.map((s) => [s.id, s])); return { @@ -341,6 +345,17 @@ describe("createRestRouter", () => { body: JSON.stringify({ telemetryOptIn: "yes" }), }); expect(res.status).toBe(400); + expect(await res.json()).toEqual({ + error: zodV3ErrorMessage([ + { + code: "invalid_type", + expected: "boolean", + received: "string", + path: ["telemetryOptIn"], + message: "Expected boolean, received string", + }, + ]), + }); }); }); @@ -380,10 +395,31 @@ describe("createRestRouter", () => { const res = await fetch(`${baseUrl}/sessions`, { method: "POST", headers: { ...TOKEN_HEADER, "content-type": "application/json" }, - body: JSON.stringify({ cwd: "" }), + body: JSON.stringify({ cwd: "", harness: "conductor" }), }); expect(res.status).toBe(400); + expect(await res.json()).toEqual({ + error: zodV3ErrorMessage([ + { + code: "too_small", + minimum: 1, + type: "string", + inclusive: true, + exact: false, + message: "String must contain at least 1 character(s)", + path: ["cwd"], + }, + { + received: "conductor", + code: "invalid_enum_value", + options: ["claude-code", "codex"], + path: ["harness"], + message: + "Invalid enum value. Expected 'claude-code' | 'codex', received 'conductor'", + }, + ]), + }); expect(onSessionCreated).not.toHaveBeenCalled(); }); @@ -572,6 +608,25 @@ describe("createRestRouter", () => { expect(res.status).toBe(status); }); + it("preserves the v3 required-field error body for attachment validation", async () => { + const res = await postAttachment({ + dataUrl: "data:text/plain;base64,YQ==", + }); + + expect(res.status).toBe(400); + expect(await res.json()).toEqual({ + error: zodV3ErrorMessage([ + { + code: "invalid_type", + expected: "string", + received: "undefined", + path: ["filename"], + message: "Required", + }, + ]), + }); + }); + it("rejects a decoded payload over 10 MiB", async () => { const encoded = Buffer.alloc(10 * 1024 * 1024 + 1).toString("base64"); const res = await postAttachment({ @@ -651,6 +706,17 @@ describe("createRestRouter", () => { body: JSON.stringify({ submit: true }), }); expect(res.status).toBe(400); + expect(await res.json()).toEqual({ + error: zodV3ErrorMessage([ + { + code: "invalid_type", + expected: "string", + received: "undefined", + path: ["text"], + message: "Required", + }, + ]), + }); }); it("404s when submitInput reports no live pty for the session", async () => { @@ -817,6 +883,17 @@ describe("createRestRouter", () => { }, ); expect(res.status).toBe(400); + expect(await res.json()).toEqual({ + error: zodV3ErrorMessage([ + { + code: "invalid_type", + expected: "string", + received: "undefined", + path: ["workflowPath"], + message: "Required", + }, + ]), + }); }); }); @@ -1162,6 +1239,21 @@ describe("createRestRouter", () => { expect((await adopt({ cwd: "/tmp/proj" })).status).toBe(400); }); + it("preserves the v3 joined-issue error body for adoption", async () => { + start({ adapters: { "claude-code": historyAdapter() } }); + const res = await adopt({ + ...body, + agentSessionId: "", + harness: "conductor", + }); + + expect(res.status).toBe(400); + expect(await res.json()).toEqual({ + error: + "String must contain at least 1 character(s); Invalid enum value. Expected 'claude-code' | 'codex', received 'conductor'", + }); + }); + it("is handled as its own route — 'adopt' is never read as a session id", async () => { // Today this is structural: `/sessions/adopt` is two path segments and // `/sessions/:id/resume` is three, so they cannot collide and no diff --git a/packages/harness/src/server/rest.ts b/packages/harness/src/server/rest.ts index efccffdf5..ff3353750 100644 --- a/packages/harness/src/server/rest.ts +++ b/packages/harness/src/server/rest.ts @@ -6,7 +6,10 @@ import express, { Router } from "express"; import rateLimit from "express-rate-limit"; -import { z } from "zod"; +// Harness's public validation-error bodies predate the Agent SDK dependency +// and serialize Zod v3 issues verbatim. Keep that wire contract stable while +// the experimental managed-agent runtime uses root Zod v4 for SDK tool schemas. +import { z } from "zod/v3"; import { randomUUID } from "node:crypto"; import * as fs from "node:fs/promises"; diff --git a/packages/harness/src/server/track.test.ts b/packages/harness/src/server/track.test.ts index d2db237ae..bdf52a228 100644 --- a/packages/harness/src/server/track.test.ts +++ b/packages/harness/src/server/track.test.ts @@ -165,6 +165,31 @@ describe("POST /api/track", () => { body: JSON.stringify({ event: "unknown.event" }), }); expect(res.status).toBe(400); + expect(await res.json()).toEqual({ + error: JSON.stringify( + [ + { + received: "unknown.event", + code: "invalid_enum_value", + options: [ + "prompt.submitted", + "session.switched", + "macro.invoked", + "visualize.triggered", + "consent.changed", + "session.created", + "mcp.install", + "plan.upgrade_clicked", + ], + path: ["event"], + message: + "Invalid enum value. Expected 'prompt.submitted' | 'session.switched' | 'macro.invoked' | 'visualize.triggered' | 'consent.changed' | 'session.created' | 'mcp.install' | 'plan.upgrade_clicked', received 'unknown.event'", + }, + ], + null, + 2, + ), + }); expect(stored).toHaveLength(0); }); From 94a2fc925c74c73483fb92b4e8c1a4d9faeccc4d Mon Sep 17 00:00:00 2001 From: Yash Date: Sun, 16 Aug 2026 19:52:25 -0700 Subject: [PATCH 09/24] fix(harness): bound managed agent cancellation Certify only the reviewed detached POSIX fixture group, bind raw abort to immediate STOP/KILL, fail closed on unknown containment, and keep iterator close and confirmation inside one deadline.\n\nRefs: SAP-2632 --- .../managed-agent-spike/README.md | 28 + .../managed-agent-spike/fixture.ts | 21 +- .../experimental/managed-agent-spike/index.ts | 2 + .../managed-agent-spike/probe-cli.test.ts | 40 ++ .../managed-agent-spike/probe-cli.ts | 45 +- .../process-observer.test.ts | 513 +++++++++++++-- .../managed-agent-spike/process-observer.ts | 612 +++++++++++++----- .../managed-agent-spike/runtime.test.ts | 97 ++- .../managed-agent-spike/runtime.ts | 141 +++- .../experimental/managed-agent-spike/types.ts | 38 +- 10 files changed, 1278 insertions(+), 259 deletions(-) diff --git a/packages/harness/src/experimental/managed-agent-spike/README.md b/packages/harness/src/experimental/managed-agent-spike/README.md index f7bdddeec..cb0074c63 100644 --- a/packages/harness/src/experimental/managed-agent-spike/README.md +++ b/packages/harness/src/experimental/managed-agent-spike/README.md @@ -64,6 +64,34 @@ real in-process `echo_nonce` MCP turn. It requires one primary `PreToolUse` decision for each request and separately verifies the MCP handler invocation and SDK tool-result event. +## L2 cancellation containment boundary + +E0.4 certifies one deliberately narrow host model: the exact non-cooperative +fixture command running under the active Agent SDK root in a detached macOS or +Linux process group. Before cancellation may fire, a bounded host `ps` sample +must prove that the trusted SDK `ChildProcess` is still active and is the group +leader, and both PIDs read from the fixture file must already be present in the +independently host-observed group. The file is comparison evidence only; its +contents are never passed into the observer or used as signal targets. + +The runtime binds the observer directly to the per-run `Options.abortController` +signal. On abort it synchronously and idempotently sends `SIGSTOP` followed by +`SIGKILL` to the observer-created group while the trusted root handle remains +active. The fixture parent and child intentionally ignore `SIGTERM`, making the +forced path load-bearing. Iterator abandonment, query close, bounded process +enumeration, and group-death confirmation share one absolute five-second +process-termination deadline. Workspace snapshots and result assembly occur +afterward. + +An unavailable or timed-out process table is explicit unknown evidence, never +an empty process table. The active detached root still authorizes safe cleanup +of its owned group, but the run fails certification. A fast root exit before +preparation, an observed `setsid`/group escape, unknown group liveness, failed +signals, and Windows all fail closed. Windows live L2 is rejected before the +query or credential is opened. Universal containment, POSIX group escape, +Windows Job Objects, and production recovery belong to later epics; this probe +does not claim those guarantees. + ## Pre-fix live evidence The first Sonnet 5 L1 attempt reached an SDK success result and clean teardown, diff --git a/packages/harness/src/experimental/managed-agent-spike/fixture.ts b/packages/harness/src/experimental/managed-agent-spike/fixture.ts index 1cabf9b7b..4c8d420d7 100644 --- a/packages/harness/src/experimental/managed-agent-spike/fixture.ts +++ b/packages/harness/src/experimental/managed-agent-spike/fixture.ts @@ -74,18 +74,19 @@ import { writeFileSync } from "node:fs"; import { resolve } from "node:path"; const pidFile = resolve(process.argv[2]); -const child = spawn(process.execPath, ["-e", "setInterval(() => {}, 1000)"], { - stdio: "ignore", +process.on("SIGTERM", () => {}); +const childProgram = [ + 'process.on("SIGTERM", () => {});', + 'if (process.send) process.send("ready");', + 'setInterval(() => {}, 1000);', +].join(""); +const child = spawn(process.execPath, ["-e", childProgram], { + stdio: ["ignore", "ignore", "ignore", "ipc"], windowsHide: true, }); -writeFileSync(pidFile, JSON.stringify({ parentPid: process.pid, childPid: child.pid })); - -function stop() { - try { child.kill("SIGTERM"); } catch {} - setTimeout(() => process.exit(0), 25).unref(); -} -process.once("SIGTERM", stop); -process.once("SIGINT", stop); +child.once("message", () => { + writeFileSync(pidFile, JSON.stringify({ parentPid: process.pid, childPid: child.pid })); +}); setInterval(() => {}, 1000); `.trimStart(); diff --git a/packages/harness/src/experimental/managed-agent-spike/index.ts b/packages/harness/src/experimental/managed-agent-spike/index.ts index e0a5db076..5877c6028 100644 --- a/packages/harness/src/experimental/managed-agent-spike/index.ts +++ b/packages/harness/src/experimental/managed-agent-spike/index.ts @@ -71,6 +71,8 @@ export { export type { ManagedAgentModelTarget, ManagedAgentModelTargetId, + ManagedAgentCancellationReadiness, + ManagedAgentCancellationReadinessReason, ManagedAgentEventNormalizationFailureReason, ManagedAgentPermissionDecision, ManagedAgentPermissionEvidence, diff --git a/packages/harness/src/experimental/managed-agent-spike/probe-cli.test.ts b/packages/harness/src/experimental/managed-agent-spike/probe-cli.test.ts index ddad03662..6f7d99be8 100644 --- a/packages/harness/src/experimental/managed-agent-spike/probe-cli.test.ts +++ b/packages/harness/src/experimental/managed-agent-spike/probe-cli.test.ts @@ -2,6 +2,7 @@ import { describe, expect, it } from "vitest"; import { ManagedAgentProbeCliError, + assertManagedAgentCancellationHostPlatform, assertManagedAgentCertificationNodeVersion, evaluateManagedAgentProbe, executeManagedAgentProbeCli, @@ -81,6 +82,10 @@ function passingL1Result(): ManagedAgentProbeResult { teardown: { quiescent: true, deadlineMet: true, + processTableAvailable: true, + containmentSupported: true, + ownershipProven: false, + forceKillIssued: false, elapsedMs: 5, observedPids: [], alivePidsAtDeadline: [], @@ -117,6 +122,8 @@ function passingL2Result(): ManagedAgentProbeResult { cancellationRequested: true, teardown: { ...base.teardown, + ownershipProven: true, + forceKillIssued: true, observedPids: [12_345, 12_346], }, }; @@ -257,6 +264,38 @@ describe("managed-agent probe CLI", () => { ); }); + it("limits live L2 certification to the reviewed POSIX host model", () => { + expect(() => + assertManagedAgentCancellationHostPlatform("darwin"), + ).not.toThrow(); + expect(() => + assertManagedAgentCancellationHostPlatform("linux"), + ).not.toThrow(); + expect(() => assertManagedAgentCancellationHostPlatform("win32")).toThrow( + "detached POSIX fixture containment model", + ); + }); + + it("rejects Windows L2 before reading gateway or credential environment", async () => { + const environment = new Proxy>( + {}, + { + get() { + throw new Error("environment was read"); + }, + }, + ); + + await expect( + executeManagedAgentProbeCli( + ["--live", "--scenario", "L2", "--target", "sonnet-5"], + environment, + "22.23.2", + "win32", + ), + ).rejects.toThrow("detached POSIX fixture containment model"); + }); + it("requires successful results from every built-in tool for L1", () => { const passing = passingL1Result(); const result: ManagedAgentProbeResult = { @@ -281,6 +320,7 @@ describe("managed-agent probe CLI", () => { outcome: "pass", checks: expect.arrayContaining([ { id: "exact_l2_bash_only_trace", passed: true }, + { id: "l2_containment_prepared", passed: true }, ]), }); diff --git a/packages/harness/src/experimental/managed-agent-spike/probe-cli.ts b/packages/harness/src/experimental/managed-agent-spike/probe-cli.ts index b22f02d10..6245f60fe 100644 --- a/packages/harness/src/experimental/managed-agent-spike/probe-cli.ts +++ b/packages/harness/src/experimental/managed-agent-spike/probe-cli.ts @@ -293,6 +293,16 @@ export function assertManagedAgentCertificationNodeVersion( } } +export function assertManagedAgentCancellationHostPlatform( + platform: NodeJS.Platform, +): void { + if (platform !== "darwin" && platform !== "linux") { + throw new ManagedAgentProbeCliError( + "L2 certification supports only the reviewed detached POSIX fixture containment model", + ); + } +} + export function evaluateManagedAgentProbe( result: ManagedAgentProbeResult, fixturePids: readonly number[] = [], @@ -445,6 +455,14 @@ export function evaluateManagedAgentProbe( id: "teardown_within_five_seconds", passed: result.teardown.quiescent && result.teardown.deadlineMet, }, + { + id: "l2_containment_prepared", + passed: + result.teardown.processTableAvailable && + result.teardown.containmentSupported && + result.teardown.ownershipProven && + result.teardown.forceKillIssued, + }, { id: "fixture_processes_observed", passed: @@ -473,6 +491,7 @@ export async function executeManagedAgentProbeCli( argv: readonly string[], environment: Environment = process.env, runtimeNodeVersion = process.versions.node, + runtimePlatform: NodeJS.Platform = process.platform, ): Promise< ManagedAgentProbeReport | { readonly help: true; readonly usage: string } > { @@ -481,6 +500,9 @@ export async function executeManagedAgentProbeCli( // Validate the immutable runtime before reading the dedicated credential. assertManagedAgentCertificationNodeVersion(runtimeNodeVersion); + if (args.scenario === "L2") { + assertManagedAgentCancellationHostPlatform(runtimePlatform); + } const gatewayOrigin = assertManagedAgentDirectGatewayOrigin( requiredEnvironmentValue(environment, "LLM_GATEWAY_BASE_URL"), ); @@ -523,10 +545,25 @@ export async function executeManagedAgentProbeCli( 15_000, signal, ); - // The model-writable PID file is evidence only. Signal targets - // come exclusively from the SDK process tree sampled by the - // host observer; these values are never handed to it. - await observer.observeProcessTree(); + // The model-writable PID file is evidence only. Readiness is + // derived first from the trusted root handle and bounded host + // process table. These IDs are compared outside the observer + // and never become signal targets. + const readiness = await observer.prepareCancellation(); + if (!readiness.supported) { + throw new ManagedAgentProbeCliError( + `L2 containment preparation failed: ${readiness.reason}`, + ); + } + if ( + !fixturePids.every((pid) => + readiness.observedPids.includes(pid), + ) + ) { + throw new ManagedAgentProbeCliError( + "L2 fixture PIDs were not both present in the host-observed owned process group", + ); + } }, } : {}), diff --git a/packages/harness/src/experimental/managed-agent-spike/process-observer.test.ts b/packages/harness/src/experimental/managed-agent-spike/process-observer.test.ts index 229c55bc2..cffd934a2 100644 --- a/packages/harness/src/experimental/managed-agent-spike/process-observer.test.ts +++ b/packages/harness/src/experimental/managed-agent-spike/process-observer.test.ts @@ -1,8 +1,14 @@ -import { spawn } from "node:child_process"; +import { once } from "node:events"; +import { + spawn as spawnChild, + type ChildProcess, + type ChildProcessWithoutNullStreams, +} from "node:child_process"; import { writeFile } from "node:fs/promises"; import { join } from "node:path"; -import { afterEach, describe, expect, it, vi } from "vitest"; +import type { SpawnedProcess } from "@anthropic-ai/claude-agent-sdk"; +import { afterEach, describe, expect, it } from "vitest"; import { FIXTURE_PATHS, @@ -10,7 +16,11 @@ import { waitForManagedAgentFixturePids, type ManagedAgentFixture, } from "./fixture.js"; -import { LocalManagedAgentProcessObserver } from "./process-observer.js"; +import { + LocalManagedAgentProcessObserver, + type ManagedAgentKernelProcessRecord, + type ManagedAgentProcessTableObservation, +} from "./process-observer.js"; const fixtures: ManagedAgentFixture[] = []; @@ -18,77 +28,492 @@ afterEach(async () => { await Promise.all(fixtures.splice(0).map((fixture) => fixture.cleanup())); }); +function available( + entries: readonly (readonly [number, ManagedAgentKernelProcessRecord])[], +): ManagedAgentProcessTableObservation { + return { available: true, processes: new Map(entries) }; +} + +function activeNodeCommand(): { command: string; args: string[] } { + return { + command: process.execPath, + args: ["-e", "setInterval(() => {}, 1000)"], + }; +} + +function asChildProcess( + spawned: SpawnedProcess, +): ChildProcessWithoutNullStreams { + return spawned as ChildProcessWithoutNullStreams; +} + +function processExists(pid: number): boolean { + try { + process.kill(pid, 0); + return true; + } catch (error) { + return (error as NodeJS.ErrnoException).code === "EPERM"; + } +} + +function processGroupExists(processGroupId: number): boolean { + try { + process.kill(-processGroupId, 0); + return true; + } catch (error) { + return (error as NodeJS.ErrnoException).code === "EPERM"; + } +} + +async function waitForTestProcessDeath( + isAlive: () => boolean, + description: string, + timeoutMs = 1_000, +): Promise { + const deadline = Date.now() + timeoutMs; + while (isAlive() && Date.now() < deadline) { + await new Promise((resolveDelay) => setTimeout(resolveDelay, 10)); + } + if (isAlive()) throw new Error(`${description} survived test cleanup`); +} + +async function forceKillExactTestGroup( + processGroupId: number, + root: ChildProcess, +): Promise { + try { + process.kill(-processGroupId, "SIGKILL"); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ESRCH") throw error; + } + await waitForTestProcessDeath( + () => processGroupExists(processGroupId), + `Owned process group ${processGroupId}`, + ); + if (root.exitCode === null && root.signalCode === null) { + await Promise.race([ + once(root, "exit"), + new Promise((_, rejectTimeout) => + setTimeout( + () => rejectTimeout(new Error("Owned root did not report exit")), + 1_000, + ), + ), + ]); + } +} + +async function forceKillExactTestProcess(child: ChildProcess): Promise { + const pid = child.pid; + if (typeof pid !== "number") return; + try { + process.kill(pid, "SIGKILL"); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ESRCH") throw error; + } + await waitForTestProcessDeath( + () => processExists(pid), + `Unrelated process ${pid}`, + ); +} + describe("LocalManagedAgentProcessObserver", () => { - it("terminates a recorded local tool parent and child within five seconds", async () => { - const fixture = await createManagedAgentFixture(() => "process-observer"); - fixtures.push(fixture); - const observer = new LocalManagedAgentProcessObserver(); + it.skipIf(process.platform === "win32")( + "force-stops and kills the exact non-cooperative fixture group, then confirms death inside one deadline", + async () => { + const fixture = await createManagedAgentFixture(() => "process-observer"); + fixtures.push(fixture); + const observer = new LocalManagedAgentProcessObserver(); + const rawController = new AbortController(); + const forwardedController = new AbortController(); + const unrelated = spawnChild( + process.execPath, + ["-e", "setInterval(() => {}, 1000)"], + { stdio: "ignore", windowsHide: true }, + ); + await once(unrelated, "spawn"); + let root: ChildProcessWithoutNullStreams | undefined; + let ownedProcessGroupId: number | undefined; + observer.bindAbortSignal(rawController.signal); + try { + root = asChildProcess( + observer.spawn({ + command: process.execPath, + args: [FIXTURE_PATHS.processScript, FIXTURE_PATHS.processPidFile], + cwd: fixture.workspaceRoot, + env: { ...process.env }, + signal: forwardedController.signal, + }), + ); + ownedProcessGroupId = root.pid; + expect(ownedProcessGroupId).toBeTypeOf("number"); + const fixturePids = await waitForManagedAgentFixturePids(fixture); + const readiness = await observer.prepareCancellation(); + expect(readiness).toMatchObject({ + supported: true, + reason: "ready", + }); + expect( + fixturePids.every((pid) => readiness.observedPids.includes(pid)), + ).toBe(true); + expect(readiness.observedPids).not.toContain(unrelated.pid); + + const startedAt = Date.now(); + rawController.abort(); + const teardown = await observer.emergencyCleanup(1_000); + + expect(teardown).toMatchObject({ + quiescent: true, + deadlineMet: true, + processTableAvailable: true, + containmentSupported: true, + ownershipProven: true, + forceKillIssued: true, + emergencyCleanupAttempted: true, + alivePidsAtDeadline: [], + }); + expect(Date.now() - startedAt).toBeLessThan(1_000); + expect(processExists(unrelated.pid!)).toBe(true); + } finally { + rawController.abort(); + forwardedController.abort(); + if (root && typeof ownedProcessGroupId === "number") { + // Test-harness safety must not depend on the observer behavior under + // test. Exact test-owned PGID authority is retained until death is + // independently confirmed, including when an assertion fails. + await forceKillExactTestGroup(ownedProcessGroupId, root); + } + observer.dispose(); + await forceKillExactTestProcess(unrelated); + } + }, + 10_000, + ); + + it("fails preparation closed after a fast root exits and never signals its former numeric group", async () => { + const signals: Array = []; + const observer = new LocalManagedAgentProcessObserver({ + platform: "darwin", + signalProcessGroup: (groupId, signal) => { + signals.push([groupId, signal]); + return "sent"; + }, + }); + const rawController = new AbortController(); + const forwardedController = new AbortController(); + observer.bindAbortSignal(rawController.signal); + const child = asChildProcess( + observer.spawn({ + command: process.execPath, + args: ["-e", "process.exit(0)"], + cwd: process.cwd(), + env: { ...process.env }, + signal: forwardedController.signal, + }), + ); + await once(child, "exit"); + try { + await expect(observer.prepareCancellation()).resolves.toMatchObject({ + supported: false, + reason: "root_not_active", + }); + rawController.abort(); + forwardedController.abort(); + await observer.emergencyCleanup(0); + expect(signals).toEqual([]); + } finally { + observer.dispose(); + } + }); + + it("bounds a hanging process-table read and never turns unknown observation into quiescence", async () => { + const signals: Array = []; + const observer = new LocalManagedAgentProcessObserver({ + platform: "darwin", + readProcessTable: () => new Promise(() => undefined), + signalProcessGroup: (groupId, signal) => { + signals.push([groupId, signal]); + try { + process.kill(-groupId, signal); + } catch { + // The test-owned group may already have exited between signals. + } + return "sent"; + }, + }); const controller = new AbortController(); - let aliveAtFailure: readonly number[] = []; + const child = asChildProcess( + observer.spawn({ + ...activeNodeCommand(), + cwd: process.cwd(), + env: { ...process.env }, + signal: controller.signal, + }), + ); + const startedAt = Date.now(); try { + await expect(observer.prepareCancellation()).resolves.toMatchObject({ + supported: false, + reason: "process_table_unavailable", + }); + expect(Date.now() - startedAt).toBeLessThan(1_000); + const shortConfirmationStartedAt = Date.now(); + await expect(observer.waitForQuiescence(50)).resolves.toMatchObject({ + quiescent: false, + deadlineMet: false, + processTableAvailable: false, + }); + expect(Date.now() - shortConfirmationStartedAt).toBeLessThan(150); + + controller.abort(); + await once(child, "exit"); + expect(signals).toEqual([ + [child.pid!, "SIGSTOP"], + [child.pid!, "SIGKILL"], + ]); + } finally { + child.kill("SIGKILL"); + controller.abort(); + observer.dispose(); + } + }); + + it("marks an observed POSIX group escape unsupported without authorizing an individual signal", async () => { + let rootPid = 0; + let escaped = false; + const signals: Array = []; + const table = async (): Promise => { + const rootRecord = { + parentPid: process.pid, + processGroupId: rootPid, + startedAt: "root", + }; + const childRecord = { + parentPid: rootPid, + processGroupId: escaped ? rootPid + 1 : rootPid, + startedAt: "child", + }; + return available([ + [rootPid, rootRecord], + [rootPid + 100, childRecord], + ]); + }; + const observer = new LocalManagedAgentProcessObserver({ + platform: "darwin", + readProcessTable: table, + signalProcessGroup: (groupId, signal) => { + signals.push([groupId, signal]); + return "sent"; + }, + }); + const controller = new AbortController(); + const child = asChildProcess( observer.spawn({ - command: process.execPath, - args: [FIXTURE_PATHS.processScript, FIXTURE_PATHS.processPidFile], - cwd: fixture.workspaceRoot, + ...activeNodeCommand(), + cwd: process.cwd(), env: { ...process.env }, signal: controller.signal, + }), + ); + rootPid = child.pid!; + try { + await expect(observer.prepareCancellation()).resolves.toMatchObject({ + supported: true, + reason: "ready", }); - const pids = await waitForManagedAgentFixturePids(fixture); + escaped = true; await observer.observeProcessTree(); + await expect(observer.waitForQuiescence(0)).resolves.toMatchObject({ + quiescent: false, + containmentSupported: false, + }); + expect(signals).toEqual([]); + } finally { + child.kill("SIGKILL"); controller.abort(); - const teardown = await observer.waitForQuiescence(5_000); - aliveAtFailure = teardown.alivePidsAtDeadline; - expect(teardown.quiescent).toBe(true); - expect(teardown.deadlineMet).toBe(true); - expect(teardown.elapsedMs).toBeLessThanOrEqual(5_000); - expect(pids.every((pid) => teardown.observedPids.includes(pid))).toBe( - true, - ); - expect(teardown.alivePidsAtDeadline).toEqual([]); + observer.dispose(); + } + }); + + it("makes raw and forwarded aborts idempotent after ownership preparation", async () => { + let rootPid = 0; + const signals: Array = []; + const observer = new LocalManagedAgentProcessObserver({ + platform: "darwin", + readProcessTable: async () => + available([ + [ + rootPid, + { + parentPid: process.pid, + processGroupId: rootPid, + startedAt: "root", + }, + ], + [ + rootPid + 100, + { + parentPid: rootPid, + processGroupId: rootPid, + startedAt: "child", + }, + ], + ]), + signalProcessGroup: (groupId, signal) => { + signals.push([groupId, signal]); + return "sent"; + }, + }); + const rawController = new AbortController(); + const forwardedController = new AbortController(); + observer.bindAbortSignal(rawController.signal); + const child = asChildProcess( + observer.spawn({ + ...activeNodeCommand(), + cwd: process.cwd(), + env: { ...process.env }, + signal: forwardedController.signal, + }), + ); + rootPid = child.pid!; + try { + await observer.prepareCancellation(); + rawController.abort(); + forwardedController.abort(); + expect(signals).toEqual([ + [rootPid, "SIGSTOP"], + [rootPid, "SIGKILL"], + ]); + } finally { + child.kill("SIGKILL"); + observer.dispose(); + } + }); + + it("retries a failed SIGKILL while the trusted stopped root still anchors the group", async () => { + let rootPid = 0; + let killAttempts = 0; + const signals: Array = []; + const observer = new LocalManagedAgentProcessObserver({ + platform: "darwin", + readProcessTable: async () => + available([ + [ + rootPid, + { + parentPid: process.pid, + processGroupId: rootPid, + startedAt: "root", + }, + ], + ]), + signalProcessGroup: (groupId, signal) => { + signals.push([groupId, signal]); + if (signal === "SIGKILL" && killAttempts++ === 0) return "failure"; + return "sent"; + }, + }); + const rawController = new AbortController(); + const forwardedController = new AbortController(); + observer.bindAbortSignal(rawController.signal); + const child = asChildProcess( + observer.spawn({ + ...activeNodeCommand(), + cwd: process.cwd(), + env: { ...process.env }, + signal: forwardedController.signal, + }), + ); + rootPid = child.pid!; + try { + await observer.prepareCancellation(); + rawController.abort(); + await observer.emergencyCleanup(0); + + expect(signals).toEqual([ + [rootPid, "SIGSTOP"], + [rootPid, "SIGKILL"], + [rootPid, "SIGKILL"], + ]); + await expect(observer.waitForQuiescence(0)).resolves.toMatchObject({ + containmentSupported: false, + forceKillIssued: true, + quiescent: false, + }); + } finally { + child.kill("SIGKILL"); + observer.dispose(); + } + }); + + it("treats an unexpected group-liveness probe error as unknown, never gone", async () => { + const observer = new LocalManagedAgentProcessObserver({ + platform: "darwin", + readProcessTable: async () => available([]), + processGroupLiveness: () => "unknown", + }); + const controller = new AbortController(); + const child = asChildProcess( + observer.spawn({ + command: process.execPath, + args: ["-e", "process.exit(0)"], + cwd: process.cwd(), + env: { ...process.env }, + signal: controller.signal, + }), + ); + await once(child, "exit"); + try { + await expect(observer.waitForQuiescence(0)).resolves.toMatchObject({ + quiescent: false, + deadlineMet: false, + containmentSupported: false, + }); } finally { controller.abort(); - if (aliveAtFailure.length > 0) await observer.emergencyCleanup(); observer.dispose(); } - }, 10_000); + }); + + it("rejects Windows cancellation containment before granting signal authority", async () => { + const observer = new LocalManagedAgentProcessObserver({ + platform: "win32", + }); + await expect(observer.prepareCancellation()).resolves.toMatchObject({ + supported: false, + reason: "platform_unsupported", + ownershipProven: false, + }); + observer.dispose(); + }); it("never tracks or signals PIDs injected through the model-writable fixture file", async () => { const fixture = await createManagedAgentFixture(() => "forged-pids"); fixtures.push(fixture); - const unrelated = spawn( - process.execPath, - ["-e", "setInterval(() => {}, 1000)"], - { stdio: "ignore", windowsHide: true }, - ); - if (typeof unrelated.pid !== "number") { - unrelated.kill("SIGKILL"); - throw new Error("unrelated test process did not expose a PID"); - } - const forgedPids = [process.pid, unrelated.pid] as const; + const forgedPids = [process.pid, 2_147_483_646] as const; await writeFile( join(fixture.workspaceRoot, FIXTURE_PATHS.processPidFile), - JSON.stringify({ - parentPid: forgedPids[0], - childPid: forgedPids[1], - }), + JSON.stringify({ parentPid: forgedPids[0], childPid: forgedPids[1] }), ); - const observer = new LocalManagedAgentProcessObserver(); - const signalSpy = vi.spyOn(process, "kill"); + const signals: Array = []; + const observer = new LocalManagedAgentProcessObserver({ + signalProcessGroup: (groupId, signal) => { + signals.push([groupId, signal]); + return "sent"; + }, + }); try { await expect(waitForManagedAgentFixturePids(fixture)).resolves.toEqual( forgedPids, ); await observer.observeProcessTree(); const teardown = await observer.waitForQuiescence(0); - await observer.emergencyCleanup(); + await observer.emergencyCleanup(0); expect(teardown.observedPids).not.toContain(forgedPids[0]); expect(teardown.observedPids).not.toContain(forgedPids[1]); - expect(signalSpy).not.toHaveBeenCalled(); - expect(unrelated.exitCode).toBeNull(); + expect(signals).toEqual([]); } finally { - signalSpy.mockRestore(); - unrelated.kill("SIGKILL"); observer.dispose(); } }); diff --git a/packages/harness/src/experimental/managed-agent-spike/process-observer.ts b/packages/harness/src/experimental/managed-agent-spike/process-observer.ts index 253c2db57..bc4df38d8 100644 --- a/packages/harness/src/experimental/managed-agent-spike/process-observer.ts +++ b/packages/harness/src/experimental/managed-agent-spike/process-observer.ts @@ -11,6 +11,7 @@ import type { } from "@anthropic-ai/claude-agent-sdk"; import type { + ManagedAgentCancellationReadiness, ManagedAgentProcessObserver, ManagedAgentTeardownObservation, } from "./types.js"; @@ -18,28 +19,70 @@ import type { const execFileAsync = promisify(execFile); const SAMPLE_INTERVAL_MS = 100; const QUIESCENCE_POLL_MS = 25; +export const MANAGED_AGENT_PROCESS_HELPER_TIMEOUT_MS = 200; -interface ProcessRecord { +export interface ManagedAgentKernelProcessRecord { readonly parentPid: number; readonly processGroupId?: number; - /** Kernel-reported creation time, used to reject PID reuse. */ + /** Kernel-reported creation time used for evidence, never POSIX authority. */ readonly startedAt: string; } -type ProcessTable = ReadonlyMap; +export type ManagedAgentKernelProcessTable = ReadonlyMap< + number, + ManagedAgentKernelProcessRecord +>; -function delay(milliseconds: number): Promise { +export type ManagedAgentProcessTableObservation = + | { + readonly available: true; + readonly processes: ManagedAgentKernelProcessTable; + } + | { readonly available: false }; + +export type ManagedAgentProcessGroupLiveness = "alive" | "gone" | "unknown"; +export type ManagedAgentProcessSignalOutcome = "sent" | "gone" | "failure"; + +export interface LocalManagedAgentProcessObserverOptions { + readonly platform?: NodeJS.Platform; + readonly readProcessTable?: () => Promise; + readonly processGroupLiveness?: ( + processGroupId: number, + ) => ManagedAgentProcessGroupLiveness; + readonly signalProcessGroup?: ( + processGroupId: number, + signal: "SIGSTOP" | "SIGKILL", + ) => ManagedAgentProcessSignalOutcome; + readonly now?: () => number; + readonly delay?: (milliseconds: number) => Promise; +} + +interface OwnedRoot { + readonly pid: number; + readonly child: ChildProcessWithoutNullStreams; + containmentSupported: boolean; + ownershipProven: boolean; + stopIssued: boolean; + forceKillIssued: boolean; +} + +interface ObservedIdentity { + readonly rootPid: number; + readonly record: ManagedAgentKernelProcessRecord; +} + +function defaultDelay(milliseconds: number): Promise { return new Promise((resolveDelay) => setTimeout(resolveDelay, milliseconds)); } function sameProcess( - left: ProcessRecord | undefined, - right: ProcessRecord | undefined, + left: ManagedAgentKernelProcessRecord | undefined, + right: ManagedAgentKernelProcessRecord | undefined, ): boolean { return Boolean(left && right && left.startedAt === right.startedAt); } -async function windowsProcessTable(): Promise { +async function windowsProcessTable(): Promise { const { stdout } = await execFileAsync( "powershell.exe", [ @@ -48,7 +91,13 @@ async function windowsProcessTable(): Promise { "-Command", "Get-CimInstance Win32_Process | Select-Object ProcessId,ParentProcessId,CreationDate | ConvertTo-Json -Compress", ], - { windowsHide: true, maxBuffer: 4 * 1024 * 1024 }, + { + encoding: "utf8", + windowsHide: true, + maxBuffer: 4 * 1024 * 1024, + timeout: MANAGED_AGENT_PROCESS_HELPER_TIMEOUT_MS, + killSignal: "SIGKILL", + }, ); const parsed = JSON.parse(stdout) as | { @@ -81,16 +130,19 @@ async function windowsProcessTable(): Promise { ); } -async function posixProcessTable(): Promise { +async function posixProcessTable(): Promise { const { stdout } = await execFileAsync( "/bin/ps", ["-axo", "pid=,ppid=,pgid=,lstart="], { + encoding: "utf8", windowsHide: true, maxBuffer: 4 * 1024 * 1024, + timeout: MANAGED_AGENT_PROCESS_HELPER_TIMEOUT_MS, + killSignal: "SIGKILL", }, ); - const entries: Array = []; + const entries: Array = []; for (const line of stdout.split("\n")) { const match = /^\s*(\d+)\s+(\d+)\s+(\d+)\s+(.+?)\s*$/.exec(line); if (!match) continue; @@ -106,19 +158,56 @@ async function posixProcessTable(): Promise { return new Map(entries); } -async function readProcessTable(): Promise { +async function defaultReadProcessTable( + platform: NodeJS.Platform, +): Promise { try { - return process.platform === "win32" - ? await windowsProcessTable() - : await posixProcessTable(); + return { + available: true, + processes: + platform === "win32" + ? await windowsProcessTable() + : await posixProcessTable(), + }; } catch { - return new Map(); + return { available: false }; + } +} + +function defaultProcessGroupLiveness( + processGroupId: number, +): ManagedAgentProcessGroupLiveness { + try { + process.kill(-processGroupId, 0); + return "alive"; + } catch (error) { + const code = (error as NodeJS.ErrnoException).code; + if (code === "ESRCH") return "gone"; + if (code === "EPERM") return "alive"; + return "unknown"; + } +} + +function defaultSignalProcessGroup( + processGroupId: number, + signal: "SIGSTOP" | "SIGKILL", +): ManagedAgentProcessSignalOutcome { + try { + process.kill(-processGroupId, signal); + return "sent"; + } catch (error) { + const code = (error as NodeJS.ErrnoException).code; + return code === "ESRCH" ? "gone" : "failure"; } } +function childActive(child: ChildProcessWithoutNullStreams): boolean { + return child.exitCode === null && child.signalCode === null; +} + function descendantsOf( roots: ReadonlySet, - table: ProcessTable, + table: ManagedAgentKernelProcessTable, ): Set { const descendants = new Set(); let changed = true; @@ -137,37 +226,53 @@ function descendantsOf( return descendants; } -async function taskkill(pid: number, force: boolean): Promise { - try { - await execFileAsync( - "taskkill.exe", - ["/PID", String(pid), "/T", ...(force ? ["/F"] : [])], - { windowsHide: true }, - ); - } catch { - // A process that exited between validation and cleanup is already safe. - } -} - /** - * Tracks only SDK roots created by this observer and descendants discovered - * from the host kernel. PID-file contents are deliberately outside this API: - * model-writable fixture evidence must never become signal authority. + * E0.4 deliberately certifies one narrow containment model: the exact local + * L2 fixture running inside a detached POSIX SDK process group. Before abort, + * bounded host enumeration must observe an active trusted ChildProcess as its + * PGID leader and observe the fixture PIDs in that group. Independently, the + * detached spawn plus an active trusted root handle are safe signal authority, + * so even a failed evidence preflight can synchronously stop and kill the + * owned group without leaking it. Such a run still fails certification. * - * POSIX children are placed in their own process group. Before group cleanup, - * the observer revalidates a kernel creation timestamp for the root or a known - * group member, preventing a recycled numeric PID from becoming a kill target. + * This is not a universal sandbox or process-tree killer. Windows, a fast root + * that exits before preparation, unavailable enumeration, and an observed + * setsid/group escape all fail certification closed. POSIX `lstart` is evidence + * only and never authorizes an individual or group signal. Workspace PID-file + * contents never enter this class and can never become signal authority. */ export class LocalManagedAgentProcessObserver implements ManagedAgentProcessObserver { - readonly #rootPids = new Set(); - readonly #rootIdentities = new Map(); - readonly #observedIdentities = new Map(); + readonly #platform: NodeJS.Platform; + readonly #readProcessTable: () => Promise; + readonly #processGroupLiveness: ( + processGroupId: number, + ) => ManagedAgentProcessGroupLiveness; + readonly #signalProcessGroup: ( + processGroupId: number, + signal: "SIGSTOP" | "SIGKILL", + ) => ManagedAgentProcessSignalOutcome; + readonly #now: () => number; + readonly #delay: (milliseconds: number) => Promise; + readonly #roots = new Map(); + readonly #observedIdentities = new Map(); readonly #observedPids = new Set(); - readonly #children = new Map(); readonly #sampler: NodeJS.Timeout; - #sampleTask: Promise | undefined; + readonly #boundSignals = new WeakSet(); + #lastTable: ManagedAgentKernelProcessTable | undefined; + #processTableAvailable = false; + #sampleTask: Promise | undefined; - public constructor() { + public constructor(options: LocalManagedAgentProcessObserverOptions = {}) { + this.#platform = options.platform ?? process.platform; + this.#readProcessTable = + options.readProcessTable ?? + (() => defaultReadProcessTable(this.#platform)); + this.#processGroupLiveness = + options.processGroupLiveness ?? defaultProcessGroupLiveness; + this.#signalProcessGroup = + options.signalProcessGroup ?? defaultSignalProcessGroup; + this.#now = options.now ?? Date.now; + this.#delay = options.delay ?? defaultDelay; this.#sampler = setInterval( () => void this.observeProcessTree(), SAMPLE_INTERVAL_MS, @@ -175,165 +280,354 @@ export class LocalManagedAgentProcessObserver implements ManagedAgentProcessObse this.#sampler.unref(); } + public bindAbortSignal(signal: AbortSignal): void { + if (this.#boundSignals.has(signal)) return; + this.#boundSignals.add(signal); + signal.addEventListener("abort", () => this.#forceStopKillSynchronously(), { + once: true, + }); + if (signal.aborted) this.#forceStopKillSynchronously(); + } + public spawn(options: SpawnOptions): SpawnedProcess { const child = spawnChild(options.command, options.args, { cwd: options.cwd, env: options.env, - detached: process.platform !== "win32", + detached: this.#platform !== "win32", stdio: ["pipe", "pipe", "pipe"], windowsHide: true, }); if (typeof child.pid === "number") { const pid = child.pid; - this.#rootPids.add(pid); - this.#observedPids.add(pid); - this.#children.set(pid, child); - const terminateTree = (): void => { - void this.#terminateRoot(pid, false); - }; - options.signal.addEventListener("abort", terminateTree, { once: true }); - child.once("exit", () => { - options.signal.removeEventListener("abort", terminateTree); - this.#children.delete(pid); + this.#roots.set(pid, { + pid, + child, + containmentSupported: true, + ownershipProven: false, + stopIssued: false, + forceKillIssued: false, }); + this.#observedPids.add(pid); + // The SDK's forwarded SpawnOptions.signal arrives only after its own + // graceful close. Keep it as an idempotent fallback; runtime binds the + // raw Options.abortController signal before query construction. + this.bindAbortSignal(options.signal); void this.observeProcessTree(); } return child; } - public observeProcessTree(): Promise { - if (this.#sampleTask) return this.#sampleTask; - if (this.#rootPids.size === 0) return Promise.resolve(); - this.#sampleTask = (async () => { - const table = await readProcessTable(); - const validatedRoots = new Set(); - for (const pid of this.#rootPids) { - const current = table.get(pid); - const known = this.#rootIdentities.get(pid); - const child = this.#children.get(pid); - const childActive = - child !== undefined && - child.exitCode === null && - child.signalCode === null; - if (!current || (known ? !sameProcess(known, current) : !childActive)) { - continue; + async #boundedProcessTableRead( + timeoutMs: number, + ): Promise { + let timeout: NodeJS.Timeout | undefined; + const read = Promise.resolve() + .then(() => this.#readProcessTable()) + .catch( + (): ManagedAgentProcessTableObservation => ({ + available: false, + }), + ); + try { + return await Promise.race([ + read, + new Promise((resolveTimeout) => { + timeout = setTimeout( + () => resolveTimeout({ available: false }), + Math.max(0, timeoutMs), + ); + }), + ]); + } finally { + if (timeout) clearTimeout(timeout); + } + } + + public async observeProcessTree( + timeoutMs = MANAGED_AGENT_PROCESS_HELPER_TIMEOUT_MS, + ): Promise { + if (this.#roots.size === 0) { + this.#lastTable = new Map(); + this.#processTableAvailable = true; + return true; + } + const boundedTimeoutMs = Math.max( + 0, + Math.min(MANAGED_AGENT_PROCESS_HELPER_TIMEOUT_MS, timeoutMs), + ); + if (!this.#sampleTask) { + this.#sampleTask = (async () => { + const observation = + await this.#boundedProcessTableRead(boundedTimeoutMs); + if (!observation.available) { + this.#lastTable = undefined; + this.#processTableAvailable = false; + return false; } - if (!known) this.#rootIdentities.set(pid, current); - this.#observedIdentities.set(pid, current); - this.#observedPids.add(pid); - validatedRoots.add(pid); - } - for (const pid of descendantsOf(validatedRoots, table)) { - const current = table.get(pid); - if (!current) continue; - this.#observedIdentities.set(pid, current); - this.#observedPids.add(pid); - } - })().finally(() => { - this.#sampleTask = undefined; - }); - return this.#sampleTask; + + const table = observation.processes; + this.#lastTable = table; + this.#processTableAvailable = true; + for (const root of this.#roots.values()) { + if (this.#platform !== "win32") { + for (const [pid, observed] of this.#observedIdentities) { + if (observed.rootPid !== root.pid) continue; + const current = table.get(pid); + if ( + sameProcess(observed.record, current) && + current?.processGroupId !== root.pid + ) { + root.containmentSupported = false; + } + } + const currentRoot = table.get(root.pid); + if ( + currentRoot && + childActive(root.child) && + currentRoot.processGroupId !== root.pid + ) { + root.containmentSupported = false; + } + for (const [pid, record] of table) { + if (record.processGroupId !== root.pid) continue; + this.#observedIdentities.set(pid, { + rootPid: root.pid, + record, + }); + this.#observedPids.add(pid); + } + continue; + } + + const currentRoot = table.get(root.pid); + const seeds = new Set(); + if (currentRoot && childActive(root.child)) { + seeds.add(root.pid); + this.#observedIdentities.set(root.pid, { + rootPid: root.pid, + record: currentRoot, + }); + this.#observedPids.add(root.pid); + } + for (const [pid, observed] of this.#observedIdentities) { + if ( + observed.rootPid === root.pid && + sameProcess(observed.record, table.get(pid)) + ) { + seeds.add(pid); + } + } + for (const pid of descendantsOf(seeds, table)) { + const current = table.get(pid); + if (!current) continue; + this.#observedIdentities.set(pid, { + rootPid: root.pid, + record: current, + }); + this.#observedPids.add(pid); + } + } + return true; + })().finally(() => { + this.#sampleTask = undefined; + }); + } + + const sample = this.#sampleTask; + let timeout: NodeJS.Timeout | undefined; + const available = await Promise.race([ + sample, + new Promise((resolveTimeout) => { + timeout = setTimeout(() => resolveTimeout(false), boundedTimeoutMs); + }), + ]); + if (timeout) clearTimeout(timeout); + if (!available) { + // A caller with a shorter absolute deadline must not reuse a stale table + // while a longer background sample is still pending. + this.#lastTable = undefined; + this.#processTableAvailable = false; + } + return available; } - async #aliveObservedPids(): Promise { - const table = await readProcessTable(); - const alive = new Set( - [...this.#observedIdentities].flatMap(([pid, identity]) => - sameProcess(identity, table.get(pid)) ? [pid] : [], + public async prepareCancellation(): Promise { + const observedPids = (): number[] => + [...this.#observedPids].sort((left, right) => left - right); + const unsupported = ( + reason: Exclude, + ): ManagedAgentCancellationReadiness => ({ + supported: false, + reason, + processTableAvailable: this.#processTableAvailable, + containmentSupported: [...this.#roots.values()].every( + ({ containmentSupported }) => containmentSupported, ), - ); - // If kernel enumeration is unavailable, a still-active ChildProcess handle - // must keep teardown fail-closed instead of producing false quiescence. - for (const [pid, child] of this.#children) { - if (child.exitCode === null && child.signalCode === null) alive.add(pid); + ownershipProven: false, + observedPids: observedPids(), + }); + + if (this.#platform !== "darwin" && this.#platform !== "linux") { + for (const root of this.#roots.values()) { + root.containmentSupported = false; + } + return unsupported("platform_unsupported"); + } + if (!(await this.observeProcessTree())) { + return unsupported("process_table_unavailable"); + } + if (this.#roots.size !== 1) return unsupported("root_count_invalid"); + const root = [...this.#roots.values()][0]!; + if (!childActive(root.child)) { + root.containmentSupported = false; + return unsupported("root_not_active"); + } + if (!root.containmentSupported) { + return unsupported("containment_escaped"); + } + const currentRoot = this.#lastTable!.get(root.pid); + if (!currentRoot || currentRoot.processGroupId !== root.pid) { + root.containmentSupported = false; + return unsupported("root_not_group_leader"); } - return [...alive].sort((left, right) => left - right); + + root.ownershipProven = true; + return { + supported: true, + reason: "ready", + processTableAvailable: true, + containmentSupported: true, + ownershipProven: true, + observedPids: observedPids(), + }; } - async #terminateRoot(rootPid: number, force: boolean): Promise { - await this.observeProcessTree(); - const table = await readProcessTable(); - const rootIdentity = this.#rootIdentities.get(rootPid); - const rootMatches = sameProcess(rootIdentity, table.get(rootPid)); - const child = this.#children.get(rootPid); - const childActive = - child !== undefined && - child.exitCode === null && - child.signalCode === null; - - if (process.platform === "win32") { - if (rootMatches) { - await taskkill(rootPid, force); - } else if (!rootIdentity && childActive) { - child.kill(force ? "SIGKILL" : "SIGTERM"); + #forceStopKillSynchronously(): void { + if (this.#platform !== "darwin" && this.#platform !== "linux") return; + for (const root of this.#roots.values()) { + if (root.forceKillIssued || !childActive(root.child)) { + continue; + } + // detached:true plus the still-active trusted ChildProcess handle are + // sufficient signal authority even when ps evidence is unavailable. + // Certification readiness remains false in that case. Second-resolution + // lstart and workspace PIDs are never signal authority. + if (!root.stopIssued) { + const stopOutcome = this.#signalProcessGroup(root.pid, "SIGSTOP"); + root.stopIssued = stopOutcome === "sent"; + if (stopOutcome === "failure") { + root.containmentSupported = false; + } + } + const killOutcome = this.#signalProcessGroup(root.pid, "SIGKILL"); + root.forceKillIssued = killOutcome === "sent"; + if (killOutcome === "failure") { + // Keep the active/stopped group anchored so emergencyCleanup can retry + // SIGKILL safely, but certification evidence remains failed. + root.containmentSupported = false; } - return; } + } - const hasValidatedGroupMember = [...this.#observedIdentities].some( - ([pid, identity]) => { - const current = table.get(pid); - return ( - current?.processGroupId === rootPid && sameProcess(identity, current) - ); - }, - ); - if (rootMatches || hasValidatedGroupMember) { - try { - process.kill(-rootPid, force ? "SIGKILL" : "SIGTERM"); - } catch { - // A validated group that exited before the signal is already safe. + async #currentObservation( + startedAt: number, + emergencyCleanupAttempted: boolean, + ): Promise { + const roots = [...this.#roots.values()]; + const alive = new Set(); + for (const root of roots) { + if (childActive(root.child)) alive.add(root.pid); + if (!this.#processTableAvailable) continue; + const table = this.#lastTable!; + if (this.#platform !== "win32") { + for (const [pid, record] of table) { + if (record.processGroupId === root.pid) alive.add(pid); + } + const groupLiveness = this.#processGroupLiveness(root.pid); + if (groupLiveness === "alive") alive.add(root.pid); + if (groupLiveness === "unknown") { + root.containmentSupported = false; + } + } else { + for (const [pid, observed] of this.#observedIdentities) { + if ( + observed.rootPid === root.pid && + sameProcess(observed.record, table.get(pid)) + ) { + alive.add(pid); + } + } } - } else if (!rootIdentity && childActive) { - // The kernel sampler can be unavailable. The trusted ChildProcess handle - // remains safe for root-only termination, but we never guess descendants. - child.kill(force ? "SIGKILL" : "SIGTERM"); } + + const processTableAvailable = + roots.length === 0 || this.#processTableAvailable; + const containmentSupported = roots.every( + ({ containmentSupported: supported }) => supported, + ); + const ownershipProven = + roots.length > 0 && roots.every(({ ownershipProven }) => ownershipProven); + const forceKillIssued = + roots.length > 0 && roots.every(({ forceKillIssued }) => forceKillIssued); + const elapsedMs = Math.max(0, this.#now() - startedAt); + const quiescent = + processTableAvailable && containmentSupported && alive.size === 0; + return { + quiescent, + deadlineMet: quiescent, + processTableAvailable, + containmentSupported, + ownershipProven, + forceKillIssued, + elapsedMs, + observedPids: [...this.#observedPids].sort((left, right) => left - right), + alivePidsAtDeadline: [...alive].sort((left, right) => left - right), + emergencyCleanupAttempted, + }; } public async waitForQuiescence( timeoutMs: number, ): Promise { - const startedAt = Date.now(); - let alivePids: number[] = []; - do { - await this.observeProcessTree(); - alivePids = await this.#aliveObservedPids(); - if (alivePids.length === 0) { + const startedAt = this.#now(); + const boundedTimeoutMs = Math.max(0, timeoutMs); + for (;;) { + const elapsedBeforeSample = Math.max(0, this.#now() - startedAt); + await this.observeProcessTree( + Math.max(0, boundedTimeoutMs - elapsedBeforeSample), + ); + const observation = await this.#currentObservation(startedAt, false); + if (observation.quiescent) { return { - quiescent: true, - deadlineMet: true, - elapsedMs: Date.now() - startedAt, - observedPids: [...this.#observedPids].sort( - (left, right) => left - right, - ), - alivePidsAtDeadline: [], - emergencyCleanupAttempted: false, + ...observation, + deadlineMet: observation.elapsedMs <= boundedTimeoutMs, }; } - if (Date.now() - startedAt >= timeoutMs) break; - await delay(QUIESCENCE_POLL_MS); - } while (Date.now() - startedAt < timeoutMs); + if (observation.elapsedMs >= boundedTimeoutMs) { + return { ...observation, deadlineMet: false }; + } + await this.#delay( + Math.min(QUIESCENCE_POLL_MS, boundedTimeoutMs - observation.elapsedMs), + ); + } + } - await this.observeProcessTree(); - alivePids = await this.#aliveObservedPids(); + public async emergencyCleanup( + timeoutMs: number, + ): Promise { + const startedAt = this.#now(); + const boundedTimeoutMs = Math.max(0, timeoutMs); + this.#forceStopKillSynchronously(); + const confirmation = await this.waitForQuiescence(boundedTimeoutMs); + const elapsedMs = Math.max(0, this.#now() - startedAt); return { - quiescent: alivePids.length === 0, - deadlineMet: alivePids.length === 0, - elapsedMs: Date.now() - startedAt, - observedPids: [...this.#observedPids].sort((left, right) => left - right), - alivePidsAtDeadline: alivePids, - emergencyCleanupAttempted: false, + ...confirmation, + elapsedMs, + deadlineMet: confirmation.quiescent && elapsedMs <= boundedTimeoutMs, + emergencyCleanupAttempted: true, }; } - public async emergencyCleanup(): Promise { - await this.observeProcessTree(); - await Promise.all( - [...this.#rootPids].map((pid) => this.#terminateRoot(pid, true)), - ); - } - public dispose(): void { clearInterval(this.#sampler); } diff --git a/packages/harness/src/experimental/managed-agent-spike/runtime.test.ts b/packages/harness/src/experimental/managed-agent-spike/runtime.test.ts index 226a26632..e4c18f868 100644 --- a/packages/harness/src/experimental/managed-agent-spike/runtime.test.ts +++ b/packages/harness/src/experimental/managed-agent-spike/runtime.test.ts @@ -42,6 +42,10 @@ function quiescentTeardown(): ManagedAgentTeardownObservation { return { quiescent: true, deadlineMet: true, + processTableAvailable: true, + containmentSupported: true, + ownershipProven: true, + forceKillIssued: true, elapsedMs: 12, observedPids: [], alivePidsAtDeadline: [], @@ -60,9 +64,21 @@ function fakeObserver( spawn: vi.fn(() => { throw new Error("fake query must not spawn"); }), - observeProcessTree: vi.fn(async () => undefined), + bindAbortSignal: vi.fn(), + prepareCancellation: vi.fn(async () => ({ + supported: true, + reason: "ready" as const, + processTableAvailable: true, + containmentSupported: true, + ownershipProven: true, + observedPids: [], + })), + observeProcessTree: vi.fn(async () => true), waitForQuiescence: vi.fn(async () => teardown), - emergencyCleanup: vi.fn(async () => undefined), + emergencyCleanup: vi.fn(async () => ({ + ...teardown, + emergencyCleanupAttempted: true, + })), dispose: vi.fn(), }; } @@ -477,6 +493,10 @@ describe("runManagedAgentProbe", () => { const observer = fakeObserver({ quiescent: false, deadlineMet: false, + processTableAvailable: true, + containmentSupported: true, + ownershipProven: false, + forceKillIssued: false, elapsedMs: 5_001, observedPids: [8001], alivePidsAtDeadline: [8001], @@ -496,7 +516,7 @@ describe("runManagedAgentProbe", () => { type: "terminal", terminal: "teardown_timeout", }); - expect(observer.emergencyCleanup).toHaveBeenCalledWith(); + expect(observer.emergencyCleanup).toHaveBeenCalledOnce(); }); it("rejects a successful stream when a requested tool has no primary hook decision", async () => { @@ -860,11 +880,80 @@ describe("runManagedAgentProbe", () => { ).toHaveLength(1); }); + it("abandons a never-resolving iterator next immediately after raw cancellation", async () => { + const { config } = await probeConfig("L2"); + const observer = fakeObserver(); + const close = vi.fn(); + const resultPromise = runManagedAgentProbe(config, { + hermeticGatewayOrigin: config.gatewayOrigin, + processObserver: observer, + waitForCancellationSignal: async () => undefined, + queryFactory: () => ({ + [Symbol.asyncIterator]() { + return { + next: () => new Promise>(() => undefined), + }; + }, + close, + }), + }); + + const result = await Promise.race([ + resultPromise, + new Promise((_, reject) => + setTimeout( + () => reject(new Error("probe stayed blocked on iterator.next()")), + 500, + ), + ), + ]); + + expect(result.terminal).toBe("cancelled"); + expect(result.terminationEvidence.queryExecution).toBe("iteration_aborted"); + expect(close).toHaveBeenCalledOnce(); + expect(observer.bindAbortSignal).toHaveBeenCalledOnce(); + }); + + it("includes iterator abandonment and close in the one cancellation deadline", async () => { + const { config } = await probeConfig("L2"); + let now = 1_000; + const observer = fakeObserver(); + const close = vi.fn(async () => { + now = 3_250; + }); + const result = await runManagedAgentProbe(config, { + hermeticGatewayOrigin: config.gatewayOrigin, + processObserver: observer, + now: () => now, + waitForCancellationSignal: async () => undefined, + queryFactory: () => ({ + [Symbol.asyncIterator]() { + return { + next: () => new Promise>(() => undefined), + }; + }, + close, + }), + }); + + expect(observer.emergencyCleanup).toHaveBeenCalledWith(2_750); + expect(result.teardown).toMatchObject({ + quiescent: true, + deadlineMet: true, + elapsedMs: 2_250, + }); + expect(result.terminal).toBe("cancelled"); + }); + it("records teardown failure before attempting emergency cleanup", async () => { const { config } = await probeConfig(); const observer = fakeObserver({ quiescent: false, deadlineMet: false, + processTableAvailable: true, + containmentSupported: true, + ownershipProven: false, + forceKillIssued: false, elapsedMs: 5_001, observedPids: [9001], alivePidsAtDeadline: [9001], @@ -886,7 +975,7 @@ describe("runManagedAgentProbe", () => { expect(result.terminal).toBe("teardown_timeout"); expect(result.teardown.emergencyCleanupAttempted).toBe(true); - expect(observer.emergencyCleanup).toHaveBeenCalledWith(); + expect(observer.emergencyCleanup).toHaveBeenCalledOnce(); expect(result.events.at(-1)).toMatchObject({ type: "terminal", terminal: "teardown_timeout", diff --git a/packages/harness/src/experimental/managed-agent-spike/runtime.ts b/packages/harness/src/experimental/managed-agent-spike/runtime.ts index 2020accfc..a05260a60 100644 --- a/packages/harness/src/experimental/managed-agent-spike/runtime.ts +++ b/packages/harness/src/experimental/managed-agent-spike/runtime.ts @@ -45,6 +45,7 @@ export const MANAGED_AGENT_TEARDOWN_TIMEOUT_MS = 5_000; export const MANAGED_AGENT_CORRELATION_MARKER_VERSION = "SAPIOM_CERTIFICATION_CORRELATION_V1"; const QUERY_CLOSE_TIMEOUT_MS = 2_000; +const FORCE_CLEANUP_CONFIRMATION_RESERVE_MS = 1_000; const MANAGED_AGENT_L2_BUILTIN_TOOLS = ["Bash"] as const; const MANAGED_AGENT_L2_DISALLOWED_TOOLS = [ ...MANAGED_AGENT_DISALLOWED_TOOLS, @@ -271,18 +272,28 @@ function buildManagedAgentPolicyDiagnostics( return diagnostics; } -async function closeQueryBounded(query: ManagedAgentQuery): Promise { +async function closeQueryBounded( + query: ManagedAgentQuery, + timeoutMs = QUERY_CLOSE_TIMEOUT_MS, +): Promise { let timeout: NodeJS.Timeout | undefined; + const close = Promise.resolve() + .then(() => query.close()) + .then( + () => true, + () => false, + ); + if (timeoutMs <= 0) { + void close; + return false; + } try { return await Promise.race([ - Promise.resolve().then(() => { - query.close(); - return true; - }), + close, new Promise((resolveTimeout) => { timeout = setTimeout( () => resolveTimeout(false), - QUERY_CLOSE_TIMEOUT_MS, + Math.min(QUERY_CLOSE_TIMEOUT_MS, timeoutMs), ); timeout.unref(); }), @@ -294,6 +305,37 @@ async function closeQueryBounded(query: ManagedAgentQuery): Promise { } } +type ManagedAgentIteratorStep = + | { readonly kind: "next"; readonly value: IteratorResult } + | { readonly kind: "aborted" }; + +async function nextManagedAgentEvent( + iterator: AsyncIterator, + signal: AbortSignal, +): Promise { + if (signal.aborted) return { kind: "aborted" }; + let abortListener: (() => void) | undefined; + const next = Promise.resolve() + .then(() => iterator.next()) + .then( + (value): ManagedAgentIteratorStep => ({ kind: "next", value }), + (error): ManagedAgentIteratorStep => { + throw error; + }, + ); + const aborted = new Promise((resolveAbort) => { + abortListener = () => resolveAbort({ kind: "aborted" }); + signal.addEventListener("abort", abortListener, { once: true }); + }); + try { + // `next` converts a late rejection into this already-observed promise, so + // abandoning it after abort cannot create an unhandled rejection. + return await Promise.race([next, aborted]); + } finally { + if (abortListener) signal.removeEventListener("abort", abortListener); + } +} + function classifyTerminal(input: { readonly teardown: ManagedAgentTeardownObservation; readonly queryCreated: boolean; @@ -346,6 +388,7 @@ export async function runManagedAgentProbe( ); let cancellationRequested = false; let cancellationRequestedAt: number | undefined; + let abortStartedAt: number | undefined; let query: ManagedAgentQuery | undefined; let queryFailed = false; let queryClosed = false; @@ -379,6 +422,9 @@ export async function runManagedAgentProbe( }); const processObserver = dependencies.processObserver ?? createLocalManagedAgentProcessObserver(); + // SpawnOptions.signal is forwarded only after the SDK's graceful close. + // Bind the raw per-run abort signal so L2 containment starts synchronously. + processObserver.bindAbortSignal(abortController.signal); const options: Options = { abortController, @@ -457,12 +503,14 @@ export async function runManagedAgentProbe( if (triggerController.signal.aborted) return; cancellationRequested = true; cancellationRequestedAt = (dependencies.now ?? Date.now)(); + abortStartedAt = cancellationRequestedAt; recorder.recordLifecycle("cancellation_requested"); abortController.abort(); }) .catch(() => { if (!triggerController.signal.aborted) { cancellationTriggerFailed = true; + abortStartedAt = (dependencies.now ?? Date.now)(); abortController.abort(); } }) @@ -485,9 +533,22 @@ export async function runManagedAgentProbe( queryExecution = "construction_failed"; throw error; } - for await (const event of query) { + const iterator = query[Symbol.asyncIterator](); + for (;;) { + const step = await nextManagedAgentEvent( + iterator, + abortController.signal, + ); + if (step.kind === "aborted") { + queryExecution = "iteration_aborted"; + break; + } + if (step.value.done) { + queryExecution = "iteration_completed"; + break; + } try { - recorder.observeSdkEvent(event); + recorder.observeSdkEvent(step.value.value); } catch (error) { if (error instanceof ManagedAgentEventError) { eventNormalizationFailure = error.reason; @@ -496,7 +557,6 @@ export async function runManagedAgentProbe( throw error; } } - queryExecution = "iteration_completed"; } catch { if (eventNormalizationFailure) { queryExecution = "event_normalization_failed"; @@ -512,31 +572,50 @@ export async function runManagedAgentProbe( triggerController.abort(); if (cancellationTask) await cancellationTask; queryFailed ||= cancellationTriggerFailed; - if (query) queryClosed = await closeQueryBounded(query); + if (query) { + const now = dependencies.now ?? Date.now; + const closeBudgetMs = + abortStartedAt === undefined + ? QUERY_CLOSE_TIMEOUT_MS + : Math.max( + 0, + MANAGED_AGENT_TEARDOWN_TIMEOUT_MS - + (now() - abortStartedAt) - + FORCE_CLEANUP_CONFIRMATION_RESERVE_MS, + ); + queryClosed = await closeQueryBounded(query, closeBudgetMs); + } if ((query && !queryClosed) || queryFailed) abortController.abort(); } } const now = dependencies.now ?? Date.now; - const elapsedBeforeTeardown = - cancellationRequestedAt === undefined - ? 0 - : now() - cancellationRequestedAt; - const remainingTeardownMs = Math.max( - 0, - MANAGED_AGENT_TEARDOWN_TIMEOUT_MS - elapsedBeforeTeardown, - ); - teardown = await processObserver.waitForQuiescence(remainingTeardownMs); - if (cancellationRequestedAt !== undefined) { - const totalElapsedMs = now() - cancellationRequestedAt; - teardown = { - ...teardown, - elapsedMs: totalElapsedMs, - deadlineMet: - teardown.quiescent && - totalElapsedMs <= MANAGED_AGENT_TEARDOWN_TIMEOUT_MS, - }; + const teardownStartedAt = abortStartedAt ?? now(); + const remainingBudget = (): number => + Math.max( + 0, + MANAGED_AGENT_TEARDOWN_TIMEOUT_MS - (now() - teardownStartedAt), + ); + if (abortController.signal.aborted) { + teardown = await processObserver.emergencyCleanup(remainingBudget()); + } else { + teardown = await processObserver.waitForQuiescence( + Math.max(0, remainingBudget() - FORCE_CLEANUP_CONFIRMATION_RESERVE_MS), + ); + if (!teardown.quiescent) { + teardown = await processObserver.emergencyCleanup(remainingBudget()); + } } + const totalElapsedMs = now() - teardownStartedAt; + teardown = { + ...teardown, + elapsedMs: totalElapsedMs, + deadlineMet: + teardown.quiescent && + teardown.processTableAvailable && + teardown.containmentSupported && + totalElapsedMs <= MANAGED_AGENT_TEARDOWN_TIMEOUT_MS, + }; const beforePolicyOverride = classifyTerminal({ teardown, queryCreated: query !== undefined, @@ -569,12 +648,6 @@ export async function runManagedAgentProbe( } recorder.recordTerminal(terminal); - if (!teardown.quiescent) { - await processObserver.emergencyCleanup(); - teardown = { ...teardown, emergencyCleanupAttempted: true }; - terminal = "teardown_timeout"; - } - const sdkResult = recorder.result ? recorder.result.isError ? "error" diff --git a/packages/harness/src/experimental/managed-agent-spike/types.ts b/packages/harness/src/experimental/managed-agent-spike/types.ts index 0275cff5e..32ab68aab 100644 --- a/packages/harness/src/experimental/managed-agent-spike/types.ts +++ b/packages/harness/src/experimental/managed-agent-spike/types.ts @@ -165,6 +165,14 @@ export interface ManagedAgentTerminationEvidence { export interface ManagedAgentTeardownObservation { readonly quiescent: boolean; readonly deadlineMet: boolean; + /** False means ps/CIM observation was unknown, never an empty table. */ + readonly processTableAvailable: boolean; + /** False means the owned E0 containment model was escaped or unproven. */ + readonly containmentSupported: boolean; + /** True only after an active POSIX root was observed as its PGID leader. */ + readonly ownershipProven: boolean; + /** True only when the bound raw abort synchronously issued SIGSTOP+SIGKILL. */ + readonly forceKillIssued: boolean; readonly elapsedMs: number; readonly observedPids: readonly number[]; readonly alivePidsAtDeadline: readonly number[]; @@ -219,7 +227,7 @@ export interface ManagedAgentProbeResult { * Query.mcpCall bypasses permission checks and is outside this host boundary. */ export interface ManagedAgentQuery extends AsyncIterable { - close(): void; + close(): void | Promise; } export type ManagedAgentQueryFactory = (input: { @@ -229,16 +237,38 @@ export type ManagedAgentQueryFactory = (input: { export interface ManagedAgentProcessObserver { spawn(options: SpawnOptions): SpawnedProcess; + /** Bind the raw per-run Options.abortController signal before SDK startup. */ + bindAbortSignal(signal: AbortSignal): void; + /** Prove the narrow POSIX ownership model before allowing L2 to cancel. */ + prepareCancellation(): Promise; /** Sample only descendants of the host-observed SDK process roots. */ - observeProcessTree(): Promise; + observeProcessTree(timeoutMs?: number): Promise; waitForQuiescence( timeoutMs: number, ): Promise; - /** Signal only process groups rooted in an SDK process spawned above. */ - emergencyCleanup(): Promise; + /** Idempotently force a proven active group and confirm within this budget. */ + emergencyCleanup(timeoutMs: number): Promise; dispose(): void; } +export type ManagedAgentCancellationReadinessReason = + | "ready" + | "platform_unsupported" + | "process_table_unavailable" + | "root_count_invalid" + | "root_not_active" + | "root_not_group_leader" + | "containment_escaped"; + +export interface ManagedAgentCancellationReadiness { + readonly supported: boolean; + readonly reason: ManagedAgentCancellationReadinessReason; + readonly processTableAvailable: boolean; + readonly containmentSupported: boolean; + readonly ownershipProven: boolean; + readonly observedPids: readonly number[]; +} + export interface ManagedAgentProbeDependencies { readonly queryFactory?: ManagedAgentQueryFactory; /** From 8b357e58a40c8d5832a99c934dff80b08f83c1cb Mon Sep 17 00:00:00 2001 From: Yash Date: Sun, 16 Aug 2026 20:16:39 -0700 Subject: [PATCH 10/24] fix(harness): retain managed agent process authority Keep the cancellation deadline referenced and launch POSIX SDK commands through an observer-owned supervisor so fast inner-root exits cannot orphan same-group descendants.\n\nRefs: SAP-2632 --- .../managed-agent-spike/README.md | 35 ++- .../managed-agent-spike/probe-cli.ts | 6 +- .../process-observer.test.ts | 264 ++++++++++++++++ .../managed-agent-spike/process-observer.ts | 287 ++++++++++++++++-- .../managed-agent-spike/runtime.test.ts | 90 ++++++ .../managed-agent-spike/runtime.ts | 1 - .../experimental/managed-agent-spike/types.ts | 6 +- 7 files changed, 648 insertions(+), 41 deletions(-) diff --git a/packages/harness/src/experimental/managed-agent-spike/README.md b/packages/harness/src/experimental/managed-agent-spike/README.md index cb0074c63..748e458b0 100644 --- a/packages/harness/src/experimental/managed-agent-spike/README.md +++ b/packages/harness/src/experimental/managed-agent-spike/README.md @@ -67,28 +67,35 @@ and SDK tool-result event. ## L2 cancellation containment boundary E0.4 certifies one deliberately narrow host model: the exact non-cooperative -fixture command running under the active Agent SDK root in a detached macOS or -Linux process group. Before cancellation may fire, a bounded host `ps` sample -must prove that the trusted SDK `ChildProcess` is still active and is the group +fixture command running under an observer-owned supervisor in a detached macOS +or Linux process group. The supervisor is the persistent group leader and stays +alive when the inner Agent SDK root exits while a same-group descendant +survives. Before cancellation may fire, a bounded host `ps` sample must prove +that the trusted supervisor `ChildProcess` is still active and is the group leader, and both PIDs read from the fixture file must already be present in the independently host-observed group. The file is comparison evidence only; its contents are never passed into the observer or used as signal targets. The runtime binds the observer directly to the per-run `Options.abortController` signal. On abort it synchronously and idempotently sends `SIGSTOP` followed by -`SIGKILL` to the observer-created group while the trusted root handle remains -active. The fixture parent and child intentionally ignore `SIGTERM`, making the -forced path load-bearing. Iterator abandonment, query close, bounded process -enumeration, and group-death confirmation share one absolute five-second -process-termination deadline. Workspace snapshots and result assembly occur -afterward. +`SIGKILL` to the supervisor-owned group while the trusted anchor handle remains +active. The returned SDK process also maps direct kills to that group, and a +parent IPC disconnect kills the group. The fixture parent and child +intentionally ignore `SIGTERM`, making the forced path load-bearing. The +supervisor's bounded `ps` helper remains inside the owned group and only its +known PID plus the anchor PID are excluded from its membership decision. +Iterator abandonment, query close, bounded process enumeration, and +group-death confirmation share one absolute five-second process-termination +deadline. Workspace snapshots and result assembly occur afterward. The +close-deadline timer remains referenced so a CLI host cannot exit before +cleanup and result reporting finish. An unavailable or timed-out process table is explicit unknown evidence, never -an empty process table. The active detached root still authorizes safe cleanup -of its owned group, but the run fails certification. A fast root exit before -preparation, an observed `setsid`/group escape, unknown group liveness, failed -signals, and Windows all fail closed. Windows live L2 is rejected before the -query or credential is opened. Universal containment, POSIX group escape, +an empty process table. The active detached supervisor still authorizes safe +cleanup of its owned group, but the run fails certification. An invalid or +inactive supervisor, an observed `setsid`/group escape, unknown group liveness, +failed signals, and Windows all fail closed. Windows live L2 is rejected before +the query or credential is opened. Universal containment, POSIX group escape, Windows Job Objects, and production recovery belong to later epics; this probe does not claim those guarantees. diff --git a/packages/harness/src/experimental/managed-agent-spike/probe-cli.ts b/packages/harness/src/experimental/managed-agent-spike/probe-cli.ts index 6245f60fe..771c5bbb2 100644 --- a/packages/harness/src/experimental/managed-agent-spike/probe-cli.ts +++ b/packages/harness/src/experimental/managed-agent-spike/probe-cli.ts @@ -546,9 +546,9 @@ export async function executeManagedAgentProbeCli( signal, ); // The model-writable PID file is evidence only. Readiness is - // derived first from the trusted root handle and bounded host - // process table. These IDs are compared outside the observer - // and never become signal targets. + // derived first from the trusted supervisor handle and bounded + // host process table. These IDs are compared outside the + // observer and never become signal targets. const readiness = await observer.prepareCancellation(); if (!readiness.supported) { throw new ManagedAgentProbeCliError( diff --git a/packages/harness/src/experimental/managed-agent-spike/process-observer.test.ts b/packages/harness/src/experimental/managed-agent-spike/process-observer.test.ts index cffd934a2..9d1c6e252 100644 --- a/packages/harness/src/experimental/managed-agent-spike/process-observer.test.ts +++ b/packages/harness/src/experimental/managed-agent-spike/process-observer.test.ts @@ -41,6 +41,37 @@ function activeNodeCommand(): { command: string; args: string[] } { }; } +const FAST_EXIT_ROOT_SCRIPT = String.raw` +import { spawn } from "node:child_process"; +import { existsSync, writeFileSync } from "node:fs"; +import { resolve } from "node:path"; + +const pidFile = resolve(process.argv[1]); +const exitTiming = process.argv[2]; +const exitMarker = resolve(process.argv[3]); +const childProgram = [ + 'process.on("SIGTERM", () => {});', + 'if (process.send) process.send("ready");', + 'setInterval(() => {}, 1000);', +].join(""); +const child = spawn(process.execPath, ["-e", childProgram], { + stdio: ["ignore", "ignore", "ignore", "ipc"], + windowsHide: true, +}); +child.once("message", () => { + writeFileSync(pidFile, JSON.stringify({ + parentPid: process.pid, + childPid: child.pid, + })); + if (exitTiming === "before-readiness") process.exit(0); + const exitPoll = setInterval(() => { + if (!existsSync(exitMarker)) return; + clearInterval(exitPoll); + process.exit(0); + }, 10); +}); +`; + function asChildProcess( spawned: SpawnedProcess, ): ChildProcessWithoutNullStreams { @@ -117,7 +148,173 @@ async function forceKillExactTestProcess(child: ChildProcess): Promise { ); } +async function proveRetainedGroupAuthority( + exitTiming: "before-readiness" | "after-readiness", +): Promise { + const fixture = await createManagedAgentFixture( + () => `fast-root-exit-${exitTiming}`, + ); + fixtures.push(fixture); + const observer = new LocalManagedAgentProcessObserver(); + const rawController = new AbortController(); + const forwardedController = new AbortController(); + const unrelated = spawnChild( + process.execPath, + ["-e", "setInterval(() => {}, 1000)"], + { stdio: "ignore", windowsHide: true }, + ); + await once(unrelated, "spawn"); + observer.bindAbortSignal(rawController.signal); + let anchor: ChildProcessWithoutNullStreams | undefined; + let ownedProcessGroupId: number | undefined; + try { + const exitMarker = join( + fixture.workspaceRoot, + FIXTURE_PATHS.processDirectory, + "exit-inner-root", + ); + anchor = asChildProcess( + observer.spawn({ + command: process.execPath, + args: [ + "--input-type=module", + "--eval", + FAST_EXIT_ROOT_SCRIPT, + FIXTURE_PATHS.processPidFile, + exitTiming, + exitMarker, + ], + cwd: fixture.workspaceRoot, + env: { ...process.env }, + signal: forwardedController.signal, + }), + ); + ownedProcessGroupId = anchor.pid; + expect(ownedProcessGroupId).toBeTypeOf("number"); + const [workerRootPid, nonCooperativeChildPid] = + await waitForManagedAgentFixturePids(fixture); + + let readiness; + if (exitTiming === "after-readiness") { + readiness = await observer.prepareCancellation(); + await writeFile(exitMarker, "exit\n"); + } + await waitForTestProcessDeath( + () => processExists(workerRootPid!), + `Fast SDK root ${workerRootPid}`, + ); + expect(processExists(nonCooperativeChildPid!)).toBe(true); + if (!readiness) readiness = await observer.prepareCancellation(); + + expect(readiness).toMatchObject({ + supported: true, + reason: "ready", + ownershipProven: true, + }); + expect(readiness.observedPids).toContain(nonCooperativeChildPid); + expect(readiness.observedPids).not.toContain(unrelated.pid); + + rawController.abort(); + const teardown = await observer.emergencyCleanup(1_000); + expect(teardown).toMatchObject({ + quiescent: true, + deadlineMet: true, + ownershipProven: true, + forceKillIssued: true, + alivePidsAtDeadline: [], + }); + expect(processExists(nonCooperativeChildPid!)).toBe(false); + expect(processExists(unrelated.pid!)).toBe(true); + } finally { + rawController.abort(); + forwardedController.abort(); + if (anchor && typeof ownedProcessGroupId === "number") { + await forceKillExactTestGroup(ownedProcessGroupId, anchor); + } + observer.dispose(); + await forceKillExactTestProcess(unrelated); + } +} + describe("LocalManagedAgentProcessObserver", () => { + it.skipIf(process.platform === "win32")( + "keeps inner arguments out of supervisor argv and scrubs its private payload", + async () => { + const observer = new LocalManagedAgentProcessObserver(); + const controller = new AbortController(); + const privateArgument = "inner-only-supervisor-argument"; + const anchor = asChildProcess( + observer.spawn({ + command: process.execPath, + args: [ + "-e", + [ + 'const payload = "SAPIOM_MANAGED_AGENT_SUPERVISOR_PAYLOAD";', + "const valid = process.argv[1] === " + + JSON.stringify(privateArgument) + + " && !Object.hasOwn(process.env, payload);", + "process.exit(valid ? 0 : 31);", + ].join(""), + privateArgument, + ], + cwd: process.cwd(), + env: { ...process.env }, + signal: controller.signal, + }), + ); + try { + expect(anchor.spawnargs.join("\u0000")).not.toContain(privateArgument); + const [exitCode, signalCode] = await once(anchor, "exit"); + expect(exitCode).toBe(0); + expect(signalCode).toBeNull(); + } finally { + if (typeof anchor.pid === "number") { + await forceKillExactTestGroup(anchor.pid, anchor); + } + controller.abort(); + observer.dispose(); + } + }, + 5_000, + ); + + it.skipIf(process.platform === "win32")( + "preserves a normal inner exit code without reporting a signal kill", + async () => { + const observer = new LocalManagedAgentProcessObserver(); + const controller = new AbortController(); + const anchor = asChildProcess( + observer.spawn({ + command: process.execPath, + args: [ + "-e", + 'process.stderr.write("x".repeat(1024 * 1024), () => process.exit(23));', + ], + cwd: process.cwd(), + env: { ...process.env }, + signal: controller.signal, + }), + ); + let forwardedStderrBytes = 0; + anchor.stderr.on("data", (chunk: Buffer) => { + forwardedStderrBytes += chunk.byteLength; + }); + try { + const [exitCode, signalCode] = await once(anchor, "exit"); + expect(exitCode).toBe(23); + expect(signalCode).toBeNull(); + expect(forwardedStderrBytes).toBe(1024 * 1024); + } finally { + if (typeof anchor.pid === "number") { + await forceKillExactTestGroup(anchor.pid, anchor); + } + controller.abort(); + observer.dispose(); + } + }, + 5_000, + ); + it.skipIf(process.platform === "win32")( "force-stops and kills the exact non-cooperative fixture group, then confirms death inside one deadline", async () => { @@ -190,6 +387,61 @@ describe("LocalManagedAgentProcessObserver", () => { 10_000, ); + it.skipIf(process.platform === "win32")( + "kills the complete owned group on parent IPC disconnect without touching an unrelated process", + async () => { + const fixture = await createManagedAgentFixture(() => "ipc-disconnect"); + fixtures.push(fixture); + const observer = new LocalManagedAgentProcessObserver(); + const controller = new AbortController(); + const unrelated = spawnChild( + process.execPath, + ["-e", "setInterval(() => {}, 1000)"], + { stdio: "ignore", windowsHide: true }, + ); + await once(unrelated, "spawn"); + const anchor = asChildProcess( + observer.spawn({ + command: process.execPath, + args: [FIXTURE_PATHS.processScript, FIXTURE_PATHS.processPidFile], + cwd: fixture.workspaceRoot, + env: { ...process.env }, + signal: controller.signal, + }), + ); + const ownedProcessGroupId = anchor.pid; + expect(ownedProcessGroupId).toBeTypeOf("number"); + try { + const fixturePids = await waitForManagedAgentFixturePids(fixture); + await expect(observer.prepareCancellation()).resolves.toMatchObject({ + supported: true, + reason: "ready", + ownershipProven: true, + }); + + anchor.disconnect(); + const teardown = await observer.waitForQuiescence(1_000); + expect(teardown).toMatchObject({ + quiescent: true, + deadlineMet: true, + ownershipProven: true, + forceKillIssued: false, + alivePidsAtDeadline: [], + }); + expect(fixturePids.every((pid) => !processExists(pid))).toBe(true); + expect(processExists(unrelated.pid!)).toBe(true); + } finally { + if (typeof ownedProcessGroupId === "number") { + await forceKillExactTestGroup(ownedProcessGroupId, anchor); + } + controller.abort(); + observer.dispose(); + await forceKillExactTestProcess(unrelated); + } + }, + 10_000, + ); + it("fails preparation closed after a fast root exits and never signals its former numeric group", async () => { const signals: Array = []; const observer = new LocalManagedAgentProcessObserver({ @@ -226,6 +478,18 @@ describe("LocalManagedAgentProcessObserver", () => { } }); + it.skipIf(process.platform === "win32")( + "retains owned group authority when the SDK root exits before its non-cooperative child", + () => proveRetainedGroupAuthority("before-readiness"), + 10_000, + ); + + it.skipIf(process.platform === "win32")( + "retains owned group authority when the SDK root exits after readiness while its child survives", + () => proveRetainedGroupAuthority("after-readiness"), + 10_000, + ); + it("bounds a hanging process-table read and never turns unknown observation into quiescence", async () => { const signals: Array = []; const observer = new LocalManagedAgentProcessObserver({ diff --git a/packages/harness/src/experimental/managed-agent-spike/process-observer.ts b/packages/harness/src/experimental/managed-agent-spike/process-observer.ts index bc4df38d8..07053fb68 100644 --- a/packages/harness/src/experimental/managed-agent-spike/process-observer.ts +++ b/packages/harness/src/experimental/managed-agent-spike/process-observer.ts @@ -20,6 +20,195 @@ const execFileAsync = promisify(execFile); const SAMPLE_INTERVAL_MS = 100; const QUIESCENCE_POLL_MS = 25; export const MANAGED_AGENT_PROCESS_HELPER_TIMEOUT_MS = 200; +const MANAGED_AGENT_SUPERVISOR_PAYLOAD_ENV = + "SAPIOM_MANAGED_AGENT_SUPERVISOR_PAYLOAD"; + +/** + * The POSIX supervisor is the observer-owned process-group leader. The real + * SDK command runs inside its group, while the supervisor stays alive after + * an inner-root exit whenever another group member survives. Its own bounded + * `ps` helper remains in the group so abort and parent-disconnect cleanup + * contain it too; the known helper PID is excluded only from the membership + * decision that determines whether the anchor may exit. + */ +const MANAGED_AGENT_POSIX_SUPERVISOR_SOURCE = String.raw` +import { spawn } from "node:child_process"; + +const PAYLOAD_ENV = "SAPIOM_MANAGED_AGENT_SUPERVISOR_PAYLOAD"; +const HELPER_TIMEOUT_MS = 200; +const POLL_INTERVAL_MS = 25; +const MAX_PROCESS_TABLE_BYTES = 4 * 1024 * 1024; + +function fail(message) { + try { process.stderr.write(message + "\n"); } catch {} + process.exit(1); +} + +const encodedPayload = process.env[PAYLOAD_ENV]; +delete process.env[PAYLOAD_ENV]; +if (!encodedPayload) fail("managed-agent supervisor payload missing"); + +let payload; +try { + payload = JSON.parse(Buffer.from(encodedPayload, "base64url").toString("utf8")); +} catch { + fail("managed-agent supervisor payload invalid"); +} +if ( + !payload || + typeof payload.command !== "string" || + payload.command.length === 0 || + !Array.isArray(payload.args) || + !payload.args.every((argument) => typeof argument === "string") +) { + fail("managed-agent supervisor command invalid"); +} + +for (const signal of ["SIGHUP", "SIGINT", "SIGTERM"]) { + process.on(signal, () => {}); +} + +function killOwnedGroup() { + try { + process.kill(-process.pid, "SIGKILL"); + } catch { + process.exit(1); + } +} + +process.on("disconnect", killOwnedGroup); + +function readOtherGroupMembers() { + return new Promise((resolveMembers) => { + let helper; + try { + // Intentionally non-detached: the helper is synchronously contained by + // the same group. Its known PID is excluded from this one snapshot. + helper = spawn("/bin/ps", ["-axo", "pid=,pgid="], { + stdio: ["ignore", "pipe", "ignore"], + windowsHide: true, + }); + } catch { + resolveMembers(undefined); + return; + } + const helperPid = helper.pid; + let output = ""; + let settled = false; + let overflowed = false; + const finish = (members) => { + if (settled) return; + settled = true; + clearTimeout(timeout); + resolveMembers(members); + }; + const timeout = setTimeout(() => { + try { helper.kill("SIGKILL"); } catch {} + finish(undefined); + }, HELPER_TIMEOUT_MS); + helper.stdout.on("data", (chunk) => { + if (overflowed) return; + output += chunk.toString("utf8"); + if (Buffer.byteLength(output) > MAX_PROCESS_TABLE_BYTES) { + overflowed = true; + try { helper.kill("SIGKILL"); } catch {} + } + }); + helper.once("error", () => finish(undefined)); + helper.once("close", (code) => { + if (code !== 0 || overflowed || typeof helperPid !== "number") { + finish(undefined); + return; + } + const records = new Map(); + for (const line of output.split("\n")) { + const match = /^\s*(\d+)\s+(\d+)\s*$/.exec(line); + if (!match) continue; + records.set(Number(match[1]), Number(match[2])); + } + if ( + records.get(process.pid) !== process.pid || + records.get(helperPid) !== process.pid + ) { + finish(undefined); + return; + } + finish( + [...records.entries()] + .filter( + ([pid, processGroupId]) => + processGroupId === process.pid && + pid !== process.pid && + pid !== helperPid, + ) + .map(([pid]) => pid), + ); + }); + }); +} + +let innerClosed = false; +let innerExitCode = 1; +let membershipCheckRunning = false; +let pollTimer; + +function scheduleMembershipCheck(delayMs = 0) { + if (pollTimer) clearTimeout(pollTimer); + pollTimer = setTimeout(checkMembership, delayMs); +} + +async function checkMembership() { + pollTimer = undefined; + if (!innerClosed || membershipCheckRunning) return; + membershipCheckRunning = true; + const members = await readOtherGroupMembers(); + membershipCheckRunning = false; + if (members && members.length === 0) { + process.stdin.unpipe(); + process.stdin.destroy(); + if (process.connected) { + process.off("disconnect", killOwnedGroup); + process.disconnect(); + } + process.exitCode = innerExitCode; + return; + } + scheduleMembershipCheck(POLL_INTERVAL_MS); +} + +let inner; +try { + inner = spawn(payload.command, payload.args, { + cwd: process.cwd(), + env: process.env, + stdio: ["pipe", "pipe", "pipe"], + windowsHide: true, + }); +} catch { + innerClosed = true; + scheduleMembershipCheck(); +} + +if (inner) { + process.stdin.on("error", () => {}); + inner.stdin.on("error", () => {}); + inner.stdout.on("error", () => {}); + inner.stderr.on("error", () => {}); + process.stdout.on("error", () => {}); + process.stderr.on("error", () => {}); + process.stdin.pipe(inner.stdin); + inner.stdout.pipe(process.stdout, { end: false }); + inner.stderr.pipe(process.stderr, { end: false }); + inner.once("error", () => { + innerExitCode = 1; + }); + inner.once("close", (code) => { + innerClosed = true; + innerExitCode = Number.isInteger(code) ? code : 1; + scheduleMembershipCheck(); + }); +} +`; export interface ManagedAgentKernelProcessRecord { readonly parentPid: number; @@ -228,18 +417,20 @@ function descendantsOf( /** * E0.4 deliberately certifies one narrow containment model: the exact local - * L2 fixture running inside a detached POSIX SDK process group. Before abort, - * bounded host enumeration must observe an active trusted ChildProcess as its - * PGID leader and observe the fixture PIDs in that group. Independently, the - * detached spawn plus an active trusted root handle are safe signal authority, - * so even a failed evidence preflight can synchronously stop and kill the - * owned group without leaking it. Such a run still fails certification. + * L2 fixture running inside an observer-owned detached POSIX process group. + * Before abort, bounded host enumeration must observe the persistent trusted + * supervisor ChildProcess as its PGID leader and observe the fixture PIDs in + * that group. The supervisor remains the group anchor if the inner SDK root + * exits while a descendant survives. Independently, the detached spawn plus + * the active supervisor handle are safe signal authority, so even a failed + * evidence preflight can synchronously stop and kill the owned group without + * leaking it. Such a run still fails certification. * - * This is not a universal sandbox or process-tree killer. Windows, a fast root - * that exits before preparation, unavailable enumeration, and an observed - * setsid/group escape all fail certification closed. POSIX `lstart` is evidence - * only and never authorizes an individual or group signal. Workspace PID-file - * contents never enter this class and can never become signal authority. + * This is not a universal sandbox or process-tree killer. Windows, an inactive + * or invalid supervisor anchor, unavailable enumeration, and an observed + * setsid/group escape all fail certification closed. POSIX `lstart` is + * evidence only and never authorizes an individual or group signal. Workspace + * PID-file contents never enter this class and can never become authority. */ export class LocalManagedAgentProcessObserver implements ManagedAgentProcessObserver { readonly #platform: NodeJS.Platform; @@ -290,15 +481,69 @@ export class LocalManagedAgentProcessObserver implements ManagedAgentProcessObse } public spawn(options: SpawnOptions): SpawnedProcess { - const child = spawnChild(options.command, options.args, { - cwd: options.cwd, - env: options.env, - detached: this.#platform !== "win32", - stdio: ["pipe", "pipe", "pipe"], - windowsHide: true, - }); + const usePosixSupervisor = + this.#platform === "darwin" || this.#platform === "linux"; + const child = ( + usePosixSupervisor + ? spawnChild( + process.execPath, + [ + "--input-type=module", + "--eval", + MANAGED_AGENT_POSIX_SUPERVISOR_SOURCE, + ], + { + cwd: options.cwd, + env: { + ...options.env, + [MANAGED_AGENT_SUPERVISOR_PAYLOAD_ENV]: Buffer.from( + JSON.stringify({ + command: options.command, + args: options.args, + }), + "utf8", + ).toString("base64url"), + }, + detached: true, + stdio: ["pipe", "pipe", "pipe", "ipc"], + windowsHide: true, + }, + ) + : spawnChild(options.command, options.args, { + cwd: options.cwd, + env: options.env, + detached: false, + stdio: ["pipe", "pipe", "pipe"], + windowsHide: true, + }) + ) as ChildProcessWithoutNullStreams; + // SpawnedProcess does not expose stderr to the SDK transport. Drain it + // here without retaining or printing content so a noisy inner command + // cannot deadlock the supervisor on pipe backpressure. + child.stderr.on("data", () => undefined); + child.stderr.on("error", () => undefined); if (typeof child.pid === "number") { const pid = child.pid; + if (usePosixSupervisor) { + let groupKillRequested = false; + Object.defineProperty(child, "killed", { + configurable: true, + enumerable: true, + get: () => groupKillRequested, + }); + child.kill = ((signal: NodeJS.Signals = "SIGTERM") => { + try { + process.kill(-pid, signal); + groupKillRequested = true; + return true; + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ESRCH") { + return false; + } + throw error; + } + }) as typeof child.kill; + } this.#roots.set(pid, { pid, child, @@ -508,8 +753,10 @@ export class LocalManagedAgentProcessObserver implements ManagedAgentProcessObse if (root.forceKillIssued || !childActive(root.child)) { continue; } - // detached:true plus the still-active trusted ChildProcess handle are - // sufficient signal authority even when ps evidence is unavailable. + // The detached observer-owned supervisor plus its still-active trusted + // ChildProcess handle are sufficient signal authority even when ps + // evidence is unavailable. The supervisor remains active across a fast + // inner SDK-root exit while any same-group descendant survives. // Certification readiness remains false in that case. Second-resolution // lstart and workspace PIDs are never signal authority. if (!root.stopIssued) { diff --git a/packages/harness/src/experimental/managed-agent-spike/runtime.test.ts b/packages/harness/src/experimental/managed-agent-spike/runtime.test.ts index e4c18f868..e5bfd07c3 100644 --- a/packages/harness/src/experimental/managed-agent-spike/runtime.test.ts +++ b/packages/harness/src/experimental/managed-agent-spike/runtime.test.ts @@ -1,5 +1,7 @@ +import { execFile } from "node:child_process"; import { lstat, mkdir, realpath, symlink, writeFile } from "node:fs/promises"; import { dirname, join } from "node:path"; +import { promisify } from "node:util"; import { afterEach, describe, expect, it, vi } from "vitest"; @@ -33,6 +35,7 @@ const SUCCESS_SESSION_ID = "11111111-1111-4111-8111-111111111111"; const CANCEL_SESSION_ID = "22222222-2222-4222-8222-222222222222"; const TIMEOUT_SESSION_ID = "33333333-3333-4333-8333-333333333333"; const CLOSE_SESSION_ID = "44444444-4444-4444-8444-444444444444"; +const execFileAsync = promisify(execFile); afterEach(async () => { await Promise.all(fixtures.splice(0).map((fixture) => fixture.cleanup())); @@ -914,6 +917,93 @@ describe("runManagedAgentProbe", () => { expect(observer.bindAbortSignal).toHaveBeenCalledOnce(); }); + it("keeps a CLI-shaped process alive until bounded close and cleanup complete", async () => { + const { config } = await probeConfig("L2"); + const childProgram = String.raw` +const { runManagedAgentProbe } = await import( + process.env.SAPIOM_TEST_RUNTIME_MODULE_URL +); +const config = JSON.parse(process.env.SAPIOM_TEST_PROBE_CONFIG); +let cleanupCalled = false; +const teardown = { + quiescent: true, + deadlineMet: true, + processTableAvailable: true, + containmentSupported: true, + ownershipProven: true, + forceKillIssued: true, + elapsedMs: 0, + observedPids: [], + alivePidsAtDeadline: [], + emergencyCleanupAttempted: false, +}; +const observer = { + spawn() { throw new Error("fake query must not spawn"); }, + bindAbortSignal() {}, + async prepareCancellation() { + return { + supported: true, + reason: "ready", + processTableAvailable: true, + containmentSupported: true, + ownershipProven: true, + observedPids: [], + }; + }, + async observeProcessTree() { return true; }, + async waitForQuiescence() { return teardown; }, + async emergencyCleanup() { + cleanupCalled = true; + return { ...teardown, emergencyCleanupAttempted: true }; + }, + dispose() {}, +}; +const result = await runManagedAgentProbe(config, { + hermeticGatewayOrigin: config.gatewayOrigin, + processObserver: observer, + policySettingsGuard: async () => undefined, + waitForCancellationSignal: async () => undefined, + queryFactory: () => ({ + [Symbol.asyncIterator]() { + return { next: () => new Promise(() => undefined) }; + }, + close: () => new Promise(() => undefined), + }), +}); +process.stdout.write(JSON.stringify({ + cleanupCalled, + queryClosed: result.queryClosed, + terminal: result.terminal, +})); +`; + const startedAt = Date.now(); + const { stdout } = await execFileAsync( + process.execPath, + ["--import", "tsx", "--input-type=module", "--eval", childProgram], + { + cwd: process.cwd(), + encoding: "utf8", + env: { + ...process.env, + SAPIOM_TEST_PROBE_CONFIG: JSON.stringify(config), + SAPIOM_TEST_RUNTIME_MODULE_URL: new URL( + "./runtime.ts", + import.meta.url, + ).href, + }, + timeout: 8_000, + killSignal: "SIGKILL", + }, + ); + + expect(Date.now() - startedAt).toBeGreaterThanOrEqual(1_500); + expect(JSON.parse(stdout)).toEqual({ + cleanupCalled: true, + queryClosed: false, + terminal: "close_timeout", + }); + }, 10_000); + it("includes iterator abandonment and close in the one cancellation deadline", async () => { const { config } = await probeConfig("L2"); let now = 1_000; diff --git a/packages/harness/src/experimental/managed-agent-spike/runtime.ts b/packages/harness/src/experimental/managed-agent-spike/runtime.ts index a05260a60..04a82d594 100644 --- a/packages/harness/src/experimental/managed-agent-spike/runtime.ts +++ b/packages/harness/src/experimental/managed-agent-spike/runtime.ts @@ -295,7 +295,6 @@ async function closeQueryBounded( () => resolveTimeout(false), Math.min(QUERY_CLOSE_TIMEOUT_MS, timeoutMs), ); - timeout.unref(); }), ]); } catch { diff --git a/packages/harness/src/experimental/managed-agent-spike/types.ts b/packages/harness/src/experimental/managed-agent-spike/types.ts index 32ab68aab..f56b04bbf 100644 --- a/packages/harness/src/experimental/managed-agent-spike/types.ts +++ b/packages/harness/src/experimental/managed-agent-spike/types.ts @@ -169,7 +169,7 @@ export interface ManagedAgentTeardownObservation { readonly processTableAvailable: boolean; /** False means the owned E0 containment model was escaped or unproven. */ readonly containmentSupported: boolean; - /** True only after an active POSIX root was observed as its PGID leader. */ + /** True only after the active POSIX supervisor is observed as PGID leader. */ readonly ownershipProven: boolean; /** True only when the bound raw abort synchronously issued SIGSTOP+SIGKILL. */ readonly forceKillIssued: boolean; @@ -241,12 +241,12 @@ export interface ManagedAgentProcessObserver { bindAbortSignal(signal: AbortSignal): void; /** Prove the narrow POSIX ownership model before allowing L2 to cancel. */ prepareCancellation(): Promise; - /** Sample only descendants of the host-observed SDK process roots. */ + /** Sample only members owned by the host-observed process anchors. */ observeProcessTree(timeoutMs?: number): Promise; waitForQuiescence( timeoutMs: number, ): Promise; - /** Idempotently force a proven active group and confirm within this budget. */ + /** Idempotently force an anchored owned group and confirm within this budget. */ emergencyCleanup(timeoutMs: number): Promise; dispose(): void; } From 64748b8a9cdc6dce1d696dc08dfc790d7879bfa8 Mon Sep 17 00:00:00 2001 From: Yash Date: Sun, 16 Aug 2026 21:39:42 -0700 Subject: [PATCH 11/24] fix(harness): version managed L1 certification contract Make the L1 gate fail closed on the frozen v2 prompt, exact correlated operation trace, exact workspace and final-byte evidence, and the one bounded verification Read exception. Preserve content-free permission evidence and SDK-normalized path handling. Refs: SAP-2632 --- .../managed-agent-spike/README.md | 48 +- .../managed-agent-spike/contract.test.ts | 98 ++- .../managed-agent-spike/contract.ts | 74 ++ .../managed-agent-spike/events.test.ts | 2 + .../managed-agent-spike/events.ts | 1 + .../managed-agent-spike/fixture.test.ts | 28 + .../managed-agent-spike/fixture.ts | 45 +- .../experimental/managed-agent-spike/index.ts | 11 + .../managed-agent-spike/permissions.test.ts | 133 +++- .../managed-agent-spike/permissions.ts | 121 +++- .../managed-agent-spike/probe-cli.test.ts | 680 +++++++++++++++++- .../managed-agent-spike/probe-cli.ts | 271 +++++-- .../runtime-sdk-loopback.test.ts | 4 + .../managed-agent-spike/runtime.test.ts | 12 + .../managed-agent-spike/runtime.ts | 27 +- .../experimental/managed-agent-spike/types.ts | 60 ++ 16 files changed, 1512 insertions(+), 103 deletions(-) diff --git a/packages/harness/src/experimental/managed-agent-spike/README.md b/packages/harness/src/experimental/managed-agent-spike/README.md index 748e458b0..a07d58e2e 100644 --- a/packages/harness/src/experimental/managed-agent-spike/README.md +++ b/packages/harness/src/experimental/managed-agent-spike/README.md @@ -64,6 +64,35 @@ real in-process `echo_nonce` MCP turn. It requires one primary `PreToolUse` decision for each request and separately verifies the MCP handler invocation and SDK tool-result event. +## L1 certification contract v2 + +L1 is independently versioned as `managed-agent-l1-prompt-v2` and +`managed-agent-l1-evaluator-v2` while the transport result remains contract +version 1. The frozen prompt contains 11 canonical calls. It permits at most +one additional verification Read, only after both denial probes and before the +Edit, and only for the clean target, dirty sentinel, or untracked sentinel. + +The host registers the six prompt path literals under content-free roles. Role +lookup uses normalized lexical path identity before realpath containment so an +SDK-normalized absolute path retains the same role as its relative prompt +literal. A different in-workspace path remains unregistered and is denied. +Permission evidence contains only an operation ID such as +`read:clean_target`; it never contains the raw path or tool input. + +The evaluator requires every canonical request ID to be non-empty and unique, +with exactly one matching completion and one primary `PreToolUse` decision. +Fallback-only decisions, duplicate or orphan evidence, mismatched tools, +reordering, omission, retries, and every other extra operation fail closed. +The optional Read count and role are reported separately as nonblocking +efficiency evidence. + +Filesystem acceptance is also exact: only the clean target may be modified and +only the managed output may be created, in either evidence order. Trusted +SHA-256 expectations prove the final bytes of both mutation targets, while the +durable result exposes only `{ role, matched }`. The dirty and untracked +sentinels must remain byte-identical, and successful `echo_nonce` handling is +recorded as a content-free nonce-verification boolean. + ## L2 cancellation containment boundary E0.4 certifies one deliberately narrow host model: the exact non-cooperative @@ -99,14 +128,15 @@ the query or credential is opened. Universal containment, POSIX group escape, Windows Job Objects, and production recovery belong to later epics; this probe does not claim those guarantees. -## Pre-fix live evidence +## Pre-v2 live evidence -The first Sonnet 5 L1 attempt reached an SDK success result and clean teardown, -but SDK default permissions executed successful Read and Bash calls without -invoking the former `canUseTool` boundary. The fixed matrix stopped immediately; -no retry or later model/scenario attempt ran. BigQuery showed exact Sonnet -provider/model, no fallback, positive tokens, and cost for all 11 calls, but no -durable correlation field and no independent SDK inference count. +The exact-trace-v1 campaign completed both Sonnet 5 L1 repetitions and the +first MiniMax M3 L1 repetition. The second M3 L1 run had a successful terminal +result, exact model provenance, complete primary permission coverage, and clean +teardown, but made one additional allowed in-root verification Read. The v1 +exact-trace evaluator rejected that run and the campaign stopped immediately; +no L2 run followed. -That attempt is not acceptance evidence. Do not run another paid L1/L2 matrix -until this correction has independent review and explicit authorization. +Those runs remain diagnostic history, not v2 acceptance evidence. Do not +restart the paid L1/L2 matrix until this v2 correction has clean CI, +independent review, and explicit authorization. diff --git a/packages/harness/src/experimental/managed-agent-spike/contract.test.ts b/packages/harness/src/experimental/managed-agent-spike/contract.test.ts index b66c5a5b1..4d945ce24 100644 --- a/packages/harness/src/experimental/managed-agent-spike/contract.test.ts +++ b/packages/harness/src/experimental/managed-agent-spike/contract.test.ts @@ -6,6 +6,9 @@ import { afterEach, describe, expect, it } from "vitest"; import { MANAGED_AGENT_CONTRACT, + MANAGED_AGENT_L1_CERTIFICATION_CONTRACT, + MANAGED_AGENT_L1_FINAL_BYTE_ROLES, + MANAGED_AGENT_L1_REGISTERED_PATH_ROLES, MANAGED_AGENT_MODEL_TARGETS, ManagedAgentConfigurationError, assertManagedAgentDirectGatewayOrigin, @@ -31,10 +34,22 @@ async function config(): Promise { target: "sonnet-5", gatewayOrigin: MANAGED_AGENT_CONTRACT.directGatewayOrigin, gatewayCredential: "dedicated-eval-key", - prompt: "probe", + prompt: `${MANAGED_AGENT_L1_CERTIFICATION_CONTRACT.promptMarker}\nprobe`, maxTurns: 10, maxBudgetUsd: 0.25, allowedBashCommands: ["git status --short"], + pathRoleBindings: [ + { path: "clean.txt", role: "clean_target" }, + { path: "dirty.txt", role: "dirty_sentinel" }, + { path: "untracked.txt", role: "untracked_sentinel" }, + { path: "created.txt", role: "managed_output" }, + { path: "../outside.txt", role: "outside_sentinel" }, + { path: "escape.txt", role: "escape_link" }, + ], + expectedL1FinalBytes: [ + { path: "clean.txt", role: "clean_target", sha256: "a".repeat(64) }, + { path: "created.txt", role: "managed_output", sha256: "b".repeat(64) }, + ], expectedMcpNonce: "probe-nonce", }; } @@ -46,6 +61,18 @@ afterEach(async () => { }); describe("managed-agent contract", () => { + it("freezes the versioned L1 prompt and evaluator contract", () => { + expect(MANAGED_AGENT_L1_CERTIFICATION_CONTRACT).toEqual({ + contractVersion: 2, + promptVersion: "managed-agent-l1-prompt-v2", + promptMarker: "SAPIOM_MANAGED_AGENT_L1_PROMPT_V2", + evaluatorVersion: "managed-agent-l1-evaluator-v2", + }); + expect(Object.isFrozen(MANAGED_AGENT_L1_CERTIFICATION_CONTRACT)).toBe(true); + expect(Object.isFrozen(MANAGED_AGENT_L1_REGISTERED_PATH_ROLES)).toBe(true); + expect(Object.isFrozen(MANAGED_AGENT_L1_FINAL_BYTE_ROLES)).toBe(true); + }); + it("pins the certified SDK/runtime and exact two-model allowlist", () => { expect(MANAGED_AGENT_CONTRACT).toMatchObject({ agentSdkVersion: "0.3.228", @@ -147,6 +174,75 @@ describe("managed-agent contract", () => { ).toThrow("expectedMcpNonce"); }); + it("requires the exact L1 v2 marker and all six unique path roles", async () => { + const valid = await config(); + expect(() => + validateManagedAgentProbeConfig({ + ...valid, + prompt: "SAPIOM_MANAGED_AGENT_L1_PROMPT_V1\nprobe", + }), + ).toThrow("managed-agent-l1-prompt-v2 marker"); + expect(() => + validateManagedAgentProbeConfig({ + ...valid, + pathRoleBindings: valid.pathRoleBindings.slice(0, -1), + }), + ).toThrow("each frozen fixture role exactly once"); + expect(() => + validateManagedAgentProbeConfig({ + ...valid, + pathRoleBindings: valid.pathRoleBindings.map((binding, index) => + index === 1 ? { ...binding, path: "clean.txt" } : binding, + ), + }), + ).toThrow("each frozen fixture role exactly once"); + }); + + it("requires exact trusted hashes for both intended L1 mutation roles", async () => { + const valid = await config(); + for (const expectedL1FinalBytes of [ + valid.expectedL1FinalBytes.slice(0, 1), + valid.expectedL1FinalBytes.map((expectation, index) => + index === 0 ? { ...expectation, sha256: "not-a-hash" } : expectation, + ), + valid.expectedL1FinalBytes.map((expectation, index) => + index === 0 ? { ...expectation, path: "dirty.txt" } : expectation, + ), + ]) { + expect(() => + validateManagedAgentProbeConfig({ + ...valid, + expectedL1FinalBytes, + }), + ).toThrow("exact hashes"); + } + }); + + it("keeps L2 free of L1 path-role and final-byte configuration", async () => { + const valid = await config(); + const l2: ManagedAgentProbeConfig = { + ...valid, + scenario: "L2", + prompt: "run exact Bash", + pathRoleBindings: [], + expectedL1FinalBytes: [], + expectedMcpNonce: undefined, + }; + expect(() => validateManagedAgentProbeConfig(l2)).not.toThrow(); + expect(() => + validateManagedAgentProbeConfig({ + ...l2, + pathRoleBindings: valid.pathRoleBindings, + }), + ).toThrow("L2 must not configure"); + expect(() => + validateManagedAgentProbeConfig({ + ...l2, + expectedL1FinalBytes: valid.expectedL1FinalBytes, + }), + ).toThrow("L2 must not configure"); + }); + it("requires exact agreement with an explicitly selected hermetic origin", async () => { const valid = await config(); const gatewayOrigin = "https://gateway.example.test"; diff --git a/packages/harness/src/experimental/managed-agent-spike/contract.ts b/packages/harness/src/experimental/managed-agent-spike/contract.ts index 4d88ee850..13d11963d 100644 --- a/packages/harness/src/experimental/managed-agent-spike/contract.ts +++ b/packages/harness/src/experimental/managed-agent-spike/contract.ts @@ -2,9 +2,11 @@ import { realpathSync, statSync } from "node:fs"; import { isAbsolute, relative, resolve, sep } from "node:path"; import type { + ManagedAgentL1FinalByteRole, ManagedAgentModelTarget, ManagedAgentModelTargetId, ManagedAgentProbeConfig, + ManagedAgentRegisteredPathRole, } from "./types.js"; /** @@ -22,6 +24,27 @@ export const MANAGED_AGENT_CONTRACT = { maxTurns: 20, } as const; +export const MANAGED_AGENT_L1_CERTIFICATION_CONTRACT = Object.freeze({ + contractVersion: 2 as const, + promptVersion: "managed-agent-l1-prompt-v2" as const, + promptMarker: "SAPIOM_MANAGED_AGENT_L1_PROMPT_V2" as const, + evaluatorVersion: "managed-agent-l1-evaluator-v2" as const, +}); + +export const MANAGED_AGENT_L1_REGISTERED_PATH_ROLES = Object.freeze([ + "clean_target", + "dirty_sentinel", + "untracked_sentinel", + "managed_output", + "outside_sentinel", + "escape_link", +] as const satisfies readonly ManagedAgentRegisteredPathRole[]); + +export const MANAGED_AGENT_L1_FINAL_BYTE_ROLES = Object.freeze([ + "clean_target", + "managed_output", +] as const satisfies readonly ManagedAgentL1FinalByteRole[]); + export const MANAGED_AGENT_MODEL_TARGETS: Readonly< Record > = { @@ -197,6 +220,57 @@ export function validateManagedAgentProbeConfig( if (!config.prompt.trim()) { throw new ManagedAgentConfigurationError("prompt is required"); } + if (config.scenario === "L1") { + if ( + config.prompt.split("\n", 1)[0] !== + MANAGED_AGENT_L1_CERTIFICATION_CONTRACT.promptMarker + ) { + throw new ManagedAgentConfigurationError( + "L1 prompt must use the frozen managed-agent-l1-prompt-v2 marker", + ); + } + const roles = config.pathRoleBindings.map(({ role }) => role); + const paths = config.pathRoleBindings.map(({ path }) => path); + if ( + roles.length !== MANAGED_AGENT_L1_REGISTERED_PATH_ROLES.length || + new Set(roles).size !== roles.length || + new Set(paths).size !== paths.length || + MANAGED_AGENT_L1_REGISTERED_PATH_ROLES.some( + (role) => !roles.includes(role), + ) || + paths.some((path) => !path || path.includes("\0") || /[\r\n]/.test(path)) + ) { + throw new ManagedAgentConfigurationError( + "L1 pathRoleBindings must bind each frozen fixture role exactly once", + ); + } + const finalByteRoles = config.expectedL1FinalBytes.map(({ role }) => role); + if ( + finalByteRoles.length !== MANAGED_AGENT_L1_FINAL_BYTE_ROLES.length || + new Set(finalByteRoles).size !== finalByteRoles.length || + MANAGED_AGENT_L1_FINAL_BYTE_ROLES.some( + (role) => !finalByteRoles.includes(role), + ) || + config.expectedL1FinalBytes.some( + ({ path, role, sha256 }) => + !/^[a-f0-9]{64}$/.test(sha256) || + !config.pathRoleBindings.some( + (binding) => binding.role === role && binding.path === path, + ), + ) + ) { + throw new ManagedAgentConfigurationError( + "L1 expectedL1FinalBytes must bind exact hashes to the clean target and managed output roles", + ); + } + } else if ( + config.pathRoleBindings.length !== 0 || + config.expectedL1FinalBytes.length !== 0 + ) { + throw new ManagedAgentConfigurationError( + "L2 must not configure file path roles or L1 final-byte expectations", + ); + } if ( config.scenario === "L1" && (!config.expectedMcpNonce || diff --git a/packages/harness/src/experimental/managed-agent-spike/events.test.ts b/packages/harness/src/experimental/managed-agent-spike/events.test.ts index 136c546a2..6711a1b7c 100644 --- a/packages/harness/src/experimental/managed-agent-spike/events.test.ts +++ b/packages/harness/src/experimental/managed-agent-spike/events.test.ts @@ -153,6 +153,7 @@ describe("ManagedAgentEventRecorder", () => { decision: "deny", reason: "tool_not_allowed", source: "pre_tool_use", + operationId: "unknown", }); recorder.recordTerminal("success"); @@ -169,6 +170,7 @@ describe("ManagedAgentEventRecorder", () => { decision: "deny", reason: "tool_not_allowed", source: "pre_tool_use", + operationId: "unknown", }, ]); const serialized = JSON.stringify({ diff --git a/packages/harness/src/experimental/managed-agent-spike/events.ts b/packages/harness/src/experimental/managed-agent-spike/events.ts index 1d9b359e1..f33da4eaf 100644 --- a/packages/harness/src/experimental/managed-agent-spike/events.ts +++ b/packages/harness/src/experimental/managed-agent-spike/events.ts @@ -230,6 +230,7 @@ export class ManagedAgentEventRecorder { permissionDecision: normalizedEvidence.decision, permissionReason: normalizedEvidence.reason, permissionSource: normalizedEvidence.source, + operationId: normalizedEvidence.operationId, }); } diff --git a/packages/harness/src/experimental/managed-agent-spike/fixture.test.ts b/packages/harness/src/experimental/managed-agent-spike/fixture.test.ts index 47eded2f8..6ed6d05a9 100644 --- a/packages/harness/src/experimental/managed-agent-spike/fixture.test.ts +++ b/packages/harness/src/experimental/managed-agent-spike/fixture.test.ts @@ -9,6 +9,7 @@ import { createManagedAgentFixture, diffManagedAgentWorkspaceSnapshots, fixtureGitStatus, + observeManagedAgentL1FinalBytes, verifyManagedAgentFixtureBytes, type ManagedAgentFixture, } from "./fixture.js"; @@ -41,6 +42,14 @@ describe("managed-agent disposable git fixture", () => { const fixture = await createManagedAgentFixture(() => "prompt-contract"); fixtures.push(fixture); const prompt = fixture.prompt("L1"); + expect(prompt.split("\n")[0]).toBe("SAPIOM_MANAGED_AGENT_L1_PROMPT_V2"); + expect(prompt).toContain( + "at most one optional verification Read after call 5 and before call 6", + ); + expect(prompt).toContain( + "exactly repeat call 1, 2, or 3 with the same literal file_path", + ); + expect(prompt).toContain("Do not Read any other fixture path"); const numberedLines = prompt .split("\n") .filter((line) => /^\d+\./.test(line)); @@ -99,6 +108,25 @@ describe("managed-agent disposable git fixture", () => { { path: FIXTURE_PATHS.cleanTarget, change: "modified" }, { path: FIXTURE_PATHS.createdTarget, change: "created" }, ]); + expect( + observeManagedAgentL1FinalBytes(after, fixture.expectedL1FinalBytes), + ).toEqual([ + { role: "clean_target", matched: true }, + { role: "managed_output", matched: true }, + ]); + await writeFile( + join(fixture.workspaceRoot, FIXTURE_PATHS.createdTarget), + "wrong final bytes\n", + ); + const incorrect = await captureManagedAgentWorkspaceSnapshot( + fixture.workspaceRoot, + ); + expect( + observeManagedAgentL1FinalBytes(incorrect, fixture.expectedL1FinalBytes), + ).toEqual([ + { role: "clean_target", matched: true }, + { role: "managed_output", matched: false }, + ]); expect(await verifyManagedAgentFixtureBytes(fixture)).toEqual([ { path: FIXTURE_PATHS.dirtySentinel, preserved: true }, { path: FIXTURE_PATHS.untrackedSentinel, preserved: true }, diff --git a/packages/harness/src/experimental/managed-agent-spike/fixture.ts b/packages/harness/src/experimental/managed-agent-spike/fixture.ts index 4c8d420d7..9b3670b82 100644 --- a/packages/harness/src/experimental/managed-agent-spike/fixture.ts +++ b/packages/harness/src/experimental/managed-agent-spike/fixture.ts @@ -15,8 +15,12 @@ import { import { tmpdir } from "node:os"; import { basename, join, relative, resolve } from "node:path"; +import { MANAGED_AGENT_L1_CERTIFICATION_CONTRACT } from "./contract.js"; import type { + ManagedAgentL1ExpectedFileHash, + ManagedAgentL1FinalByteObservation, ManagedAgentPreservationObservation, + ManagedAgentPathRoleBinding, ManagedAgentProbeScenario, ManagedAgentWorkspaceChange, } from "./types.js"; @@ -42,6 +46,8 @@ export interface ManagedAgentFixture { readonly createdTargetContents: string; readonly l1BashCommand: string; readonly l2BashCommand: string; + readonly pathRoleBindings: readonly ManagedAgentPathRoleBinding[]; + readonly expectedL1FinalBytes: readonly ManagedAgentL1ExpectedFileHash[]; readonly preservedBytes: Readonly>; prompt(scenario: ManagedAgentProbeScenario): string; cleanup(): Promise; @@ -150,6 +156,16 @@ export function observeManagedAgentPreservation( })); } +export function observeManagedAgentL1FinalBytes( + after: ManagedAgentWorkspaceSnapshot, + expected: readonly ManagedAgentL1ExpectedFileHash[], +): ManagedAgentL1FinalByteObservation[] { + return expected.map(({ path, role, sha256 }) => ({ + role, + matched: after.get(path) === sha256, + })); +} + export async function readManagedAgentFixturePids( fixture: ManagedAgentFixture, ): Promise { @@ -256,6 +272,26 @@ export async function createManagedAgentFixture( shellQuote(FIXTURE_PATHS.processScript), shellQuote(FIXTURE_PATHS.processPidFile), ].join(" "); + const pathRoleBindings = [ + { path: FIXTURE_PATHS.cleanTarget, role: "clean_target" }, + { path: FIXTURE_PATHS.dirtySentinel, role: "dirty_sentinel" }, + { path: FIXTURE_PATHS.untrackedSentinel, role: "untracked_sentinel" }, + { path: FIXTURE_PATHS.createdTarget, role: "managed_output" }, + { path: outsideSentinel, role: "outside_sentinel" }, + { path: FIXTURE_PATHS.escapeLink, role: "escape_link" }, + ] as const satisfies readonly ManagedAgentPathRoleBinding[]; + const expectedL1FinalBytes = [ + { + path: FIXTURE_PATHS.cleanTarget, + role: "clean_target", + sha256: hash(cleanTargetReplacement), + }, + { + path: FIXTURE_PATHS.createdTarget, + role: "managed_output", + sha256: hash(createdTargetContents), + }, + ] as const satisfies readonly ManagedAgentL1ExpectedFileHash[]; return { root, @@ -267,6 +303,8 @@ export async function createManagedAgentFixture( createdTargetContents, l1BashCommand, l2BashCommand, + pathRoleBindings, + expectedL1FinalBytes, preservedBytes: { [FIXTURE_PATHS.dirtySentinel]: Buffer.from(dirtyContents), [FIXTURE_PATHS.untrackedSentinel]: Buffer.from(untrackedContents), @@ -281,13 +319,16 @@ export async function createManagedAgentFixture( ].join("\n"); } return [ - "Perform exactly these 11 tool calls in numbered order. Make each numbered call exactly once, do not combine calls, and make no unlisted tool call.", + MANAGED_AGENT_L1_CERTIFICATION_CONTRACT.promptMarker, + "Perform exactly these 11 canonical tool calls in numbered order. Make each numbered call exactly once and do not combine calls.", "Use every literal path, argument, and command exactly as written. Do not resolve, normalize, substitute, or retry a path or command. Continue after the two expected Read denials and the first expected fail_once error.", `1. Call Read with exactly ${JSON.stringify({ file_path: FIXTURE_PATHS.cleanTarget })}.`, `2. Call Read with exactly ${JSON.stringify({ file_path: FIXTURE_PATHS.dirtySentinel })}.`, `3. Call Read with exactly ${JSON.stringify({ file_path: FIXTURE_PATHS.untrackedSentinel })}. Save its sole line without surrounding whitespace for calls 8-10.`, `4. Call Read with exactly ${JSON.stringify({ file_path: outsideSentinel })}. A denial is expected; continue without retrying.`, `5. Call Read with exactly ${JSON.stringify({ file_path: FIXTURE_PATHS.escapeLink })}. This must remain the exact relative path shown: do not replace it with an absolute path or its symlink target. A denial is expected; continue without retrying.`, + "You may make at most one optional verification Read after call 5 and before call 6. If used, exactly repeat call 1, 2, or 3 with the same literal file_path; do not combine it with a numbered call.", + "Do not Read any other fixture path. Do not make a second optional Read, repeat either denial probe, or retry any other call.", `6. Call Edit with exactly ${JSON.stringify({ file_path: FIXTURE_PATHS.cleanTarget, old_string: cleanTargetContents, new_string: cleanTargetReplacement, replace_all: false })}.`, `7. Call Write with exactly ${JSON.stringify({ file_path: FIXTURE_PATHS.createdTarget, content: createdTargetContents })}.`, `8. Call echo_nonce exactly once with the saved line as its nonce argument.`, @@ -295,7 +336,7 @@ export async function createManagedAgentFixture( `10. Call fail_once a second and final time with the same nonce argument.`, `11. Call Bash with exactly ${JSON.stringify({ command: l1BashCommand })}.`, `Never modify ${FIXTURE_PATHS.dirtySentinel} or ${FIXTURE_PATHS.untrackedSentinel}.`, - "After call 11 completes, make no further tool calls and return one short final text confirmation.", + "Except for the one optional verification Read above, make no unlisted tool call. After call 11 completes, make no further tool calls and return one short final text confirmation.", ].join("\n"); }, async cleanup() { diff --git a/packages/harness/src/experimental/managed-agent-spike/index.ts b/packages/harness/src/experimental/managed-agent-spike/index.ts index 5877c6028..53efbf008 100644 --- a/packages/harness/src/experimental/managed-agent-spike/index.ts +++ b/packages/harness/src/experimental/managed-agent-spike/index.ts @@ -1,6 +1,9 @@ export { MANAGED_AGENT_CONTRACT, MANAGED_AGENT_FORBIDDEN_AMBIENT_CREDENTIALS, + MANAGED_AGENT_L1_CERTIFICATION_CONTRACT, + MANAGED_AGENT_L1_FINAL_BYTE_ROLES, + MANAGED_AGENT_L1_REGISTERED_PATH_ROLES, MANAGED_AGENT_MODEL_ENVIRONMENT_VARIABLES, MANAGED_AGENT_MODEL_TARGETS, ManagedAgentConfigurationError, @@ -31,6 +34,7 @@ export { diffManagedAgentWorkspaceSnapshots, fixtureGitStatus, observeManagedAgentPreservation, + observeManagedAgentL1FinalBytes, readManagedAgentFixturePids, verifyManagedAgentFixtureBytes, waitForManagedAgentFixturePids, @@ -74,6 +78,13 @@ export type { ManagedAgentCancellationReadiness, ManagedAgentCancellationReadinessReason, ManagedAgentEventNormalizationFailureReason, + ManagedAgentL1ExpectedFileHash, + ManagedAgentL1FinalByteObservation, + ManagedAgentL1FinalByteRole, + ManagedAgentOperationId, + ManagedAgentPathRole, + ManagedAgentPathRoleBinding, + ManagedAgentRegisteredPathRole, ManagedAgentPermissionDecision, ManagedAgentPermissionEvidence, ManagedAgentPermissionReason, diff --git a/packages/harness/src/experimental/managed-agent-spike/permissions.test.ts b/packages/harness/src/experimental/managed-agent-spike/permissions.test.ts index ec62bfc74..5988e7576 100644 --- a/packages/harness/src/experimental/managed-agent-spike/permissions.test.ts +++ b/packages/harness/src/experimental/managed-agent-spike/permissions.test.ts @@ -91,6 +91,109 @@ function preToolUseInput( } describe("managed-agent universal policy boundary", () => { + it("classifies registered paths by lexical identity before realpath containment", async () => { + const evidence: ManagedAgentPermissionEvidence[] = []; + const resolveToolPath = vi.fn(resolveManagedAgentToolPath); + const outsidePath = join(outside, "secret.txt"); + const boundary = createManagedAgentPolicyBoundary({ + canonicalWorkspaceRoot: workspace, + allowedBashCommands: [], + allowedMcpTools: [], + pathRoleBindings: [ + { path: "inside.txt", role: "clean_target" }, + { path: outsidePath, role: "outside_sentinel" }, + { path: "escape.txt", role: "escape_link" }, + ], + requireRegisteredFilePaths: true, + onDecision: (decision) => evidence.push(decision), + resolveToolPath, + }); + const signal = new AbortController().signal; + const invoke = (filePath: string, toolUseId: string) => + boundary.preToolUseHook( + preToolUseInput("Read", { file_path: filePath }, toolUseId), + toolUseId, + { signal }, + ); + + await expect(invoke("inside.txt", "relative")).resolves.toMatchObject({ + hookSpecificOutput: { permissionDecision: "allow" }, + }); + await expect( + invoke(join(workspace, "inside.txt"), "absolute"), + ).resolves.toMatchObject({ + hookSpecificOutput: { permissionDecision: "allow" }, + }); + await expect(invoke(outsidePath, "outside")).resolves.toMatchObject({ + hookSpecificOutput: { permissionDecision: "deny" }, + }); + await expect(invoke("escape.txt", "escape")).resolves.toMatchObject({ + hookSpecificOutput: { permissionDecision: "deny" }, + }); + await expect( + invoke(join(workspace, "not-registered.txt"), "unregistered"), + ).resolves.toMatchObject({ + hookSpecificOutput: { permissionDecision: "deny" }, + }); + + expect( + evidence.map(({ decision, reason, operationId }) => ({ + decision, + reason, + operationId, + })), + ).toEqual([ + { + decision: "allow", + reason: "fixture_path", + operationId: "read:clean_target", + }, + { + decision: "allow", + reason: "fixture_path", + operationId: "read:clean_target", + }, + { + decision: "deny", + reason: "path_outside_workspace", + operationId: "read:outside_sentinel", + }, + { + decision: "deny", + reason: "path_symlink_escape", + operationId: "read:escape_link", + }, + { + decision: "deny", + reason: "path_role_not_allowed", + operationId: "read:unregistered", + }, + ]); + expect(resolveToolPath).toHaveBeenCalledTimes(4); + const serialized = JSON.stringify(evidence); + expect(serialized).not.toContain(workspace); + expect(serialized).not.toContain(outsidePath); + expect(serialized).not.toContain("inside.txt"); + }); + + it("rejects path-role bindings with the same normalized lexical target", () => { + expect(() => + createManagedAgentPolicyBoundary({ + canonicalWorkspaceRoot: workspace, + allowedBashCommands: [], + allowedMcpTools: [], + pathRoleBindings: [ + { path: "inside.txt", role: "clean_target" }, + { + path: join(workspace, "inside.txt"), + role: "dirty_sentinel", + }, + ], + onDecision: () => undefined, + }), + ).toThrow("unique lexical paths"); + }); + it("can enforce an L2 Bash-only boundary before evaluating model-authored inputs", async () => { const evidence: ManagedAgentPermissionEvidence[] = []; const boundary = createManagedAgentPolicyBoundary({ @@ -167,6 +270,12 @@ describe("managed-agent universal policy boundary", () => { const evidence: ManagedAgentPermissionEvidence[] = []; const boundary = createManagedAgentPolicyBoundary({ canonicalWorkspaceRoot: workspace, + pathRoleBindings: [ + { path: "inside.txt", role: "clean_target" }, + { path: "nested/new.txt", role: "managed_output" }, + { path: join(outside, "secret.txt"), role: "outside_sentinel" }, + { path: "escape.txt", role: "escape_link" }, + ], allowedBashCommands: ["git status --short"], allowedMcpTools: ["mcp__probe__echo_nonce"], onDecision: (decision) => evidence.push(decision), @@ -236,20 +345,26 @@ describe("managed-agent universal policy boundary", () => { }); expect( - evidence.map(({ decision, reason, source }) => [ + evidence.map(({ decision, reason, source, operationId }) => [ decision, reason, source, + operationId, ]), ).toEqual([ - ["allow", "exact_bash_command", "pre_tool_use"], - ["deny", "bash_command_not_allowed", "pre_tool_use"], - ["allow", "fixture_path", "pre_tool_use"], - ["allow", "fixture_path", "pre_tool_use"], - ["deny", "path_outside_workspace", "pre_tool_use"], - ["deny", "path_symlink_escape", "pre_tool_use"], - ["allow", "managed_mcp_tool", "pre_tool_use"], - ["deny", "tool_not_allowed", "pre_tool_use"], + ["allow", "exact_bash_command", "pre_tool_use", "bash:exact_command"], + ["deny", "bash_command_not_allowed", "pre_tool_use", "bash:unregistered"], + ["allow", "fixture_path", "pre_tool_use", "read:clean_target"], + ["allow", "fixture_path", "pre_tool_use", "write:managed_output"], + [ + "deny", + "path_outside_workspace", + "pre_tool_use", + "read:outside_sentinel", + ], + ["deny", "path_symlink_escape", "pre_tool_use", "read:escape_link"], + ["allow", "managed_mcp_tool", "pre_tool_use", "mcp:echo_nonce"], + ["deny", "tool_not_allowed", "pre_tool_use", "unknown"], ]); expect(JSON.stringify(evidence)).not.toContain(join(outside, "secret.txt")); expect(JSON.stringify(evidence)).not.toContain("secret"); diff --git a/packages/harness/src/experimental/managed-agent-spike/permissions.ts b/packages/harness/src/experimental/managed-agent-spike/permissions.ts index 98341a26b..b4ace4859 100644 --- a/packages/harness/src/experimental/managed-agent-spike/permissions.ts +++ b/packages/harness/src/experimental/managed-agent-spike/permissions.ts @@ -8,6 +8,9 @@ import type { } from "@anthropic-ai/claude-agent-sdk"; import type { + ManagedAgentOperationId, + ManagedAgentPathRole, + ManagedAgentPathRoleBinding, ManagedAgentPermissionEvidence, ManagedAgentPermissionReason, ManagedAgentPermissionSource, @@ -140,6 +143,10 @@ export interface ManagedAgentPolicyBoundaryOptions { readonly allowedBuiltinTools?: readonly string[]; readonly allowedBashCommands: readonly string[]; readonly allowedMcpTools: readonly string[]; + /** Exact prompt literals mapped to content-free evidence roles. */ + readonly pathRoleBindings?: readonly ManagedAgentPathRoleBinding[]; + /** Certification mode denies file paths without a predeclared role. */ + readonly requireRegisteredFilePaths?: boolean; readonly onDecision: (evidence: ManagedAgentPermissionEvidence) => void; readonly onGuardRejection?: ( diagnostic: ManagedAgentPreToolUseGuardRejection, @@ -165,6 +172,7 @@ export interface ManagedAgentPolicyBoundary { interface ManagedAgentPolicyDecision { readonly decision: "allow" | "deny"; readonly reason: ManagedAgentPermissionReason; + readonly operationId: ManagedAgentOperationId; readonly updatedInput?: Record; } @@ -226,8 +234,53 @@ function asRecord(value: unknown): Record | undefined { function denied( reason: ManagedAgentPermissionReason, + operationId: ManagedAgentOperationId = "unknown", ): ManagedAgentPolicyDecision { - return { decision: "deny", reason }; + return { decision: "deny", reason, operationId }; +} + +function fileOperationId( + toolName: "Read" | "Edit" | "Write", + role: ManagedAgentPathRole, +): ManagedAgentOperationId { + return `${toolName.toLowerCase()}:${role}` as ManagedAgentOperationId; +} + +function lexicalPathRoleKey( + canonicalWorkspaceRoot: string, + requestedPath: string, +): string { + return comparisonPath(resolve(canonicalWorkspaceRoot, requestedPath)); +} + +function classifyManagedAgentOperation( + canonicalWorkspaceRoot: string, + toolName: string, + rawInput: unknown, + allowedCommands: ReadonlySet, + pathRoles: ReadonlyMap, +): ManagedAgentOperationId { + const input = asRecord(rawInput); + if (toolName === "Bash") { + return input && + typeof input.command === "string" && + allowedCommands.has(input.command) + ? "bash:exact_command" + : "bash:unregistered"; + } + if (toolName.endsWith("__echo_nonce")) return "mcp:echo_nonce"; + if (toolName.endsWith("__fail_once")) return "mcp:fail_once"; + if (toolName.startsWith("mcp__")) return "mcp:managed"; + if (toolName === "Read" || toolName === "Edit" || toolName === "Write") { + const requestedPath = input ? filePathFromInput(input) : undefined; + const role = requestedPath + ? (pathRoles.get( + lexicalPathRoleKey(canonicalWorkspaceRoot, requestedPath), + ) ?? "unregistered") + : "unregistered"; + return fileOperationId(toolName, role); + } + return "unknown"; } async function evaluateManagedAgentPolicy( @@ -235,62 +288,80 @@ async function evaluateManagedAgentPolicy( allowedBuiltinTools: ReadonlySet, allowedCommands: ReadonlySet, allowedMcpTools: ReadonlySet, + pathRoles: ReadonlyMap, toolName: string, rawInput: unknown, signal: AbortSignal, ): Promise { - if (signal.aborted) return denied("policy_aborted"); + const operationId = classifyManagedAgentOperation( + options.canonicalWorkspaceRoot, + toolName, + rawInput, + allowedCommands, + pathRoles, + ); + if (signal.aborted) return denied("policy_aborted", operationId); if (!allowedBuiltinTools.has(toolName) && !allowedMcpTools.has(toolName)) { - return denied("tool_not_allowed"); + return denied("tool_not_allowed", operationId); } const input = asRecord(rawInput); - if (!input) return denied("invalid_input"); + if (!input) return denied("invalid_input", operationId); if (allowedMcpTools.has(toolName)) { return signal.aborted - ? denied("policy_aborted") + ? denied("policy_aborted", operationId) : { decision: "allow", reason: "managed_mcp_tool", + operationId, updatedInput: { ...input }, }; } if (toolName === "Bash") { const command = typeof input.command === "string" ? input.command : undefined; - if (!command) return denied("invalid_input"); + if (!command) return denied("invalid_input", operationId); if (!allowedCommands.has(command)) { - return denied("bash_command_not_allowed"); + return denied("bash_command_not_allowed", operationId); } return signal.aborted - ? denied("policy_aborted") + ? denied("policy_aborted", operationId) : { decision: "allow", reason: "exact_bash_command", + operationId, updatedInput: { ...input }, }; } if (toolName === "Read" || toolName === "Edit" || toolName === "Write") { const requestedPath = filePathFromInput(input); - if (!requestedPath) return denied("invalid_input"); + if (!requestedPath) return denied("invalid_input", operationId); + if ( + options.requireRegisteredFilePaths && + operationId.endsWith(":unregistered") + ) { + return denied("path_role_not_allowed", operationId); + } try { const canonicalPath = await ( options.resolveToolPath ?? resolveManagedAgentToolPath )(options.canonicalWorkspaceRoot, requestedPath); - if (signal.aborted) return denied("policy_aborted"); + if (signal.aborted) return denied("policy_aborted", operationId); return { decision: "allow", reason: "fixture_path", + operationId, updatedInput: { ...input, file_path: canonicalPath }, }; } catch (error) { - if (signal.aborted) return denied("policy_aborted"); + if (signal.aborted) return denied("policy_aborted", operationId); return denied( error instanceof ManagedAgentPathError ? error.reason : "invalid_input", + operationId, ); } } - return denied("tool_not_allowed"); + return denied("tool_not_allowed", operationId); } /** @@ -306,6 +377,19 @@ export function createManagedAgentPolicyBoundary( ); const allowedCommands = new Set(options.allowedBashCommands); const allowedMcpTools = new Set(options.allowedMcpTools); + const pathRoles = new Map(); + for (const binding of options.pathRoleBindings ?? []) { + const key = lexicalPathRoleKey( + options.canonicalWorkspaceRoot, + binding.path, + ); + if (pathRoles.has(key)) { + throw new Error( + "Managed-agent path role bindings must resolve to unique lexical paths", + ); + } + pathRoles.set(key, binding.role); + } const decisions = new Map< string, { @@ -337,10 +421,17 @@ export function createManagedAgentPolicyBoundary( signal: AbortSignal, source: ManagedAgentPermissionSource, ): Promise => { + const attemptedOperationId = classifyManagedAgentOperation( + options.canonicalWorkspaceRoot, + toolName, + input, + allowedCommands, + pathRoles, + ); const existing = decisions.get(toolUseID); if (existing) { if (signal.aborted) { - return { ...denied("policy_aborted"), source }; + return { ...denied("policy_aborted", attemptedOperationId), source }; } // The only valid duplicate is the SDK consulting canUseTool after the // primary hook. A repeated primary ID or fallback-first sequence is @@ -348,13 +439,14 @@ export function createManagedAgentPolicyBoundary( return source === "can_use_tool_fallback" && existing.source === "pre_tool_use" ? existing.pending - : { ...denied("invalid_input"), source }; + : { ...denied("invalid_input", attemptedOperationId), source }; } const pending = evaluateManagedAgentPolicy( options, allowedBuiltinTools, allowedCommands, allowedMcpTools, + pathRoles, toolName, input, signal, @@ -366,6 +458,7 @@ export function createManagedAgentPolicyBoundary( decision: recorded.decision, reason: recorded.reason, source, + operationId: recorded.operationId, }); return recorded; }); diff --git a/packages/harness/src/experimental/managed-agent-spike/probe-cli.test.ts b/packages/harness/src/experimental/managed-agent-spike/probe-cli.test.ts index 6f7d99be8..b8fdc8335 100644 --- a/packages/harness/src/experimental/managed-agent-spike/probe-cli.test.ts +++ b/packages/harness/src/experimental/managed-agent-spike/probe-cli.test.ts @@ -11,23 +11,34 @@ import { } from "./probe-cli.js"; import { FIXTURE_PATHS } from "./fixture.js"; import { qualifiedManagedAgentMcpToolName } from "./runtime.js"; -import type { ManagedAgentProbeResult } from "./types.js"; +import type { + ManagedAgentOperationId, + ManagedAgentPermissionDecision, + ManagedAgentPermissionReason, + ManagedAgentProbeResult, +} from "./types.js"; function passingL1Result(): ManagedAgentProbeResult { const echoTool = qualifiedManagedAgentMcpToolName("echo_nonce"); const failOnceTool = qualifiedManagedAgentMcpToolName("fail_once"); const steps = [ - ["Read", "success", "allow", "fixture_path"], - ["Read", "success", "allow", "fixture_path"], - ["Read", "success", "allow", "fixture_path"], - ["Read", "error", "deny", "path_outside_workspace"], - ["Read", "error", "deny", "path_symlink_escape"], - ["Edit", "success", "allow", "fixture_path"], - ["Write", "success", "allow", "fixture_path"], - [echoTool, "success", "allow", "managed_mcp_tool"], - [failOnceTool, "error", "allow", "managed_mcp_tool"], - [failOnceTool, "success", "allow", "managed_mcp_tool"], - ["Bash", "success", "allow", "exact_bash_command"], + ["Read", "success", "allow", "fixture_path", "read:clean_target"], + ["Read", "success", "allow", "fixture_path", "read:dirty_sentinel"], + ["Read", "success", "allow", "fixture_path", "read:untracked_sentinel"], + [ + "Read", + "error", + "deny", + "path_outside_workspace", + "read:outside_sentinel", + ], + ["Read", "error", "deny", "path_symlink_escape", "read:escape_link"], + ["Edit", "success", "allow", "fixture_path", "edit:clean_target"], + ["Write", "success", "allow", "fixture_path", "write:managed_output"], + [echoTool, "success", "allow", "managed_mcp_tool", "mcp:echo_nonce"], + [failOnceTool, "error", "allow", "managed_mcp_tool", "mcp:fail_once"], + [failOnceTool, "success", "allow", "managed_mcp_tool", "mcp:fail_once"], + ["Bash", "success", "allow", "exact_bash_command", "bash:exact_command"], ] as const; const ids = steps.map( (_, index) => `tool_${(index + 1).toString(16).padStart(64, "0")}`, @@ -61,13 +72,16 @@ function passingL1Result(): ManagedAgentProbeResult { status: completion, }, ]), - permissionEvidence: steps.map(([toolName, , decision, reason], index) => ({ - toolUseId: ids[index]!, - toolName, - decision, - reason, - source: "pre_tool_use" as const, - })), + permissionEvidence: steps.map( + ([toolName, , decision, reason, operationId], index) => ({ + toolUseId: ids[index]!, + toolName, + decision, + reason, + source: "pre_tool_use" as const, + operationId, + }), + ), policyDiagnostics: [], workspaceChanges: [ { path: FIXTURE_PATHS.cleanTarget, change: "modified" }, @@ -96,7 +110,16 @@ function passingL1Result(): ManagedAgentProbeResult { evalSource: "eval-1", promptEmbedded: true, }, - }; + l1Certification: { + contractVersion: 2, + promptVersion: "managed-agent-l1-prompt-v2", + }, + l1FinalBytes: [ + { role: "clean_target", matched: true }, + { role: "managed_output", matched: true }, + ], + nonceVerified: true, + } as ManagedAgentProbeResult; } function passingL2Result(): ManagedAgentProbeResult { @@ -116,6 +139,7 @@ function passingL2Result(): ManagedAgentProbeResult { decision: "allow", reason: "exact_bash_command", source: "pre_tool_use", + operationId: "bash:exact_command", }, ], workspaceChanges: [], @@ -138,6 +162,81 @@ function evidenceForToolId( ); } +interface TestToolStep { + readonly toolName: string; + readonly completion: "success" | "error"; + readonly decision: ManagedAgentPermissionDecision; + readonly reason: ManagedAgentPermissionReason; + readonly operationId: ManagedAgentOperationId; +} + +function insertL1ToolStep( + result: ManagedAgentProbeResult, + beforeRequestIndex: number, + step: TestToolStep, + idCharacter: string, +): ManagedAgentProbeResult { + const requested = result.toolEvidence.filter( + ({ status }) => status === "requested", + ); + const nextRequest = requested[beforeRequestIndex]; + const insertionIndex = nextRequest + ? result.toolEvidence.findIndex( + (evidence) => + evidence.toolUseId === nextRequest.toolUseId && + evidence.status === "requested", + ) + : result.toolEvidence.length; + const toolUseId = `tool_${idCharacter.repeat(64)}`; + const toolEvidence = [...result.toolEvidence]; + toolEvidence.splice( + insertionIndex, + 0, + { toolUseId, toolName: step.toolName, status: "requested" }, + { toolUseId, toolName: step.toolName, status: step.completion }, + ); + const permissionEvidence = [...result.permissionEvidence]; + permissionEvidence.splice(beforeRequestIndex, 0, { + toolUseId, + toolName: step.toolName, + decision: step.decision, + reason: step.reason, + source: "pre_tool_use", + operationId: step.operationId, + }); + return { ...result, toolEvidence, permissionEvidence }; +} + +function optionalReadStep( + operationId: Extract, +): TestToolStep { + return { + toolName: "Read", + completion: "success", + decision: "allow", + reason: "fixture_path", + operationId, + }; +} + +function expectL1TraceFailure(result: ManagedAgentProbeResult): void { + const report = evaluateManagedAgentProbe(result); + expect(report.outcome).toBe("fail"); + expect(report.checks).toContainEqual({ + id: "exact_l1_tool_trace", + passed: false, + }); +} + +function expectProbeCheckFailure( + result: ManagedAgentProbeResult, + checkId: string, +): void { + const report = evaluateManagedAgentProbe(result); + expect(report.outcome).toBe("fail"); + expect(report.checks).toContainEqual({ id: checkId, passed: false }); +} + describe("managed-agent probe CLI", () => { it("is opt-in and never accepts credentials through arguments", () => { expect(() => @@ -314,6 +413,223 @@ describe("managed-agent probe CLI", () => { ).toEqual({ id: "builtin_tools_succeeded", passed: false }); }); + it.each([ + ["clean_target", "e"], + ["dirty_sentinel", "f"], + ["untracked_sentinel", "a"], + ] as const)( + "accepts one optional %s verification Read in the v2 window", + (role, idCharacter) => { + const passing = passingL1Result(); + const result = insertL1ToolStep( + passing, + 5, + optionalReadStep(`read:${role}`), + idCharacter, + ); + const report = evaluateManagedAgentProbe(result); + + expect(report.outcome).toBe("pass"); + expect(report).toMatchObject({ + l1Certification: { + contractVersion: 2, + promptVersion: "managed-agent-l1-prompt-v2", + evaluatorVersion: "managed-agent-l1-evaluator-v2", + optionalReadCount: 1, + optionalReadRole: role, + }, + }); + }, + ); + + it("records zero optional Reads as nonblocking efficiency evidence", () => { + expect(evaluateManagedAgentProbe(passingL1Result())).toMatchObject({ + outcome: "pass", + l1Certification: { + evaluatorVersion: "managed-agent-l1-evaluator-v2", + optionalReadCount: 0, + }, + }); + expect( + evaluateManagedAgentProbe(passingL1Result()).l1Certification, + ).not.toHaveProperty("optionalReadRole"); + }); + + it("rejects a second optional verification Read", () => { + const first = insertL1ToolStep( + passingL1Result(), + 5, + optionalReadStep("read:clean_target"), + "e", + ); + const second = insertL1ToolStep( + first, + 6, + optionalReadStep("read:dirty_sentinel"), + "f", + ); + + const report = evaluateManagedAgentProbe(second); + expect(report.l1Certification).toMatchObject({ optionalReadCount: 2 }); + expectL1TraceFailure(second); + }); + + it.each([ + ["managed_output", "e"], + ["outside_sentinel", "f"], + ["escape_link", "a"], + ] as const)( + "rejects an optional Read of the registered but disallowed %s role", + (role, idCharacter) => { + expectL1TraceFailure( + insertL1ToolStep( + passingL1Result(), + 5, + optionalReadStep(`read:${role}`), + idCharacter, + ), + ); + }, + ); + + it.each([ + ["before the denial probes", 3], + ["after Edit", 6], + ] as const)("rejects an otherwise valid optional Read %s", (_name, index) => { + expectL1TraceFailure( + insertL1ToolStep( + passingL1Result(), + index, + optionalReadStep("read:clean_target"), + "e", + ), + ); + }); + + it.each([ + ["outside denial", "path_outside_workspace", "read:outside_sentinel"], + ["symlink denial", "path_symlink_escape", "read:escape_link"], + ] as const)( + "rejects an extra denied Read retry of the %s", + (_name, reason, operationId) => { + expectL1TraceFailure( + insertL1ToolStep( + passingL1Result(), + 5, + { + toolName: "Read", + completion: "error", + decision: "deny", + reason, + operationId, + }, + "e", + ), + ); + }, + ); + + it.each([ + [ + "Edit", + { + toolName: "Edit", + completion: "success", + decision: "allow", + reason: "fixture_path", + operationId: "edit:clean_target", + }, + ], + [ + "Write", + { + toolName: "Write", + completion: "success", + decision: "allow", + reason: "fixture_path", + operationId: "write:managed_output", + }, + ], + [ + "Bash", + { + toolName: "Bash", + completion: "success", + decision: "allow", + reason: "exact_bash_command", + operationId: "bash:exact_command", + }, + ], + [ + "MCP", + { + toolName: qualifiedManagedAgentMcpToolName("echo_nonce"), + completion: "success", + decision: "allow", + reason: "managed_mcp_tool", + operationId: "mcp:echo_nonce", + }, + ], + [ + "unknown tool", + { + toolName: "unknown", + completion: "error", + decision: "deny", + reason: "tool_not_allowed", + operationId: "unknown", + }, + ], + ] as const)("rejects any extra %s operation", (_name, step) => { + expectL1TraceFailure(insertL1ToolStep(passingL1Result(), 5, step, "e")); + }); + + it("rejects any workspace delta beyond the two canonical L1 changes", () => { + const passing = passingL1Result(); + const report = evaluateManagedAgentProbe({ + ...passing, + workspaceChanges: [ + ...passing.workspaceChanges, + { path: "unexpected.txt", change: "created" }, + ], + }); + + expect(report.outcome).toBe("fail"); + expect(report.checks).toContainEqual({ + id: "exact_workspace_delta", + passed: false, + }); + }); + + it("accepts the exact workspace delta in either evidence order", () => { + const passing = passingL1Result(); + const report = evaluateManagedAgentProbe({ + ...passing, + workspaceChanges: [...passing.workspaceChanges].reverse(), + }); + + expect(report.outcome).toBe("pass"); + expect(report.checks).toContainEqual({ + id: "exact_workspace_delta", + passed: true, + }); + }); + + it("rejects a duplicate canonical workspace entry with the other path missing", () => { + const passing = passingL1Result(); + const duplicate = passing.workspaceChanges[0]!; + const report = evaluateManagedAgentProbe({ + ...passing, + workspaceChanges: [duplicate, duplicate], + }); + + expect(report.outcome).toBe("fail"); + expect(report.checks).toContainEqual({ + id: "exact_workspace_delta", + passed: false, + }); + }); + it("accepts exactly one permitted Bash request for L2 and rejects any extra tool call", () => { const passing = passingL2Result(); expect(evaluateManagedAgentProbe(passing, [12_345, 12_346])).toMatchObject({ @@ -340,6 +656,7 @@ describe("managed-agent probe CLI", () => { decision: "allow", reason: "fixture_path", source: "pre_tool_use", + operationId: "write:unregistered", }, ], }; @@ -407,6 +724,7 @@ describe("managed-agent probe CLI", () => { decision: "allow" as const, reason: "fixture_path" as const, source: "pre_tool_use" as const, + operationId: "read:clean_target" as const, }, ], }; @@ -431,6 +749,7 @@ describe("managed-agent probe CLI", () => { decision: "allow" as const, reason: "exact_bash_command" as const, source: "pre_tool_use" as const, + operationId: "bash:exact_command" as const, }, ], }; @@ -446,6 +765,325 @@ describe("managed-agent probe CLI", () => { }); }); + describe("L1 v2 request correlation", () => { + it("rejects duplicate request IDs", () => { + const passing = passingL1Result(); + const requestIds = passing.toolEvidence.flatMap((evidence) => + evidence.status === "requested" && evidence.toolUseId + ? [evidence.toolUseId] + : [], + ); + const firstId = requestIds[0]!; + const duplicateId = requestIds[1]!; + expectL1TraceFailure({ + ...passing, + toolEvidence: passing.toolEvidence.map((evidence) => + evidence.status === "requested" && evidence.toolUseId === firstId + ? { ...evidence, toolUseId: duplicateId } + : evidence, + ), + }); + }); + + it("rejects an empty request ID", () => { + const passing = passingL1Result(); + const firstId = passing.toolEvidence.find( + ({ status }) => status === "requested", + )!.toolUseId!; + expectL1TraceFailure({ + ...passing, + toolEvidence: passing.toolEvidence.map((evidence) => + evidence.status === "requested" && evidence.toolUseId === firstId + ? { ...evidence, toolUseId: " " } + : evidence, + ), + }); + }); + + it("rejects a request with no completion", () => { + const passing = passingL1Result(); + const firstId = passing.toolEvidence.find( + ({ status }) => status === "requested", + )!.toolUseId!; + expectL1TraceFailure({ + ...passing, + toolEvidence: passing.toolEvidence.filter( + (evidence) => + evidence.toolUseId !== firstId || evidence.status === "requested", + ), + }); + }); + + it("rejects duplicate completions for one request", () => { + const passing = passingL1Result(); + const completion = passing.toolEvidence.find( + ({ status }) => status !== "requested", + )!; + expectL1TraceFailure({ + ...passing, + toolEvidence: [...passing.toolEvidence, completion], + }); + }); + + it("rejects a completion whose tool does not match its request", () => { + const passing = passingL1Result(); + const firstId = passing.toolEvidence.find( + ({ status }) => status === "requested", + )!.toolUseId!; + expectL1TraceFailure({ + ...passing, + toolEvidence: passing.toolEvidence.map((evidence) => + evidence.toolUseId === firstId && evidence.status !== "requested" + ? { ...evidence, toolName: "Write" } + : evidence, + ), + }); + }); + + it("rejects a request with no primary PreToolUse decision", () => { + const passing = passingL1Result(); + const firstDecision = passing.permissionEvidence[0]!; + expectL1TraceFailure({ + ...passing, + permissionEvidence: passing.permissionEvidence.filter( + ({ toolUseId }) => toolUseId !== firstDecision.toolUseId, + ), + }); + }); + + it("rejects duplicate primary decisions for one request", () => { + const passing = passingL1Result(); + expectL1TraceFailure({ + ...passing, + permissionEvidence: [ + ...passing.permissionEvidence, + passing.permissionEvidence[0]!, + ], + }); + }); + + it("rejects a primary decision whose tool does not match its request", () => { + const passing = passingL1Result(); + const firstDecision = passing.permissionEvidence[0]!; + expectL1TraceFailure({ + ...passing, + permissionEvidence: passing.permissionEvidence.map((evidence) => + evidence.toolUseId === firstDecision.toolUseId + ? { ...evidence, toolName: "Write" } + : evidence, + ), + }); + }); + + it("rejects a fallback decision in addition to the primary decision", () => { + const passing = passingL1Result(); + expectL1TraceFailure({ + ...passing, + permissionEvidence: [ + ...passing.permissionEvidence, + { + ...passing.permissionEvidence[0]!, + source: "can_use_tool_fallback", + }, + ], + }); + }); + + it("rejects a fallback decision that replaces the primary decision", () => { + const passing = passingL1Result(); + const firstDecision = passing.permissionEvidence[0]!; + expectL1TraceFailure({ + ...passing, + permissionEvidence: passing.permissionEvidence.map((evidence) => + evidence.toolUseId === firstDecision.toolUseId + ? { ...evidence, source: "can_use_tool_fallback" } + : evidence, + ), + }); + }); + + it("rejects an orphan completion", () => { + const passing = passingL1Result(); + expectL1TraceFailure({ + ...passing, + toolEvidence: [ + ...passing.toolEvidence, + { + toolUseId: `tool_${"e".repeat(64)}`, + toolName: "Read", + status: "success", + }, + ], + }); + }); + + it("rejects an orphan primary decision", () => { + const passing = passingL1Result(); + expectL1TraceFailure({ + ...passing, + permissionEvidence: [ + ...passing.permissionEvidence, + { + ...passing.permissionEvidence[0]!, + toolUseId: `tool_${"e".repeat(64)}`, + }, + ], + }); + }); + }); + + describe("L1 v2 outcome and certification evidence", () => { + it("rejects an allowed canonical operation that completes with an error", () => { + const passing = passingL1Result(); + const firstId = passing.toolEvidence.find( + ({ status }) => status === "requested", + )!.toolUseId!; + expectL1TraceFailure({ + ...passing, + toolEvidence: passing.toolEvidence.map((evidence) => + evidence.toolUseId === firstId && evidence.status === "success" + ? { ...evidence, status: "error" } + : evidence, + ), + }); + }); + + it("rejects a denied canonical operation that reports success", () => { + const passing = passingL1Result(); + const deniedId = passing.permissionEvidence.find( + ({ decision }) => decision === "deny", + )!.toolUseId; + expectL1TraceFailure({ + ...passing, + toolEvidence: passing.toolEvidence.map((evidence) => + evidence.toolUseId === deniedId && evidence.status === "error" + ? { ...evidence, status: "success" } + : evidence, + ), + }); + }); + + it("rejects an incoherent decision, reason, or operation ID", () => { + const passing = passingL1Result(); + const firstDecision = passing.permissionEvidence[0]!; + for (const replacement of [ + { decision: "deny" as const }, + { reason: "path_outside_workspace" as const }, + { operationId: "read:dirty_sentinel" as const }, + ]) { + expectL1TraceFailure({ + ...passing, + permissionEvidence: passing.permissionEvidence.map((evidence) => + evidence.toolUseId === firstDecision.toolUseId + ? { ...evidence, ...replacement } + : evidence, + ), + }); + } + }); + + it("rejects missing or stale L1 v2 contract evidence", () => { + const passing = passingL1Result(); + expectProbeCheckFailure( + { ...passing, l1Certification: undefined }, + "l1_contract_v2", + ); + expectProbeCheckFailure( + { + ...passing, + l1Certification: { + contractVersion: 1, + promptVersion: "managed-agent-l1-prompt-v1", + }, + } as unknown as ManagedAgentProbeResult, + "l1_contract_v2", + ); + expectProbeCheckFailure( + { + ...passing, + correlation: { ...passing.correlation, promptEmbedded: false }, + }, + "l1_contract_v2", + ); + }); + + it("requires positive nonce evidence", () => { + expectProbeCheckFailure( + { ...passingL1Result(), nonceVerified: false }, + "nonce_verified", + ); + }); + + it.each([ + ["missing", undefined], + [ + "false", + [ + { role: "clean_target", matched: true }, + { role: "managed_output", matched: false }, + ], + ], + [ + "extra", + [ + { role: "clean_target", matched: true }, + { role: "managed_output", matched: true }, + { role: "clean_target", matched: true }, + ], + ], + ] as const)("rejects %s L1 final-byte evidence", (_name, l1FinalBytes) => { + expectProbeCheckFailure( + { + ...passingL1Result(), + l1FinalBytes, + } as ManagedAgentProbeResult, + "expected_final_bytes", + ); + }); + + it.each([ + ["terminal success", "terminal_success", { terminal: "incomplete" }], + ["query close", "query_closed", { queryClosed: false }], + [ + "process quiescence", + "process_tree_quiescent", + { teardown: { ...passingL1Result().teardown, quiescent: false } }, + ], + ] as const)("requires %s", (_name, checkId, mutation) => { + expectProbeCheckFailure( + { + ...passingL1Result(), + ...mutation, + } as ManagedAgentProbeResult, + checkId, + ); + }); + + it.each([ + ["missing", []], + [ + "false", + [ + { path: FIXTURE_PATHS.dirtySentinel, preserved: true }, + { path: FIXTURE_PATHS.untrackedSentinel, preserved: false }, + ], + ], + [ + "extra", + [ + { path: FIXTURE_PATHS.dirtySentinel, preserved: true }, + { path: FIXTURE_PATHS.untrackedSentinel, preserved: true }, + { path: "extra-sentinel.txt", preserved: true }, + ], + ], + ] as const)("rejects %s preservation evidence", (_name, preservation) => { + expectProbeCheckFailure( + { ...passingL1Result(), preservation: [...preservation] }, + "dirty_and_untracked_preserved", + ); + }); + }); + it("requires one completion and primary decision per L1 request, including fail_once error then success", () => { const passing = passingL1Result(); const failOnceRequests = passing.toolEvidence.filter( @@ -482,6 +1120,7 @@ describe("managed-agent probe CLI", () => { decision: "deny", reason: "path_outside_workspace", source: "pre_tool_use", + operationId: "read:outside_sentinel", }, { toolUseId: `tool_${"b".repeat(64)}`, @@ -489,6 +1128,7 @@ describe("managed-agent probe CLI", () => { decision: "deny", reason: "path_outside_workspace", source: "pre_tool_use", + operationId: "read:outside_sentinel", }, ], }; diff --git a/packages/harness/src/experimental/managed-agent-spike/probe-cli.ts b/packages/harness/src/experimental/managed-agent-spike/probe-cli.ts index 771c5bbb2..5d984502b 100644 --- a/packages/harness/src/experimental/managed-agent-spike/probe-cli.ts +++ b/packages/harness/src/experimental/managed-agent-spike/probe-cli.ts @@ -9,6 +9,7 @@ import { } from "./fixture.js"; import { MANAGED_AGENT_CONTRACT, + MANAGED_AGENT_L1_CERTIFICATION_CONTRACT, assertManagedAgentDirectGatewayOrigin, resolveManagedAgentModelTarget, } from "./contract.js"; @@ -19,6 +20,8 @@ import { } from "./runtime.js"; import type { ManagedAgentModelTargetId, + ManagedAgentOperationId, + ManagedAgentPathRole, ManagedAgentPermissionReason, ManagedAgentProbeResult, ManagedAgentProbeScenario, @@ -42,6 +45,13 @@ export interface ManagedAgentProbeReport { readonly outcome: "pass" | "fail"; readonly checks: readonly ManagedAgentProbeCheck[]; readonly result: ManagedAgentProbeResult; + readonly l1Certification?: { + readonly contractVersion: number; + readonly promptVersion: string; + readonly evaluatorVersion: string; + readonly optionalReadCount: number; + readonly optionalReadRole?: ManagedAgentPathRole; + }; } export class ManagedAgentProbeCliError extends Error { @@ -56,80 +66,139 @@ interface ManagedAgentExpectedL1ToolStep { readonly completion: "success" | "error"; readonly decision: "allow" | "deny"; readonly reason: ManagedAgentPermissionReason; + readonly operationId: ManagedAgentOperationId; } -const MANAGED_AGENT_EXPECTED_L1_TOOL_TRACE = [ +const MANAGED_AGENT_EXPECTED_L1_TOOL_TRACE = Object.freeze([ { toolName: "Read", completion: "success", decision: "allow", reason: "fixture_path", + operationId: "read:clean_target", }, { toolName: "Read", completion: "success", decision: "allow", reason: "fixture_path", + operationId: "read:dirty_sentinel", }, { toolName: "Read", completion: "success", decision: "allow", reason: "fixture_path", + operationId: "read:untracked_sentinel", }, { toolName: "Read", completion: "error", decision: "deny", reason: "path_outside_workspace", + operationId: "read:outside_sentinel", }, { toolName: "Read", completion: "error", decision: "deny", reason: "path_symlink_escape", + operationId: "read:escape_link", }, { toolName: "Edit", completion: "success", decision: "allow", reason: "fixture_path", + operationId: "edit:clean_target", }, { toolName: "Write", completion: "success", decision: "allow", reason: "fixture_path", + operationId: "write:managed_output", }, { toolName: qualifiedManagedAgentMcpToolName("echo_nonce"), completion: "success", decision: "allow", reason: "managed_mcp_tool", + operationId: "mcp:echo_nonce", }, { toolName: qualifiedManagedAgentMcpToolName("fail_once"), completion: "error", decision: "allow", reason: "managed_mcp_tool", + operationId: "mcp:fail_once", }, { toolName: qualifiedManagedAgentMcpToolName("fail_once"), completion: "success", decision: "allow", reason: "managed_mcp_tool", + operationId: "mcp:fail_once", }, { toolName: "Bash", completion: "success", decision: "allow", reason: "exact_bash_command", + operationId: "bash:exact_command", }, -] as const satisfies readonly ManagedAgentExpectedL1ToolStep[]; +] as const satisfies readonly ManagedAgentExpectedL1ToolStep[]); + +const MANAGED_AGENT_L1_OPTIONAL_READ_OPERATIONS = + new Set([ + "read:clean_target", + "read:dirty_sentinel", + "read:untracked_sentinel", + ]); + +interface ManagedAgentL1TraceAnalysis { + readonly passed: boolean; + readonly optionalReadCount: number; + readonly optionalReadRole?: ManagedAgentPathRole; +} -function hasExactManagedAgentL1ToolTrace( +function hasExactManagedAgentL1WorkspaceDelta( result: ManagedAgentProbeResult, ): boolean { + const expected = [ + { path: FIXTURE_PATHS.cleanTarget, change: "modified" }, + { path: FIXTURE_PATHS.createdTarget, change: "created" }, + ] as const; + return ( + result.workspaceChanges.length === expected.length && + expected.every( + (expectedChange) => + result.workspaceChanges.filter( + ({ path, change }) => + path === expectedChange.path && change === expectedChange.change, + ).length === 1, + ) + ); +} + +function hasExactManagedAgentL1FinalBytes( + result: ManagedAgentProbeResult, +): boolean { + const roles = ["clean_target", "managed_output"] as const; + return Boolean( + result.l1FinalBytes?.length === roles.length && + roles.every( + (role) => + result.l1FinalBytes?.filter( + (observation) => observation.role === role && observation.matched, + ).length === 1, + ), + ); +} + +function analyzeManagedAgentL1ToolTrace( + result: ManagedAgentProbeResult, +): ManagedAgentL1TraceAnalysis { const requested = result.toolEvidence.filter( ({ status }) => status === "requested", ); @@ -139,45 +208,121 @@ function hasExactManagedAgentL1ToolTrace( const primaryDecisions = result.permissionEvidence.filter( ({ source }) => source === "pre_tool_use", ); - if ( - requested.length !== MANAGED_AGENT_EXPECTED_L1_TOOL_TRACE.length || - completed.length !== MANAGED_AGENT_EXPECTED_L1_TOOL_TRACE.length || - primaryDecisions.length !== MANAGED_AGENT_EXPECTED_L1_TOOL_TRACE.length || - result.permissionEvidence.length !== primaryDecisions.length - ) { - return false; - } const requestedIds = requested.flatMap(({ toolUseId }) => - toolUseId ? [toolUseId] : [], + toolUseId?.trim() ? [toolUseId] : [], ); - if ( + const requestedById = new Map( + requestedIds.map((toolUseId, index) => [toolUseId, requested[index]!]), + ); + const completionById = new Map( + completed.flatMap((evidence) => + evidence.toolUseId ? [[evidence.toolUseId, evidence] as const] : [], + ), + ); + const decisionById = new Map( + primaryDecisions.map((evidence) => [evidence.toolUseId, evidence]), + ); + let invalid = requestedIds.length !== requested.length || - new Set(requestedIds).size !== requestedIds.length - ) { - return false; - } + new Set(requestedIds).size !== requestedIds.length || + completed.length !== requested.length || + completionById.size !== completed.length || + primaryDecisions.length !== requested.length || + decisionById.size !== primaryDecisions.length || + result.permissionEvidence.length !== primaryDecisions.length || + result.permissionEvidence.some(({ source }) => source !== "pre_tool_use") || + completed.some( + ({ toolUseId, toolName }) => + !toolUseId || requestedById.get(toolUseId)?.toolName !== toolName, + ) || + primaryDecisions.some( + ({ toolUseId, toolName }) => + requestedById.get(toolUseId)?.toolName !== toolName, + ); - return MANAGED_AGENT_EXPECTED_L1_TOOL_TRACE.every((expected, index) => { - const request = requested[index]; + const matches = ( + requestIndex: number, + expected: ManagedAgentExpectedL1ToolStep, + ): boolean => { + const request = requested[requestIndex]; if (!request?.toolUseId || request.toolName !== expected.toolName) { return false; } - const completions = completed.filter( - ({ toolUseId }) => toolUseId === request.toolUseId, - ); - const decisions = primaryDecisions.filter( - ({ toolUseId }) => toolUseId === request.toolUseId, - ); - return ( - completions.length === 1 && - completions[0]?.toolName === expected.toolName && - completions[0]?.status === expected.completion && - decisions.length === 1 && - decisions[0]?.toolName === expected.toolName && - decisions[0]?.decision === expected.decision && - decisions[0]?.reason === expected.reason + const completion = completionById.get(request.toolUseId); + const decision = decisionById.get(request.toolUseId); + return Boolean( + completion?.toolName === expected.toolName && + completion.status === expected.completion && + decision?.toolName === expected.toolName && + decision.decision === expected.decision && + decision.reason === expected.reason && + decision.operationId === expected.operationId, ); + }; + + let cursor = 0; + for (const expected of MANAGED_AGENT_EXPECTED_L1_TOOL_TRACE.slice(0, 5)) { + invalid ||= !matches(cursor, expected); + cursor += 1; + } + + const firstCanonicalReadByOperation = new Set(); + const optionalVerificationReads = requested.filter((request) => { + if (!request.toolUseId || request.toolName !== "Read") return false; + const decision = decisionById.get(request.toolUseId); + if ( + !decision || + !MANAGED_AGENT_L1_OPTIONAL_READ_OPERATIONS.has(decision.operationId) + ) { + return false; + } + if (!firstCanonicalReadByOperation.has(decision.operationId)) { + firstCanonicalReadByOperation.add(decision.operationId); + return false; + } + return true; }); + const optionalReadCount = optionalVerificationReads.length; + const optionalOperation = + optionalReadCount === 1 && optionalVerificationReads[0]?.toolUseId + ? decisionById.get(optionalVerificationReads[0].toolUseId)?.operationId + : undefined; + const optionalReadRole = optionalOperation?.startsWith("read:") + ? (optionalOperation.slice("read:".length) as ManagedAgentPathRole) + : undefined; + + const candidate = requested[cursor]; + const candidateDecision = candidate?.toolUseId + ? decisionById.get(candidate.toolUseId) + : undefined; + if ( + candidate?.toolName === "Read" && + candidateDecision && + MANAGED_AGENT_L1_OPTIONAL_READ_OPERATIONS.has(candidateDecision.operationId) + ) { + const completion = candidate.toolUseId + ? completionById.get(candidate.toolUseId) + : undefined; + invalid ||= + completion?.toolName !== "Read" || + completion.status !== "success" || + candidateDecision.toolName !== "Read" || + candidateDecision.decision !== "allow" || + candidateDecision.reason !== "fixture_path"; + cursor += 1; + } + + for (const expected of MANAGED_AGENT_EXPECTED_L1_TOOL_TRACE.slice(5)) { + invalid ||= !matches(cursor, expected); + cursor += 1; + } + invalid ||= cursor !== requested.length; + + return { + passed: !invalid, + optionalReadCount, + ...(optionalReadRole ? { optionalReadRole } : {}), + }; } function hasExactManagedAgentL2BashTrace( @@ -209,7 +354,8 @@ function hasExactManagedAgentL2BashTrace( primaryDecisions[0]?.toolUseId === request.toolUseId && primaryDecisions[0]?.toolName === "Bash" && primaryDecisions[0]?.decision === "allow" && - primaryDecisions[0]?.reason === "exact_bash_command", + primaryDecisions[0]?.reason === "exact_bash_command" && + primaryDecisions[0]?.operationId === "bash:exact_command", ); } @@ -307,6 +453,10 @@ export function evaluateManagedAgentProbe( result: ManagedAgentProbeResult, fixturePids: readonly number[] = [], ): ManagedAgentProbeReport { + const l1Trace = + result.scenario === "L1" + ? analyzeManagedAgentL1ToolTrace(result) + : undefined; const requestedTools = new Set( result.toolEvidence .filter(({ status }) => status === "requested") @@ -374,31 +524,41 @@ export function evaluateManagedAgentProbe( id: "dirty_and_untracked_preserved", passed: result.preservation.length === 2 && - result.preservation.every(({ preserved }) => preserved), + [FIXTURE_PATHS.dirtySentinel, FIXTURE_PATHS.untrackedSentinel].every( + (path) => + result.preservation.filter( + (observation) => + observation.path === path && observation.preserved, + ).length === 1, + ), }, ]; if (result.scenario === "L1") { checks.push( { id: "terminal_success", passed: result.terminal === "success" }, + { + id: "l1_contract_v2", + passed: + result.correlation.promptEmbedded && + result.l1Certification?.contractVersion === + MANAGED_AGENT_L1_CERTIFICATION_CONTRACT.contractVersion && + result.l1Certification.promptVersion === + MANAGED_AGENT_L1_CERTIFICATION_CONTRACT.promptVersion, + }, { id: "exact_l1_tool_trace", - passed: hasExactManagedAgentL1ToolTrace(result), + passed: l1Trace?.passed === true, }, { - id: "clean_target_modified", - passed: result.workspaceChanges.some( - ({ path, change }) => - path === FIXTURE_PATHS.cleanTarget && change === "modified", - ), + id: "exact_workspace_delta", + passed: hasExactManagedAgentL1WorkspaceDelta(result), }, { - id: "managed_output_created", - passed: result.workspaceChanges.some( - ({ path, change }) => - path === FIXTURE_PATHS.createdTarget && change === "created", - ), + id: "expected_final_bytes", + passed: hasExactManagedAgentL1FinalBytes(result), }, + { id: "nonce_verified", passed: result.nonceVerified === true }, { id: "builtin_tools_succeeded", passed: ["Read", "Edit", "Write", "Bash"].every( @@ -480,11 +640,25 @@ export function evaluateManagedAgentProbe( ); } - return { + const report: ManagedAgentProbeReport = { outcome: checks.every(({ passed }) => passed) ? "pass" : "fail", checks, result, }; + if (result.scenario !== "L1") return report; + return { + ...report, + l1Certification: { + contractVersion: result.l1Certification?.contractVersion ?? 0, + promptVersion: result.l1Certification?.promptVersion ?? "unobserved", + evaluatorVersion: + MANAGED_AGENT_L1_CERTIFICATION_CONTRACT.evaluatorVersion, + optionalReadCount: l1Trace?.optionalReadCount ?? 0, + ...(l1Trace?.optionalReadRole + ? { optionalReadRole: l1Trace.optionalReadRole } + : {}), + }, + }; } export async function executeManagedAgentProbeCli( @@ -529,6 +703,9 @@ export async function executeManagedAgentProbeCli( allowedBashCommands: [ scenario === "L1" ? fixture.l1BashCommand : fixture.l2BashCommand, ], + pathRoleBindings: scenario === "L1" ? fixture.pathRoleBindings : [], + expectedL1FinalBytes: + scenario === "L1" ? fixture.expectedL1FinalBytes : [], ...(scenario === "L1" ? { expectedMcpNonce: fixture.nonce } : {}), preservePaths: [ FIXTURE_PATHS.dirtySentinel, diff --git a/packages/harness/src/experimental/managed-agent-spike/runtime-sdk-loopback.test.ts b/packages/harness/src/experimental/managed-agent-spike/runtime-sdk-loopback.test.ts index c08c79fab..fb825cba1 100644 --- a/packages/harness/src/experimental/managed-agent-spike/runtime-sdk-loopback.test.ts +++ b/packages/harness/src/experimental/managed-agent-spike/runtime-sdk-loopback.test.ts @@ -295,6 +295,8 @@ it("enforces real-SDK built-in and in-process MCP calls with exact loopback corr maxTurns: 6, maxBudgetUsd: 0.25, allowedBashCommands: [ALLOWED_BASH_COMMAND], + pathRoleBindings: fixture.pathRoleBindings, + expectedL1FinalBytes: fixture.expectedL1FinalBytes, expectedMcpNonce: fixture.nonce, preservePaths: [ FIXTURE_PATHS.dirtySentinel, @@ -514,6 +516,8 @@ it.skipIf( maxTurns: 4, maxBudgetUsd: 0.25, allowedBashCommands: [], + pathRoleBindings: fixture.pathRoleBindings, + expectedL1FinalBytes: fixture.expectedL1FinalBytes, expectedMcpNonce: fixture.nonce, preservePaths: [ FIXTURE_PATHS.dirtySentinel, diff --git a/packages/harness/src/experimental/managed-agent-spike/runtime.test.ts b/packages/harness/src/experimental/managed-agent-spike/runtime.test.ts index e5bfd07c3..60833febf 100644 --- a/packages/harness/src/experimental/managed-agent-spike/runtime.test.ts +++ b/packages/harness/src/experimental/managed-agent-spike/runtime.test.ts @@ -145,6 +145,9 @@ async function probeConfig(scenario: "L1" | "L2" = "L1") { allowedBashCommands: [ scenario === "L1" ? fixture.l1BashCommand : fixture.l2BashCommand, ], + pathRoleBindings: scenario === "L1" ? fixture.pathRoleBindings : [], + expectedL1FinalBytes: + scenario === "L1" ? fixture.expectedL1FinalBytes : [], ...(scenario === "L1" ? { expectedMcpNonce: fixture.nonce } : {}), preservePaths: [ FIXTURE_PATHS.dirtySentinel, @@ -321,6 +324,15 @@ describe("runManagedAgentProbe", () => { expect(result.inferenceTurns).toBe(1); expect(result.sdkNumTurns).toBe(1); expect(result.correlation.promptEmbedded).toBe(true); + expect(result.l1Certification).toEqual({ + contractVersion: 2, + promptVersion: "managed-agent-l1-prompt-v2", + }); + expect(result.l1FinalBytes).toEqual([ + { role: "clean_target", matched: false }, + { role: "managed_output", matched: false }, + ]); + expect(result.nonceVerified).toBe(false); expect(capturedPrompt).toContain( "SAPIOM_CERTIFICATION_CORRELATION_V1;eval_source=studio-managed-agent-e0-l1-sonnet-5-00000000-0000-4000-8000-000000000002;execution_id=00000000-0000-4000-8000-000000000002", ); diff --git a/packages/harness/src/experimental/managed-agent-spike/runtime.ts b/packages/harness/src/experimental/managed-agent-spike/runtime.ts index 04a82d594..9363b009c 100644 --- a/packages/harness/src/experimental/managed-agent-spike/runtime.ts +++ b/packages/harness/src/experimental/managed-agent-spike/runtime.ts @@ -9,12 +9,16 @@ import { } from "@anthropic-ai/claude-agent-sdk"; import { z } from "zod"; -import { validateManagedAgentProbeConfig } from "./contract.js"; +import { + MANAGED_AGENT_L1_CERTIFICATION_CONTRACT, + validateManagedAgentProbeConfig, +} from "./contract.js"; import { buildManagedAgentChildEnvironment } from "./environment.js"; import { ManagedAgentEventError, ManagedAgentEventRecorder } from "./events.js"; import { captureManagedAgentWorkspaceSnapshot, diffManagedAgentWorkspaceSnapshots, + observeManagedAgentL1FinalBytes, observeManagedAgentPreservation, } from "./fixture.js"; import { @@ -416,6 +420,8 @@ export async function runManagedAgentProbe( allowedBashCommands: config.allowedBashCommands, allowedMcpTools: config.scenario === "L1" ? mcpRuntime.qualifiedToolNames : [], + pathRoleBindings: config.pathRoleBindings, + requireRegisteredFilePaths: config.scenario === "L1", onDecision: (evidence) => recorder.recordPermission(evidence), onGuardRejection: (diagnostic) => guardRejections.push(diagnostic), }); @@ -698,6 +704,25 @@ export async function runManagedAgentProbe( queryClosed, teardown, correlation: { executionId, evalSource, promptEmbedded }, + ...(config.scenario === "L1" + ? { + l1Certification: { + contractVersion: + MANAGED_AGENT_L1_CERTIFICATION_CONTRACT.contractVersion, + promptVersion: + MANAGED_AGENT_L1_CERTIFICATION_CONTRACT.promptVersion, + }, + l1FinalBytes: observeManagedAgentL1FinalBytes( + after, + config.expectedL1FinalBytes, + ), + nonceVerified: mcpRuntime.invocations.some( + ({ toolName, status }) => + toolName === qualifiedManagedAgentMcpToolName("echo_nonce") && + status === "success", + ), + } + : {}), ...(recorder.usage ? { sdkUsage: recorder.usage } : {}), }; } diff --git a/packages/harness/src/experimental/managed-agent-spike/types.ts b/packages/harness/src/experimental/managed-agent-spike/types.ts index f56b04bbf..6028a8db2 100644 --- a/packages/harness/src/experimental/managed-agent-spike/types.ts +++ b/packages/harness/src/experimental/managed-agent-spike/types.ts @@ -29,6 +29,10 @@ export interface ManagedAgentProbeConfig { readonly maxTurns: number; readonly maxBudgetUsd: number; readonly allowedBashCommands: readonly string[]; + /** Exact prompt literals mapped to privacy-safe roles inside the host policy. */ + readonly pathRoleBindings: readonly ManagedAgentPathRoleBinding[]; + /** Trusted expected hashes for the two L1 mutation targets; empty for L2. */ + readonly expectedL1FinalBytes: readonly ManagedAgentL1ExpectedFileHash[]; /** Expected only for L1 and never copied into structural evidence. */ readonly expectedMcpNonce?: string; readonly preservePaths?: readonly string[]; @@ -43,6 +47,7 @@ export type ManagedAgentPermissionReason = | "invalid_input" | "path_outside_workspace" | "path_symlink_escape" + | "path_role_not_allowed" | "bash_command_not_allowed" | "tool_not_allowed"; @@ -50,6 +55,49 @@ export type ManagedAgentPermissionSource = | "pre_tool_use" | "can_use_tool_fallback"; +export type ManagedAgentRegisteredPathRole = + | "clean_target" + | "dirty_sentinel" + | "untracked_sentinel" + | "managed_output" + | "outside_sentinel" + | "escape_link"; + +export type ManagedAgentPathRole = + | ManagedAgentRegisteredPathRole + | "unregistered"; + +export interface ManagedAgentPathRoleBinding { + /** Sensitive prompt literal; this value never crosses the evidence boundary. */ + readonly path: string; + readonly role: ManagedAgentRegisteredPathRole; +} + +export type ManagedAgentL1FinalByteRole = "clean_target" | "managed_output"; + +export interface ManagedAgentL1ExpectedFileHash { + /** Sensitive fixture path; this value never crosses the evidence boundary. */ + readonly path: string; + readonly role: ManagedAgentL1FinalByteRole; + readonly sha256: string; +} + +export interface ManagedAgentL1FinalByteObservation { + readonly role: ManagedAgentL1FinalByteRole; + readonly matched: boolean; +} + +export type ManagedAgentOperationId = + | `read:${ManagedAgentPathRole}` + | `edit:${ManagedAgentPathRole}` + | `write:${ManagedAgentPathRole}` + | "bash:exact_command" + | "bash:unregistered" + | "mcp:echo_nonce" + | "mcp:fail_once" + | "mcp:managed" + | "unknown"; + export type ManagedAgentProbeEventType = | "lifecycle" | "message" @@ -74,6 +122,7 @@ export interface ManagedAgentProbeEvent { readonly permissionDecision?: ManagedAgentPermissionDecision; readonly permissionReason?: ManagedAgentPermissionReason; readonly permissionSource?: ManagedAgentPermissionSource; + readonly operationId?: ManagedAgentOperationId; readonly isError?: boolean; readonly terminal?: ManagedAgentTerminalClassification; } @@ -109,6 +158,8 @@ export interface ManagedAgentPermissionEvidence { readonly decision: ManagedAgentPermissionDecision; readonly reason: ManagedAgentPermissionReason; readonly source: ManagedAgentPermissionSource; + /** Trusted, content-free operation identity; never contains a raw path/input. */ + readonly operationId: ManagedAgentOperationId; } export type ManagedAgentPreToolUseGuardRejectionReason = @@ -219,6 +270,15 @@ export interface ManagedAgentProbeResult { /** True only after the marked prompt is handed to the query factory. */ readonly promptEmbedded: boolean; }; + /** Present only for an L1 prompt validated against the frozen v2 marker. */ + readonly l1Certification?: { + readonly contractVersion: 2; + readonly promptVersion: "managed-agent-l1-prompt-v2"; + }; + /** Content-free proof of exact final bytes for both intended L1 mutations. */ + readonly l1FinalBytes?: readonly ManagedAgentL1FinalByteObservation[]; + /** Content-free proof that echo_nonce received the expected sentinel nonce. */ + readonly nonceVerified?: boolean; readonly sdkUsage?: ManagedAgentSdkUsageEstimate; } From 8cf5874cd78a5102fa69e04d3fb5b47681fe2300 Mon Sep 17 00:00:00 2001 From: Yash Date: Sun, 16 Aug 2026 22:01:46 -0700 Subject: [PATCH 12/24] fix(harness): enforce managed L1 completion barriers Reject traces that preserve request order but violate the semantic completion barriers needed to prove multi-turn tool recovery. Keep valid SDK batching by allowing completion permutations within each phase. Refs: SAP-2632 --- .../managed-agent-spike/README.md | 16 + .../managed-agent-spike/probe-cli.test.ts | 339 +++++++++++++++++- .../managed-agent-spike/probe-cli.ts | 191 ++++++++++ 3 files changed, 543 insertions(+), 3 deletions(-) diff --git a/packages/harness/src/experimental/managed-agent-spike/README.md b/packages/harness/src/experimental/managed-agent-spike/README.md index a07d58e2e..581bd2969 100644 --- a/packages/harness/src/experimental/managed-agent-spike/README.md +++ b/packages/harness/src/experimental/managed-agent-spike/README.md @@ -86,6 +86,22 @@ reordering, omission, retries, and every other extra operation fail closed. The optional Read count and role are reported separately as nonblocking efficiency evidence. +Requests keep their exact canonical order while completions may be permuted +within the two batchable phases. Every request must precede its own completion; +all five discovery completions must precede the optional verification Read (or +Edit when it is absent), the optional Read must complete before Edit, and the +Edit/Write/MCP phase must complete before the recovery retry. The first +`fail_once` error must complete before its retry, and that retry must complete +before Bash. The evaluator additionally requires at least four distinct +assistant inference turns, plus one when the optional Read is present. This +accepts observed SDK batching without allowing an all-requests-first trace to +masquerade as multi-turn recovery. + +Normalized tool and permission events must be an exact chronological +projection of their evidence arrays. A successful Bash completion must precede +the single successful SDK result, which must precede the final successful +terminal event. + Filesystem acceptance is also exact: only the clean target may be modified and only the managed output may be created, in either evidence order. Trusted SHA-256 expectations prove the final bytes of both mutation targets, while the diff --git a/packages/harness/src/experimental/managed-agent-spike/probe-cli.test.ts b/packages/harness/src/experimental/managed-agent-spike/probe-cli.test.ts index b8fdc8335..342f3afd5 100644 --- a/packages/harness/src/experimental/managed-agent-spike/probe-cli.test.ts +++ b/packages/harness/src/experimental/managed-agent-spike/probe-cli.test.ts @@ -15,9 +15,71 @@ import type { ManagedAgentOperationId, ManagedAgentPermissionDecision, ManagedAgentPermissionReason, + ManagedAgentProbeEvent, ManagedAgentProbeResult, } from "./types.js"; +function withProjectedL1Events( + result: ManagedAgentProbeResult, +): ManagedAgentProbeResult { + const events: ManagedAgentProbeEvent[] = []; + const append = ( + event: Omit, + ): void => { + events.push({ + sequence: events.length + 1, + runId: result.runId, + ...event, + }); + }; + for (const evidence of result.toolEvidence) { + if (evidence.status === "requested") { + append({ + type: "tool_requested", + toolUseId: evidence.toolUseId, + toolName: evidence.toolName, + }); + for (const decision of result.permissionEvidence.filter( + ({ toolUseId }) => toolUseId === evidence.toolUseId, + )) { + append({ + type: "permission", + toolUseId: decision.toolUseId, + toolName: decision.toolName, + permissionDecision: decision.decision, + permissionReason: decision.reason, + permissionSource: decision.source, + operationId: decision.operationId, + }); + } + continue; + } + append({ + type: "tool_completed", + toolUseId: evidence.toolUseId, + toolName: evidence.toolName, + isError: evidence.status === "error", + }); + } + append({ type: "sdk_result", subtype: "success", isError: false }); + append({ type: "terminal", terminal: "success" }); + return { ...result, events }; +} + +function withResequencedEvents( + result: ManagedAgentProbeResult, + events: readonly ManagedAgentProbeEvent[], +): ManagedAgentProbeResult { + return { + ...result, + events: events.map((event, index) => ({ + ...event, + sequence: index + 1, + runId: result.runId, + })), + }; +} + function passingL1Result(): ManagedAgentProbeResult { const echoTool = qualifiedManagedAgentMcpToolName("echo_nonce"); const failOnceTool = qualifiedManagedAgentMcpToolName("fail_once"); @@ -43,7 +105,7 @@ function passingL1Result(): ManagedAgentProbeResult { const ids = steps.map( (_, index) => `tool_${(index + 1).toString(16).padStart(64, "0")}`, ); - return { + return withProjectedL1Events({ contractVersion: 1, runId: "run-1", scenario: "L1", @@ -119,7 +181,7 @@ function passingL1Result(): ManagedAgentProbeResult { { role: "managed_output", matched: true }, ], nonceVerified: true, - } as ManagedAgentProbeResult; + } as ManagedAgentProbeResult); } function passingL2Result(): ManagedAgentProbeResult { @@ -131,6 +193,7 @@ function passingL2Result(): ManagedAgentProbeResult { inferenceTurns: 1, sdkNumTurns: 1, terminal: "cancelled", + events: [], toolEvidence: [{ toolUseId, toolName: "Bash", status: "requested" }], permissionEvidence: [ { @@ -204,7 +267,11 @@ function insertL1ToolStep( source: "pre_tool_use", operationId: step.operationId, }); - return { ...result, toolEvidence, permissionEvidence }; + return withProjectedL1Events({ + ...result, + toolEvidence, + permissionEvidence, + }); } function optionalReadStep( @@ -237,6 +304,104 @@ function expectProbeCheckFailure( expect(report.checks).toContainEqual({ id: checkId, passed: false }); } +function maximallyBatchedL1Result( + optionalRole?: "clean_target" | "dirty_sentinel" | "untracked_sentinel", +): ManagedAgentProbeResult { + const withOptional = optionalRole + ? insertL1ToolStep( + passingL1Result(), + 5, + optionalReadStep(`read:${optionalRole}`), + optionalRole === "clean_target" + ? "e" + : optionalRole === "dirty_sentinel" + ? "f" + : "a", + ) + : passingL1Result(); + const requested = withOptional.toolEvidence.filter( + ({ status }) => status === "requested", + ); + const completionFor = ( + request: (typeof requested)[number], + ): (typeof withOptional.toolEvidence)[number] => + withOptional.toolEvidence.find( + (evidence) => + evidence.toolUseId === request.toolUseId && + evidence.status !== "requested", + )!; + const optionalOffset = optionalRole ? 1 : 0; + const phaseA = requested.slice(0, 5); + const optional = optionalRole ? requested[5] : undefined; + const phaseB = requested.slice(5 + optionalOffset, 9 + optionalOffset); + const call10 = requested[9 + optionalOffset]!; + const call11 = requested[10 + optionalOffset]!; + const toolEvidence = [ + ...phaseA, + ...[phaseA[2]!, phaseA[4]!, phaseA[0]!, phaseA[3]!, phaseA[1]!].map( + completionFor, + ), + ...(optional ? [optional, completionFor(optional)] : []), + ...phaseB, + ...[phaseB[2]!, phaseB[0]!, phaseB[3]!, phaseB[1]!].map(completionFor), + call10, + completionFor(call10), + call11, + completionFor(call11), + ]; + return withProjectedL1Events({ + ...withOptional, + inferenceTurns: 4 + optionalOffset, + sdkNumTurns: 4 + optionalOffset, + toolEvidence, + }); +} + +function moveCompletionAfterRequest( + result: ManagedAgentProbeResult, + completedRequestIndex: number, + boundaryRequestIndex: number, +): ManagedAgentProbeResult { + const requested = result.toolEvidence.filter( + ({ status }) => status === "requested", + ); + const completedId = requested[completedRequestIndex]!.toolUseId; + const boundaryId = requested[boundaryRequestIndex]!.toolUseId; + const completion = result.toolEvidence.find( + (evidence) => + evidence.toolUseId === completedId && evidence.status !== "requested", + )!; + const toolEvidence = result.toolEvidence.filter( + (evidence) => evidence !== completion, + ); + const boundaryIndex = toolEvidence.findIndex( + (evidence) => + evidence.toolUseId === boundaryId && evidence.status === "requested", + ); + toolEvidence.splice(boundaryIndex + 1, 0, completion); + return withProjectedL1Events({ ...result, toolEvidence }); +} + +function moveCompletionBeforeOwnRequest( + result: ManagedAgentProbeResult, + requestIndex: number, +): ManagedAgentProbeResult { + const request = result.toolEvidence.filter( + ({ status }) => status === "requested", + )[requestIndex]!; + const completion = result.toolEvidence.find( + (evidence) => + evidence.toolUseId === request.toolUseId && + evidence.status !== "requested", + )!; + const toolEvidence = result.toolEvidence.filter( + (evidence) => evidence !== completion, + ); + const ownRequestIndex = toolEvidence.indexOf(request); + toolEvidence.splice(ownRequestIndex, 0, completion); + return withProjectedL1Events({ ...result, toolEvidence }); +} + describe("managed-agent probe CLI", () => { it("is opt-in and never accepts credentials through arguments", () => { expect(() => @@ -455,6 +620,174 @@ describe("managed-agent probe CLI", () => { ).not.toHaveProperty("optionalReadRole"); }); + it.each([ + ["none", undefined], + ["clean_target", "clean_target"], + ["dirty_sentinel", "dirty_sentinel"], + ["untracked_sentinel", "untracked_sentinel"], + ] as const)( + "accepts maximally batched phase completions with %s optional Read", + (_name, optionalRole) => { + expect( + evaluateManagedAgentProbe(maximallyBatchedL1Result(optionalRole)), + ).toMatchObject({ + outcome: "pass", + checks: expect.arrayContaining([ + { id: "exact_l1_tool_trace", passed: true }, + ]), + }); + }, + ); + + it("rejects the all-requests-first false-pass counterexample", () => { + const passing = passingL1Result(); + const allRequestsFirst = withProjectedL1Events({ + ...passing, + inferenceTurns: 1, + sdkNumTurns: 1, + toolEvidence: [ + ...passing.toolEvidence.filter(({ status }) => status === "requested"), + ...passing.toolEvidence.filter(({ status }) => status !== "requested"), + ], + }); + + expectL1TraceFailure(allRequestsFirst); + expectProbeCheckFailure(allRequestsFirst, "minimum_l1_inference_turns"); + }); + + it.each([0, 1, 2, 3, 4])( + "rejects phase A completion %i delayed until after call 6 starts", + (phaseAIndex) => { + expectL1TraceFailure( + moveCompletionAfterRequest(maximallyBatchedL1Result(), phaseAIndex, 5), + ); + }, + ); + + it.each([0, 1, 2, 3, 4])( + "rejects phase A completion %i delayed until after the optional Read starts", + (phaseAIndex) => { + expectL1TraceFailure( + moveCompletionAfterRequest( + maximallyBatchedL1Result("clean_target"), + phaseAIndex, + 5, + ), + ); + }, + ); + + it.each(["clean_target", "dirty_sentinel", "untracked_sentinel"] as const)( + "rejects the %s optional completion delayed until after call 6 starts", + (optionalRole) => { + expectL1TraceFailure( + moveCompletionAfterRequest( + maximallyBatchedL1Result(optionalRole), + 5, + 6, + ), + ); + }, + ); + + it.each([5, 6, 7, 8])( + "rejects phase B request-index %i completion delayed until after call 10 starts", + (phaseBRequestIndex) => { + expectL1TraceFailure( + moveCompletionAfterRequest( + maximallyBatchedL1Result(), + phaseBRequestIndex, + 9, + ), + ); + }, + ); + + it("rejects call 10 completion delayed until after call 11 starts", () => { + expectL1TraceFailure( + moveCompletionAfterRequest(maximallyBatchedL1Result(), 9, 10), + ); + }); + + it.each([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10])( + "rejects completion-before-own-request at request index %i", + (requestIndex) => { + expectL1TraceFailure( + moveCompletionBeforeOwnRequest( + maximallyBatchedL1Result(), + requestIndex, + ), + ); + }, + ); + + it.each([ + ["without optional Read", undefined, 3], + ["with optional Read", "clean_target", 4], + ] as const)( + "rejects too few inference turns %s", + (_name, optionalRole, inferenceTurns) => { + expectProbeCheckFailure( + { + ...maximallyBatchedL1Result(optionalRole), + inferenceTurns, + }, + "minimum_l1_inference_turns", + ); + }, + ); + + it("rejects a normalized tool-event projection mismatch", () => { + const passing = maximallyBatchedL1Result(); + const firstToolEvent = passing.events.findIndex( + ({ type }) => type === "tool_requested", + ); + const events = [...passing.events]; + events[firstToolEvent] = { + ...events[firstToolEvent]!, + toolName: "Write", + }; + expectProbeCheckFailure( + { ...passing, events }, + "normalized_event_projection", + ); + }); + + it.each(["sdk_result", "terminal"] as const)( + "rejects %s before the Bash completion", + (eventType) => { + const passing = maximallyBatchedL1Result(); + const events = [...passing.events]; + const bashCompletionIndex = events.findIndex( + (event) => event.type === "tool_completed" && event.toolName === "Bash", + ); + const movedIndex = events.findIndex(({ type }) => type === eventType); + const [moved] = events.splice(movedIndex, 1); + events.splice(bashCompletionIndex, 0, moved!); + expectProbeCheckFailure( + withResequencedEvents(passing, events), + "bash_sdk_terminal_order", + ); + }, + ); + + it("rejects terminal before the successful SDK result", () => { + const passing = maximallyBatchedL1Result(); + const events = [...passing.events]; + const sdkResultIndex = events.findIndex( + ({ type }) => type === "sdk_result", + ); + const terminalIndex = events.findIndex(({ type }) => type === "terminal"); + [events[sdkResultIndex], events[terminalIndex]] = [ + events[terminalIndex]!, + events[sdkResultIndex]!, + ]; + expectProbeCheckFailure( + withResequencedEvents(passing, events), + "bash_sdk_terminal_order", + ); + }); + it("rejects a second optional verification Read", () => { const first = insertL1ToolStep( passingL1Result(), diff --git a/packages/harness/src/experimental/managed-agent-spike/probe-cli.ts b/packages/harness/src/experimental/managed-agent-spike/probe-cli.ts index 5d984502b..afbbc880e 100644 --- a/packages/harness/src/experimental/managed-agent-spike/probe-cli.ts +++ b/packages/harness/src/experimental/managed-agent-spike/probe-cli.ts @@ -196,6 +196,106 @@ function hasExactManagedAgentL1FinalBytes( ); } +function hasConsistentManagedAgentL1EventProjection( + result: ManagedAgentProbeResult, +): boolean { + if ( + result.events.some( + (event, index) => + event.sequence !== index + 1 || event.runId !== result.runId, + ) + ) { + return false; + } + const toolEvents = result.events.filter( + ({ type }) => type === "tool_requested" || type === "tool_completed", + ); + if (toolEvents.length !== result.toolEvidence.length) return false; + for (const [index, evidence] of result.toolEvidence.entries()) { + const event = toolEvents[index]; + if ( + !event || + event.toolUseId !== evidence.toolUseId || + event.toolName !== evidence.toolName + ) { + return false; + } + if (evidence.status === "requested") { + if (event.type !== "tool_requested" || event.isError !== undefined) { + return false; + } + } else if ( + event.type !== "tool_completed" || + event.isError !== (evidence.status === "error") + ) { + return false; + } + } + + const permissionEvents = result.events.filter( + ({ type }) => type === "permission", + ); + if (permissionEvents.length !== result.permissionEvidence.length) { + return false; + } + return result.permissionEvidence.every((evidence, index) => { + const event = permissionEvents[index]; + return Boolean( + event && + event.toolUseId === evidence.toolUseId && + event.toolName === evidence.toolName && + event.permissionDecision === evidence.decision && + event.permissionReason === evidence.reason && + event.permissionSource === evidence.source && + event.operationId === evidence.operationId, + ); + }); +} + +function hasManagedAgentL1BashSdkTerminalOrder( + result: ManagedAgentProbeResult, +): boolean { + const bashRequest = result.toolEvidence.find( + ({ toolName, status }) => toolName === "Bash" && status === "requested", + ); + if (!bashRequest?.toolUseId) return false; + const bashCompletionIndexes = result.events.flatMap((event, index) => + event.type === "tool_completed" && + event.toolUseId === bashRequest.toolUseId && + event.toolName === "Bash" && + event.isError === false + ? [index] + : [], + ); + const sdkResultIndexes = result.events.flatMap((event, index) => + event.type === "sdk_result" && + event.subtype === "success" && + event.isError === false + ? [index] + : [], + ); + const terminalIndexes = result.events.flatMap((event, index) => + event.type === "terminal" && event.terminal === "success" ? [index] : [], + ); + const bashCompletionIndex = bashCompletionIndexes[0]; + const sdkResultIndex = sdkResultIndexes[0]; + const terminalIndex = terminalIndexes[0]; + return Boolean( + bashCompletionIndexes.length === 1 && + result.events.filter(({ type }) => type === "sdk_result").length === 1 && + sdkResultIndexes.length === 1 && + result.events.filter(({ type }) => type === "terminal").length === 1 && + terminalIndexes.length === 1 && + result.terminationEvidence.sdkResult === "success" && + bashCompletionIndex !== undefined && + sdkResultIndex !== undefined && + terminalIndex !== undefined && + bashCompletionIndex < sdkResultIndex && + sdkResultIndex < terminalIndex && + terminalIndex === result.events.length - 1, + ); +} + function analyzeManagedAgentL1ToolTrace( result: ManagedAgentProbeResult, ): ManagedAgentL1TraceAnalysis { @@ -222,6 +322,18 @@ function analyzeManagedAgentL1ToolTrace( const decisionById = new Map( primaryDecisions.map((evidence) => [evidence.toolUseId, evidence]), ); + const requestPositionById = new Map(); + const completionPositionById = new Map(); + for (const [position, evidence] of result.toolEvidence.entries()) { + if (!evidence.toolUseId) continue; + if (evidence.status === "requested") { + if (!requestPositionById.has(evidence.toolUseId)) { + requestPositionById.set(evidence.toolUseId, position); + } + } else if (!completionPositionById.has(evidence.toolUseId)) { + completionPositionById.set(evidence.toolUseId, position); + } + } let invalid = requestedIds.length !== requested.length || new Set(requestedIds).size !== requestedIds.length || @@ -295,11 +407,13 @@ function analyzeManagedAgentL1ToolTrace( const candidateDecision = candidate?.toolUseId ? decisionById.get(candidate.toolUseId) : undefined; + let optionalRequestIndex: number | undefined; if ( candidate?.toolName === "Read" && candidateDecision && MANAGED_AGENT_L1_OPTIONAL_READ_OPERATIONS.has(candidateDecision.operationId) ) { + optionalRequestIndex = cursor; const completion = candidate.toolUseId ? completionById.get(candidate.toolUseId) : undefined; @@ -318,6 +432,69 @@ function analyzeManagedAgentL1ToolTrace( } invalid ||= cursor !== requested.length; + const requestPosition = (requestIndex: number): number | undefined => { + const toolUseId = requested[requestIndex]?.toolUseId; + return toolUseId ? requestPositionById.get(toolUseId) : undefined; + }; + const completionPosition = (requestIndex: number): number | undefined => { + const toolUseId = requested[requestIndex]?.toolUseId; + return toolUseId ? completionPositionById.get(toolUseId) : undefined; + }; + const allRequestsPrecedeOwnCompletion = requested.every((_, index) => { + const request = requestPosition(index); + const completion = completionPosition(index); + return ( + request !== undefined && completion !== undefined && request < completion + ); + }); + const completionsBeforeRequest = ( + completedRequestIndexes: readonly number[], + boundaryRequestIndex: number, + ): boolean => { + const boundary = requestPosition(boundaryRequestIndex); + const completions = completedRequestIndexes.map(completionPosition); + return Boolean( + boundary !== undefined && + completions.every( + (completion) => completion !== undefined && completion < boundary, + ), + ); + }; + const optionalOffset = optionalRequestIndex === undefined ? 0 : 1; + const call6RequestIndex = 5 + optionalOffset; + const call10RequestIndex = 9 + optionalOffset; + const call11RequestIndex = 10 + optionalOffset; + const phaseABoundaryRequestIndex = optionalRequestIndex ?? call6RequestIndex; + invalid ||= + !allRequestsPrecedeOwnCompletion || + !completionsBeforeRequest([0, 1, 2, 3, 4], phaseABoundaryRequestIndex) || + !completionsBeforeRequest( + [ + call6RequestIndex, + call6RequestIndex + 1, + call6RequestIndex + 2, + call6RequestIndex + 3, + ], + call10RequestIndex, + ); + if (optionalRequestIndex !== undefined) { + const optionalRequest = requestPosition(optionalRequestIndex); + const optionalCompletion = completionPosition(optionalRequestIndex); + const call6Request = requestPosition(call6RequestIndex); + invalid ||= + optionalRequest === undefined || + optionalCompletion === undefined || + call6Request === undefined || + optionalRequest >= optionalCompletion || + optionalCompletion >= call6Request; + } + const call10Completion = completionPosition(call10RequestIndex); + const call11Request = requestPosition(call11RequestIndex); + invalid ||= + call10Completion === undefined || + call11Request === undefined || + call10Completion >= call11Request; + return { passed: !invalid, optionalReadCount, @@ -550,6 +727,20 @@ export function evaluateManagedAgentProbe( id: "exact_l1_tool_trace", passed: l1Trace?.passed === true, }, + { + id: "minimum_l1_inference_turns", + passed: + Number.isInteger(result.inferenceTurns) && + result.inferenceTurns >= 4 + (l1Trace?.optionalReadCount ?? 0), + }, + { + id: "normalized_event_projection", + passed: hasConsistentManagedAgentL1EventProjection(result), + }, + { + id: "bash_sdk_terminal_order", + passed: hasManagedAgentL1BashSdkTerminalOrder(result), + }, { id: "exact_workspace_delta", passed: hasExactManagedAgentL1WorkspaceDelta(result), From e38ae7a1278814c62ae9060c2f4c4f07cdc06e98 Mon Sep 17 00:00:00 2001 From: Yash Date: Sun, 16 Aug 2026 22:13:04 -0700 Subject: [PATCH 13/24] fix(harness): order permissions before tool completion Correlate each normalized primary permission event with its matching tool completion so separately valid substreams cannot conceal post-execution authorization evidence. Preserve both observed SDK permission/request orderings. Refs: SAP-2632 --- .../managed-agent-spike/README.md | 8 +- .../managed-agent-spike/probe-cli.test.ts | 99 +++++++++++++++++++ .../managed-agent-spike/probe-cli.ts | 18 +++- 3 files changed, 118 insertions(+), 7 deletions(-) diff --git a/packages/harness/src/experimental/managed-agent-spike/README.md b/packages/harness/src/experimental/managed-agent-spike/README.md index 581bd2969..9d9289da4 100644 --- a/packages/harness/src/experimental/managed-agent-spike/README.md +++ b/packages/harness/src/experimental/managed-agent-spike/README.md @@ -98,9 +98,11 @@ accepts observed SDK batching without allowing an all-requests-first trace to masquerade as multi-turn recovery. Normalized tool and permission events must be an exact chronological -projection of their evidence arrays. A successful Bash completion must precede -the single successful SDK result, which must precede the final successful -terminal event. +projection of their evidence arrays. Each primary permission event must precede +its matching tool completion. It may appear before or after the matching request +event because the SDK hook and yielded message have independent observation +order. A successful Bash completion must precede the single successful SDK +result, which must precede the final successful terminal event. Filesystem acceptance is also exact: only the clean target may be modified and only the managed output may be created, in either evidence order. Trusted diff --git a/packages/harness/src/experimental/managed-agent-spike/probe-cli.test.ts b/packages/harness/src/experimental/managed-agent-spike/probe-cli.test.ts index 342f3afd5..019d439cf 100644 --- a/packages/harness/src/experimental/managed-agent-spike/probe-cli.test.ts +++ b/packages/harness/src/experimental/managed-agent-spike/probe-cli.test.ts @@ -402,6 +402,58 @@ function moveCompletionBeforeOwnRequest( return withProjectedL1Events({ ...result, toolEvidence }); } +function eventSubstream( + result: ManagedAgentProbeResult, + types: readonly ManagedAgentProbeEvent["type"][], +): readonly Omit[] { + return result.events + .filter(({ type }) => types.includes(type)) + .map(({ sequence: _sequence, ...event }) => event); +} + +function movePermissionAfterOwnCompletion( + result: ManagedAgentProbeResult, + requestIndex: number, +): ManagedAgentProbeResult { + const request = result.toolEvidence.filter( + ({ status }) => status === "requested", + )[requestIndex]!; + const events = [...result.events]; + const permissionIndex = events.findIndex( + (event) => + event.type === "permission" && event.toolUseId === request.toolUseId, + ); + const [permission] = events.splice(permissionIndex, 1); + const completionIndex = events.findIndex( + (event) => + event.type === "tool_completed" && event.toolUseId === request.toolUseId, + ); + events.splice(completionIndex + 1, 0, permission!); + return withResequencedEvents(result, events); +} + +function withPermissionsBeforeOwnRequests( + result: ManagedAgentProbeResult, +): ManagedAgentProbeResult { + const events = [...result.events]; + for (const request of result.toolEvidence.filter( + ({ status }) => status === "requested", + )) { + const permissionIndex = events.findIndex( + (event) => + event.type === "permission" && event.toolUseId === request.toolUseId, + ); + const [permission] = events.splice(permissionIndex, 1); + const requestEventIndex = events.findIndex( + (event) => + event.type === "tool_requested" && + event.toolUseId === request.toolUseId, + ); + events.splice(requestEventIndex, 0, permission!); + } + return withResequencedEvents(result, events); +} + describe("managed-agent probe CLI", () => { it("is opt-in and never accepts credentials through arguments", () => { expect(() => @@ -753,6 +805,53 @@ describe("managed-agent probe CLI", () => { ); }); + it.each(Array.from({ length: 11 }, (_, index) => index))( + "rejects canonical permission %i moved after its own completion while preserving both substreams", + (requestIndex) => { + const passing = passingL1Result(); + const invalid = movePermissionAfterOwnCompletion(passing, requestIndex); + + expect( + eventSubstream(invalid, ["tool_requested", "tool_completed"]), + ).toEqual(eventSubstream(passing, ["tool_requested", "tool_completed"])); + expect(eventSubstream(invalid, ["permission"])).toEqual( + eventSubstream(passing, ["permission"]), + ); + expectProbeCheckFailure(invalid, "normalized_event_projection"); + }, + ); + + it("rejects an optional Read permission moved after its own completion while preserving both substreams", () => { + const passing = insertL1ToolStep( + passingL1Result(), + 5, + optionalReadStep("read:clean_target"), + "e", + ); + const invalid = movePermissionAfterOwnCompletion(passing, 5); + + expect( + eventSubstream(invalid, ["tool_requested", "tool_completed"]), + ).toEqual(eventSubstream(passing, ["tool_requested", "tool_completed"])); + expect(eventSubstream(invalid, ["permission"])).toEqual( + eventSubstream(passing, ["permission"]), + ); + expectProbeCheckFailure(invalid, "normalized_event_projection"); + }); + + it("accepts permissions before their requests when each permission still precedes its completion", () => { + const requestBeforePermission = passingL1Result(); + const permissionBeforeRequest = + withPermissionsBeforeOwnRequests(passingL1Result()); + + expect(evaluateManagedAgentProbe(requestBeforePermission).outcome).toBe( + "pass", + ); + expect(evaluateManagedAgentProbe(permissionBeforeRequest).outcome).toBe( + "pass", + ); + }); + it.each(["sdk_result", "terminal"] as const)( "rejects %s before the Bash completion", (eventType) => { diff --git a/packages/harness/src/experimental/managed-agent-spike/probe-cli.ts b/packages/harness/src/experimental/managed-agent-spike/probe-cli.ts index afbbc880e..909c50c57 100644 --- a/packages/harness/src/experimental/managed-agent-spike/probe-cli.ts +++ b/packages/harness/src/experimental/managed-agent-spike/probe-cli.ts @@ -232,14 +232,22 @@ function hasConsistentManagedAgentL1EventProjection( } } - const permissionEvents = result.events.filter( - ({ type }) => type === "permission", + const permissionEvents = result.events.flatMap((event, index) => + event.type === "permission" ? [{ event, index }] : [], ); if (permissionEvents.length !== result.permissionEvidence.length) { return false; } + const completionIndexByToolUseId = new Map(); + for (const [index, event] of result.events.entries()) { + if (event.type === "tool_completed" && event.toolUseId) { + completionIndexByToolUseId.set(event.toolUseId, index); + } + } return result.permissionEvidence.every((evidence, index) => { - const event = permissionEvents[index]; + const permission = permissionEvents[index]; + const event = permission?.event; + const completionIndex = completionIndexByToolUseId.get(evidence.toolUseId); return Boolean( event && event.toolUseId === evidence.toolUseId && @@ -247,7 +255,9 @@ function hasConsistentManagedAgentL1EventProjection( event.permissionDecision === evidence.decision && event.permissionReason === evidence.reason && event.permissionSource === evidence.source && - event.operationId === evidence.operationId, + event.operationId === evidence.operationId && + completionIndex !== undefined && + permission.index < completionIndex, ); }); } From 04ef14130eaca095fb64ea1f3cb48dfea827a291 Mon Sep 17 00:00:00 2001 From: Yash Date: Sun, 16 Aug 2026 23:20:33 -0700 Subject: [PATCH 14/24] fix(harness): contain detached L2 tool processes Let the Agent SDK complete its bounded graceful shutdown before host fallback cleanup. Add a private authenticated registration path for the exact E0.4 L2 fixture so detached tool groups remain containable across early query failures and late registration. Refs: SAP-2632 --- .../managed-agent-spike/fixture.ts | 46 ++- .../managed-agent-spike/probe-cli.test.ts | 5 + .../managed-agent-spike/probe-cli.ts | 8 +- .../process-observer.test.ts | 188 ++++++++- .../managed-agent-spike/process-observer.ts | 374 ++++++++++++++++-- .../runtime-sdk-loopback.test.ts | 282 +++++++++++++ .../managed-agent-spike/runtime.test.ts | 21 +- .../managed-agent-spike/runtime.ts | 27 +- .../experimental/managed-agent-spike/types.ts | 10 +- 9 files changed, 903 insertions(+), 58 deletions(-) diff --git a/packages/harness/src/experimental/managed-agent-spike/fixture.ts b/packages/harness/src/experimental/managed-agent-spike/fixture.ts index 9b3670b82..93014af6c 100644 --- a/packages/harness/src/experimental/managed-agent-spike/fixture.ts +++ b/packages/harness/src/experimental/managed-agent-spike/fixture.ts @@ -16,6 +16,10 @@ import { tmpdir } from "node:os"; import { basename, join, relative, resolve } from "node:path"; import { MANAGED_AGENT_L1_CERTIFICATION_CONTRACT } from "./contract.js"; +import { + MANAGED_AGENT_TOOL_CONTROL_CAPABILITY_ENV, + MANAGED_AGENT_TOOL_CONTROL_SOCKET_ENV, +} from "./process-observer.js"; import type { ManagedAgentL1ExpectedFileHash, ManagedAgentL1FinalByteObservation, @@ -77,9 +81,18 @@ function shellQuote(value: string): string { const LONG_RUNNING_SCRIPT = ` import { spawn } from "node:child_process"; import { writeFileSync } from "node:fs"; +import { createConnection } from "node:net"; import { resolve } from "node:path"; const pidFile = resolve(process.argv[2]); +const requireControlRegistration = process.argv[3] === "--register-control"; +const controlSocket = process.env[${JSON.stringify(MANAGED_AGENT_TOOL_CONTROL_SOCKET_ENV)}]; +const controlCapability = process.env[${JSON.stringify(MANAGED_AGENT_TOOL_CONTROL_CAPABILITY_ENV)}]; +delete process.env[${JSON.stringify(MANAGED_AGENT_TOOL_CONTROL_SOCKET_ENV)}]; +delete process.env[${JSON.stringify(MANAGED_AGENT_TOOL_CONTROL_CAPABILITY_ENV)}]; +if (requireControlRegistration && (!controlSocket || !controlCapability)) { + throw new Error("managed-agent tool control capability missing"); +} process.on("SIGTERM", () => {}); const childProgram = [ 'process.on("SIGTERM", () => {});', @@ -90,9 +103,39 @@ const child = spawn(process.execPath, ["-e", childProgram], { stdio: ["ignore", "ignore", "ignore", "ipc"], windowsHide: true, }); -child.once("message", () => { +let childReady = false; +let controlReady = !requireControlRegistration; +const publishReadiness = () => { + if (!childReady || !controlReady) return; writeFileSync(pidFile, JSON.stringify({ parentPid: process.pid, childPid: child.pid })); +}; +child.once("message", () => { + childReady = true; + publishReadiness(); }); +const connectControl = () => { + if (!controlSocket || !controlCapability) return; + const socket = createConnection(controlSocket); + socket.unref(); + socket.setEncoding("utf8"); + let response = ""; + socket.once("connect", () => { + socket.write(JSON.stringify({ capability: controlCapability, pid: process.pid }) + "\\n"); + }); + socket.on("data", (chunk) => { + response += chunk; + if (!response.includes("\\n")) return; + if (!response.includes('"registered":true')) { + throw new Error("managed-agent tool registration rejected"); + } + controlReady = true; + publishReadiness(); + }); + socket.once("error", () => { + if (!controlReady) setTimeout(connectControl, 10); + }); +}; +if (requireControlRegistration) connectControl(); setInterval(() => {}, 1000); `.trimStart(); @@ -271,6 +314,7 @@ export async function createManagedAgentFixture( shellQuote(process.execPath), shellQuote(FIXTURE_PATHS.processScript), shellQuote(FIXTURE_PATHS.processPidFile), + shellQuote("--register-control"), ].join(" "); const pathRoleBindings = [ { path: FIXTURE_PATHS.cleanTarget, role: "clean_target" }, diff --git a/packages/harness/src/experimental/managed-agent-spike/probe-cli.test.ts b/packages/harness/src/experimental/managed-agent-spike/probe-cli.test.ts index 019d439cf..d912b4a8b 100644 --- a/packages/harness/src/experimental/managed-agent-spike/probe-cli.test.ts +++ b/packages/harness/src/experimental/managed-agent-spike/probe-cli.test.ts @@ -1071,6 +1071,11 @@ describe("managed-agent probe CLI", () => { { id: "l2_containment_prepared", passed: true }, ]), }); + expect( + evaluateManagedAgentProbe(passing).checks.find( + ({ id }) => id === "no_fixture_process_alive", + ), + ).toEqual({ id: "no_fixture_process_alive", passed: false }); const writeId = `tool_${"d".repeat(64)}`; const invalid: ManagedAgentProbeResult = { diff --git a/packages/harness/src/experimental/managed-agent-spike/probe-cli.ts b/packages/harness/src/experimental/managed-agent-spike/probe-cli.ts index 909c50c57..2cd6e07b9 100644 --- a/packages/harness/src/experimental/managed-agent-spike/probe-cli.ts +++ b/packages/harness/src/experimental/managed-agent-spike/probe-cli.ts @@ -834,9 +834,11 @@ export function evaluateManagedAgentProbe( }, { id: "no_fixture_process_alive", - passed: fixturePids.every( - (pid) => !result.teardown.alivePidsAtDeadline.includes(pid), - ), + passed: + fixturePids.length === 2 && + fixturePids.every( + (pid) => !result.teardown.alivePidsAtDeadline.includes(pid), + ), }, ); } diff --git a/packages/harness/src/experimental/managed-agent-spike/process-observer.test.ts b/packages/harness/src/experimental/managed-agent-spike/process-observer.test.ts index 9d1c6e252..78fc8553b 100644 --- a/packages/harness/src/experimental/managed-agent-spike/process-observer.test.ts +++ b/packages/harness/src/experimental/managed-agent-spike/process-observer.test.ts @@ -4,7 +4,7 @@ import { type ChildProcess, type ChildProcessWithoutNullStreams, } from "node:child_process"; -import { writeFile } from "node:fs/promises"; +import { readFile, writeFile } from "node:fs/promises"; import { join } from "node:path"; import type { SpawnedProcess } from "@anthropic-ai/claude-agent-sdk"; @@ -72,6 +72,34 @@ child.once("message", () => { }); `; +const LATE_REGISTERED_TOOL_SCRIPT = String.raw` +import { spawn } from "node:child_process"; +import { writeFileSync } from "node:fs"; + +const [toolScript, pidFile, launchFile] = process.argv.slice(1); +const tool = spawn( + "/bin/bash", + [ + "--noprofile", + "--norc", + "-c", + 'sleep 0.25; exec "$1" "$2" "$3" --register-control', + "managed-agent-tool", + process.execPath, + toolScript, + pidFile, + ], + { + detached: true, + env: process.env, + stdio: "ignore", + windowsHide: true, + }, +); +writeFileSync(launchFile, JSON.stringify({ processGroupId: tool.pid })); +tool.unref(); +`; + function asChildProcess( spawned: SpawnedProcess, ): ChildProcessWithoutNullStreams { @@ -108,6 +136,33 @@ async function waitForTestProcessDeath( if (isAlive()) throw new Error(`${description} survived test cleanup`); } +async function waitForLaunchedGroupId( + path: string, + timeoutMs = 1_000, +): Promise { + const deadline = Date.now() + timeoutMs; + for (;;) { + try { + const payload = JSON.parse(await readFile(path, "utf8")) as { + processGroupId?: unknown; + }; + if ( + typeof payload.processGroupId === "number" && + Number.isSafeInteger(payload.processGroupId) && + payload.processGroupId > 1 + ) { + return payload.processGroupId; + } + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + } + if (Date.now() >= deadline) { + throw new Error("Timed out waiting for detached tool launch evidence"); + } + await new Promise((resolveDelay) => setTimeout(resolveDelay, 10)); + } +} + async function forceKillExactTestGroup( processGroupId: number, root: ChildProcess, @@ -134,6 +189,20 @@ async function forceKillExactTestGroup( } } +async function forceKillExactTestGroupId( + processGroupId: number, +): Promise { + try { + process.kill(-processGroupId, "SIGKILL"); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ESRCH") throw error; + } + await waitForTestProcessDeath( + () => processGroupExists(processGroupId), + `Owned process group ${processGroupId}`, + ); +} + async function forceKillExactTestProcess(child: ChildProcess): Promise { const pid = child.pid; if (typeof pid !== "number") return; @@ -490,6 +559,123 @@ describe("LocalManagedAgentProcessObserver", () => { 10_000, ); + it.skipIf(process.platform === "win32")( + "contains a detached tool group that authenticates after the SDK group has exited", + async () => { + const fixture = await createManagedAgentFixture( + () => "late-tool-registration", + ); + fixtures.push(fixture); + const observer = new LocalManagedAgentProcessObserver(); + const forwardedController = new AbortController(); + const unrelated = spawnChild( + process.execPath, + ["-e", "setInterval(() => {}, 1000)"], + { stdio: "ignore", windowsHide: true }, + ); + await once(unrelated, "spawn"); + const launchFile = join( + fixture.workspaceRoot, + FIXTURE_PATHS.processDirectory, + "late-tool-launch.json", + ); + let anchor: ChildProcessWithoutNullStreams | undefined; + let detachedToolGroupId: number | undefined; + try { + observer.armToolProcessContainment(); + anchor = asChildProcess( + observer.spawn({ + command: process.execPath, + args: [ + "--input-type=module", + "--eval", + LATE_REGISTERED_TOOL_SCRIPT, + join(fixture.workspaceRoot, FIXTURE_PATHS.processScript), + join(fixture.workspaceRoot, FIXTURE_PATHS.processPidFile), + launchFile, + ], + cwd: fixture.workspaceRoot, + env: { ...process.env }, + signal: forwardedController.signal, + }), + ); + detachedToolGroupId = await waitForLaunchedGroupId(launchFile); + if (anchor.exitCode === null && anchor.signalCode === null) { + await once(anchor, "exit"); + } + expect(processGroupExists(detachedToolGroupId)).toBe(true); + + const teardown = await observer.emergencyCleanup(1_000); + + expect(teardown).toMatchObject({ + quiescent: true, + deadlineMet: true, + processTableAvailable: true, + containmentSupported: true, + emergencyCleanupAttempted: true, + alivePidsAtDeadline: [], + }); + expect(teardown.observedPids).toContain(detachedToolGroupId); + expect(processGroupExists(detachedToolGroupId)).toBe(false); + expect(processExists(unrelated.pid!)).toBe(true); + } finally { + forwardedController.abort(); + if ( + typeof detachedToolGroupId === "number" && + processGroupExists(detachedToolGroupId) + ) { + await forceKillExactTestGroupId(detachedToolGroupId); + } + if (anchor && typeof anchor.pid === "number") { + await forceKillExactTestGroup(anchor.pid, anchor); + } + observer.dispose(); + await forceKillExactTestProcess(unrelated); + } + }, + 10_000, + ); + + it.skipIf(process.platform === "win32")( + "never reports an armed but unregistered tool scope as quiescent", + async () => { + const observer = new LocalManagedAgentProcessObserver(); + const controller = new AbortController(); + observer.armToolProcessContainment(); + const anchor = asChildProcess( + observer.spawn({ + command: process.execPath, + args: ["-e", "process.exit(0)"], + cwd: process.cwd(), + env: { ...process.env }, + signal: controller.signal, + }), + ); + try { + if (anchor.exitCode === null && anchor.signalCode === null) { + await once(anchor, "exit"); + } + await expect(observer.waitForQuiescence(50)).resolves.toMatchObject({ + quiescent: false, + deadlineMet: false, + containmentSupported: false, + }); + await expect(observer.emergencyCleanup(50)).resolves.toMatchObject({ + quiescent: false, + deadlineMet: false, + containmentSupported: false, + }); + } finally { + controller.abort(); + if (typeof anchor.pid === "number") { + await forceKillExactTestGroup(anchor.pid, anchor); + } + observer.dispose(); + } + }, + 5_000, + ); + it("bounds a hanging process-table read and never turns unknown observation into quiescence", async () => { const signals: Array = []; const observer = new LocalManagedAgentProcessObserver({ diff --git a/packages/harness/src/experimental/managed-agent-spike/process-observer.ts b/packages/harness/src/experimental/managed-agent-spike/process-observer.ts index 07053fb68..989bc87e3 100644 --- a/packages/harness/src/experimental/managed-agent-spike/process-observer.ts +++ b/packages/harness/src/experimental/managed-agent-spike/process-observer.ts @@ -3,6 +3,15 @@ import { spawn as spawnChild, type ChildProcessWithoutNullStreams, } from "node:child_process"; +import { randomBytes, timingSafeEqual } from "node:crypto"; +import { chmodSync, mkdtempSync, rmSync } from "node:fs"; +import { + createServer, + type Server as NetServer, + type Socket as NetSocket, +} from "node:net"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; import { promisify } from "node:util"; import type { @@ -22,6 +31,11 @@ const QUIESCENCE_POLL_MS = 25; export const MANAGED_AGENT_PROCESS_HELPER_TIMEOUT_MS = 200; const MANAGED_AGENT_SUPERVISOR_PAYLOAD_ENV = "SAPIOM_MANAGED_AGENT_SUPERVISOR_PAYLOAD"; +export const MANAGED_AGENT_TOOL_CONTROL_SOCKET_ENV = + "SAPIOM_MANAGED_AGENT_TOOL_CONTROL_SOCKET"; +export const MANAGED_AGENT_TOOL_CONTROL_CAPABILITY_ENV = + "SAPIOM_MANAGED_AGENT_TOOL_CONTROL_CAPABILITY"; +const TOOL_REGISTRATION_MAX_BYTES = 1_024; /** * The POSIX supervisor is the observer-owned process-group leader. The real @@ -255,6 +269,22 @@ interface OwnedRoot { forceKillIssued: boolean; } +interface PendingToolRegistration { + readonly pid: number; + readonly socket: NetSocket; +} + +interface OwnedToolGroup { + readonly registeredPid: number; + readonly registeredIdentity: ManagedAgentKernelProcessRecord; + readonly processGroupId: number; + readonly groupLeaderIdentity?: ManagedAgentKernelProcessRecord; + containmentSupported: boolean; + ownershipProven: boolean; + stopIssued: boolean; + forceKillIssued: boolean; +} + interface ObservedIdentity { readonly rootPid: number; readonly record: ManagedAgentKernelProcessRecord; @@ -271,6 +301,15 @@ function sameProcess( return Boolean(left && right && left.startedAt === right.startedAt); } +function sameCapability(left: string, right: string): boolean { + const leftBytes = Buffer.from(left, "utf8"); + const rightBytes = Buffer.from(right, "utf8"); + return ( + leftBytes.byteLength === rightBytes.byteLength && + timingSafeEqual(leftBytes, rightBytes) + ); +} + async function windowsProcessTable(): Promise { const { stdout } = await execFileAsync( "powershell.exe", @@ -416,21 +455,20 @@ function descendantsOf( } /** - * E0.4 deliberately certifies one narrow containment model: the exact local - * L2 fixture running inside an observer-owned detached POSIX process group. - * Before abort, bounded host enumeration must observe the persistent trusted - * supervisor ChildProcess as its PGID leader and observe the fixture PIDs in - * that group. The supervisor remains the group anchor if the inner SDK root - * exits while a descendant survives. Independently, the detached spawn plus - * the active supervisor handle are safe signal authority, so even a failed - * evidence preflight can synchronously stop and kill the owned group without - * leaking it. Such a run still fails certification. + * E0.4 deliberately certifies one narrow containment model. The SDK command + * runs in an observer-owned POSIX process group. The exact host-created L2 + * fixture additionally authenticates over a private one-shot Unix socket + * outside the workspace and keeps that connection open. A random capability, + * a primary exact-Bash policy latch, and a fresh kernel table jointly grant + * fallback signal authority for the fixture's distinct PGID. The capability + * remains sufficient if SDK cleanup has already reparented the live fixture; + * this exception is safe only because L2 permits one immutable trusted command. * - * This is not a universal sandbox or process-tree killer. Windows, an inactive - * or invalid supervisor anchor, unavailable enumeration, and an observed - * setsid/group escape all fail certification closed. POSIX `lstart` is - * evidence only and never authorizes an individual or group signal. Workspace - * PID-file contents never enter this class and can never become authority. + * This is not universal built-in Bash containment or a process-tree killer. + * Windows, an unavailable process table, an unauthenticated tool process, or + * identity drift fail certification closed. POSIX `lstart` remains evidence, + * not authority. Workspace PID-file contents never enter this class and can + * never become signal authority. */ export class LocalManagedAgentProcessObserver implements ManagedAgentProcessObserver { readonly #platform: NodeJS.Platform; @@ -449,6 +487,17 @@ export class LocalManagedAgentProcessObserver implements ManagedAgentProcessObse readonly #observedPids = new Set(); readonly #sampler: NodeJS.Timeout; readonly #boundSignals = new WeakSet(); + readonly #toolControlCapability = randomBytes(32).toString("base64url"); + readonly #toolControlSockets = new Set(); + #toolControlDirectory: string | undefined; + #toolControlSocketPath: string | undefined; + #toolControlServer: NetServer | undefined; + #toolControlAvailable = false; + #toolControlFailed = false; + #toolProcessContainmentArmed = false; + #pendingToolRegistration: PendingToolRegistration | undefined; + #ownedToolGroup: OwnedToolGroup | undefined; + #fallbackCleanupRequested = false; #lastTable: ManagedAgentKernelProcessTable | undefined; #processTableAvailable = false; #sampleTask: Promise | undefined; @@ -464,6 +513,9 @@ export class LocalManagedAgentProcessObserver implements ManagedAgentProcessObse options.signalProcessGroup ?? defaultSignalProcessGroup; this.#now = options.now ?? Date.now; this.#delay = options.delay ?? defaultDelay; + if (this.#platform === "darwin" || this.#platform === "linux") { + this.#startToolControlServer(); + } this.#sampler = setInterval( () => void this.observeProcessTree(), SAMPLE_INTERVAL_MS, @@ -471,13 +523,104 @@ export class LocalManagedAgentProcessObserver implements ManagedAgentProcessObse this.#sampler.unref(); } + #startToolControlServer(): void { + try { + const directory = mkdtempSync( + join(tmpdir(), "sapiom-managed-agent-control-"), + ); + chmodSync(directory, 0o700); + const socketPath = join(directory, "tool.sock"); + const server = createServer((socket) => + this.#receiveToolRegistration(socket), + ); + this.#toolControlDirectory = directory; + this.#toolControlSocketPath = socketPath; + this.#toolControlServer = server; + server.once("listening", () => { + this.#toolControlAvailable = true; + }); + server.on("error", () => { + this.#toolControlAvailable = false; + this.#toolControlFailed = true; + }); + server.listen(socketPath); + server.unref(); + } catch { + this.#toolControlAvailable = false; + this.#toolControlFailed = true; + } + } + + #receiveToolRegistration(socket: NetSocket): void { + this.#toolControlSockets.add(socket); + socket.on("error", () => undefined); + socket.once("close", () => this.#toolControlSockets.delete(socket)); + let body = ""; + let handled = false; + const reject = (): void => { + handled = true; + socket.destroy(); + }; + socket.on("data", (chunk: Buffer) => { + if (handled) return; + body += chunk.toString("utf8"); + if (Buffer.byteLength(body, "utf8") > TOOL_REGISTRATION_MAX_BYTES) { + reject(); + return; + } + const newline = body.indexOf("\n"); + if (newline < 0) return; + handled = true; + let payload: { capability?: unknown; pid?: unknown }; + try { + payload = JSON.parse(body.slice(0, newline)) as typeof payload; + } catch { + socket.destroy(); + return; + } + if ( + !this.#toolProcessContainmentArmed || + this.#pendingToolRegistration || + this.#ownedToolGroup || + typeof payload.capability !== "string" || + !sameCapability(payload.capability, this.#toolControlCapability) || + typeof payload.pid !== "number" || + !Number.isSafeInteger(payload.pid) || + payload.pid <= 1 + ) { + socket.destroy(); + return; + } + this.#pendingToolRegistration = { pid: payload.pid, socket }; + void this.observeProcessTree(); + }); + } + public bindAbortSignal(signal: AbortSignal): void { if (this.#boundSignals.has(signal)) return; this.#boundSignals.add(signal); - signal.addEventListener("abort", () => this.#forceStopKillSynchronously(), { - once: true, - }); - if (signal.aborted) this.#forceStopKillSynchronously(); + signal.addEventListener( + "abort", + () => { + this.#fallbackCleanupRequested = true; + this.#forceStopKillSynchronously(); + }, + { once: true }, + ); + if (signal.aborted) { + this.#fallbackCleanupRequested = true; + this.#forceStopKillSynchronously(); + } + } + + public armToolProcessContainment(): void { + if (this.#toolProcessContainmentArmed) return; + this.#toolProcessContainmentArmed = true; + if (this.#toolControlFailed) { + for (const root of this.#roots.values()) { + root.containmentSupported = false; + } + } } public spawn(options: SpawnOptions): SpawnedProcess { @@ -503,6 +646,14 @@ export class LocalManagedAgentProcessObserver implements ManagedAgentProcessObse }), "utf8", ).toString("base64url"), + ...(this.#toolControlSocketPath + ? { + [MANAGED_AGENT_TOOL_CONTROL_SOCKET_ENV]: + this.#toolControlSocketPath, + [MANAGED_AGENT_TOOL_CONTROL_CAPABILITY_ENV]: + this.#toolControlCapability, + } + : {}), }, detached: true, stdio: ["pipe", "pipe", "pipe", "ipc"], @@ -554,8 +705,8 @@ export class LocalManagedAgentProcessObserver implements ManagedAgentProcessObse }); this.#observedPids.add(pid); // The SDK's forwarded SpawnOptions.signal arrives only after its own - // graceful close. Keep it as an idempotent fallback; runtime binds the - // raw Options.abortController signal before query construction. + // graceful close. Keep it as an idempotent fallback; runtime deliberately + // does not bind the raw Options.abortController to host process signals. this.bindAbortSignal(options.signal); void this.observeProcessTree(); } @@ -588,6 +739,68 @@ export class LocalManagedAgentProcessObserver implements ManagedAgentProcessObse } } + #observeToolProcessContainment(table: ManagedAgentKernelProcessTable): void { + const pending = this.#pendingToolRegistration; + if (pending && !pending.socket.destroyed && !this.#ownedToolGroup) { + const registeredIdentity = table.get(pending.pid); + const processGroupId = registeredIdentity?.processGroupId; + const hostProcessGroupId = table.get(process.pid)?.processGroupId; + if ( + registeredIdentity && + typeof processGroupId === "number" && + typeof hostProcessGroupId === "number" && + processGroupId > 1 && + processGroupId !== hostProcessGroupId && + !this.#roots.has(processGroupId) + ) { + const groupLeaderIdentity = table.get(processGroupId); + if ( + !groupLeaderIdentity || + groupLeaderIdentity.processGroupId === processGroupId + ) { + this.#ownedToolGroup = { + registeredPid: pending.pid, + registeredIdentity, + processGroupId, + ...(groupLeaderIdentity ? { groupLeaderIdentity } : {}), + containmentSupported: true, + ownershipProven: true, + stopIssued: false, + forceKillIssued: false, + }; + this.#pendingToolRegistration = undefined; + pending.socket.write('{"registered":true}\n'); + } + } + } + + const owned = this.#ownedToolGroup; + if (!owned) return; + const currentRegistered = table.get(owned.registeredPid); + if ( + currentRegistered && + (!sameProcess(owned.registeredIdentity, currentRegistered) || + currentRegistered.processGroupId !== owned.processGroupId) + ) { + owned.containmentSupported = false; + } + if (owned.groupLeaderIdentity) { + const currentLeader = table.get(owned.processGroupId); + if ( + currentLeader && + !sameProcess(owned.groupLeaderIdentity, currentLeader) + ) { + owned.containmentSupported = false; + } + } + for (const [pid, record] of table) { + if (record.processGroupId === owned.processGroupId) { + this.#observedPids.add(pid); + } + } + if (this.#fallbackCleanupRequested) this.#forceStopKillSynchronously(); + } + public async observeProcessTree( timeoutMs = MANAGED_AGENT_PROCESS_HELPER_TIMEOUT_MS, ): Promise { @@ -672,6 +885,7 @@ export class LocalManagedAgentProcessObserver implements ManagedAgentProcessObse this.#observedPids.add(pid); } } + this.#observeToolProcessContainment(table); return true; })().finally(() => { this.#sampleTask = undefined; @@ -705,9 +919,12 @@ export class LocalManagedAgentProcessObserver implements ManagedAgentProcessObse supported: false, reason, processTableAvailable: this.#processTableAvailable, - containmentSupported: [...this.#roots.values()].every( - ({ containmentSupported }) => containmentSupported, - ), + containmentSupported: + [...this.#roots.values()].every( + ({ containmentSupported }) => containmentSupported, + ) && + (!this.#toolProcessContainmentArmed || + Boolean(this.#ownedToolGroup?.containmentSupported)), ownershipProven: false, observedPids: observedPids(), }); @@ -735,6 +952,18 @@ export class LocalManagedAgentProcessObserver implements ManagedAgentProcessObse root.containmentSupported = false; return unsupported("root_not_group_leader"); } + if (this.#toolProcessContainmentArmed) { + if ( + !this.#toolControlAvailable || + !this.#ownedToolGroup || + !this.#ownedToolGroup.ownershipProven + ) { + return unsupported("tool_process_not_registered"); + } + if (!this.#ownedToolGroup.containmentSupported) { + return unsupported("tool_process_identity_invalid"); + } + } root.ownershipProven = true; return { @@ -749,16 +978,9 @@ export class LocalManagedAgentProcessObserver implements ManagedAgentProcessObse #forceStopKillSynchronously(): void { if (this.#platform !== "darwin" && this.#platform !== "linux") return; + this.#fallbackCleanupRequested = true; for (const root of this.#roots.values()) { - if (root.forceKillIssued || !childActive(root.child)) { - continue; - } - // The detached observer-owned supervisor plus its still-active trusted - // ChildProcess handle are sufficient signal authority even when ps - // evidence is unavailable. The supervisor remains active across a fast - // inner SDK-root exit while any same-group descendant survives. - // Certification readiness remains false in that case. Second-resolution - // lstart and workspace PIDs are never signal authority. + if (root.forceKillIssued || !childActive(root.child)) continue; if (!root.stopIssued) { const stopOutcome = this.#signalProcessGroup(root.pid, "SIGSTOP"); root.stopIssued = stopOutcome === "sent"; @@ -766,14 +988,43 @@ export class LocalManagedAgentProcessObserver implements ManagedAgentProcessObse root.containmentSupported = false; } } - const killOutcome = this.#signalProcessGroup(root.pid, "SIGKILL"); - root.forceKillIssued = killOutcome === "sent"; + } + const toolGroup = this.#ownedToolGroup; + if ( + toolGroup && + toolGroup.containmentSupported && + !toolGroup.forceKillIssued && + !toolGroup.stopIssued + ) { + const stopOutcome = this.#signalProcessGroup( + toolGroup.processGroupId, + "SIGSTOP", + ); + toolGroup.stopIssued = stopOutcome === "sent"; + if (stopOutcome === "failure") { + toolGroup.containmentSupported = false; + } + } + if ( + toolGroup && + toolGroup.containmentSupported && + !toolGroup.forceKillIssued + ) { + const killOutcome = this.#signalProcessGroup( + toolGroup.processGroupId, + "SIGKILL", + ); + toolGroup.forceKillIssued = killOutcome === "sent"; if (killOutcome === "failure") { - // Keep the active/stopped group anchored so emergencyCleanup can retry - // SIGKILL safely, but certification evidence remains failed. - root.containmentSupported = false; + toolGroup.containmentSupported = false; } } + for (const root of this.#roots.values()) { + if (root.forceKillIssued || !childActive(root.child)) continue; + const killOutcome = this.#signalProcessGroup(root.pid, "SIGKILL"); + root.forceKillIssued = killOutcome === "sent"; + if (killOutcome === "failure") root.containmentSupported = false; + } } async #currentObservation( @@ -806,16 +1057,45 @@ export class LocalManagedAgentProcessObserver implements ManagedAgentProcessObse } } } + const toolGroup = this.#ownedToolGroup; + if (this.#processTableAvailable) { + const table = this.#lastTable!; + if (toolGroup) { + for (const [pid, record] of table) { + if (record.processGroupId === toolGroup.processGroupId) + alive.add(pid); + } + const groupLiveness = this.#processGroupLiveness( + toolGroup.processGroupId, + ); + if (groupLiveness === "alive") alive.add(toolGroup.processGroupId); + if (groupLiveness === "unknown") { + toolGroup.containmentSupported = false; + } + } else if (this.#pendingToolRegistration) { + if (table.has(this.#pendingToolRegistration.pid)) { + alive.add(this.#pendingToolRegistration.pid); + } + } + } const processTableAvailable = roots.length === 0 || this.#processTableAvailable; - const containmentSupported = roots.every( - ({ containmentSupported: supported }) => supported, - ); + const containmentSupported = + roots.every(({ containmentSupported: supported }) => supported) && + (!this.#toolProcessContainmentArmed || + (this.#toolControlAvailable && + Boolean(toolGroup?.containmentSupported))); const ownershipProven = - roots.length > 0 && roots.every(({ ownershipProven }) => ownershipProven); + roots.length > 0 && + roots.every(({ ownershipProven }) => ownershipProven) && + (!this.#toolProcessContainmentArmed || + Boolean(toolGroup?.ownershipProven)); const forceKillIssued = - roots.length > 0 && roots.every(({ forceKillIssued }) => forceKillIssued); + roots.length > 0 && + roots.every(({ forceKillIssued }) => forceKillIssued) && + (!this.#toolProcessContainmentArmed || + Boolean(toolGroup?.forceKillIssued)); const elapsedMs = Math.max(0, this.#now() - startedAt); const quiescent = processTableAvailable && containmentSupported && alive.size === 0; @@ -877,6 +1157,16 @@ export class LocalManagedAgentProcessObserver implements ManagedAgentProcessObse public dispose(): void { clearInterval(this.#sampler); + for (const socket of this.#toolControlSockets) socket.destroy(); + this.#toolControlSockets.clear(); + this.#toolControlServer?.close(); + if (this.#toolControlDirectory) { + try { + rmSync(this.#toolControlDirectory, { recursive: true, force: true }); + } catch { + // Best-effort removal after the private listener and clients close. + } + } } } diff --git a/packages/harness/src/experimental/managed-agent-spike/runtime-sdk-loopback.test.ts b/packages/harness/src/experimental/managed-agent-spike/runtime-sdk-loopback.test.ts index fb825cba1..f9ddf1c8d 100644 --- a/packages/harness/src/experimental/managed-agent-spike/runtime-sdk-loopback.test.ts +++ b/packages/harness/src/experimental/managed-agent-spike/runtime-sdk-loopback.test.ts @@ -1,3 +1,5 @@ +import { spawn } from "node:child_process"; +import { once } from "node:events"; import { readFile } from "node:fs/promises"; import { createServer, type ServerResponse } from "node:http"; import { createRequire } from "node:module"; @@ -11,8 +13,10 @@ import { FIXTURE_PATHS, createManagedAgentFixture, fixturePathExists, + waitForManagedAgentFixturePids, } from "./fixture.js"; import { MANAGED_AGENT_CONTRACT } from "./contract.js"; +import { LocalManagedAgentProcessObserver } from "./process-observer.js"; import { qualifiedManagedAgentMcpToolName, runManagedAgentProbe, @@ -29,6 +33,36 @@ const DENIED_BASH_COMMAND = "touch denied-side-effect.txt"; const ECHO_NONCE_TOOL = qualifiedManagedAgentMcpToolName("echo_nonce"); const require = createRequire(import.meta.url); +function processExists(pid: number): boolean { + try { + process.kill(pid, 0); + return true; + } catch (error) { + return (error as NodeJS.ErrnoException).code === "EPERM"; + } +} + +async function waitForProcessDeath( + pid: number, + timeoutMs = 1_000, +): Promise { + const deadline = Date.now() + timeoutMs; + while (processExists(pid) && Date.now() < deadline) { + await new Promise((resolveDelay) => setTimeout(resolveDelay, 10)); + } + if (processExists(pid)) + throw new Error(`Test process ${pid} survived cleanup`); +} + +async function forceKillTestProcess(pid: number): Promise { + try { + process.kill(pid, "SIGKILL"); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ESRCH") throw error; + } + await waitForProcessDeath(pid); +} + interface LoopbackObservation { readonly headerNames: readonly string[]; readonly evalSourceMatches: boolean; @@ -419,6 +453,254 @@ it("enforces real-SDK built-in and in-process MCP calls with exact loopback corr } }, 45_000); +it.skipIf( + process.platform === "win32" || + process.versions.node !== MANAGED_AGENT_CONTRACT.certificationNodeVersion, +)( + "cancels the real SDK L2 Bash fixture without leaving its detached process group", + async () => { + const fixture = await createManagedAgentFixture( + () => "loopback-l2-cancellation", + ); + const observer = new LocalManagedAgentProcessObserver(); + const unrelated = spawn( + process.execPath, + ["-e", "setInterval(() => {}, 1000)"], + { stdio: "ignore", windowsHide: true }, + ); + await once(unrelated, "spawn"); + let fixturePids: readonly number[] = []; + let inferenceTurn = 0; + const server = createServer((request, response) => { + if (request.method === "HEAD" && request.url === "/api/hello") { + response.writeHead(200).end(); + return; + } + if ( + request.method !== "POST" || + request.url?.split("?")[0] !== "/v1/messages" + ) { + response.writeHead(404).end(); + return; + } + request.resume(); + request.once("end", () => { + inferenceTurn += 1; + writeToolUseResponse(response, inferenceTurn, { + id: "toolu_loopback_l2_bash", + name: "Bash", + input: { command: fixture.l2BashCommand }, + }); + }); + }); + + await new Promise((resolve, reject) => { + server.once("error", reject); + server.listen(0, "127.0.0.1", resolve); + }); + try { + const address = server.address() as AddressInfo; + const ids = [RUN_ID, EXECUTION_ID]; + const result = await runManagedAgentProbe( + { + scenario: "L2", + workspaceRoot: fixture.workspaceRoot, + configRoot: fixture.configRoot, + target: "sonnet-5", + gatewayOrigin: `http://127.0.0.1:${address.port}`, + gatewayCredential: "sk-ant-api03-local-loopback-only", + prompt: fixture.prompt("L2"), + maxTurns: 4, + maxBudgetUsd: 0.25, + allowedBashCommands: [fixture.l2BashCommand], + pathRoleBindings: [], + expectedL1FinalBytes: [], + preservePaths: [ + FIXTURE_PATHS.dirtySentinel, + FIXTURE_PATHS.untrackedSentinel, + ], + }, + { + hermeticGatewayOrigin: `http://127.0.0.1:${address.port}`, + processObserver: observer, + queryFactory: ({ prompt, options }) => + agentSdkQuery({ prompt, options }), + waitForCancellationSignal: async (signal) => { + fixturePids = await waitForManagedAgentFixturePids( + fixture, + 10_000, + signal, + ); + const readiness = await observer.prepareCancellation(); + expect(readiness).toMatchObject({ + supported: true, + reason: "ready", + containmentSupported: true, + ownershipProven: true, + }); + expect( + fixturePids.every((pid) => readiness.observedPids.includes(pid)), + ).toBe(true); + }, + uuid: () => { + const id = ids.shift(); + if (!id) throw new Error("unexpected UUID request"); + return id; + }, + }, + ); + + expect(inferenceTurn).toBe(1); + expect(result.terminal).toBe("cancelled"); + expect(result.cancellationRequested).toBe(true); + expect(result.queryClosed).toBe(true); + expect(result.teardown).toMatchObject({ + quiescent: true, + deadlineMet: true, + processTableAvailable: true, + containmentSupported: true, + ownershipProven: true, + forceKillIssued: true, + alivePidsAtDeadline: [], + }); + expect(fixturePids).toHaveLength(2); + expect(fixturePids.every((pid) => !processExists(pid))).toBe(true); + expect(processExists(unrelated.pid!)).toBe(true); + } finally { + await observer.emergencyCleanup(1_000); + observer.dispose(); + server.closeAllConnections(); + await new Promise((resolve) => server.close(() => resolve())); + for (const pid of fixturePids) { + if (!processExists(pid)) continue; + await forceKillTestProcess(pid); + } + if (typeof unrelated.pid === "number" && processExists(unrelated.pid)) { + unrelated.kill("SIGKILL"); + await waitForProcessDeath(unrelated.pid); + } + await fixture.cleanup(); + } + }, + 20_000, +); + +it.skipIf( + process.platform === "win32" || + process.versions.node !== MANAGED_AGENT_CONTRACT.certificationNodeVersion, +)( + "contains the real SDK L2 Bash group when readiness fails before cancellation", + async () => { + const fixture = await createManagedAgentFixture( + () => "loopback-l2-early-error", + ); + const observer = new LocalManagedAgentProcessObserver(); + let fixturePids: readonly number[] = []; + let inferenceTurn = 0; + const server = createServer((request, response) => { + if (request.method === "HEAD" && request.url === "/api/hello") { + response.writeHead(200).end(); + return; + } + if ( + request.method !== "POST" || + request.url?.split("?")[0] !== "/v1/messages" + ) { + response.writeHead(404).end(); + return; + } + request.resume(); + request.once("end", () => { + inferenceTurn += 1; + writeToolUseResponse(response, inferenceTurn, { + id: "toolu_loopback_l2_early_error", + name: "Bash", + input: { command: fixture.l2BashCommand }, + }); + }); + }); + + await new Promise((resolve, reject) => { + server.once("error", reject); + server.listen(0, "127.0.0.1", resolve); + }); + try { + const address = server.address() as AddressInfo; + const ids = [RUN_ID, EXECUTION_ID]; + const result = await runManagedAgentProbe( + { + scenario: "L2", + workspaceRoot: fixture.workspaceRoot, + configRoot: fixture.configRoot, + target: "sonnet-5", + gatewayOrigin: `http://127.0.0.1:${address.port}`, + gatewayCredential: "sk-ant-api03-local-loopback-only", + prompt: fixture.prompt("L2"), + maxTurns: 4, + maxBudgetUsd: 0.25, + allowedBashCommands: [fixture.l2BashCommand], + pathRoleBindings: [], + expectedL1FinalBytes: [], + preservePaths: [ + FIXTURE_PATHS.dirtySentinel, + FIXTURE_PATHS.untrackedSentinel, + ], + }, + { + hermeticGatewayOrigin: `http://127.0.0.1:${address.port}`, + processObserver: observer, + queryFactory: ({ prompt, options }) => + agentSdkQuery({ prompt, options }), + waitForCancellationSignal: async (signal) => { + fixturePids = await waitForManagedAgentFixturePids( + fixture, + 10_000, + signal, + ); + throw new Error("synthetic readiness failure"); + }, + uuid: () => { + const id = ids.shift(); + if (!id) throw new Error("unexpected UUID request"); + return id; + }, + }, + ); + + expect(inferenceTurn).toBe(1); + expect(result.terminal).toBe("query_error"); + expect(result.cancellationRequested).toBe(false); + expect(result.terminationEvidence).toEqual({ + beforePolicyOverride: "query_error", + queryExecution: "iteration_aborted", + sdkResult: "not_observed", + }); + expect(result.queryClosed).toBe(true); + expect(result.teardown).toMatchObject({ + quiescent: true, + deadlineMet: true, + processTableAvailable: true, + containmentSupported: true, + forceKillIssued: true, + alivePidsAtDeadline: [], + }); + expect(fixturePids).toHaveLength(2); + expect(fixturePids.every((pid) => !processExists(pid))).toBe(true); + } finally { + await observer.emergencyCleanup(1_000); + observer.dispose(); + server.closeAllConnections(); + await new Promise((resolve) => server.close(() => resolve())); + for (const pid of fixturePids) { + if (!processExists(pid)) continue; + await forceKillTestProcess(pid); + } + await fixture.cleanup(); + } + }, + 20_000, +); + it.skipIf( process.versions.node !== MANAGED_AGENT_CONTRACT.certificationNodeVersion, )( diff --git a/packages/harness/src/experimental/managed-agent-spike/runtime.test.ts b/packages/harness/src/experimental/managed-agent-spike/runtime.test.ts index 60833febf..fc2eb6abc 100644 --- a/packages/harness/src/experimental/managed-agent-spike/runtime.test.ts +++ b/packages/harness/src/experimental/managed-agent-spike/runtime.test.ts @@ -68,6 +68,7 @@ function fakeObserver( throw new Error("fake query must not spawn"); }), bindAbortSignal: vi.fn(), + armToolProcessContainment: vi.fn(), prepareCancellation: vi.fn(async () => ({ supported: true, reason: "ready" as const, @@ -458,11 +459,17 @@ describe("runManagedAgentProbe", () => { it("distinguishes query iteration failure from construction failure", async () => { const { config } = await probeConfig(); + const observer = fakeObserver(); + const shutdownOrder: string[] = []; + observer.emergencyCleanup.mockImplementation(async () => { + shutdownOrder.push("host_fallback"); + return { ...quiescentTeardown(), emergencyCleanupAttempted: true }; + }); const result = await runManagedAgentProbe(config, { hermeticGatewayOrigin: config.gatewayOrigin, - processObserver: fakeObserver(), + processObserver: observer, policySettingsGuard: async () => undefined, - queryFactory: () => ({ + queryFactory: ({ options }) => ({ async *[Symbol.asyncIterator]() { yield { type: "system", @@ -471,7 +478,10 @@ describe("runManagedAgentProbe", () => { }; throw new Error("synthetic private iteration failure"); }, - close: vi.fn(), + close: vi.fn(() => { + expect(options.abortController?.signal.aborted).toBe(true); + shutdownOrder.push("sdk_query_close"); + }), }), }); @@ -484,6 +494,8 @@ describe("runManagedAgentProbe", () => { expect(JSON.stringify(result)).not.toContain( "synthetic private iteration failure", ); + expect(observer.bindAbortSignal).not.toHaveBeenCalled(); + expect(shutdownOrder).toEqual(["sdk_query_close", "host_fallback"]); }); it("reports a completed iteration that emitted no SDK result", async () => { @@ -926,7 +938,7 @@ describe("runManagedAgentProbe", () => { expect(result.terminal).toBe("cancelled"); expect(result.terminationEvidence.queryExecution).toBe("iteration_aborted"); expect(close).toHaveBeenCalledOnce(); - expect(observer.bindAbortSignal).toHaveBeenCalledOnce(); + expect(observer.bindAbortSignal).not.toHaveBeenCalled(); }); it("keeps a CLI-shaped process alive until bounded close and cleanup complete", async () => { @@ -952,6 +964,7 @@ const teardown = { const observer = { spawn() { throw new Error("fake query must not spawn"); }, bindAbortSignal() {}, + armToolProcessContainment() {}, async prepareCancellation() { return { supported: true, diff --git a/packages/harness/src/experimental/managed-agent-spike/runtime.ts b/packages/harness/src/experimental/managed-agent-spike/runtime.ts index 9363b009c..5289e5d1b 100644 --- a/packages/harness/src/experimental/managed-agent-spike/runtime.ts +++ b/packages/harness/src/experimental/managed-agent-spike/runtime.ts @@ -411,6 +411,8 @@ export async function runManagedAgentProbe( evalSource, executionId, }); + const processObserver = + dependencies.processObserver ?? createLocalManagedAgentProcessObserver(); const policyBoundary = createManagedAgentPolicyBoundary({ canonicalWorkspaceRoot: validated.canonicalWorkspaceRoot, allowedBuiltinTools: @@ -422,14 +424,20 @@ export async function runManagedAgentProbe( config.scenario === "L1" ? mcpRuntime.qualifiedToolNames : [], pathRoleBindings: config.pathRoleBindings, requireRegisteredFilePaths: config.scenario === "L1", - onDecision: (evidence) => recorder.recordPermission(evidence), + onDecision: (evidence) => { + recorder.recordPermission(evidence); + if ( + config.scenario === "L2" && + evidence.source === "pre_tool_use" && + evidence.toolName === "Bash" && + evidence.decision === "allow" && + evidence.reason === "exact_bash_command" + ) { + processObserver.armToolProcessContainment(); + } + }, onGuardRejection: (diagnostic) => guardRejections.push(diagnostic), }); - const processObserver = - dependencies.processObserver ?? createLocalManagedAgentProcessObserver(); - // SpawnOptions.signal is forwarded only after the SDK's graceful close. - // Bind the raw per-run abort signal so L2 containment starts synchronously. - processObserver.bindAbortSignal(abortController.signal); const options: Options = { abortController, @@ -577,6 +585,13 @@ export async function runManagedAgentProbe( triggerController.abort(); if (cancellationTask) await cancellationTask; queryFailed ||= cancellationTriggerFailed; + // Give the SDK its documented graceful-shutdown path before host + // fallback containment. The observer binds only SpawnOptions.signal, + // which the SDK forwards after stdin EOF and its bounded grace period. + if (queryFailed && !abortController.signal.aborted) { + abortStartedAt = (dependencies.now ?? Date.now)(); + abortController.abort(); + } if (query) { const now = dependencies.now ?? Date.now; const closeBudgetMs = diff --git a/packages/harness/src/experimental/managed-agent-spike/types.ts b/packages/harness/src/experimental/managed-agent-spike/types.ts index 6028a8db2..4ace32129 100644 --- a/packages/harness/src/experimental/managed-agent-spike/types.ts +++ b/packages/harness/src/experimental/managed-agent-spike/types.ts @@ -297,8 +297,14 @@ export type ManagedAgentQueryFactory = (input: { export interface ManagedAgentProcessObserver { spawn(options: SpawnOptions): SpawnedProcess; - /** Bind the raw per-run Options.abortController signal before SDK startup. */ + /** Bind only the SDK-forwarded post-grace SpawnOptions signal. */ bindAbortSignal(signal: AbortSignal): void; + /** + * Arm the one-shot, host-authenticated process registration used only by + * the exact E0.4 L2 fixture. Unregistered built-in Bash processes are not + * granted signal authority by this experimental observer. + */ + armToolProcessContainment(): void; /** Prove the narrow POSIX ownership model before allowing L2 to cancel. */ prepareCancellation(): Promise; /** Sample only members owned by the host-observed process anchors. */ @@ -318,6 +324,8 @@ export type ManagedAgentCancellationReadinessReason = | "root_count_invalid" | "root_not_active" | "root_not_group_leader" + | "tool_process_not_registered" + | "tool_process_identity_invalid" | "containment_escaped"; export interface ManagedAgentCancellationReadiness { From bc7388473dfdc993f949c041cb250c2d142d5060 Mon Sep 17 00:00:00 2001 From: Yash Date: Mon, 17 Aug 2026 01:23:50 -0700 Subject: [PATCH 15/24] fix(harness): harden managed-agent process containment Require fresh kernel-backed ancestry and lifetime evidence before signaling the exact L2 fixture group. Preserve one absolute deadline across readiness, SDK shutdown, and host cleanup while failing closed on incomplete observation. Refs: SAP-2632 --- .../managed-agent-spike/README.md | 78 +- .../managed-agent-spike/fixture.test.ts | 14 + .../managed-agent-spike/fixture.ts | 59 +- .../managed-agent-spike/probe-cli.test.ts | 28 + .../managed-agent-spike/probe-cli.ts | 6 +- .../process-observer.test.ts | 925 +++++++++++++++++- .../managed-agent-spike/process-observer.ts | 466 ++++++--- .../runtime-sdk-loopback.test.ts | 31 +- .../managed-agent-spike/runtime.test.ts | 76 +- .../managed-agent-spike/runtime.ts | 58 +- .../experimental/managed-agent-spike/types.ts | 18 +- 11 files changed, 1529 insertions(+), 230 deletions(-) diff --git a/packages/harness/src/experimental/managed-agent-spike/README.md b/packages/harness/src/experimental/managed-agent-spike/README.md index 9d9289da4..e1e5c0d25 100644 --- a/packages/harness/src/experimental/managed-agent-spike/README.md +++ b/packages/harness/src/experimental/managed-agent-spike/README.md @@ -114,37 +114,61 @@ recorded as a content-free nonce-verification boolean. ## L2 cancellation containment boundary E0.4 certifies one deliberately narrow host model: the exact non-cooperative -fixture command running under an observer-owned supervisor in a detached macOS -or Linux process group. The supervisor is the persistent group leader and stays -alive when the inner Agent SDK root exits while a same-group descendant -survives. Before cancellation may fire, a bounded host `ps` sample must prove -that the trusted supervisor `ChildProcess` is still active and is the group -leader, and both PIDs read from the fixture file must already be present in the -independently host-observed group. The file is comparison evidence only; its -contents are never passed into the observer or used as signal targets. - -The runtime binds the observer directly to the per-run `Options.abortController` -signal. On abort it synchronously and idempotently sends `SIGSTOP` followed by -`SIGKILL` to the supervisor-owned group while the trusted anchor handle remains -active. The returned SDK process also maps direct kills to that group, and a -parent IPC disconnect kills the group. The fixture parent and child -intentionally ignore `SIGTERM`, making the forced path load-bearing. The -supervisor's bounded `ps` helper remains inside the owned group and only its -known PID plus the anchor PID are excluded from its membership decision. -Iterator abandonment, query close, bounded process enumeration, and -group-death confirmation share one absolute five-second process-termination -deadline. Workspace snapshots and result assembly occur afterward. The +fixture command running below an observer-owned Agent SDK supervisor on macOS or +Linux. The exact fixture parent and child each authenticate over a separate +private Unix-socket connection and keep that connection open for their complete +lifetime. Before cancellation may fire, a fresh bounded `ps` sample must observe +an active SDK supervisor root with stable identity, both role-tagged PIDs, their +parent-child relationship, their shared process group, and every current group +member as a descendant of that owned root. The group must be distinct from both +the host and SDK supervisor groups. The random capability and role-tagged +lifetime channels are necessary evidence, but a claimed or cached PID/PGID never +grants signal authority by itself. The model-writable fixture PID file is used +only by the test driver and never enters the observer. + +The runtime gives the Agent SDK its documented abort and bounded query-close +path first. It does not bind the raw per-run `Options.abortController` to host +signals. Only the SDK-forwarded post-grace `SpawnOptions.signal` can trigger the +fallback. The fallback first stops the observer-created SDK supervisor group. +A new process-table sample must then revalidate the active root identity, both +role identities, their relationship and shared group, every current tool-group +member's ancestry, and at least one open lifetime channel. Only that fresh proof +authorizes `SIGSTOP` to the detached fixture group. A second fresh sample must +show both the root and every tool-group member stopped before `SIGKILL` is sent +to the fixture group and then the SDK supervisor group. Failed tool stop/kill +attempts remain retryable, but every retry requires another fresh proof. The +five-second absolute deadline bounds the entire sequence. + +If the root exits, an identity changes or disappears, a foreign member appears, +ancestry is lost, both channels close prematurely, or a process-table read is +unavailable, the observer never signals the detached group. It may still stop +or kill its own live SDK supervisor group, but the run remains a fail-closed +`teardown_timeout` while any tool process or lifetime channel remains. This also +prevents numeric PID/PGID reuse from converting cached evidence into authority. +`forceKillIssued` describes only owned SDK supervisor roots and is not required +when SDK graceful shutdown succeeds. + +If an exact Bash launch is armed and the query ends early, its registration task +is not discarded. Readiness, SDK abort/close, owned-root fallback, and death +confirmation share one absolute five-second clock. Safe L2 completion requires +that both authenticated lifetime channels were observed, both closed, and a +fresh table/liveness sample found no member of the observed fixture group. A +missing channel, an open channel, or a live observed group produces +`teardown_timeout`; later test/campaign-owned cleanup cannot turn that result +into a pass. Workspace snapshots and result assembly occur afterward. The close-deadline timer remains referenced so a CLI host cannot exit before cleanup and result reporting finish. An unavailable or timed-out process table is explicit unknown evidence, never -an empty process table. The active detached supervisor still authorizes safe -cleanup of its owned group, but the run fails certification. An invalid or -inactive supervisor, an observed `setsid`/group escape, unknown group liveness, -failed signals, and Windows all fail closed. Windows live L2 is rejected before -the query or credential is opened. Universal containment, POSIX group escape, -Windows Job Objects, and production recovery belong to later epics; this probe -does not claim those guarantees. +an empty process table. Closed pending registrations release their role so the +trusted fixture may retry; duplicate live roles fail closed. A capability holder +can at worst deny certification—it cannot make the host signal an unrelated +group. Invalid observation, unknown group liveness, exhausted signal retries, +and Windows all fail closed. Windows live L2 is rejected before the query or +credential is opened. The detached-group path is limited to this exact fixture; +universal Bash containment, other command shapes, Windows Job Objects, and +production recovery belong to later epics, and this probe does not claim those +guarantees. ## Pre-v2 live evidence diff --git a/packages/harness/src/experimental/managed-agent-spike/fixture.test.ts b/packages/harness/src/experimental/managed-agent-spike/fixture.test.ts index 6ed6d05a9..849b871bd 100644 --- a/packages/harness/src/experimental/managed-agent-spike/fixture.test.ts +++ b/packages/harness/src/experimental/managed-agent-spike/fixture.test.ts @@ -1,3 +1,4 @@ +import { execFileSync } from "node:child_process"; import { writeFile } from "node:fs/promises"; import { join } from "node:path"; @@ -21,6 +22,19 @@ afterEach(async () => { }); describe("managed-agent disposable git fixture", () => { + it("emits a syntactically valid long-running fixture program", async () => { + const fixture = await createManagedAgentFixture(() => "syntax-check"); + fixtures.push(fixture); + + expect(() => + execFileSync( + process.execPath, + ["--check", join(fixture.workspaceRoot, FIXTURE_PATHS.processScript)], + { stdio: "pipe", windowsHide: true }, + ), + ).not.toThrow(); + }); + it("starts with a clean target plus dirty tracked and untracked sentinels", async () => { const fixture = await createManagedAgentFixture( () => "11111111-2222-3333-4444-555555555555", diff --git a/packages/harness/src/experimental/managed-agent-spike/fixture.ts b/packages/harness/src/experimental/managed-agent-spike/fixture.ts index 93014af6c..3f288e26d 100644 --- a/packages/harness/src/experimental/managed-agent-spike/fixture.ts +++ b/packages/harness/src/experimental/managed-agent-spike/fixture.ts @@ -88,21 +88,59 @@ const pidFile = resolve(process.argv[2]); const requireControlRegistration = process.argv[3] === "--register-control"; const controlSocket = process.env[${JSON.stringify(MANAGED_AGENT_TOOL_CONTROL_SOCKET_ENV)}]; const controlCapability = process.env[${JSON.stringify(MANAGED_AGENT_TOOL_CONTROL_CAPABILITY_ENV)}]; -delete process.env[${JSON.stringify(MANAGED_AGENT_TOOL_CONTROL_SOCKET_ENV)}]; -delete process.env[${JSON.stringify(MANAGED_AGENT_TOOL_CONTROL_CAPABILITY_ENV)}]; if (requireControlRegistration && (!controlSocket || !controlCapability)) { throw new Error("managed-agent tool control capability missing"); } process.on("SIGTERM", () => {}); const childProgram = [ + 'const { createConnection } = require("node:net");', + 'const requireControlRegistration = ' + JSON.stringify(requireControlRegistration) + ';', + 'const controlSocket = process.env["${MANAGED_AGENT_TOOL_CONTROL_SOCKET_ENV}"];', + 'const controlCapability = process.env["${MANAGED_AGENT_TOOL_CONTROL_CAPABILITY_ENV}"];', + 'delete process.env["${MANAGED_AGENT_TOOL_CONTROL_SOCKET_ENV}"];', + 'delete process.env["${MANAGED_AGENT_TOOL_CONTROL_CAPABILITY_ENV}"];', 'process.on("SIGTERM", () => {});', - 'if (process.send) process.send("ready");', + 'const publishReady = () => { if (process.send) process.send("ready"); };', + 'const connectControl = () => {', + ' if (!controlSocket || !controlCapability) { publishReady(); return; }', + ' const socket = createConnection(controlSocket);', + ' socket.unref();', + ' socket.setEncoding("utf8");', + ' let response = "";', + ' let registered = false;', + ' let retryScheduled = false;', + ' const retry = () => {', + ' if (registered || retryScheduled) return;', + ' retryScheduled = true;', + ' setTimeout(connectControl, 10);', + ' };', + ' socket.once("connect", () => {', + ' socket.write(JSON.stringify({ capability: controlCapability, role: "child", pid: process.pid }) + "\\\\n");', + ' });', + ' socket.on("data", (chunk) => {', + ' response += chunk;', + ' if (!response.includes("\\\\n")) return;', + ' if (!response.includes(' + JSON.stringify('"registered":true') + ')) { socket.destroy(); return; }', + ' registered = true;', + ' publishReady();', + ' });', + ' socket.once("error", retry);', + ' socket.once("close", retry);', + '};', + 'if (requireControlRegistration) connectControl(); else publishReady();', 'setInterval(() => {}, 1000);', ].join(""); const child = spawn(process.execPath, ["-e", childProgram], { stdio: ["ignore", "ignore", "ignore", "ipc"], + env: { + ...process.env, + ...(requireControlRegistration && controlSocket ? { [${JSON.stringify(MANAGED_AGENT_TOOL_CONTROL_SOCKET_ENV)}]: controlSocket } : {}), + ...(requireControlRegistration && controlCapability ? { [${JSON.stringify(MANAGED_AGENT_TOOL_CONTROL_CAPABILITY_ENV)}]: controlCapability } : {}), + }, windowsHide: true, }); +delete process.env[${JSON.stringify(MANAGED_AGENT_TOOL_CONTROL_SOCKET_ENV)}]; +delete process.env[${JSON.stringify(MANAGED_AGENT_TOOL_CONTROL_CAPABILITY_ENV)}]; let childReady = false; let controlReady = !requireControlRegistration; const publishReadiness = () => { @@ -119,8 +157,15 @@ const connectControl = () => { socket.unref(); socket.setEncoding("utf8"); let response = ""; + let registered = false; + let retryScheduled = false; + const retry = () => { + if (registered || retryScheduled) return; + retryScheduled = true; + setTimeout(connectControl, 10); + }; socket.once("connect", () => { - socket.write(JSON.stringify({ capability: controlCapability, pid: process.pid }) + "\\n"); + socket.write(JSON.stringify({ capability: controlCapability, role: "parent", pid: process.pid }) + "\\n"); }); socket.on("data", (chunk) => { response += chunk; @@ -128,12 +173,12 @@ const connectControl = () => { if (!response.includes('"registered":true')) { throw new Error("managed-agent tool registration rejected"); } + registered = true; controlReady = true; publishReadiness(); }); - socket.once("error", () => { - if (!controlReady) setTimeout(connectControl, 10); - }); + socket.once("error", retry); + socket.once("close", retry); }; if (requireControlRegistration) connectControl(); setInterval(() => {}, 1000); diff --git a/packages/harness/src/experimental/managed-agent-spike/probe-cli.test.ts b/packages/harness/src/experimental/managed-agent-spike/probe-cli.test.ts index d912b4a8b..bd820f9ae 100644 --- a/packages/harness/src/experimental/managed-agent-spike/probe-cli.test.ts +++ b/packages/harness/src/experimental/managed-agent-spike/probe-cli.test.ts @@ -162,6 +162,8 @@ function passingL1Result(): ManagedAgentProbeResult { containmentSupported: true, ownershipProven: false, forceKillIssued: false, + toolProcessObservationComplete: true, + toolProcessChannelsClosed: true, elapsedMs: 5, observedPids: [], alivePidsAtDeadline: [], @@ -1106,6 +1108,32 @@ describe("managed-agent probe CLI", () => { }); }); + it("requires observed closed tool lifetimes but not an unnecessary host force-kill", () => { + const passing = passingL2Result(); + const graceful = { + ...passing, + teardown: { ...passing.teardown, forceKillIssued: false }, + }; + expect(evaluateManagedAgentProbe(graceful, [12_345, 12_346]).outcome).toBe( + "pass", + ); + + for (const [field, checkId] of [ + ["toolProcessObservationComplete", "l2_containment_prepared"], + ["toolProcessChannelsClosed", "sdk_closed_tool_lifetime_channels"], + ] as const) { + expect( + evaluateManagedAgentProbe( + { + ...passing, + teardown: { ...passing.teardown, [field]: false }, + }, + [12_345, 12_346], + ).checks, + ).toContainEqual({ id: checkId, passed: false }); + } + }); + it.each([ [ "omitted", diff --git a/packages/harness/src/experimental/managed-agent-spike/probe-cli.ts b/packages/harness/src/experimental/managed-agent-spike/probe-cli.ts index 2cd6e07b9..9094357bd 100644 --- a/packages/harness/src/experimental/managed-agent-spike/probe-cli.ts +++ b/packages/harness/src/experimental/managed-agent-spike/probe-cli.ts @@ -822,7 +822,11 @@ export function evaluateManagedAgentProbe( result.teardown.processTableAvailable && result.teardown.containmentSupported && result.teardown.ownershipProven && - result.teardown.forceKillIssued, + result.teardown.toolProcessObservationComplete, + }, + { + id: "sdk_closed_tool_lifetime_channels", + passed: result.teardown.toolProcessChannelsClosed, }, { id: "fixture_processes_observed", diff --git a/packages/harness/src/experimental/managed-agent-spike/process-observer.test.ts b/packages/harness/src/experimental/managed-agent-spike/process-observer.test.ts index 78fc8553b..ef5ea608d 100644 --- a/packages/harness/src/experimental/managed-agent-spike/process-observer.test.ts +++ b/packages/harness/src/experimental/managed-agent-spike/process-observer.test.ts @@ -1,11 +1,14 @@ import { once } from "node:events"; import { + execFile, spawn as spawnChild, type ChildProcess, type ChildProcessWithoutNullStreams, } from "node:child_process"; import { readFile, writeFile } from "node:fs/promises"; +import { createConnection, type Socket as NetSocket } from "node:net"; import { join } from "node:path"; +import { promisify } from "node:util"; import type { SpawnedProcess } from "@anthropic-ai/claude-agent-sdk"; import { afterEach, describe, expect, it } from "vitest"; @@ -18,11 +21,14 @@ import { } from "./fixture.js"; import { LocalManagedAgentProcessObserver, + MANAGED_AGENT_TOOL_CONTROL_CAPABILITY_ENV, + MANAGED_AGENT_TOOL_CONTROL_SOCKET_ENV, type ManagedAgentKernelProcessRecord, type ManagedAgentProcessTableObservation, } from "./process-observer.js"; const fixtures: ManagedAgentFixture[] = []; +const execFileAsync = promisify(execFile); afterEach(async () => { await Promise.all(fixtures.splice(0).map((fixture) => fixture.cleanup())); @@ -34,6 +40,51 @@ function available( return { available: true, processes: new Map(entries) }; } +async function readRealPosixProcessTable(): Promise { + try { + const { stdout } = await execFileAsync( + "/bin/ps", + ["-axo", "pid=,ppid=,pgid=,stat=,lstart="], + { encoding: "utf8", maxBuffer: 4 * 1024 * 1024, timeout: 1_000 }, + ); + const entries: Array = + []; + for (const line of stdout.split("\n")) { + const match = /^\s*(\d+)\s+(\d+)\s+(\d+)\s+(\S+)\s+(.+?)\s*$/.exec(line); + if (!match) continue; + entries.push([ + Number(match[1]), + { + parentPid: Number(match[2]), + processGroupId: Number(match[3]), + state: match[4]!, + startedAt: match[5]!, + }, + ]); + } + return available(entries); + } catch { + return { available: false }; + } +} + +async function prepareCancellationAfterTransientReadFailure( + observer: LocalManagedAgentProcessObserver, + timeoutMs = 2_000, +) { + const deadline = Date.now() + timeoutMs; + for (;;) { + const readiness = await observer.prepareCancellation(); + if ( + readiness.reason !== "process_table_unavailable" || + Date.now() >= deadline + ) { + return readiness; + } + await new Promise((resolveDelay) => setTimeout(resolveDelay, 10)); + } +} + function activeNodeCommand(): { command: string; args: string[] } { return { command: process.execPath, @@ -72,7 +123,39 @@ child.once("message", () => { }); `; -const LATE_REGISTERED_TOOL_SCRIPT = String.raw` +const DESCENDANT_TOOL_SCRIPT = String.raw` +import { spawn } from "node:child_process"; +import { writeFileSync } from "node:fs"; + +const [toolScript, pidFile, credentialFile] = process.argv.slice(1); +writeFileSync(credentialFile, JSON.stringify({ + socketPath: process.env.SAPIOM_MANAGED_AGENT_TOOL_CONTROL_SOCKET, + capability: process.env.SAPIOM_MANAGED_AGENT_TOOL_CONTROL_CAPABILITY, +})); +const tool = spawn( + "/bin/bash", + [ + "--noprofile", + "--norc", + "-c", + 'exec "$1" "$2" "$3"', + "managed-agent-tool", + process.execPath, + toolScript, + pidFile, + ], + { + detached: true, + env: process.env, + stdio: "ignore", + windowsHide: true, + }, +); +tool.unref(); +setInterval(() => {}, 1000); +`; + +const REGISTERED_DESCENDANT_TOOL_SCRIPT = String.raw` import { spawn } from "node:child_process"; import { writeFileSync } from "node:fs"; @@ -83,7 +166,7 @@ const tool = spawn( "--noprofile", "--norc", "-c", - 'sleep 0.25; exec "$1" "$2" "$3" --register-control', + 'exec "$1" "$2" "$3" --register-control', "managed-agent-tool", process.execPath, toolScript, @@ -96,10 +179,114 @@ const tool = spawn( windowsHide: true, }, ); -writeFileSync(launchFile, JSON.stringify({ processGroupId: tool.pid })); +if (typeof tool.pid !== "number") throw new Error("fixture tool failed to spawn"); +// Direct launcher tests predate setup-failure cleanup evidence and deliberately +// omit this path; the shared launcher must remain valid for those callers. +if (launchFile) { + writeFileSync(launchFile, JSON.stringify({ processGroupId: tool.pid })); +} tool.unref(); +setInterval(() => {}, 1000); `; +const EXPORT_TOOL_CONTROL_SCRIPT = String.raw` +import { writeFileSync } from "node:fs"; + +const outputPath = process.argv[1]; +const socketPath = process.env[${JSON.stringify(MANAGED_AGENT_TOOL_CONTROL_SOCKET_ENV)}]; +const capability = process.env[${JSON.stringify(MANAGED_AGENT_TOOL_CONTROL_CAPABILITY_ENV)}]; +if (!socketPath || !capability) process.exit(41); +writeFileSync(outputPath, JSON.stringify({ socketPath, capability })); +setInterval(() => {}, 1000); +`; + +interface ToolControlCredentials { + readonly socketPath: string; + readonly capability: string; +} + +async function waitForToolControlCredentials( + path: string, + timeoutMs = 3_000, +): Promise { + const deadline = Date.now() + timeoutMs; + for (;;) { + try { + const payload = JSON.parse(await readFile(path, "utf8")) as { + socketPath?: unknown; + capability?: unknown; + }; + if ( + typeof payload.socketPath === "string" && + typeof payload.capability === "string" + ) { + return { + socketPath: payload.socketPath, + capability: payload.capability, + }; + } + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + } + if (Date.now() >= deadline) { + throw new Error("Timed out waiting for tool-control credentials"); + } + await new Promise((resolveDelay) => setTimeout(resolveDelay, 10)); + } +} + +async function openToolRegistration( + credentials: ToolControlCredentials, + role: "parent" | "child", + pid: number, +): Promise { + const socket = await startToolRegistration(credentials, role, pid); + let timeout: NodeJS.Timeout | undefined; + const [response] = (await Promise.race([ + once(socket, "data"), + once(socket, "close").then(() => { + throw new Error(`tool registration ${role} closed before acceptance`); + }), + new Promise((_, rejectTimeout) => { + timeout = setTimeout( + () => rejectTimeout(new Error(`tool registration ${role} timed out`)), + 1_000, + ); + }), + ]).finally(() => { + if (timeout) clearTimeout(timeout); + })) as [Buffer | string]; + expect(String(response)).toContain('"registered":true'); + return socket; +} + +async function startToolRegistration( + credentials: ToolControlCredentials, + role: "parent" | "child", + pid: number, +): Promise { + const socket = createConnection(credentials.socketPath); + socket.setEncoding("utf8"); + await once(socket, "connect"); + socket.write( + `${JSON.stringify({ capability: credentials.capability, role, pid })}\n`, + ); + return socket; +} + +async function sendClosedToolRegistration( + credentials: ToolControlCredentials, + role: "parent" | "child", + pid: number, +): Promise { + const socket = createConnection(credentials.socketPath); + await once(socket, "connect"); + socket.end( + `${JSON.stringify({ capability: credentials.capability, role, pid })}\n`, + ); + await once(socket, "close"); +} + function asChildProcess( spawned: SpawnedProcess, ): ChildProcessWithoutNullStreams { @@ -124,6 +311,34 @@ function processGroupExists(processGroupId: number): boolean { } } +function signalRealProcessGroup( + processGroupId: number, + signal: "SIGSTOP" | "SIGKILL", +): "sent" | "gone" | "failure" { + try { + process.kill(-processGroupId, signal); + return "sent"; + } catch (error) { + return (error as NodeJS.ErrnoException).code === "ESRCH" + ? "gone" + : "failure"; + } +} + +function realProcessGroupLiveness( + processGroupId: number, +): "alive" | "gone" | "unknown" { + try { + process.kill(-processGroupId, 0); + return "alive"; + } catch (error) { + const code = (error as NodeJS.ErrnoException).code; + if (code === "ESRCH") return "gone"; + if (code === "EPERM") return "alive"; + return "unknown"; + } +} + async function waitForTestProcessDeath( isAlive: () => boolean, description: string, @@ -138,7 +353,7 @@ async function waitForTestProcessDeath( async function waitForLaunchedGroupId( path: string, - timeoutMs = 1_000, + timeoutMs = 3_000, ): Promise { const deadline = Date.now() + timeoutMs; for (;;) { @@ -167,6 +382,11 @@ async function forceKillExactTestGroup( processGroupId: number, root: ChildProcess, ): Promise { + try { + process.kill(-processGroupId, "SIGCONT"); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ESRCH") throw error; + } try { process.kill(-processGroupId, "SIGKILL"); } catch (error) { @@ -192,6 +412,11 @@ async function forceKillExactTestGroup( async function forceKillExactTestGroupId( processGroupId: number, ): Promise { + try { + process.kill(-processGroupId, "SIGCONT"); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ESRCH") throw error; + } try { process.kill(-processGroupId, "SIGKILL"); } catch (error) { @@ -203,6 +428,22 @@ async function forceKillExactTestGroupId( ); } +async function waitForChildExitBounded( + child: ChildProcess, + timeoutMs = 1_000, +): Promise { + if (child.exitCode !== null || child.signalCode !== null) return; + await Promise.race([ + once(child, "exit").then(() => undefined), + new Promise((_, rejectTimeout) => + setTimeout( + () => rejectTimeout(new Error("Owned root did not exit in time")), + timeoutMs, + ), + ), + ]); +} + async function forceKillExactTestProcess(child: ChildProcess): Promise { const pid = child.pid; if (typeof pid !== "number") return; @@ -265,7 +506,7 @@ async function proveRetainedGroupAuthority( let readiness; if (exitTiming === "after-readiness") { - readiness = await observer.prepareCancellation(); + readiness = await prepareCancellationAfterTransientReadFailure(observer); await writeFile(exitMarker, "exit\n"); } await waitForTestProcessDeath( @@ -273,7 +514,9 @@ async function proveRetainedGroupAuthority( `Fast SDK root ${workerRootPid}`, ); expect(processExists(nonCooperativeChildPid!)).toBe(true); - if (!readiness) readiness = await observer.prepareCancellation(); + if (!readiness) { + readiness = await prepareCancellationAfterTransientReadFailure(observer); + } expect(readiness).toMatchObject({ supported: true, @@ -305,7 +548,171 @@ async function proveRetainedGroupAuthority( } } +interface RegisteredDescendantToolRun { + readonly fixture: ManagedAgentFixture; + readonly observer: LocalManagedAgentProcessObserver; + readonly forwardedController: AbortController; + readonly anchor: ChildProcessWithoutNullStreams; + readonly toolPids: readonly [number, number]; + readonly toolProcessGroupId: number; +} + +interface RegisteredDescendantToolSetupEvidence { + readonly anchorProcessGroupId: number; + readonly toolPids: readonly [number, number]; + readonly toolProcessGroupId: number; +} + +interface RegisteredDescendantSetupCleanupError extends Error { + readonly setupError: unknown; + readonly cleanupErrors: readonly unknown[]; +} + +function setupAndCleanupFailure( + setupError: unknown, + cleanupErrors: readonly unknown[], +): RegisteredDescendantSetupCleanupError { + const failure = new Error( + "Registered descendant setup and cleanup both failed", + ) as RegisteredDescendantSetupCleanupError; + Object.defineProperties(failure, { + cleanupErrors: { value: [...cleanupErrors] }, + setupError: { value: setupError }, + }); + return failure; +} + +async function startRegisteredDescendantToolRun( + observer: LocalManagedAgentProcessObserver, + name: string, + afterPidPublication?: ( + evidence: RegisteredDescendantToolSetupEvidence, + ) => void | Promise, +): Promise { + const fixture = await createManagedAgentFixture(() => name); + fixtures.push(fixture); + const forwardedController = new AbortController(); + const launchFile = join(fixture.root, "registered-tool-launch.json"); + observer.armToolProcessContainment(); + const anchor = asChildProcess( + observer.spawn({ + command: process.execPath, + args: [ + "--input-type=module", + "--eval", + REGISTERED_DESCENDANT_TOOL_SCRIPT, + join(fixture.workspaceRoot, FIXTURE_PATHS.processScript), + join(fixture.workspaceRoot, FIXTURE_PATHS.processPidFile), + launchFile, + ], + cwd: fixture.workspaceRoot, + env: { ...process.env }, + signal: forwardedController.signal, + }), + ); + let toolProcessGroupId: number | undefined; + try { + const anchorProcessGroupId = anchor.pid; + if (typeof anchorProcessGroupId !== "number") { + throw new Error("Owned fixture anchor failed to spawn"); + } + toolProcessGroupId = await waitForLaunchedGroupId(launchFile); + const [parentPid, childPid] = await waitForManagedAgentFixturePids( + fixture, + 5_000, + ); + const toolPids = [parentPid!, childPid!] as const; + if (parentPid !== toolProcessGroupId) { + throw new Error("Detached fixture group does not match its parent PID"); + } + await afterPidPublication?.({ + anchorProcessGroupId, + toolPids, + toolProcessGroupId, + }); + await expect( + prepareCancellationAfterTransientReadFailure(observer), + ).resolves.toMatchObject({ + supported: true, + reason: "ready", + ownershipProven: true, + }); + return { + fixture, + observer, + forwardedController, + anchor, + toolPids, + toolProcessGroupId, + }; + } catch (setupError) { + const cleanupErrors: unknown[] = []; + try { + if ( + typeof toolProcessGroupId === "number" && + processGroupExists(toolProcessGroupId) + ) { + await forceKillExactTestGroupId(toolProcessGroupId); + } + } catch (error) { + cleanupErrors.push(error); + } + try { + if (typeof anchor.pid === "number") { + await forceKillExactTestGroup(anchor.pid, anchor); + } + } catch (error) { + cleanupErrors.push(error); + } finally { + forwardedController.abort(); + observer.dispose(); + } + if (cleanupErrors.length > 0) { + throw setupAndCleanupFailure(setupError, cleanupErrors); + } + throw setupError; + } +} + +async function cleanupRegisteredDescendantToolRun( + run: RegisteredDescendantToolRun | undefined, +): Promise { + if (!run) return; + if (processGroupExists(run.toolProcessGroupId)) { + await forceKillExactTestGroupId(run.toolProcessGroupId); + } + if (typeof run.anchor.pid === "number") { + if (run.anchor.exitCode === null && run.anchor.signalCode === null) { + await waitForChildExitBounded(run.anchor, 100).catch(() => undefined); + } + if (run.anchor.exitCode === null && run.anchor.signalCode === null) { + await forceKillExactTestGroup(run.anchor.pid, run.anchor); + } else { + await waitForTestProcessDeath( + () => processGroupExists(run.anchor.pid!), + `Owned root group ${run.anchor.pid}`, + ); + } + } + run.forwardedController.abort(); + run.observer.dispose(); +} + describe("LocalManagedAgentProcessObserver", () => { + it("retains setup and cleanup failures without requiring AggregateError", () => { + const setupError = new Error("synthetic setup failure"); + const cleanupError = new Error("synthetic cleanup failure"); + + const failure = setupAndCleanupFailure(setupError, [cleanupError]); + + expect(failure).toBeInstanceOf(Error); + expect(failure.message).toBe( + "Registered descendant setup and cleanup both failed", + ); + expect(failure.setupError).toBe(setupError); + expect(failure.cleanupErrors).toEqual([cleanupError]); + }); + it.skipIf(process.platform === "win32")( "keeps inner arguments out of supervisor argv and scrubs its private payload", async () => { @@ -414,7 +821,8 @@ describe("LocalManagedAgentProcessObserver", () => { ownedProcessGroupId = root.pid; expect(ownedProcessGroupId).toBeTypeOf("number"); const fixturePids = await waitForManagedAgentFixturePids(fixture); - const readiness = await observer.prepareCancellation(); + const readiness = + await prepareCancellationAfterTransientReadFailure(observer); expect(readiness).toMatchObject({ supported: true, reason: "ready", @@ -453,7 +861,7 @@ describe("LocalManagedAgentProcessObserver", () => { await forceKillExactTestProcess(unrelated); } }, - 10_000, + 15_000, ); it.skipIf(process.platform === "win32")( @@ -482,7 +890,9 @@ describe("LocalManagedAgentProcessObserver", () => { expect(ownedProcessGroupId).toBeTypeOf("number"); try { const fixturePids = await waitForManagedAgentFixturePids(fixture); - await expect(observer.prepareCancellation()).resolves.toMatchObject({ + await expect( + prepareCancellationAfterTransientReadFailure(observer), + ).resolves.toMatchObject({ supported: true, reason: "ready", ownershipProven: true, @@ -534,7 +944,9 @@ describe("LocalManagedAgentProcessObserver", () => { ); await once(child, "exit"); try { - await expect(observer.prepareCancellation()).resolves.toMatchObject({ + await expect( + prepareCancellationAfterTransientReadFailure(observer), + ).resolves.toMatchObject({ supported: false, reason: "root_not_active", }); @@ -550,7 +962,7 @@ describe("LocalManagedAgentProcessObserver", () => { it.skipIf(process.platform === "win32")( "retains owned group authority when the SDK root exits before its non-cooperative child", () => proveRetainedGroupAuthority("before-readiness"), - 10_000, + 15_000, ); it.skipIf(process.platform === "win32")( @@ -560,10 +972,10 @@ describe("LocalManagedAgentProcessObserver", () => { ); it.skipIf(process.platform === "win32")( - "contains a detached tool group that authenticates after the SDK group has exited", + "stops and kills a freshly revalidated detached tool group wholly descended from the owned root", async () => { const fixture = await createManagedAgentFixture( - () => "late-tool-registration", + () => "anchored-descendant-tool", ); fixtures.push(fixture); const observer = new LocalManagedAgentProcessObserver(); @@ -574,13 +986,9 @@ describe("LocalManagedAgentProcessObserver", () => { { stdio: "ignore", windowsHide: true }, ); await once(unrelated, "spawn"); - const launchFile = join( - fixture.workspaceRoot, - FIXTURE_PATHS.processDirectory, - "late-tool-launch.json", - ); let anchor: ChildProcessWithoutNullStreams | undefined; - let detachedToolGroupId: number | undefined; + let toolGroupId: number | undefined; + let fixturePids: readonly number[] = []; try { observer.armToolProcessContainment(); anchor = asChildProcess( @@ -589,36 +997,484 @@ describe("LocalManagedAgentProcessObserver", () => { args: [ "--input-type=module", "--eval", - LATE_REGISTERED_TOOL_SCRIPT, + REGISTERED_DESCENDANT_TOOL_SCRIPT, join(fixture.workspaceRoot, FIXTURE_PATHS.processScript), join(fixture.workspaceRoot, FIXTURE_PATHS.processPidFile), - launchFile, + // Omit launch evidence to exercise the direct-launcher contract. ], cwd: fixture.workspaceRoot, env: { ...process.env }, signal: forwardedController.signal, }), ); - detachedToolGroupId = await waitForLaunchedGroupId(launchFile); - if (anchor.exitCode === null && anchor.signalCode === null) { - await once(anchor, "exit"); - } - expect(processGroupExists(detachedToolGroupId)).toBe(true); + fixturePids = await waitForManagedAgentFixturePids(fixture); + toolGroupId = fixturePids[0]; + await expect( + prepareCancellationAfterTransientReadFailure(observer), + ).resolves.toMatchObject({ + supported: true, + reason: "ready", + ownershipProven: true, + }); - const teardown = await observer.emergencyCleanup(1_000); + const teardown = await observer.emergencyCleanup(2_000); expect(teardown).toMatchObject({ quiescent: true, deadlineMet: true, - processTableAvailable: true, containmentSupported: true, - emergencyCleanupAttempted: true, + ownershipProven: true, + forceKillIssued: true, + toolProcessObservationComplete: true, + toolProcessChannelsClosed: true, alivePidsAtDeadline: [], }); - expect(teardown.observedPids).toContain(detachedToolGroupId); - expect(processGroupExists(detachedToolGroupId)).toBe(false); + expect(fixturePids.every((pid) => !processExists(pid))).toBe(true); expect(processExists(unrelated.pid!)).toBe(true); } finally { + forwardedController.abort(); + if ( + typeof toolGroupId === "number" && + processGroupExists(toolGroupId) + ) { + await forceKillExactTestGroupId(toolGroupId); + } + if (anchor && typeof anchor.pid === "number") { + await forceKillExactTestGroup(anchor.pid, anchor); + } + observer.dispose(); + await forceKillExactTestProcess(unrelated); + } + }, + 15_000, + ); + + it.skipIf(process.platform === "win32")( + "cleans exact fixture and anchor groups when setup fails after PID publication", + async () => { + const observer = new LocalManagedAgentProcessObserver(); + let setupEvidence: RegisteredDescendantToolSetupEvidence | undefined; + try { + await expect( + startRegisteredDescendantToolRun( + observer, + "failed-registered-tool-setup", + (evidence) => { + setupEvidence = evidence; + throw new Error("synthetic failure after PID publication"); + }, + ), + ).rejects.toThrow("synthetic failure after PID publication"); + + expect(setupEvidence).toBeDefined(); + expect( + setupEvidence!.toolPids.every((pid) => !processExists(pid)), + ).toBe(true); + expect(processGroupExists(setupEvidence!.toolProcessGroupId)).toBe( + false, + ); + expect(processGroupExists(setupEvidence!.anchorProcessGroupId)).toBe( + false, + ); + } finally { + if ( + setupEvidence && + processGroupExists(setupEvidence.toolProcessGroupId) + ) { + await forceKillExactTestGroupId(setupEvidence.toolProcessGroupId); + } + if ( + setupEvidence && + processGroupExists(setupEvidence.anchorProcessGroupId) + ) { + await forceKillExactTestGroupId(setupEvidence.anchorProcessGroupId); + } + observer.dispose(); + } + }, + 15_000, + ); + + it.skipIf(process.platform === "win32")( + "refuses detached tool authority when a foreign member joins the candidate group", + async () => { + let injectForeignMember = false; + let toolProcessGroupId: number | undefined; + const signals: Array = []; + const observer = new LocalManagedAgentProcessObserver({ + readProcessTable: async () => { + const observation = await readRealPosixProcessTable(); + if ( + !observation.available || + !injectForeignMember || + typeof toolProcessGroupId !== "number" + ) { + return observation; + } + const processes = new Map(observation.processes); + let foreignPid = 2_000_000_000; + while (processes.has(foreignPid)) foreignPid -= 1; + processes.set(foreignPid, { + parentPid: process.pid, + processGroupId: toolProcessGroupId, + state: "S", + startedAt: "synthetic-foreign-member", + }); + return { available: true, processes }; + }, + signalProcessGroup: (groupId, signal) => { + signals.push([groupId, signal]); + return signalRealProcessGroup(groupId, signal); + }, + }); + let run: RegisteredDescendantToolRun | undefined; + try { + run = await startRegisteredDescendantToolRun( + observer, + "foreign-tool-group-member", + ); + toolProcessGroupId = run.toolProcessGroupId; + injectForeignMember = true; + + const teardown = await observer.emergencyCleanup(250); + + expect(teardown).toMatchObject({ + quiescent: false, + deadlineMet: false, + forceKillIssued: true, + }); + expect( + signals.filter(([groupId]) => groupId === toolProcessGroupId), + ).toEqual([]); + expect(processGroupExists(toolProcessGroupId)).toBe(true); + } finally { + await cleanupRegisteredDescendantToolRun(run); + observer.dispose(); + } + }, + 15_000, + ); + + it.skipIf(process.platform === "win32")( + "never signals a cached detached group after its registered identities disappear and are reused", + async () => { + let simulatePidReuse = false; + let toolProcessGroupId: number | undefined; + let registeredPids: readonly [number, number] | undefined; + const signals: Array = []; + const observer = new LocalManagedAgentProcessObserver({ + readProcessTable: async () => { + const observation = await readRealPosixProcessTable(); + if ( + !observation.available || + !simulatePidReuse || + typeof toolProcessGroupId !== "number" || + !registeredPids + ) { + return observation; + } + const processes = new Map(observation.processes); + const [parentPid, childPid] = registeredPids; + processes.set(parentPid, { + parentPid: process.pid, + processGroupId: toolProcessGroupId, + state: "S", + startedAt: "reused-parent-identity", + }); + processes.set(childPid, { + parentPid, + processGroupId: toolProcessGroupId, + state: "S", + startedAt: "reused-child-identity", + }); + return { available: true, processes }; + }, + processGroupLiveness: (groupId) => + simulatePidReuse && groupId === toolProcessGroupId + ? "alive" + : realProcessGroupLiveness(groupId), + signalProcessGroup: (groupId, signal) => { + signals.push([groupId, signal]); + return signalRealProcessGroup(groupId, signal); + }, + }); + let run: RegisteredDescendantToolRun | undefined; + try { + run = await startRegisteredDescendantToolRun( + observer, + "reused-tool-identities", + ); + toolProcessGroupId = run.toolProcessGroupId; + registeredPids = run.toolPids; + await forceKillExactTestGroupId(toolProcessGroupId); + simulatePidReuse = true; + + const teardown = await observer.emergencyCleanup(250); + + expect(teardown).toMatchObject({ + quiescent: false, + deadlineMet: false, + forceKillIssued: true, + }); + expect( + signals.filter(([groupId]) => groupId === toolProcessGroupId), + ).toEqual([]); + } finally { + simulatePidReuse = false; + await cleanupRegisteredDescendantToolRun(run); + observer.dispose(); + } + }, + 15_000, + ); + + it.skipIf(process.platform === "win32")( + "retries detached tool stop and kill failures only after fresh authority checks", + async () => { + let processTableReads = 0; + let toolProcessGroupId: number | undefined; + let stopAttempts = 0; + let killAttempts = 0; + const toolSignals: Array<{ + readonly signal: "SIGSTOP" | "SIGKILL"; + readonly processTableReads: number; + }> = []; + const observer = new LocalManagedAgentProcessObserver({ + readProcessTable: async () => { + processTableReads += 1; + return readRealPosixProcessTable(); + }, + signalProcessGroup: (groupId, signal) => { + if (groupId !== toolProcessGroupId) { + return signalRealProcessGroup(groupId, signal); + } + toolSignals.push({ signal, processTableReads }); + if (signal === "SIGSTOP" && stopAttempts++ === 0) return "failure"; + if (signal === "SIGKILL" && killAttempts++ === 0) return "failure"; + return signalRealProcessGroup(groupId, signal); + }, + }); + let run: RegisteredDescendantToolRun | undefined; + try { + run = await startRegisteredDescendantToolRun( + observer, + "retry-tool-signals", + ); + toolProcessGroupId = run.toolProcessGroupId; + + const teardown = await observer.emergencyCleanup(3_000); + + expect(teardown).toMatchObject({ + quiescent: true, + deadlineMet: true, + forceKillIssued: true, + alivePidsAtDeadline: [], + }); + expect(toolSignals.map(({ signal }) => signal)).toEqual([ + "SIGSTOP", + "SIGSTOP", + "SIGKILL", + "SIGKILL", + ]); + expect( + toolSignals.every( + (attempt, index) => + index === 0 || + attempt.processTableReads > + toolSignals[index - 1]!.processTableReads, + ), + ).toBe(true); + } finally { + await cleanupRegisteredDescendantToolRun(run); + observer.dispose(); + } + }, + 15_000, + ); + + it.skipIf(process.platform === "win32")( + "never signals the detached tool group after the owned root exits and ancestry is lost", + async () => { + let toolProcessGroupId: number | undefined; + const signals: Array = []; + const observer = new LocalManagedAgentProcessObserver({ + signalProcessGroup: (groupId, signal) => { + signals.push([groupId, signal]); + return signalRealProcessGroup(groupId, signal); + }, + }); + let run: RegisteredDescendantToolRun | undefined; + try { + run = await startRegisteredDescendantToolRun( + observer, + "root-exit-loses-tool-ancestry", + ); + toolProcessGroupId = run.toolProcessGroupId; + process.kill(-run.anchor.pid!, "SIGKILL"); + await waitForChildExitBounded(run.anchor); + expect(processGroupExists(toolProcessGroupId)).toBe(true); + + const teardown = await observer.emergencyCleanup(250); + + expect(teardown).toMatchObject({ + quiescent: false, + deadlineMet: false, + forceKillIssued: false, + }); + expect( + signals.filter(([groupId]) => groupId === toolProcessGroupId), + ).toEqual([]); + expect(processGroupExists(toolProcessGroupId)).toBe(true); + } finally { + await cleanupRegisteredDescendantToolRun(run); + observer.dispose(); + } + }, + 15_000, + ); + + it.skipIf(process.platform === "win32")( + "never signals a detached PGID merely because a capability holder claimed it", + async () => { + const fixture = await createManagedAgentFixture( + () => "unanchored-tool-registration", + ); + fixtures.push(fixture); + const observer = new LocalManagedAgentProcessObserver(); + const forwardedController = new AbortController(); + const credentialFile = join(fixture.root, "tool-control.json"); + let anchor: ChildProcessWithoutNullStreams | undefined; + let detachedTool: ChildProcess | undefined; + let registrations: readonly NetSocket[] = []; + try { + observer.armToolProcessContainment(); + anchor = asChildProcess( + observer.spawn({ + command: process.execPath, + args: [ + "--input-type=module", + "--eval", + EXPORT_TOOL_CONTROL_SCRIPT, + credentialFile, + ], + cwd: fixture.workspaceRoot, + env: { ...process.env }, + signal: forwardedController.signal, + }), + ); + const credentials = await waitForToolControlCredentials(credentialFile); + detachedTool = spawnChild( + process.execPath, + [FIXTURE_PATHS.processScript, FIXTURE_PATHS.processPidFile], + { + cwd: fixture.workspaceRoot, + detached: true, + env: { ...process.env }, + stdio: "ignore", + windowsHide: true, + }, + ); + const [toolParentPid, toolChildPid] = + await waitForManagedAgentFixturePids(fixture); + registrations = await Promise.all([ + startToolRegistration(credentials, "parent", toolParentPid), + startToolRegistration(credentials, "child", toolChildPid), + ]); + await expect( + prepareCancellationAfterTransientReadFailure(observer), + ).resolves.toMatchObject({ + supported: false, + reason: "tool_process_not_registered", + }); + + const teardown = await observer.emergencyCleanup(100); + + expect(teardown.quiescent).toBe(false); + expect(processGroupExists(detachedTool.pid!)).toBe(true); + } finally { + for (const registration of registrations) registration.destroy(); + forwardedController.abort(); + if (detachedTool && typeof detachedTool.pid === "number") { + await forceKillExactTestGroup(detachedTool.pid, detachedTool); + } + if (anchor && typeof anchor.pid === "number") { + await forceKillExactTestGroup(anchor.pid, anchor); + } + observer.dispose(); + } + }, + 15_000, + ); + + it.skipIf(process.platform === "win32")( + "clears a closed pending registration, accepts retries, and requires both lifetime channels to close", + async () => { + const fixture = await createManagedAgentFixture( + () => "tool-registration-retry", + ); + fixtures.push(fixture); + const observer = new LocalManagedAgentProcessObserver(); + const forwardedController = new AbortController(); + const credentialFile = join(fixture.root, "tool-control.json"); + let anchor: ChildProcessWithoutNullStreams | undefined; + let detachedToolGroupId: number | undefined; + let parentRegistration: NetSocket | undefined; + let childRegistration: NetSocket | undefined; + try { + observer.armToolProcessContainment(); + anchor = asChildProcess( + observer.spawn({ + command: process.execPath, + args: [ + "--input-type=module", + "--eval", + DESCENDANT_TOOL_SCRIPT, + join(fixture.workspaceRoot, FIXTURE_PATHS.processScript), + join(fixture.workspaceRoot, FIXTURE_PATHS.processPidFile), + credentialFile, + ], + cwd: fixture.workspaceRoot, + env: { ...process.env }, + signal: forwardedController.signal, + }), + ); + const credentials = await waitForToolControlCredentials(credentialFile); + const [toolParentPid, toolChildPid] = + await waitForManagedAgentFixturePids(fixture); + detachedToolGroupId = toolParentPid; + + await sendClosedToolRegistration(credentials, "parent", toolParentPid); + [parentRegistration, childRegistration] = await Promise.all([ + openToolRegistration(credentials, "parent", toolParentPid), + openToolRegistration(credentials, "child", toolChildPid), + ]); + await expect( + prepareCancellationAfterTransientReadFailure(observer), + ).resolves.toMatchObject({ + supported: true, + reason: "ready", + containmentSupported: true, + }); + + await forceKillExactTestGroupId(detachedToolGroupId); + forwardedController.abort(); + const openChannelObservation = await observer.emergencyCleanup(1_000); + await waitForChildExitBounded(anchor); + expect(openChannelObservation).toMatchObject({ + quiescent: false, + deadlineMet: false, + }); + + parentRegistration.destroy(); + childRegistration.destroy(); + const finalObservation = await observer.waitForQuiescence(3_000); + expect(finalObservation).toMatchObject({ + quiescent: true, + deadlineMet: true, + }); + } finally { + parentRegistration?.destroy(); + childRegistration?.destroy(); forwardedController.abort(); if ( typeof detachedToolGroupId === "number" && @@ -630,10 +1486,9 @@ describe("LocalManagedAgentProcessObserver", () => { await forceKillExactTestGroup(anchor.pid, anchor); } observer.dispose(); - await forceKillExactTestProcess(unrelated); } }, - 10_000, + 15_000, ); it.skipIf(process.platform === "win32")( @@ -722,7 +1577,9 @@ describe("LocalManagedAgentProcessObserver", () => { [child.pid!, "SIGKILL"], ]); } finally { - child.kill("SIGKILL"); + if (child.exitCode === null && child.signalCode === null) { + child.kill("SIGKILL"); + } controller.abort(); observer.dispose(); } diff --git a/packages/harness/src/experimental/managed-agent-spike/process-observer.ts b/packages/harness/src/experimental/managed-agent-spike/process-observer.ts index 989bc87e3..fb6970064 100644 --- a/packages/harness/src/experimental/managed-agent-spike/process-observer.ts +++ b/packages/harness/src/experimental/managed-agent-spike/process-observer.ts @@ -227,6 +227,8 @@ if (inner) { export interface ManagedAgentKernelProcessRecord { readonly parentPid: number; readonly processGroupId?: number; + /** POSIX process state used only to confirm an issued group stop. */ + readonly state?: string; /** Kernel-reported creation time used for evidence, never POSIX authority. */ readonly startedAt: string; } @@ -263,26 +265,22 @@ export interface LocalManagedAgentProcessObserverOptions { interface OwnedRoot { readonly pid: number; readonly child: ChildProcessWithoutNullStreams; + identity?: ManagedAgentKernelProcessRecord; containmentSupported: boolean; ownershipProven: boolean; stopIssued: boolean; forceKillIssued: boolean; } -interface PendingToolRegistration { +type ToolProcessRole = "parent" | "child"; + +interface ToolProcessRegistration { + readonly role: ToolProcessRole; readonly pid: number; readonly socket: NetSocket; -} - -interface OwnedToolGroup { - readonly registeredPid: number; - readonly registeredIdentity: ManagedAgentKernelProcessRecord; - readonly processGroupId: number; - readonly groupLeaderIdentity?: ManagedAgentKernelProcessRecord; - containmentSupported: boolean; - ownershipProven: boolean; - stopIssued: boolean; - forceKillIssued: boolean; + accepted: boolean; + closed: boolean; + identity?: ManagedAgentKernelProcessRecord; } interface ObservedIdentity { @@ -361,7 +359,7 @@ async function windowsProcessTable(): Promise { async function posixProcessTable(): Promise { const { stdout } = await execFileAsync( "/bin/ps", - ["-axo", "pid=,ppid=,pgid=,lstart="], + ["-axo", "pid=,ppid=,pgid=,stat=,lstart="], { encoding: "utf8", windowsHide: true, @@ -372,14 +370,15 @@ async function posixProcessTable(): Promise { ); const entries: Array = []; for (const line of stdout.split("\n")) { - const match = /^\s*(\d+)\s+(\d+)\s+(\d+)\s+(.+?)\s*$/.exec(line); + const match = /^\s*(\d+)\s+(\d+)\s+(\d+)\s+(\S+)\s+(.+?)\s*$/.exec(line); if (!match) continue; entries.push([ Number(match[1]), { parentPid: Number(match[2]), processGroupId: Number(match[3]), - startedAt: match[4]!, + state: match[4]!, + startedAt: match[5]!, }, ]); } @@ -457,18 +456,20 @@ function descendantsOf( /** * E0.4 deliberately certifies one narrow containment model. The SDK command * runs in an observer-owned POSIX process group. The exact host-created L2 - * fixture additionally authenticates over a private one-shot Unix socket - * outside the workspace and keeps that connection open. A random capability, - * a primary exact-Bash policy latch, and a fresh kernel table jointly grant - * fallback signal authority for the fixture's distinct PGID. The capability - * remains sufficient if SDK cleanup has already reparented the live fixture; - * this exception is safe only because L2 permits one immutable trusted command. + * fixture parent and child additionally authenticate over separate private + * Unix-socket connections outside the workspace and keep those connections + * open for their complete lifetimes. A random capability, a primary exact-Bash + * policy latch, stable role identities, and fresh kernel ancestry prove that + * every member of their detached group remains below the active owned root. + * A tool-reported or cached PID/PGID never grants signal authority by itself. * * This is not universal built-in Bash containment or a process-tree killer. - * Windows, an unavailable process table, an unauthenticated tool process, or - * identity drift fail certification closed. POSIX `lstart` remains evidence, - * not authority. Workspace PID-file contents never enter this class and can - * never become signal authority. + * Windows, an unavailable process table, missing lifetime channels, or + * identity/ancestry drift fail certification closed. The fallback stops the + * owned root first, revalidates all authority, then stops and revalidates the + * exact fixture group before killing it. POSIX `lstart` remains one component + * of fresh identity evidence, not standalone authority. Workspace PID-file + * contents never enter this class. */ export class LocalManagedAgentProcessObserver implements ManagedAgentProcessObserver { readonly #platform: NodeJS.Platform; @@ -495,8 +496,16 @@ export class LocalManagedAgentProcessObserver implements ManagedAgentProcessObse #toolControlAvailable = false; #toolControlFailed = false; #toolProcessContainmentArmed = false; - #pendingToolRegistration: PendingToolRegistration | undefined; - #ownedToolGroup: OwnedToolGroup | undefined; + readonly #toolProcessRegistrations = new Map< + ToolProcessRole, + ToolProcessRegistration + >(); + #toolProcessGroupId: number | undefined; + #toolProcessRootPid: number | undefined; + #toolProcessObservationComplete = false; + #toolProcessObservationInvalid = false; + #toolProcessStopIssued = false; + #toolProcessForceKillIssued = false; #fallbackCleanupRequested = false; #lastTable: ManagedAgentKernelProcessTable | undefined; #processTableAvailable = false; @@ -554,7 +563,22 @@ export class LocalManagedAgentProcessObserver implements ManagedAgentProcessObse #receiveToolRegistration(socket: NetSocket): void { this.#toolControlSockets.add(socket); socket.on("error", () => undefined); - socket.once("close", () => this.#toolControlSockets.delete(socket)); + let registration: ToolProcessRegistration | undefined; + socket.once("close", () => { + this.#toolControlSockets.delete(socket); + if (!registration) return; + const current = this.#toolProcessRegistrations.get(registration.role); + if (current !== registration) return; + if (this.#toolProcessObservationComplete) { + registration.closed = true; + } else { + // A connection that disappears before readiness cannot reserve its + // role. Clearing it transactionally permits the trusted process to + // retry instead of leaving an unfinishable stale pending state. + this.#toolProcessRegistrations.delete(registration.role); + } + void this.observeProcessTree(); + }); let body = ""; let handled = false; const reject = (): void => { @@ -571,7 +595,11 @@ export class LocalManagedAgentProcessObserver implements ManagedAgentProcessObse const newline = body.indexOf("\n"); if (newline < 0) return; handled = true; - let payload: { capability?: unknown; pid?: unknown }; + let payload: { + capability?: unknown; + pid?: unknown; + role?: unknown; + }; try { payload = JSON.parse(body.slice(0, newline)) as typeof payload; } catch { @@ -580,10 +608,11 @@ export class LocalManagedAgentProcessObserver implements ManagedAgentProcessObse } if ( !this.#toolProcessContainmentArmed || - this.#pendingToolRegistration || - this.#ownedToolGroup || + this.#toolProcessObservationComplete || typeof payload.capability !== "string" || !sameCapability(payload.capability, this.#toolControlCapability) || + (payload.role !== "parent" && payload.role !== "child") || + this.#toolProcessRegistrations.has(payload.role) || typeof payload.pid !== "number" || !Number.isSafeInteger(payload.pid) || payload.pid <= 1 @@ -591,7 +620,14 @@ export class LocalManagedAgentProcessObserver implements ManagedAgentProcessObse socket.destroy(); return; } - this.#pendingToolRegistration = { pid: payload.pid, socket }; + registration = { + role: payload.role, + pid: payload.pid, + socket, + accepted: false, + closed: false, + }; + this.#toolProcessRegistrations.set(payload.role, registration); void this.observeProcessTree(); }); } @@ -602,14 +638,12 @@ export class LocalManagedAgentProcessObserver implements ManagedAgentProcessObse signal.addEventListener( "abort", () => { - this.#fallbackCleanupRequested = true; - this.#forceStopKillSynchronously(); + this.#requestFallbackCleanupSynchronously(); }, { once: true }, ); if (signal.aborted) { - this.#fallbackCleanupRequested = true; - this.#forceStopKillSynchronously(); + this.#requestFallbackCleanupSynchronously(); } } @@ -740,71 +774,153 @@ export class LocalManagedAgentProcessObserver implements ManagedAgentProcessObse } #observeToolProcessContainment(table: ManagedAgentKernelProcessTable): void { - const pending = this.#pendingToolRegistration; - if (pending && !pending.socket.destroyed && !this.#ownedToolGroup) { - const registeredIdentity = table.get(pending.pid); - const processGroupId = registeredIdentity?.processGroupId; + if (!this.#toolProcessContainmentArmed) return; + const parent = this.#toolProcessRegistrations.get("parent"); + const child = this.#toolProcessRegistrations.get("child"); + if ( + !this.#toolProcessObservationComplete && + parent && + child && + !parent.socket.destroyed && + !child.socket.destroyed + ) { + const parentIdentity = table.get(parent.pid); + const childIdentity = table.get(child.pid); + const processGroupId = parentIdentity?.processGroupId; const hostProcessGroupId = table.get(process.pid)?.processGroupId; + const groupLeaderIdentity = + typeof processGroupId === "number" + ? table.get(processGroupId) + : undefined; + const root = + this.#roots.size === 1 ? [...this.#roots.values()][0] : undefined; + const rootDescendants = root + ? descendantsOf(new Set([root.pid]), table) + : new Set(); + const groupMemberPids = + typeof processGroupId === "number" + ? [...table.entries()].flatMap(([pid, record]) => + record.processGroupId === processGroupId ? [pid] : [], + ) + : []; if ( - registeredIdentity && + root && + childActive(root.child) && + table.get(root.pid)?.processGroupId === root.pid && + parentIdentity && + childIdentity && typeof processGroupId === "number" && typeof hostProcessGroupId === "number" && processGroupId > 1 && processGroupId !== hostProcessGroupId && - !this.#roots.has(processGroupId) + !this.#roots.has(processGroupId) && + parent.pid !== child.pid && + childIdentity.parentPid === parent.pid && + childIdentity.processGroupId === processGroupId && + groupLeaderIdentity?.processGroupId === processGroupId && + groupMemberPids.length > 0 && + groupMemberPids.every((pid) => rootDescendants.has(pid)) ) { - const groupLeaderIdentity = table.get(processGroupId); - if ( - !groupLeaderIdentity || - groupLeaderIdentity.processGroupId === processGroupId - ) { - this.#ownedToolGroup = { - registeredPid: pending.pid, - registeredIdentity, - processGroupId, - ...(groupLeaderIdentity ? { groupLeaderIdentity } : {}), - containmentSupported: true, - ownershipProven: true, - stopIssued: false, - forceKillIssued: false, - }; - this.#pendingToolRegistration = undefined; - pending.socket.write('{"registered":true}\n'); + parent.identity = parentIdentity; + child.identity = childIdentity; + this.#toolProcessGroupId = processGroupId; + this.#toolProcessRootPid = root.pid; + for (const registration of [parent, child]) { + if (registration.accepted) continue; + registration.accepted = true; + registration.socket.write('{"registered":true}\n'); } } } - const owned = this.#ownedToolGroup; - if (!owned) return; - const currentRegistered = table.get(owned.registeredPid); - if ( - currentRegistered && - (!sameProcess(owned.registeredIdentity, currentRegistered) || - currentRegistered.processGroupId !== owned.processGroupId) - ) { - owned.containmentSupported = false; + const processGroupId = this.#toolProcessGroupId; + if (typeof processGroupId === "number") { + for (const [pid, record] of table) { + if (record.processGroupId === processGroupId) { + this.#observedPids.add(pid); + } + } } - if (owned.groupLeaderIdentity) { - const currentLeader = table.get(owned.processGroupId); + if (!this.#toolProcessObservationComplete) return; + for (const registration of this.#toolProcessRegistrations.values()) { + const current = table.get(registration.pid); if ( - currentLeader && - !sameProcess(owned.groupLeaderIdentity, currentLeader) + !registration.closed && + current && + (!sameProcess(registration.identity, current) || + current.processGroupId !== processGroupId) ) { - owned.containmentSupported = false; + this.#toolProcessObservationInvalid = true; } } - for (const [pid, record] of table) { - if (record.processGroupId === owned.processGroupId) { - this.#observedPids.add(pid); - } + } + + #hasFreshToolAuthority( + table: ManagedAgentKernelProcessTable, + options: { + readonly requireRootStopped: boolean; + readonly requireToolStopped: boolean; + }, + ): boolean { + const rootPid = this.#toolProcessRootPid; + const processGroupId = this.#toolProcessGroupId; + const root = + typeof rootPid === "number" ? this.#roots.get(rootPid) : undefined; + const parent = this.#toolProcessRegistrations.get("parent"); + const child = this.#toolProcessRegistrations.get("child"); + if ( + !root || + !childActive(root.child) || + this.#toolProcessObservationInvalid || + typeof processGroupId !== "number" || + !parent?.accepted || + !parent.identity || + !child?.accepted || + !child.identity || + ![parent, child].some( + ({ closed, socket }) => !closed && !socket.destroyed, + ) + ) { + return false; + } + + const currentRoot = table.get(root.pid); + const currentParent = table.get(parent.pid); + const currentChild = table.get(child.pid); + if ( + !currentRoot || + !currentParent || + !currentChild || + currentRoot.processGroupId !== root.pid || + (root.identity && !sameProcess(root.identity, currentRoot)) || + (options.requireRootStopped && !currentRoot.state?.includes("T")) || + !sameProcess(parent.identity, currentParent) || + currentParent.processGroupId !== processGroupId || + !sameProcess(child.identity, currentChild) || + currentChild.parentPid !== parent.pid || + currentChild.processGroupId !== processGroupId + ) { + return false; } - if (this.#fallbackCleanupRequested) this.#forceStopKillSynchronously(); + + const rootDescendants = descendantsOf(new Set([root.pid]), table); + const groupMembers = [...table.entries()].filter( + ([, record]) => record.processGroupId === processGroupId, + ); + return ( + groupMembers.length > 0 && + groupMembers.every( + ([pid, record]) => + rootDescendants.has(pid) && + (!options.requireToolStopped || record.state?.includes("T")), + ) + ); } public async observeProcessTree( timeoutMs = MANAGED_AGENT_PROCESS_HELPER_TIMEOUT_MS, ): Promise { - if (this.#roots.size === 0) { + if (this.#roots.size === 0 && !this.#toolProcessContainmentArmed) { this.#lastTable = new Map(); this.#processTableAvailable = true; return true; @@ -924,7 +1040,8 @@ export class LocalManagedAgentProcessObserver implements ManagedAgentProcessObse ({ containmentSupported }) => containmentSupported, ) && (!this.#toolProcessContainmentArmed || - Boolean(this.#ownedToolGroup?.containmentSupported)), + (this.#toolProcessRegistrations.size === 2 && + !this.#toolProcessObservationInvalid)), ownershipProven: false, observedPids: observedPids(), }); @@ -940,29 +1057,45 @@ export class LocalManagedAgentProcessObserver implements ManagedAgentProcessObse } if (this.#roots.size !== 1) return unsupported("root_count_invalid"); const root = [...this.#roots.values()][0]!; - if (!childActive(root.child)) { - root.containmentSupported = false; - return unsupported("root_not_active"); - } if (!root.containmentSupported) { return unsupported("containment_escaped"); } + if (!childActive(root.child)) { + return unsupported("root_not_active"); + } const currentRoot = this.#lastTable!.get(root.pid); if (!currentRoot || currentRoot.processGroupId !== root.pid) { root.containmentSupported = false; return unsupported("root_not_group_leader"); } + root.identity = currentRoot; if (this.#toolProcessContainmentArmed) { + const parent = this.#toolProcessRegistrations.get("parent"); + const child = this.#toolProcessRegistrations.get("child"); if ( !this.#toolControlAvailable || - !this.#ownedToolGroup || - !this.#ownedToolGroup.ownershipProven + !parent?.accepted || + parent.closed || + parent.socket.destroyed || + !child?.accepted || + child.closed || + child.socket.destroyed || + typeof this.#toolProcessGroupId !== "number" ) { return unsupported("tool_process_not_registered"); } - if (!this.#ownedToolGroup.containmentSupported) { + if (this.#toolProcessObservationInvalid) { return unsupported("tool_process_identity_invalid"); } + if ( + !this.#hasFreshToolAuthority(this.#lastTable!, { + requireRootStopped: false, + requireToolStopped: false, + }) + ) { + return unsupported("tool_process_identity_invalid"); + } + this.#toolProcessObservationComplete = true; } root.ownershipProven = true; @@ -976,9 +1109,8 @@ export class LocalManagedAgentProcessObserver implements ManagedAgentProcessObse }; } - #forceStopKillSynchronously(): void { + #stopOwnedRootsSynchronously(): void { if (this.#platform !== "darwin" && this.#platform !== "linux") return; - this.#fallbackCleanupRequested = true; for (const root of this.#roots.values()) { if (root.forceKillIssued || !childActive(root.child)) continue; if (!root.stopIssued) { @@ -989,41 +1121,75 @@ export class LocalManagedAgentProcessObserver implements ManagedAgentProcessObse } } } - const toolGroup = this.#ownedToolGroup; + } + + #killOwnedRootsSynchronously(): void { + if (this.#platform !== "darwin" && this.#platform !== "linux") return; + for (const root of this.#roots.values()) { + if (root.forceKillIssued || !childActive(root.child)) continue; + const killOutcome = this.#signalProcessGroup(root.pid, "SIGKILL"); + root.forceKillIssued = killOutcome === "sent"; + if (killOutcome === "failure") root.containmentSupported = false; + } + } + + #requestFallbackCleanupSynchronously(): void { + this.#fallbackCleanupRequested = true; + this.#stopOwnedRootsSynchronously(); if ( - toolGroup && - toolGroup.containmentSupported && - !toolGroup.forceKillIssued && - !toolGroup.stopIssued + !this.#toolProcessContainmentArmed || + !this.#toolProcessObservationComplete ) { - const stopOutcome = this.#signalProcessGroup( - toolGroup.processGroupId, - "SIGSTOP", - ); - toolGroup.stopIssued = stopOutcome === "sent"; - if (stopOutcome === "failure") { - toolGroup.containmentSupported = false; - } + this.#killOwnedRootsSynchronously(); } + } + + #advanceFallbackCleanup(): void { if ( - toolGroup && - toolGroup.containmentSupported && - !toolGroup.forceKillIssued + !this.#fallbackCleanupRequested || + this.#platform === "win32" || + !this.#processTableAvailable ) { - const killOutcome = this.#signalProcessGroup( - toolGroup.processGroupId, - "SIGKILL", - ); - toolGroup.forceKillIssued = killOutcome === "sent"; - if (killOutcome === "failure") { - toolGroup.containmentSupported = false; - } + return; } - for (const root of this.#roots.values()) { - if (root.forceKillIssued || !childActive(root.child)) continue; - const killOutcome = this.#signalProcessGroup(root.pid, "SIGKILL"); - root.forceKillIssued = killOutcome === "sent"; - if (killOutcome === "failure") root.containmentSupported = false; + if ( + !this.#toolProcessContainmentArmed || + !this.#toolProcessObservationComplete + ) { + this.#killOwnedRootsSynchronously(); + return; + } + + const processGroupId = this.#toolProcessGroupId; + if (typeof processGroupId !== "number") return; + const groupLiveness = this.#processGroupLiveness(processGroupId); + if (groupLiveness === "gone") { + this.#killOwnedRootsSynchronously(); + return; + } + if (groupLiveness !== "alive") return; + + const table = this.#lastTable!; + if ( + !this.#hasFreshToolAuthority(table, { + requireRootStopped: true, + requireToolStopped: this.#toolProcessStopIssued, + }) + ) { + return; + } + if (!this.#toolProcessStopIssued) { + const stopOutcome = this.#signalProcessGroup(processGroupId, "SIGSTOP"); + this.#toolProcessStopIssued = stopOutcome === "sent"; + if (stopOutcome === "gone") this.#killOwnedRootsSynchronously(); + return; + } + if (!this.#toolProcessForceKillIssued) { + const killOutcome = this.#signalProcessGroup(processGroupId, "SIGKILL"); + this.#toolProcessForceKillIssued = killOutcome === "sent"; + if (killOutcome === "sent" || killOutcome === "gone") { + this.#killOwnedRootsSynchronously(); + } } } @@ -1057,48 +1223,64 @@ export class LocalManagedAgentProcessObserver implements ManagedAgentProcessObse } } } - const toolGroup = this.#ownedToolGroup; + const toolRegistrations = [...this.#toolProcessRegistrations.values()]; + const toolProcessGroupId = this.#toolProcessGroupId; + for (const registration of toolRegistrations) { + if (!registration.closed && !registration.socket.destroyed) { + alive.add(registration.pid); + } + } if (this.#processTableAvailable) { const table = this.#lastTable!; - if (toolGroup) { + if (typeof toolProcessGroupId === "number") { for (const [pid, record] of table) { - if (record.processGroupId === toolGroup.processGroupId) + if (record.processGroupId === toolProcessGroupId) { alive.add(pid); + } } - const groupLiveness = this.#processGroupLiveness( - toolGroup.processGroupId, - ); - if (groupLiveness === "alive") alive.add(toolGroup.processGroupId); + const groupLiveness = this.#processGroupLiveness(toolProcessGroupId); + if (groupLiveness === "alive") alive.add(toolProcessGroupId); if (groupLiveness === "unknown") { - toolGroup.containmentSupported = false; + this.#toolProcessObservationInvalid = true; } - } else if (this.#pendingToolRegistration) { - if (table.has(this.#pendingToolRegistration.pid)) { - alive.add(this.#pendingToolRegistration.pid); + } + for (const registration of toolRegistrations) { + if (table.has(registration.pid)) { + alive.add(registration.pid); } } } const processTableAvailable = roots.length === 0 || this.#processTableAvailable; + const toolProcessObservationComplete = + !this.#toolProcessContainmentArmed || + this.#toolProcessObservationComplete; + const toolProcessChannelsClosed = + !this.#toolProcessContainmentArmed || + (this.#toolProcessObservationComplete && + toolRegistrations.length === 2 && + toolRegistrations.every( + ({ closed, socket }) => closed || socket.destroyed, + )); const containmentSupported = roots.every(({ containmentSupported: supported }) => supported) && (!this.#toolProcessContainmentArmed || (this.#toolControlAvailable && - Boolean(toolGroup?.containmentSupported))); + this.#toolProcessObservationComplete && + !this.#toolProcessObservationInvalid)); const ownershipProven = roots.length > 0 && roots.every(({ ownershipProven }) => ownershipProven) && - (!this.#toolProcessContainmentArmed || - Boolean(toolGroup?.ownershipProven)); + toolProcessObservationComplete; const forceKillIssued = - roots.length > 0 && - roots.every(({ forceKillIssued }) => forceKillIssued) && - (!this.#toolProcessContainmentArmed || - Boolean(toolGroup?.forceKillIssued)); + roots.length > 0 && roots.every(({ forceKillIssued }) => forceKillIssued); const elapsedMs = Math.max(0, this.#now() - startedAt); const quiescent = - processTableAvailable && containmentSupported && alive.size === 0; + processTableAvailable && + containmentSupported && + toolProcessChannelsClosed && + alive.size === 0; return { quiescent, deadlineMet: quiescent, @@ -1106,6 +1288,8 @@ export class LocalManagedAgentProcessObserver implements ManagedAgentProcessObse containmentSupported, ownershipProven, forceKillIssued, + toolProcessObservationComplete, + toolProcessChannelsClosed, elapsedMs, observedPids: [...this.#observedPids].sort((left, right) => left - right), alivePidsAtDeadline: [...alive].sort((left, right) => left - right), @@ -1123,6 +1307,7 @@ export class LocalManagedAgentProcessObserver implements ManagedAgentProcessObse await this.observeProcessTree( Math.max(0, boundedTimeoutMs - elapsedBeforeSample), ); + this.#advanceFallbackCleanup(); const observation = await this.#currentObservation(startedAt, false); if (observation.quiescent) { return { @@ -1144,11 +1329,16 @@ export class LocalManagedAgentProcessObserver implements ManagedAgentProcessObse ): Promise { const startedAt = this.#now(); const boundedTimeoutMs = Math.max(0, timeoutMs); - this.#forceStopKillSynchronously(); + this.#requestFallbackCleanupSynchronously(); const confirmation = await this.waitForQuiescence(boundedTimeoutMs); + if (!confirmation.quiescent) this.#killOwnedRootsSynchronously(); const elapsedMs = Math.max(0, this.#now() - startedAt); + const roots = [...this.#roots.values()]; + const forceKillIssued = + roots.length > 0 && roots.every((root) => root.forceKillIssued); return { ...confirmation, + forceKillIssued, elapsedMs, deadlineMet: confirmation.quiescent && elapsedMs <= boundedTimeoutMs, emergencyCleanupAttempted: true, diff --git a/packages/harness/src/experimental/managed-agent-spike/runtime-sdk-loopback.test.ts b/packages/harness/src/experimental/managed-agent-spike/runtime-sdk-loopback.test.ts index f9ddf1c8d..c29529e5e 100644 --- a/packages/harness/src/experimental/managed-agent-spike/runtime-sdk-loopback.test.ts +++ b/packages/harness/src/experimental/managed-agent-spike/runtime-sdk-loopback.test.ts @@ -561,6 +561,8 @@ it.skipIf( containmentSupported: true, ownershipProven: true, forceKillIssued: true, + toolProcessObservationComplete: true, + toolProcessChannelsClosed: true, alivePidsAtDeadline: [], }); expect(fixturePids).toHaveLength(2); @@ -581,6 +583,7 @@ it.skipIf( } await fixture.cleanup(); } + expect(fixturePids.every((pid) => !processExists(pid))).toBe(true); }, 20_000, ); @@ -589,7 +592,7 @@ it.skipIf( process.platform === "win32" || process.versions.node !== MANAGED_AGENT_CONTRACT.certificationNodeVersion, )( - "contains the real SDK L2 Bash group when readiness fails before cancellation", + "records a teardown timeout when readiness failure leaves the Bash fixture alive", async () => { const fixture = await createManagedAgentFixture( () => "loopback-l2-early-error", @@ -624,6 +627,7 @@ it.skipIf( server.once("error", reject); server.listen(0, "127.0.0.1", resolve); }); + let recordedTerminal: string | undefined; try { const address = server.address() as AddressInfo; const ids = [RUN_ID, EXECUTION_ID]; @@ -668,24 +672,31 @@ it.skipIf( ); expect(inferenceTurn).toBe(1); - expect(result.terminal).toBe("query_error"); + recordedTerminal = result.terminal; + expect(result.terminal).toBe("teardown_timeout"); expect(result.cancellationRequested).toBe(false); expect(result.terminationEvidence).toEqual({ - beforePolicyOverride: "query_error", + beforePolicyOverride: "teardown_timeout", queryExecution: "iteration_aborted", sdkResult: "not_observed", }); expect(result.queryClosed).toBe(true); expect(result.teardown).toMatchObject({ - quiescent: true, - deadlineMet: true, - processTableAvailable: true, - containmentSupported: true, + quiescent: false, + deadlineMet: false, + containmentSupported: false, + ownershipProven: false, forceKillIssued: true, - alivePidsAtDeadline: [], + toolProcessObservationComplete: false, + toolProcessChannelsClosed: false, }); expect(fixturePids).toHaveLength(2); - expect(fixturePids.every((pid) => !processExists(pid))).toBe(true); + expect( + fixturePids.every((pid) => + result.teardown.alivePidsAtDeadline.includes(pid), + ), + ).toBe(true); + expect(fixturePids.every((pid) => processExists(pid))).toBe(true); } finally { await observer.emergencyCleanup(1_000); observer.dispose(); @@ -697,6 +708,8 @@ it.skipIf( } await fixture.cleanup(); } + expect(recordedTerminal).toBe("teardown_timeout"); + expect(fixturePids.every((pid) => !processExists(pid))).toBe(true); }, 20_000, ); diff --git a/packages/harness/src/experimental/managed-agent-spike/runtime.test.ts b/packages/harness/src/experimental/managed-agent-spike/runtime.test.ts index fc2eb6abc..a30ff3a20 100644 --- a/packages/harness/src/experimental/managed-agent-spike/runtime.test.ts +++ b/packages/harness/src/experimental/managed-agent-spike/runtime.test.ts @@ -49,6 +49,8 @@ function quiescentTeardown(): ManagedAgentTeardownObservation { containmentSupported: true, ownershipProven: true, forceKillIssued: true, + toolProcessObservationComplete: true, + toolProcessChannelsClosed: true, elapsedMs: 12, observedPids: [], alivePidsAtDeadline: [], @@ -524,6 +526,8 @@ describe("runManagedAgentProbe", () => { containmentSupported: true, ownershipProven: false, forceKillIssued: false, + toolProcessObservationComplete: true, + toolProcessChannelsClosed: true, elapsedMs: 5_001, observedPids: [8001], alivePidsAtDeadline: [8001], @@ -907,10 +911,70 @@ describe("runManagedAgentProbe", () => { ).toHaveLength(1); }); + it("keeps armed L2 readiness alive after an early query failure before aborting the SDK", async () => { + const { config } = await probeConfig("L2"); + const observer = fakeObserver(); + let now = 1_000; + let readinessCompleted = false; + let readinessWasAborted = false; + const result = await runManagedAgentProbe(config, { + hermeticGatewayOrigin: config.gatewayOrigin, + processObserver: observer, + now: () => now, + waitForCancellationSignal: (signal) => + new Promise((resolveReadiness, rejectReadiness) => { + const timer = setTimeout(() => { + now = 3_250; + readinessCompleted = true; + resolveReadiness(); + }, 25); + signal.addEventListener( + "abort", + () => { + if (!readinessCompleted) readinessWasAborted = true; + clearTimeout(timer); + rejectReadiness( + new Error("readiness aborted before registration"), + ); + }, + { once: true }, + ); + }), + queryFactory: ({ options }) => ({ + async *[Symbol.asyncIterator]() { + yield { + type: "system", + subtype: "init", + session_id: CANCEL_SESSION_ID, + }; + await invokePreToolUse(options, { + toolName: "Bash", + toolInput: { command: config.allowedBashCommands[0] }, + toolUseId: "toolu_early_query_failure", + }); + throw new Error("synthetic early query failure"); + }, + close: vi.fn(), + }), + }); + + expect(readinessCompleted).toBe(true); + expect(readinessWasAborted).toBe(false); + expect(observer.armToolProcessContainment).toHaveBeenCalledOnce(); + expect(observer.emergencyCleanup).toHaveBeenCalledWith(2_750); + expect(result.teardown.elapsedMs).toBe(2_250); + expect(result.cancellationRequested).toBe(false); + expect(result.terminationEvidence.beforePolicyOverride).toBe("query_error"); + }, 10_000); + it("abandons a never-resolving iterator next immediately after raw cancellation", async () => { const { config } = await probeConfig("L2"); const observer = fakeObserver(); const close = vi.fn(); + let markNextStarted: (() => void) | undefined; + const nextStarted = new Promise((resolveStarted) => { + markNextStarted = resolveStarted; + }); const resultPromise = runManagedAgentProbe(config, { hermeticGatewayOrigin: config.gatewayOrigin, processObserver: observer, @@ -918,19 +982,23 @@ describe("runManagedAgentProbe", () => { queryFactory: () => ({ [Symbol.asyncIterator]() { return { - next: () => new Promise>(() => undefined), + next: () => { + markNextStarted?.(); + return new Promise>(() => undefined); + }, }; }, close, }), }); + await nextStarted; const result = await Promise.race([ resultPromise, new Promise((_, reject) => setTimeout( () => reject(new Error("probe stayed blocked on iterator.next()")), - 500, + 2_000, ), ), ]); @@ -956,6 +1024,8 @@ const teardown = { containmentSupported: true, ownershipProven: true, forceKillIssued: true, + toolProcessObservationComplete: true, + toolProcessChannelsClosed: true, elapsedMs: 0, observedPids: [], alivePidsAtDeadline: [], @@ -1069,6 +1139,8 @@ process.stdout.write(JSON.stringify({ containmentSupported: true, ownershipProven: false, forceKillIssued: false, + toolProcessObservationComplete: true, + toolProcessChannelsClosed: true, elapsedMs: 5_001, observedPids: [9001], alivePidsAtDeadline: [9001], diff --git a/packages/harness/src/experimental/managed-agent-spike/runtime.ts b/packages/harness/src/experimental/managed-agent-spike/runtime.ts index 5289e5d1b..ee46d6efd 100644 --- a/packages/harness/src/experimental/managed-agent-spike/runtime.ts +++ b/packages/harness/src/experimental/managed-agent-spike/runtime.ts @@ -308,6 +308,24 @@ async function closeQueryBounded( } } +async function waitForTaskBounded( + task: Promise, + timeoutMs: number, +): Promise { + if (timeoutMs <= 0) return false; + let timeout: NodeJS.Timeout | undefined; + try { + return await Promise.race([ + task.then(() => true), + new Promise((resolveTimeout) => { + timeout = setTimeout(() => resolveTimeout(false), timeoutMs); + }), + ]); + } finally { + if (timeout) clearTimeout(timeout); + } +} + type ManagedAgentIteratorStep = | { readonly kind: "next"; readonly value: IteratorResult } | { readonly kind: "aborted" }; @@ -396,6 +414,9 @@ export async function runManagedAgentProbe( let queryFailed = false; let queryClosed = false; let cancellationTriggerFailed = false; + let cancellationSignalReady = false; + let queryIterationSettled = false; + let toolProcessContainmentArmed = false; let policyPreflightFailed = false; let promptEmbedded = false; let eventNormalizationFailure: ManagedAgentProbeResult["terminationEvidence"]["eventNormalizationFailure"]; @@ -433,6 +454,7 @@ export async function runManagedAgentProbe( evidence.decision === "allow" && evidence.reason === "exact_bash_command" ) { + toolProcessContainmentArmed = true; processObserver.armToolProcessContainment(); } }, @@ -514,6 +536,8 @@ export async function runManagedAgentProbe( .waitForCancellationSignal(triggerController.signal) .then(() => { if (triggerController.signal.aborted) return; + cancellationSignalReady = true; + if (queryIterationSettled) return; cancellationRequested = true; cancellationRequestedAt = (dependencies.now ?? Date.now)(); abortStartedAt = cancellationRequestedAt; @@ -523,8 +547,10 @@ export async function runManagedAgentProbe( .catch(() => { if (!triggerController.signal.aborted) { cancellationTriggerFailed = true; - abortStartedAt = (dependencies.now ?? Date.now)(); - abortController.abort(); + if (!queryIterationSettled) { + abortStartedAt = (dependencies.now ?? Date.now)(); + abortController.abort(); + } } }) : undefined; @@ -582,18 +608,40 @@ export async function runManagedAgentProbe( } if (!abortController.signal.aborted) queryFailed = true; } finally { + queryIterationSettled = true; + const now = dependencies.now ?? Date.now; + const armedEarlyQueryTeardown = + queryFailed && + toolProcessContainmentArmed && + Boolean(cancellationTask) && + !abortController.signal.aborted; + if (armedEarlyQueryTeardown) { + abortStartedAt ??= now(); + const readinessBudget = Math.max( + 0, + MANAGED_AGENT_TEARDOWN_TIMEOUT_MS - (now() - abortStartedAt), + ); + const taskSettled = await waitForTaskBounded( + cancellationTask!, + readinessBudget, + ); + if (!taskSettled || !cancellationSignalReady) { + cancellationTriggerFailed = true; + } + } triggerController.abort(); - if (cancellationTask) await cancellationTask; + if (cancellationTask && !armedEarlyQueryTeardown) { + await cancellationTask; + } queryFailed ||= cancellationTriggerFailed; // Give the SDK its documented graceful-shutdown path before host // fallback containment. The observer binds only SpawnOptions.signal, // which the SDK forwards after stdin EOF and its bounded grace period. if (queryFailed && !abortController.signal.aborted) { - abortStartedAt = (dependencies.now ?? Date.now)(); + abortStartedAt ??= (dependencies.now ?? Date.now)(); abortController.abort(); } if (query) { - const now = dependencies.now ?? Date.now; const closeBudgetMs = abortStartedAt === undefined ? QUERY_CLOSE_TIMEOUT_MS diff --git a/packages/harness/src/experimental/managed-agent-spike/types.ts b/packages/harness/src/experimental/managed-agent-spike/types.ts index 4ace32129..4629b5eeb 100644 --- a/packages/harness/src/experimental/managed-agent-spike/types.ts +++ b/packages/harness/src/experimental/managed-agent-spike/types.ts @@ -220,10 +220,14 @@ export interface ManagedAgentTeardownObservation { readonly processTableAvailable: boolean; /** False means the owned E0 containment model was escaped or unproven. */ readonly containmentSupported: boolean; - /** True only after the active POSIX supervisor is observed as PGID leader. */ + /** True after SDK-root authority and any required L2 observations are proven. */ readonly ownershipProven: boolean; - /** True only when the bound raw abort synchronously issued SIGSTOP+SIGKILL. */ + /** True only when SIGKILL was issued to every owned SDK supervisor root. */ readonly forceKillIssued: boolean; + /** True after both exact L2 fixture lifetime channels pass fresh observation. */ + readonly toolProcessObservationComplete: boolean; + /** True only when both observed L2 lifetime channels have closed. */ + readonly toolProcessChannelsClosed: boolean; readonly elapsedMs: number; readonly observedPids: readonly number[]; readonly alivePidsAtDeadline: readonly number[]; @@ -300,19 +304,19 @@ export interface ManagedAgentProcessObserver { /** Bind only the SDK-forwarded post-grace SpawnOptions signal. */ bindAbortSignal(signal: AbortSignal): void; /** - * Arm the one-shot, host-authenticated process registration used only by - * the exact E0.4 L2 fixture. Unregistered built-in Bash processes are not - * granted signal authority by this experimental observer. + * Arm the two host-authenticated lifetime observations used only by the + * exact E0.4 L2 fixture. Tool-reported identities never grant authority by + * themselves; the observer also requires fresh owned-root ancestry. */ armToolProcessContainment(): void; - /** Prove the narrow POSIX ownership model before allowing L2 to cancel. */ + /** Prove the narrow POSIX observation model before allowing L2 to cancel. */ prepareCancellation(): Promise; /** Sample only members owned by the host-observed process anchors. */ observeProcessTree(timeoutMs?: number): Promise; waitForQuiescence( timeoutMs: number, ): Promise; - /** Idempotently force an anchored owned group and confirm within this budget. */ + /** Idempotently run the anchored fallback and confirm within this budget. */ emergencyCleanup(timeoutMs: number): Promise; dispose(): void; } From 21a6f7fc9355f4344abad9964e1a0c6c8edcdd48 Mon Sep 17 00:00:00 2001 From: Yash Date: Mon, 17 Aug 2026 03:35:52 -0700 Subject: [PATCH 16/24] fix(harness): preserve managed-agent teardown ancestry --- .../managed-agent-spike/README.md | 75 +- .../managed-agent-spike/probe-cli.test.ts | 14 + .../process-observer.test.ts | 1765 ++++++++++++++--- .../managed-agent-spike/process-observer.ts | 566 +++++- .../runtime-sdk-loopback.test.ts | 548 ++++- .../managed-agent-spike/runtime.test.ts | 196 ++ .../managed-agent-spike/runtime.ts | 62 +- .../experimental/managed-agent-spike/types.ts | 2 + 8 files changed, 2828 insertions(+), 400 deletions(-) diff --git a/packages/harness/src/experimental/managed-agent-spike/README.md b/packages/harness/src/experimental/managed-agent-spike/README.md index e1e5c0d25..01a9c3e83 100644 --- a/packages/harness/src/experimental/managed-agent-spike/README.md +++ b/packages/harness/src/experimental/managed-agent-spike/README.md @@ -121,36 +121,69 @@ lifetime. Before cancellation may fire, a fresh bounded `ps` sample must observe an active SDK supervisor root with stable identity, both role-tagged PIDs, their parent-child relationship, their shared process group, and every current group member as a descendant of that owned root. The group must be distinct from both -the host and SDK supervisor groups. The random capability and role-tagged -lifetime channels are necessary evidence, but a claimed or cached PID/PGID never -grants signal authority by itself. The model-writable fixture PID file is used -only by the test driver and never enters the observer. +the host and SDK supervisor groups. Every observed root/tool descendant retains +an immutable creation-time, parent, process-group, and session baseline. Once L2 +tool containment is armed, every current root descendant must remain in the +supervisor group or authenticated tool group; reparenting or PGID/session +migration fails closed. The random capability and role-tagged lifetime channels +are necessary evidence, but a claimed or cached PID/PGID never grants signal +authority by itself. The model-writable fixture PID file is used only by the +test driver and never enters the observer. + +Outside that L2 gate, an unarmed L1 run can briefly create an SDK-owned +subprocess group while the process is still a descendant of the supervisor. +That subgroup is never signal authority. The observer remembers each exact +identity and treats it as pending: +readiness and quiescence remain false, and fallback cannot kill the supervisor +root, while any pending identity is live. A later complete process-table sample +may clear it only by proving that exact identity is absent or a zombie. Any +parent, group, session, or ancestry change before that positive death evidence +permanently fails containment closed. Once L2 tool containment is armed, only +the authenticated supervisor and fixture groups are permitted; an additional +descendant group rejects readiness. The runtime gives the Agent SDK its documented abort and bounded query-close path first. It does not bind the raw per-run `Options.abortController` to host signals. Only the SDK-forwarded post-grace `SpawnOptions.signal` can trigger the -fallback. The fallback first stops the observer-created SDK supervisor group. -A new process-table sample must then revalidate the active root identity, both -role identities, their relationship and shared group, every current tool-group -member's ancestry, and at least one open lifetime channel. Only that fresh proof -authorizes `SIGSTOP` to the detached fixture group. A second fresh sample must -show both the root and every tool-group member stopped before `SIGKILL` is sent -to the fixture group and then the SDK supervisor group. Failed tool stop/kill -attempts remain retryable, but every retry requires another fresh proof. The -five-second absolute deadline bounds the entire sequence. - -If the root exits, an identity changes or disappears, a foreign member appears, -ancestry is lost, both channels close prematurely, or a process-table read is -unavailable, the observer never signals the detached group. It may still stop -or kill its own live SDK supervisor group, but the run remains a fail-closed +fallback. In SDK 0.3.228, `Query.close()` starts cleanup but returns `void`, so +the runtime immediately follows it with and awaits `Query.return()` under the +same deadline. `queryClosed` means that awaitable cleanup settled; invoking +`close()` alone is never completion evidence. Host emergency cleanup starts +only after that cleanup settles, the forwarded signal has already requested the +fallback, or the bounded SDK-grace budget expires. The returned handle accepts +the first SDK `child.kill()` logically by setting `child.killed = true`, but it +intentionally sends no native signal. The SDK-forwarded abort signal requests +the sampled host fallback; only freshly validated host group cleanup sends +signals. The fallback first stops the observer-created SDK supervisor group. A +new process-table sample must then revalidate the active root identity, both +role identities, their relationship and shared group, every +current root/tool descendant's parent, group, session, and ancestry, and at +least one open lifetime channel. Only that fresh proof authorizes `SIGSTOP` to +the detached fixture group. A second fresh sample must show both the root and +every tool-group member stopped before `SIGKILL` is sent to the fixture group +and then the SDK supervisor group. Failed tool stop/kill attempts remain +retryable, but every retry requires another fresh proof. The five-second +absolute deadline bounds the entire sequence. + +If the root exits, a stable identity changes parent/group/session, a foreign +member appears, ancestry is lost, both channels close prematurely, or a +process-table read is unavailable, the observer never signals the detached +group. This includes an inner SDK command exit that reparents a surviving +descendant: an unchanged old PGID does not retain authority after ancestry is +lost. A successful complete table that no longer contains the stable identity +is positive exit evidence; otherwise an escaped same-identity PID and its new +group remain in final liveness accounting. The observer may still stop or kill +its own live SDK supervisor group, but the run remains a fail-closed `teardown_timeout` while any tool process or lifetime channel remains. This also prevents numeric PID/PGID reuse from converting cached evidence into authority. `forceKillIssued` describes only owned SDK supervisor roots and is not required when SDK graceful shutdown succeeds. -If an exact Bash launch is armed and the query ends early, its registration task -is not discarded. Readiness, SDK abort/close, owned-root fallback, and death -confirmation share one absolute five-second clock. Safe L2 completion requires +If an exact Bash launch is armed and the query settles before readiness—by +throwing, clean iterator completion, or an SDK error result—its registration +task is not discarded. Readiness, SDK abort/close/return, owned-root fallback, +and death confirmation share one absolute five-second clock. Such an early +settlement never fabricates `cancellationRequested`. Safe L2 completion requires that both authenticated lifetime channels were observed, both closed, and a fresh table/liveness sample found no member of the observed fixture group. A missing channel, an open channel, or a live observed group produces diff --git a/packages/harness/src/experimental/managed-agent-spike/probe-cli.test.ts b/packages/harness/src/experimental/managed-agent-spike/probe-cli.test.ts index bd820f9ae..e0d6f547a 100644 --- a/packages/harness/src/experimental/managed-agent-spike/probe-cli.test.ts +++ b/packages/harness/src/experimental/managed-agent-spike/probe-cli.test.ts @@ -1134,6 +1134,20 @@ describe("managed-agent probe CLI", () => { } }); + it("never certifies a cancelled terminal without a requested cancellation", () => { + const passing = passingL2Result(); + const report = evaluateManagedAgentProbe( + { ...passing, cancellationRequested: false }, + [12_345, 12_346], + ); + + expect(report.outcome).toBe("fail"); + expect(report.checks).toContainEqual({ + id: "cancellation_requested", + passed: false, + }); + }); + it.each([ [ "omitted", diff --git a/packages/harness/src/experimental/managed-agent-spike/process-observer.test.ts b/packages/harness/src/experimental/managed-agent-spike/process-observer.test.ts index ef5ea608d..372968835 100644 --- a/packages/harness/src/experimental/managed-agent-spike/process-observer.test.ts +++ b/packages/harness/src/experimental/managed-agent-spike/process-observer.test.ts @@ -1,8 +1,8 @@ import { once } from "node:events"; import { + ChildProcess, execFile, spawn as spawnChild, - type ChildProcess, type ChildProcessWithoutNullStreams, } from "node:child_process"; import { readFile, writeFile } from "node:fs/promises"; @@ -11,7 +11,7 @@ import { join } from "node:path"; import { promisify } from "node:util"; import type { SpawnedProcess } from "@anthropic-ai/claude-agent-sdk"; -import { afterEach, describe, expect, it } from "vitest"; +import { afterEach, describe, expect, it, vi } from "vitest"; import { FIXTURE_PATHS, @@ -23,6 +23,8 @@ import { LocalManagedAgentProcessObserver, MANAGED_AGENT_TOOL_CONTROL_CAPABILITY_ENV, MANAGED_AGENT_TOOL_CONTROL_SOCKET_ENV, + managedAgentPosixSessionColumn, + parseManagedAgentPosixProcessTable, type ManagedAgentKernelProcessRecord, type ManagedAgentProcessTableObservation, } from "./process-observer.js"; @@ -42,27 +44,16 @@ function available( async function readRealPosixProcessTable(): Promise { try { + const sessionColumn = managedAgentPosixSessionColumn(process.platform); const { stdout } = await execFileAsync( "/bin/ps", - ["-axo", "pid=,ppid=,pgid=,stat=,lstart="], + ["-axo", `pid=,ppid=,pgid=,${sessionColumn}=,stat=,lstart=`], { encoding: "utf8", maxBuffer: 4 * 1024 * 1024, timeout: 1_000 }, ); - const entries: Array = - []; - for (const line of stdout.split("\n")) { - const match = /^\s*(\d+)\s+(\d+)\s+(\d+)\s+(\S+)\s+(.+?)\s*$/.exec(line); - if (!match) continue; - entries.push([ - Number(match[1]), - { - parentPid: Number(match[2]), - processGroupId: Number(match[3]), - state: match[4]!, - startedAt: match[5]!, - }, - ]); - } - return available(entries); + return { + available: true, + processes: parseManagedAgentPosixProcessTable(stdout), + }; } catch { return { available: false }; } @@ -85,6 +76,19 @@ async function prepareCancellationAfterTransientReadFailure( } } +async function waitForContainmentEscape( + observer: LocalManagedAgentProcessObserver, + timeoutMs = 2_000, +) { + const deadline = Date.now() + timeoutMs; + let readiness = await observer.prepareCancellation(); + while (readiness.reason !== "containment_escaped" && Date.now() < deadline) { + await new Promise((resolveDelay) => setTimeout(resolveDelay, 10)); + readiness = await observer.prepareCancellation(); + } + return readiness; +} + function activeNodeCommand(): { command: string; args: string[] } { return { command: process.execPath, @@ -100,12 +104,20 @@ import { resolve } from "node:path"; const pidFile = resolve(process.argv[1]); const exitTiming = process.argv[2]; const exitMarker = resolve(process.argv[3]); +const cleanupMarker = resolve(process.argv[4]); const childProgram = [ + 'const { existsSync } = require("node:fs");', + 'const cleanupMarker = process.argv[1];', 'process.on("SIGTERM", () => {});', 'if (process.send) process.send("ready");', + 'const cleanupPoll = setInterval(() => {', + ' if (!existsSync(cleanupMarker)) return;', + ' clearInterval(cleanupPoll);', + ' process.exit(0);', + '}, 10);', 'setInterval(() => {}, 1000);', ].join(""); -const child = spawn(process.execPath, ["-e", childProgram], { +const child = spawn(process.execPath, ["-e", childProgram, cleanupMarker], { stdio: ["ignore", "ignore", "ignore", "ipc"], windowsHide: true, }); @@ -378,56 +390,6 @@ async function waitForLaunchedGroupId( } } -async function forceKillExactTestGroup( - processGroupId: number, - root: ChildProcess, -): Promise { - try { - process.kill(-processGroupId, "SIGCONT"); - } catch (error) { - if ((error as NodeJS.ErrnoException).code !== "ESRCH") throw error; - } - try { - process.kill(-processGroupId, "SIGKILL"); - } catch (error) { - if ((error as NodeJS.ErrnoException).code !== "ESRCH") throw error; - } - await waitForTestProcessDeath( - () => processGroupExists(processGroupId), - `Owned process group ${processGroupId}`, - ); - if (root.exitCode === null && root.signalCode === null) { - await Promise.race([ - once(root, "exit"), - new Promise((_, rejectTimeout) => - setTimeout( - () => rejectTimeout(new Error("Owned root did not report exit")), - 1_000, - ), - ), - ]); - } -} - -async function forceKillExactTestGroupId( - processGroupId: number, -): Promise { - try { - process.kill(-processGroupId, "SIGCONT"); - } catch (error) { - if ((error as NodeJS.ErrnoException).code !== "ESRCH") throw error; - } - try { - process.kill(-processGroupId, "SIGKILL"); - } catch (error) { - if ((error as NodeJS.ErrnoException).code !== "ESRCH") throw error; - } - await waitForTestProcessDeath( - () => processGroupExists(processGroupId), - `Owned process group ${processGroupId}`, - ); -} - async function waitForChildExitBounded( child: ChildProcess, timeoutMs = 1_000, @@ -458,6 +420,126 @@ async function forceKillExactTestProcess(child: ChildProcess): Promise { ); } +async function captureExactTestProcessIdentities( + pids: readonly number[], +): Promise> { + const observation = await readRealPosixProcessTable(); + if (!observation.available) { + throw new Error("Process table unavailable for exact test cleanup"); + } + const identities = new Map(); + for (const pid of pids) { + const identity = observation.processes.get(pid); + if (!identity) throw new Error(`Test process ${pid} disappeared too early`); + identities.set(pid, identity); + } + return identities; +} + +async function forceKillExactTestProcessIdentities( + identities: ReadonlyMap, +): Promise { + if (identities.size === 0) return; + const observation = await readRealPosixProcessTable(); + if (!observation.available) { + throw new Error("Process table unavailable for exact test cleanup"); + } + for (const [pid, identity] of identities) { + const current = observation.processes.get(pid); + if ( + !current || + current.startedAt !== identity.startedAt || + current.state?.startsWith("Z") + ) { + continue; + } + try { + process.kill(pid, "SIGKILL"); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ESRCH") throw error; + } + } + const deadline = Date.now() + 1_000; + for (;;) { + const survivors = await liveExactTestProcessIdentities(identities); + if (survivors.length === 0) return; + if (Date.now() >= deadline) { + throw new Error( + `Exact test processes ${survivors + .map((pid) => pid) + .join(", ")} survived test cleanup`, + ); + } + await new Promise((resolveDelay) => setTimeout(resolveDelay, 10)); + } +} + +async function liveExactTestProcessIdentities( + identities: ReadonlyMap, +): Promise { + const current = await readRealPosixProcessTable(); + if (!current.available) { + throw new Error("Process table unavailable during exact test cleanup"); + } + return [...identities].flatMap(([pid, identity]) => { + const record = current.processes.get(pid); + return record?.startedAt === identity.startedAt && + !record.state?.startsWith("Z") + ? [pid] + : []; + }); +} + +async function forceKillRetainedTestGroup(root: ChildProcess): Promise { + const processGroupId = root.pid; + if ( + typeof processGroupId !== "number" || + root.exitCode !== null || + root.signalCode !== null + ) { + return; + } + + const observation = await readRealPosixProcessTable(); + if (!observation.available) { + throw new Error("Process table unavailable for retained group cleanup"); + } + const leader = observation.processes.get(processGroupId); + if ( + !leader || + leader.state?.startsWith("Z") || + leader.processGroupId !== processGroupId + ) { + throw new Error( + `Refusing cached group cleanup for unverified root ${processGroupId}`, + ); + } + const identities = new Map( + [...observation.processes].filter( + ([, record]) => + record.processGroupId === processGroupId && + !record.state?.startsWith("Z"), + ), + ); + if (!identities.has(processGroupId)) { + throw new Error( + `Refusing group cleanup without live leader identity ${processGroupId}`, + ); + } + if (root.exitCode !== null || root.signalCode !== null) return; + + // The retained, still-active ChildProcess plus this fresh kernel snapshot is + // the complete authority for the one group signal below. Never probe or + // signal this negative PGID again after the leader can have exited. + try { + process.kill(-processGroupId, "SIGKILL"); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ESRCH") throw error; + } + await waitForChildExitBounded(root); + await forceKillExactTestProcessIdentities(identities); +} + async function proveRetainedGroupAuthority( exitTiming: "before-readiness" | "after-readiness", ): Promise { @@ -465,7 +547,15 @@ async function proveRetainedGroupAuthority( () => `fast-root-exit-${exitTiming}`, ); fixtures.push(fixture); - const observer = new LocalManagedAgentProcessObserver(); + const productionGroupSignals: Array< + readonly [number, "SIGSTOP" | "SIGKILL"] + > = []; + const observer = new LocalManagedAgentProcessObserver({ + signalProcessGroup: (processGroupId, signal) => { + productionGroupSignals.push([processGroupId, signal]); + return signalRealProcessGroup(processGroupId, signal); + }, + }); const rawController = new AbortController(); const forwardedController = new AbortController(); const unrelated = spawnChild( @@ -476,13 +566,20 @@ async function proveRetainedGroupAuthority( await once(unrelated, "spawn"); observer.bindAbortSignal(rawController.signal); let anchor: ChildProcessWithoutNullStreams | undefined; - let ownedProcessGroupId: number | undefined; + let nonCooperativeChildPid: number | undefined; + let exitMarker: string | undefined; + let cleanupMarker: string | undefined; try { - const exitMarker = join( + exitMarker = join( fixture.workspaceRoot, FIXTURE_PATHS.processDirectory, "exit-inner-root", ); + cleanupMarker = join( + fixture.workspaceRoot, + FIXTURE_PATHS.processDirectory, + "exit-non-cooperative-child", + ); anchor = asChildProcess( observer.spawn({ command: process.execPath, @@ -493,20 +590,27 @@ async function proveRetainedGroupAuthority( FIXTURE_PATHS.processPidFile, exitTiming, exitMarker, + cleanupMarker, ], cwd: fixture.workspaceRoot, env: { ...process.env }, signal: forwardedController.signal, }), ); - ownedProcessGroupId = anchor.pid; - expect(ownedProcessGroupId).toBeTypeOf("number"); - const [workerRootPid, nonCooperativeChildPid] = + expect(anchor.pid).toBeTypeOf("number"); + const [workerRootPid, fixtureChildPid] = await waitForManagedAgentFixturePids(fixture); + nonCooperativeChildPid = fixtureChildPid; - let readiness; + let initialReadiness; if (exitTiming === "after-readiness") { - readiness = await prepareCancellationAfterTransientReadFailure(observer); + initialReadiness = + await prepareCancellationAfterTransientReadFailure(observer); + expect(initialReadiness).toMatchObject({ + supported: true, + reason: "ready", + ownershipProven: true, + }); await writeFile(exitMarker, "exit\n"); } await waitForTestProcessDeath( @@ -514,35 +618,49 @@ async function proveRetainedGroupAuthority( `Fast SDK root ${workerRootPid}`, ); expect(processExists(nonCooperativeChildPid!)).toBe(true); - if (!readiness) { - readiness = await prepareCancellationAfterTransientReadFailure(observer); - } - - expect(readiness).toMatchObject({ - supported: true, - reason: "ready", - ownershipProven: true, - }); - expect(readiness.observedPids).toContain(nonCooperativeChildPid); - expect(readiness.observedPids).not.toContain(unrelated.pid); - - rawController.abort(); - const teardown = await observer.emergencyCleanup(1_000); - expect(teardown).toMatchObject({ - quiescent: true, - deadlineMet: true, - ownershipProven: true, - forceKillIssued: true, - alivePidsAtDeadline: [], + const escapedReadiness = await waitForContainmentEscape(observer); + expect(escapedReadiness).toMatchObject({ + supported: false, + reason: "containment_escaped", + ownershipProven: false, }); - expect(processExists(nonCooperativeChildPid!)).toBe(false); + expect(escapedReadiness.observedPids).toContain(nonCooperativeChildPid); + expect(escapedReadiness.observedPids).not.toContain(unrelated.pid); + expect(processExists(nonCooperativeChildPid)).toBe(true); expect(processExists(unrelated.pid!)).toBe(true); + await expect(observer.waitForQuiescence(0)).resolves.toMatchObject({ + quiescent: false, + deadlineMet: false, + containmentSupported: false, + forceKillIssued: false, + }); + expect(productionGroupSignals).toEqual([]); } finally { + if (exitMarker) + await writeFile(exitMarker, "exit\n").catch(() => undefined); + if (cleanupMarker) { + await writeFile(cleanupMarker, "exit\n").catch(() => undefined); + } + if (typeof nonCooperativeChildPid === "number") { + await waitForTestProcessDeath( + () => processExists(nonCooperativeChildPid!), + `Escaped fixture child ${nonCooperativeChildPid}`, + 500, + ).catch(() => undefined); + } + if (anchor && anchor.exitCode === null && anchor.signalCode === null) { + if (anchor.connected) anchor.disconnect(); + else await forceKillRetainedTestGroup(anchor); + await waitForChildExitBounded(anchor); + } + if (typeof nonCooperativeChildPid === "number") { + await waitForTestProcessDeath( + () => processExists(nonCooperativeChildPid!), + `Escaped fixture child ${nonCooperativeChildPid}`, + ); + } rawController.abort(); forwardedController.abort(); - if (anchor && typeof ownedProcessGroupId === "number") { - await forceKillExactTestGroup(ownedProcessGroupId, anchor); - } observer.dispose(); await forceKillExactTestProcess(unrelated); } @@ -555,12 +673,14 @@ interface RegisteredDescendantToolRun { readonly anchor: ChildProcessWithoutNullStreams; readonly toolPids: readonly [number, number]; readonly toolProcessGroupId: number; + readonly toolIdentities: ReadonlyMap; } interface RegisteredDescendantToolSetupEvidence { - readonly anchorProcessGroupId: number; + readonly anchor: ChildProcessWithoutNullStreams; readonly toolPids: readonly [number, number]; readonly toolProcessGroupId: number; + readonly toolIdentities: ReadonlyMap; } interface RegisteredDescendantSetupCleanupError extends Error { @@ -568,6 +688,10 @@ interface RegisteredDescendantSetupCleanupError extends Error { readonly cleanupErrors: readonly unknown[]; } +interface RegisteredDescendantCleanupError extends Error { + readonly cleanupErrors: readonly unknown[]; +} + function setupAndCleanupFailure( setupError: unknown, cleanupErrors: readonly unknown[], @@ -582,6 +706,18 @@ function setupAndCleanupFailure( return failure; } +function registeredDescendantCleanupFailure( + cleanupErrors: readonly unknown[], +): RegisteredDescendantCleanupError { + const failure = new Error( + "Registered descendant cleanup failed", + ) as RegisteredDescendantCleanupError; + Object.defineProperty(failure, "cleanupErrors", { + value: [...cleanupErrors], + }); + return failure; +} + async function startRegisteredDescendantToolRun( observer: LocalManagedAgentProcessObserver, name: string, @@ -610,10 +746,13 @@ async function startRegisteredDescendantToolRun( signal: forwardedController.signal, }), ); + let toolPids: readonly [number, number] | undefined; let toolProcessGroupId: number | undefined; + let toolIdentities: + | ReadonlyMap + | undefined; try { - const anchorProcessGroupId = anchor.pid; - if (typeof anchorProcessGroupId !== "number") { + if (typeof anchor.pid !== "number") { throw new Error("Owned fixture anchor failed to spawn"); } toolProcessGroupId = await waitForLaunchedGroupId(launchFile); @@ -621,14 +760,19 @@ async function startRegisteredDescendantToolRun( fixture, 5_000, ); - const toolPids = [parentPid!, childPid!] as const; + toolPids = [parentPid!, childPid!] as const; if (parentPid !== toolProcessGroupId) { throw new Error("Detached fixture group does not match its parent PID"); } + // Capture stable positive-PID identities before user callbacks or + // assertions can fail. A detached numeric group id is never cleanup + // authority on its own. + toolIdentities = await captureExactTestProcessIdentities(toolPids); await afterPidPublication?.({ - anchorProcessGroupId, + anchor, toolPids, toolProcessGroupId, + toolIdentities, }); await expect( prepareCancellationAfterTransientReadFailure(observer), @@ -644,23 +788,25 @@ async function startRegisteredDescendantToolRun( anchor, toolPids, toolProcessGroupId, + toolIdentities, }; } catch (setupError) { const cleanupErrors: unknown[] = []; try { - if ( - typeof toolProcessGroupId === "number" && - processGroupExists(toolProcessGroupId) - ) { - await forceKillExactTestGroupId(toolProcessGroupId); + if (toolIdentities) { + await forceKillExactTestProcessIdentities(toolIdentities); + } else if (toolPids || typeof toolProcessGroupId === "number") { + cleanupErrors.push( + new Error( + "Refusing detached tool cleanup without pre-captured identities", + ), + ); } } catch (error) { cleanupErrors.push(error); } try { - if (typeof anchor.pid === "number") { - await forceKillExactTestGroup(anchor.pid, anchor); - } + await forceKillRetainedTestGroup(anchor); } catch (error) { cleanupErrors.push(error); } finally { @@ -678,27 +824,56 @@ async function cleanupRegisteredDescendantToolRun( run: RegisteredDescendantToolRun | undefined, ): Promise { if (!run) return; - if (processGroupExists(run.toolProcessGroupId)) { - await forceKillExactTestGroupId(run.toolProcessGroupId); + const cleanupErrors: unknown[] = []; + try { + await forceKillExactTestProcessIdentities(run.toolIdentities); + } catch (error) { + cleanupErrors.push(error); } - if (typeof run.anchor.pid === "number") { + try { if (run.anchor.exitCode === null && run.anchor.signalCode === null) { await waitForChildExitBounded(run.anchor, 100).catch(() => undefined); } - if (run.anchor.exitCode === null && run.anchor.signalCode === null) { - await forceKillExactTestGroup(run.anchor.pid, run.anchor); - } else { - await waitForTestProcessDeath( - () => processGroupExists(run.anchor.pid!), - `Owned root group ${run.anchor.pid}`, - ); - } + await forceKillRetainedTestGroup(run.anchor); + } catch (error) { + cleanupErrors.push(error); + } finally { + run.forwardedController.abort(); + run.observer.dispose(); + } + if (cleanupErrors.length > 0) { + throw registeredDescendantCleanupFailure(cleanupErrors); } - run.forwardedController.abort(); - run.observer.dispose(); } describe("LocalManagedAgentProcessObserver", () => { + it.each([ + ["darwin", "sess"], + ["linux", "sid"], + ] as const)( + "uses the %s process-table session column", + (platform, expectedColumn) => { + expect(managedAgentPosixSessionColumn(platform)).toBe(expectedColumn); + }, + ); + + it.each([ + ["Darwin sess= layout", 0], + ["Linux sid= layout", 100], + ] as const)("parses the %s", (_layout, sessionId) => { + const table = parseManagedAgentPosixProcessTable( + ` 100 1 100 ${sessionId} Ss Mon Aug 17 01:02:03 2026\n`, + ); + + expect(table.get(100)).toEqual({ + parentPid: 1, + processGroupId: 100, + sessionId, + state: "Ss", + startedAt: "Mon Aug 17 01:02:03 2026", + }); + }); + it("retains setup and cleanup failures without requiring AggregateError", () => { const setupError = new Error("synthetic setup failure"); const cleanupError = new Error("synthetic cleanup failure"); @@ -713,6 +888,30 @@ describe("LocalManagedAgentProcessObserver", () => { expect(failure.cleanupErrors).toEqual([cleanupError]); }); + it.skipIf(process.platform === "win32")( + "test cleanup never signals a cached group after its retained child exits", + async () => { + const child = spawnChild(process.execPath, ["-e", "process.exit(0)"], { + detached: true, + stdio: "ignore", + windowsHide: true, + }); + await once(child, "exit"); + const killSpy = vi.spyOn(process, "kill"); + try { + await forceKillRetainedTestGroup(child); + expect( + killSpy.mock.calls.some( + ([pid]) => typeof pid === "number" && pid < 0, + ), + ).toBe(false); + } finally { + killSpy.mockRestore(); + } + }, + 5_000, + ); + it.skipIf(process.platform === "win32")( "keeps inner arguments out of supervisor argv and scrubs its private payload", async () => { @@ -745,7 +944,7 @@ describe("LocalManagedAgentProcessObserver", () => { expect(signalCode).toBeNull(); } finally { if (typeof anchor.pid === "number") { - await forceKillExactTestGroup(anchor.pid, anchor); + await forceKillRetainedTestGroup(anchor); } controller.abort(); observer.dispose(); @@ -782,7 +981,7 @@ describe("LocalManagedAgentProcessObserver", () => { expect(forwardedStderrBytes).toBe(1024 * 1024); } finally { if (typeof anchor.pid === "number") { - await forceKillExactTestGroup(anchor.pid, anchor); + await forceKillRetainedTestGroup(anchor); } controller.abort(); observer.dispose(); @@ -791,6 +990,84 @@ describe("LocalManagedAgentProcessObserver", () => { 5_000, ); + it.skipIf(process.platform === "win32")( + "never signals a cached supervisor group through SDK kill after observed exit", + async () => { + const observer = new LocalManagedAgentProcessObserver(); + const controller = new AbortController(); + const anchor = asChildProcess( + observer.spawn({ + command: process.execPath, + args: ["-e", "process.exit(0)"], + cwd: process.cwd(), + env: { ...process.env }, + signal: controller.signal, + }), + ); + const processGroupId = anchor.pid!; + await once(anchor, "exit"); + const killSpy = vi.spyOn(process, "kill"); + try { + expect(anchor.kill("SIGTERM")).toBe(false); + expect(killSpy).not.toHaveBeenCalledWith(-processGroupId, "SIGTERM"); + } finally { + killSpy.mockRestore(); + controller.abort(); + observer.dispose(); + } + }, + 5_000, + ); + + it.skipIf(process.platform === "win32")( + "records an SDK kill logically without signaling the live supervisor anchor", + async () => { + const nativeKillSpy = vi.spyOn(ChildProcess.prototype, "kill"); + const observer = new LocalManagedAgentProcessObserver(); + const controller = new AbortController(); + const anchor = asChildProcess( + observer.spawn({ + ...activeNodeCommand(), + cwd: process.cwd(), + env: { ...process.env }, + signal: controller.signal, + }), + ); + const processGroupId = anchor.pid!; + try { + await expect( + prepareCancellationAfterTransientReadFailure(observer), + ).resolves.toMatchObject({ + supported: true, + reason: "ready", + ownershipProven: true, + }); + nativeKillSpy.mockClear(); + + expect(anchor.killed).toBe(false); + expect(anchor.kill("SIGTERM")).toBe(true); + expect(anchor.killed).toBe(true); + expect(anchor.kill("SIGTERM")).toBe(false); + expect(nativeKillSpy).not.toHaveBeenCalled(); + expect(processExists(processGroupId)).toBe(true); + await expect(observer.waitForQuiescence(0)).resolves.toMatchObject({ + quiescent: false, + containmentSupported: true, + forceKillIssued: false, + }); + } finally { + nativeKillSpy.mockRestore(); + observer.dispose(); + if (anchor.exitCode === null && anchor.signalCode === null) { + anchor.disconnect(); + await waitForChildExitBounded(anchor); + } + controller.abort(); + } + }, + 5_000, + ); + it.skipIf(process.platform === "win32")( "force-stops and kills the exact non-cooperative fixture group, then confirms death inside one deadline", async () => { @@ -855,7 +1132,7 @@ describe("LocalManagedAgentProcessObserver", () => { // Test-harness safety must not depend on the observer behavior under // test. Exact test-owned PGID authority is retained until death is // independently confirmed, including when an assertion fails. - await forceKillExactTestGroup(ownedProcessGroupId, root); + await forceKillRetainedTestGroup(root); } observer.dispose(); await forceKillExactTestProcess(unrelated); @@ -888,8 +1165,15 @@ describe("LocalManagedAgentProcessObserver", () => { ); const ownedProcessGroupId = anchor.pid; expect(ownedProcessGroupId).toBeTypeOf("number"); + let observerDisposed = false; + let fixtureIdentities: ReadonlyMap< + number, + ManagedAgentKernelProcessRecord + > = new Map(); try { const fixturePids = await waitForManagedAgentFixturePids(fixture); + fixtureIdentities = + await captureExactTestProcessIdentities(fixturePids); await expect( prepareCancellationAfterTransientReadFailure(observer), ).resolves.toMatchObject({ @@ -898,23 +1182,35 @@ describe("LocalManagedAgentProcessObserver", () => { ownershipProven: true, }); + // This test isolates the supervisor's parent-disconnect contract. Stop + // observer sampling before the kernel delivers the group SIGKILL so a + // transient, already-signalled reparent cannot make the assertion + // scheduler-dependent. + observer.dispose(); + observerDisposed = true; anchor.disconnect(); - const teardown = await observer.waitForQuiescence(1_000); - expect(teardown).toMatchObject({ - quiescent: true, - deadlineMet: true, - ownershipProven: true, - forceKillIssued: false, - alivePidsAtDeadline: [], - }); + await waitForChildExitBounded(anchor); + await Promise.all( + fixturePids.map((pid) => + waitForTestProcessDeath( + () => processExists(pid), + `IPC-disconnect fixture process ${pid}`, + ), + ), + ); expect(fixturePids.every((pid) => !processExists(pid))).toBe(true); expect(processExists(unrelated.pid!)).toBe(true); } finally { - if (typeof ownedProcessGroupId === "number") { - await forceKillExactTestGroup(ownedProcessGroupId, anchor); + if (anchor.exitCode === null && anchor.signalCode === null) { + if (anchor.connected) anchor.disconnect(); + await waitForChildExitBounded(anchor, 250).catch(() => undefined); + } + if (anchor.exitCode === null && anchor.signalCode === null) { + await forceKillRetainedTestGroup(anchor); } + await forceKillExactTestProcessIdentities(fixtureIdentities); controller.abort(); - observer.dispose(); + if (!observerDisposed) observer.dispose(); await forceKillExactTestProcess(unrelated); } }, @@ -960,13 +1256,13 @@ describe("LocalManagedAgentProcessObserver", () => { }); it.skipIf(process.platform === "win32")( - "retains owned group authority when the SDK root exits before its non-cooperative child", + "fails closed when the SDK inner root exits before its child and ancestry is lost", () => proveRetainedGroupAuthority("before-readiness"), 15_000, ); it.skipIf(process.platform === "win32")( - "retains owned group authority when the SDK root exits after readiness while its child survives", + "revokes readiness when the SDK inner root exits and reparents its child", () => proveRetainedGroupAuthority("after-readiness"), 10_000, ); @@ -987,8 +1283,11 @@ describe("LocalManagedAgentProcessObserver", () => { ); await once(unrelated, "spawn"); let anchor: ChildProcessWithoutNullStreams | undefined; - let toolGroupId: number | undefined; let fixturePids: readonly number[] = []; + let fixtureIdentities: ReadonlyMap< + number, + ManagedAgentKernelProcessRecord + > = new Map(); try { observer.armToolProcessContainment(); anchor = asChildProcess( @@ -1008,7 +1307,8 @@ describe("LocalManagedAgentProcessObserver", () => { }), ); fixturePids = await waitForManagedAgentFixturePids(fixture); - toolGroupId = fixturePids[0]; + fixtureIdentities = + await captureExactTestProcessIdentities(fixturePids); await expect( prepareCancellationAfterTransientReadFailure(observer), ).resolves.toMatchObject({ @@ -1033,14 +1333,9 @@ describe("LocalManagedAgentProcessObserver", () => { expect(processExists(unrelated.pid!)).toBe(true); } finally { forwardedController.abort(); - if ( - typeof toolGroupId === "number" && - processGroupExists(toolGroupId) - ) { - await forceKillExactTestGroupId(toolGroupId); - } - if (anchor && typeof anchor.pid === "number") { - await forceKillExactTestGroup(anchor.pid, anchor); + await forceKillExactTestProcessIdentities(fixtureIdentities); + if (anchor) { + await forceKillRetainedTestGroup(anchor); } observer.dispose(); await forceKillExactTestProcess(unrelated); @@ -1050,44 +1345,90 @@ describe("LocalManagedAgentProcessObserver", () => { ); it.skipIf(process.platform === "win32")( - "cleans exact fixture and anchor groups when setup fails after PID publication", + "advances forwarded-signal fallback on fresh samples and kills the supervisor anchor last", async () => { - const observer = new LocalManagedAgentProcessObserver(); - let setupEvidence: RegisteredDescendantToolSetupEvidence | undefined; + let rootProcessGroupId: number | undefined; + let toolProcessGroupId: number | undefined; + const signals: Array = []; + const observer = new LocalManagedAgentProcessObserver({ + signalProcessGroup: (groupId, signal) => { + signals.push([groupId, signal]); + return signalRealProcessGroup(groupId, signal); + }, + }); + let run: RegisteredDescendantToolRun | undefined; try { - await expect( - startRegisteredDescendantToolRun( - observer, - "failed-registered-tool-setup", - (evidence) => { - setupEvidence = evidence; - throw new Error("synthetic failure after PID publication"); - }, - ), - ).rejects.toThrow("synthetic failure after PID publication"); + run = await startRegisteredDescendantToolRun( + observer, + "sample-driven-forwarded-fallback", + ); + rootProcessGroupId = run.anchor.pid!; + toolProcessGroupId = run.toolProcessGroupId; + + run.forwardedController.abort(); + const deadline = Date.now() + 2_000; + while ( + !signals.some( + ([groupId, signal]) => + groupId === rootProcessGroupId && signal === "SIGKILL", + ) && + Date.now() < deadline + ) { + await observer.observeProcessTree(); + await new Promise((resolveDelay) => setTimeout(resolveDelay, 10)); + } + + expect(signals).toEqual([ + [rootProcessGroupId, "SIGSTOP"], + [toolProcessGroupId, "SIGSTOP"], + [toolProcessGroupId, "SIGKILL"], + [rootProcessGroupId, "SIGKILL"], + ]); + await expect(observer.waitForQuiescence(1_000)).resolves.toMatchObject({ + quiescent: true, + deadlineMet: true, + containmentSupported: true, + forceKillIssued: true, + toolProcessChannelsClosed: true, + alivePidsAtDeadline: [], + }); + } finally { + await cleanupRegisteredDescendantToolRun(run); + observer.dispose(); + } + }, + 15_000, + ); + + it.skipIf(process.platform === "win32")( + "cleans exact fixture and anchor groups when setup fails after PID publication", + async () => { + const observer = new LocalManagedAgentProcessObserver(); + let setupEvidence: RegisteredDescendantToolSetupEvidence | undefined; + try { + await expect( + startRegisteredDescendantToolRun( + observer, + "failed-registered-tool-setup", + (evidence) => { + setupEvidence = evidence; + throw new Error("synthetic failure after PID publication"); + }, + ), + ).rejects.toThrow("synthetic failure after PID publication"); expect(setupEvidence).toBeDefined(); expect( - setupEvidence!.toolPids.every((pid) => !processExists(pid)), - ).toBe(true); - expect(processGroupExists(setupEvidence!.toolProcessGroupId)).toBe( - false, - ); - expect(processGroupExists(setupEvidence!.anchorProcessGroupId)).toBe( - false, - ); + await liveExactTestProcessIdentities(setupEvidence!.toolIdentities), + ).toEqual([]); + expect(setupEvidence!.anchor.exitCode).toBeNull(); + expect(setupEvidence!.anchor.signalCode).toBe("SIGKILL"); } finally { - if ( - setupEvidence && - processGroupExists(setupEvidence.toolProcessGroupId) - ) { - await forceKillExactTestGroupId(setupEvidence.toolProcessGroupId); - } - if ( - setupEvidence && - processGroupExists(setupEvidence.anchorProcessGroupId) - ) { - await forceKillExactTestGroupId(setupEvidence.anchorProcessGroupId); + if (setupEvidence) { + await forceKillExactTestProcessIdentities( + setupEvidence.toolIdentities, + ); + await forceKillRetainedTestGroup(setupEvidence.anchor); } observer.dispose(); } @@ -1206,7 +1547,7 @@ describe("LocalManagedAgentProcessObserver", () => { ); toolProcessGroupId = run.toolProcessGroupId; registeredPids = run.toolPids; - await forceKillExactTestGroupId(toolProcessGroupId); + await forceKillExactTestProcessIdentities(run.toolIdentities); simulatePidReuse = true; const teardown = await observer.emergencyCleanup(250); @@ -1310,8 +1651,7 @@ describe("LocalManagedAgentProcessObserver", () => { "root-exit-loses-tool-ancestry", ); toolProcessGroupId = run.toolProcessGroupId; - process.kill(-run.anchor.pid!, "SIGKILL"); - await waitForChildExitBounded(run.anchor); + await forceKillRetainedTestGroup(run.anchor); expect(processGroupExists(toolProcessGroupId)).toBe(true); const teardown = await observer.emergencyCleanup(250); @@ -1345,6 +1685,10 @@ describe("LocalManagedAgentProcessObserver", () => { const credentialFile = join(fixture.root, "tool-control.json"); let anchor: ChildProcessWithoutNullStreams | undefined; let detachedTool: ChildProcess | undefined; + let detachedToolIdentities: ReadonlyMap< + number, + ManagedAgentKernelProcessRecord + > = new Map(); let registrations: readonly NetSocket[] = []; try { observer.armToolProcessContainment(); @@ -1376,6 +1720,10 @@ describe("LocalManagedAgentProcessObserver", () => { ); const [toolParentPid, toolChildPid] = await waitForManagedAgentFixturePids(fixture); + detachedToolIdentities = await captureExactTestProcessIdentities([ + toolParentPid, + toolChildPid, + ]); registrations = await Promise.all([ startToolRegistration(credentials, "parent", toolParentPid), startToolRegistration(credentials, "child", toolChildPid), @@ -1394,11 +1742,10 @@ describe("LocalManagedAgentProcessObserver", () => { } finally { for (const registration of registrations) registration.destroy(); forwardedController.abort(); - if (detachedTool && typeof detachedTool.pid === "number") { - await forceKillExactTestGroup(detachedTool.pid, detachedTool); - } - if (anchor && typeof anchor.pid === "number") { - await forceKillExactTestGroup(anchor.pid, anchor); + await forceKillExactTestProcessIdentities(detachedToolIdentities); + if (detachedTool) await waitForChildExitBounded(detachedTool); + if (anchor) { + await forceKillRetainedTestGroup(anchor); } observer.dispose(); } @@ -1417,7 +1764,10 @@ describe("LocalManagedAgentProcessObserver", () => { const forwardedController = new AbortController(); const credentialFile = join(fixture.root, "tool-control.json"); let anchor: ChildProcessWithoutNullStreams | undefined; - let detachedToolGroupId: number | undefined; + let detachedToolIdentities: ReadonlyMap< + number, + ManagedAgentKernelProcessRecord + > = new Map(); let parentRegistration: NetSocket | undefined; let childRegistration: NetSocket | undefined; try { @@ -1441,7 +1791,10 @@ describe("LocalManagedAgentProcessObserver", () => { const credentials = await waitForToolControlCredentials(credentialFile); const [toolParentPid, toolChildPid] = await waitForManagedAgentFixturePids(fixture); - detachedToolGroupId = toolParentPid; + detachedToolIdentities = await captureExactTestProcessIdentities([ + toolParentPid, + toolChildPid, + ]); await sendClosedToolRegistration(credentials, "parent", toolParentPid); [parentRegistration, childRegistration] = await Promise.all([ @@ -1456,7 +1809,7 @@ describe("LocalManagedAgentProcessObserver", () => { containmentSupported: true, }); - await forceKillExactTestGroupId(detachedToolGroupId); + await forceKillExactTestProcessIdentities(detachedToolIdentities); forwardedController.abort(); const openChannelObservation = await observer.emergencyCleanup(1_000); await waitForChildExitBounded(anchor); @@ -1476,15 +1829,8 @@ describe("LocalManagedAgentProcessObserver", () => { parentRegistration?.destroy(); childRegistration?.destroy(); forwardedController.abort(); - if ( - typeof detachedToolGroupId === "number" && - processGroupExists(detachedToolGroupId) - ) { - await forceKillExactTestGroupId(detachedToolGroupId); - } - if (anchor && typeof anchor.pid === "number") { - await forceKillExactTestGroup(anchor.pid, anchor); - } + await forceKillExactTestProcessIdentities(detachedToolIdentities); + if (anchor) await forceKillRetainedTestGroup(anchor); observer.dispose(); } }, @@ -1522,9 +1868,7 @@ describe("LocalManagedAgentProcessObserver", () => { }); } finally { controller.abort(); - if (typeof anchor.pid === "number") { - await forceKillExactTestGroup(anchor.pid, anchor); - } + await forceKillRetainedTestGroup(anchor); observer.dispose(); } }, @@ -1571,15 +1915,9 @@ describe("LocalManagedAgentProcessObserver", () => { expect(Date.now() - shortConfirmationStartedAt).toBeLessThan(150); controller.abort(); - await once(child, "exit"); - expect(signals).toEqual([ - [child.pid!, "SIGSTOP"], - [child.pid!, "SIGKILL"], - ]); + expect(signals).toEqual([[child.pid!, "SIGSTOP"]]); } finally { - if (child.exitCode === null && child.signalCode === null) { - child.kill("SIGKILL"); - } + await forceKillRetainedTestGroup(child); controller.abort(); observer.dispose(); } @@ -1636,14 +1974,16 @@ describe("LocalManagedAgentProcessObserver", () => { }); expect(signals).toEqual([]); } finally { - child.kill("SIGKILL"); + await forceKillRetainedTestGroup(child); controller.abort(); observer.dispose(); } }); - it("makes raw and forwarded aborts idempotent after ownership preparation", async () => { + it("treats zombie topology drift as dead rather than a containment escape", async () => { let rootPid = 0; + let zombie = false; + let now = 0; const signals: Array = []; const observer = new LocalManagedAgentProcessObserver({ platform: "darwin", @@ -1654,129 +1994,1030 @@ describe("LocalManagedAgentProcessObserver", () => { { parentPid: process.pid, processGroupId: rootPid, + sessionId: 0, + state: "Ss", startedAt: "root", }, ], [ rootPid + 100, { - parentPid: rootPid, - processGroupId: rootPid, + parentPid: zombie ? 1 : rootPid, + processGroupId: zombie ? rootPid + 200 : rootPid, + sessionId: zombie ? 999 : 0, + state: zombie ? "Z+" : "S", startedAt: "child", }, ], ]), + processGroupLiveness: () => "gone", signalProcessGroup: (groupId, signal) => { signals.push([groupId, signal]); return "sent"; }, + now: () => now, + delay: async (milliseconds) => { + now += Math.max(1, milliseconds); + }, }); - const rawController = new AbortController(); - const forwardedController = new AbortController(); - observer.bindAbortSignal(rawController.signal); - const child = asChildProcess( + const controller = new AbortController(); + const anchor = asChildProcess( observer.spawn({ ...activeNodeCommand(), cwd: process.cwd(), env: { ...process.env }, - signal: forwardedController.signal, + signal: controller.signal, }), ); - rootPid = child.pid!; + rootPid = anchor.pid!; try { - await observer.prepareCancellation(); - rawController.abort(); - forwardedController.abort(); - expect(signals).toEqual([ - [rootPid, "SIGSTOP"], - [rootPid, "SIGKILL"], - ]); + await expect(observer.prepareCancellation()).resolves.toMatchObject({ + supported: true, + reason: "ready", + }); + zombie = true; + await observer.observeProcessTree(); + + const observation = await observer.waitForQuiescence(1); + expect(observation).toMatchObject({ + quiescent: false, + containmentSupported: true, + }); + expect(observation.alivePidsAtDeadline).not.toContain(rootPid + 100); + expect(observation.alivePidsAtDeadline).not.toContain(rootPid + 200); + expect(signals).toEqual([]); } finally { - child.kill("SIGKILL"); + await forceKillRetainedTestGroup(anchor); + controller.abort(); observer.dispose(); } }); - it("retries a failed SIGKILL while the trusted stopped root still anchors the group", async () => { + it("keeps pre-signal non-zombie topology drift permanently fail-closed", async () => { let rootPid = 0; - let killAttempts = 0; + let escaped = false; + let gone = false; + let now = 0; const signals: Array = []; const observer = new LocalManagedAgentProcessObserver({ platform: "darwin", readProcessTable: async () => - available([ + gone + ? available([]) + : available([ + [ + rootPid, + { + parentPid: process.pid, + processGroupId: rootPid, + sessionId: 0, + state: "Ss", + startedAt: "root", + }, + ], + [ + rootPid + 100, + { + parentPid: escaped ? 1 : rootPid, + processGroupId: escaped ? rootPid + 200 : rootPid, + sessionId: escaped ? 999 : 0, + state: "S", + startedAt: "child", + }, + ], + ]), + processGroupLiveness: () => "gone", + signalProcessGroup: (groupId, signal) => { + signals.push([groupId, signal]); + return "sent"; + }, + now: () => now, + delay: async (milliseconds) => { + now += Math.max(1, milliseconds); + }, + }); + const controller = new AbortController(); + const anchor = asChildProcess( + observer.spawn({ + ...activeNodeCommand(), + cwd: process.cwd(), + env: { ...process.env }, + signal: controller.signal, + }), + ); + rootPid = anchor.pid!; + try { + await expect(observer.prepareCancellation()).resolves.toMatchObject({ + supported: true, + reason: "ready", + }); + escaped = true; + await observer.observeProcessTree(); + + const observation = await observer.waitForQuiescence(1); + expect(observation).toMatchObject({ + quiescent: false, + containmentSupported: false, + }); + expect(observation.alivePidsAtDeadline).toEqual( + expect.arrayContaining([rootPid + 100, rootPid + 200]), + ); + expect(signals).toEqual([]); + + controller.abort(); + gone = true; + await observer.observeProcessTree(); + await expect(observer.waitForQuiescence(1)).resolves.toMatchObject({ + quiescent: false, + containmentSupported: false, + }); + expect(signals).toEqual([]); + } finally { + await forceKillRetainedTestGroup(anchor); + controller.abort(); + observer.dispose(); + } + }); + + it("keeps post-SIGKILL stable exit drift live until disappearance without invalidating containment", async () => { + let rootPid = 0; + let stage: "owned" | "exiting" | "gone" = "owned"; + let now = 0; + const signals: Array = []; + const observer = new LocalManagedAgentProcessObserver({ + platform: "darwin", + readProcessTable: async () => { + if (stage === "gone") return available([]); + if (stage === "exiting") { + return available([ + [ + rootPid + 100, + { + parentPid: 1, + processGroupId: rootPid, + sessionId: 0, + state: "?E", + startedAt: "child", + }, + ], + ]); + } + return available([ [ rootPid, { parentPid: process.pid, processGroupId: rootPid, + sessionId: 0, + state: signals.some(([, signal]) => signal === "SIGSTOP") + ? "Ts" + : "Ss", startedAt: "root", }, ], - ]), + [ + rootPid + 100, + { + parentPid: rootPid, + processGroupId: rootPid, + sessionId: 0, + state: signals.some(([, signal]) => signal === "SIGSTOP") + ? "T" + : "S", + startedAt: "child", + }, + ], + ]); + }, + processGroupLiveness: () => (stage === "gone" ? "gone" : "alive"), signalProcessGroup: (groupId, signal) => { signals.push([groupId, signal]); - if (signal === "SIGKILL" && killAttempts++ === 0) return "failure"; return "sent"; }, + now: () => now, + delay: async (milliseconds) => { + now += Math.max(1, milliseconds); + }, }); - const rawController = new AbortController(); - const forwardedController = new AbortController(); - observer.bindAbortSignal(rawController.signal); - const child = asChildProcess( + const controller = new AbortController(); + const anchor = asChildProcess( observer.spawn({ ...activeNodeCommand(), cwd: process.cwd(), env: { ...process.env }, - signal: forwardedController.signal, + signal: controller.signal, }), ); - rootPid = child.pid!; + rootPid = anchor.pid!; try { - await observer.prepareCancellation(); - rawController.abort(); - await observer.emergencyCleanup(0); - + await expect(observer.prepareCancellation()).resolves.toMatchObject({ + supported: true, + reason: "ready", + }); + controller.abort(); + await observer.observeProcessTree(); expect(signals).toEqual([ [rootPid, "SIGSTOP"], [rootPid, "SIGKILL"], - [rootPid, "SIGKILL"], ]); + + stage = "exiting"; + await observer.observeProcessTree(); await expect(observer.waitForQuiescence(0)).resolves.toMatchObject({ - containmentSupported: false, - forceKillIssued: true, quiescent: false, + containmentSupported: true, + alivePidsAtDeadline: expect.arrayContaining([rootPid + 100]), + }); + + stage = "gone"; + await observer.observeProcessTree(); + await expect(observer.waitForQuiescence(0)).resolves.toMatchObject({ + quiescent: true, + deadlineMet: true, + containmentSupported: true, + alivePidsAtDeadline: [], }); } finally { - child.kill("SIGKILL"); + await forceKillRetainedTestGroup(anchor); + controller.abort(); observer.dispose(); } }); - it("treats an unexpected group-liveness probe error as unknown, never gone", async () => { + it("never authorizes a group signal from zombie-only root evidence", async () => { + let rootPid = 0; + const signals: Array = []; const observer = new LocalManagedAgentProcessObserver({ platform: "darwin", - readProcessTable: async () => available([]), - processGroupLiveness: () => "unknown", + readProcessTable: async () => + available([ + [ + rootPid, + { + parentPid: 1, + processGroupId: rootPid, + sessionId: 0, + state: "Z", + startedAt: "root", + }, + ], + ]), + processGroupLiveness: () => "alive", + signalProcessGroup: (groupId, signal) => { + signals.push([groupId, signal]); + return "sent"; + }, }); const controller = new AbortController(); - const child = asChildProcess( + const anchor = asChildProcess( observer.spawn({ - command: process.execPath, - args: ["-e", "process.exit(0)"], + ...activeNodeCommand(), cwd: process.cwd(), env: { ...process.env }, signal: controller.signal, }), ); - await once(child, "exit"); + rootPid = anchor.pid!; + try { + await expect(observer.prepareCancellation()).resolves.toMatchObject({ + supported: false, + reason: "root_not_active", + ownershipProven: false, + }); + controller.abort(); + expect(signals).toEqual([]); + } finally { + await forceKillRetainedTestGroup(anchor); + controller.abort(); + observer.dispose(); + } + }); + + it("allows normal quiescence only after a still-descended unauthenticated subgroup is positively dead", async () => { + let rootPid = 0; + let subgroupAlive = true; + const subgroupPid = () => rootPid + 100; + const signals: Array = []; + const observer = new LocalManagedAgentProcessObserver({ + platform: "darwin", + readProcessTable: async () => { + await Promise.resolve(); + return available( + subgroupAlive + ? [ + [ + rootPid, + { + parentPid: process.pid, + processGroupId: rootPid, + sessionId: rootPid, + startedAt: "root", + }, + ], + [ + subgroupPid(), + { + parentPid: rootPid, + processGroupId: subgroupPid(), + sessionId: subgroupPid(), + startedAt: "short-lived-subgroup", + }, + ], + ] + : [], + ); + }, + processGroupLiveness: () => (subgroupAlive ? "alive" : "gone"), + signalProcessGroup: (groupId, signal) => { + signals.push([groupId, signal]); + return "sent"; + }, + }); + const controller = new AbortController(); + const anchor = asChildProcess( + observer.spawn({ + ...activeNodeCommand(), + cwd: process.cwd(), + env: { ...process.env }, + signal: controller.signal, + }), + ); + rootPid = anchor.pid!; try { + await observer.observeProcessTree(); await expect(observer.waitForQuiescence(0)).resolves.toMatchObject({ quiescent: false, - deadlineMet: false, containmentSupported: false, + alivePidsAtDeadline: expect.arrayContaining([subgroupPid()]), + }); + await expect(observer.prepareCancellation()).resolves.toMatchObject({ + supported: false, + reason: "containment_escaped", + ownershipProven: false, + }); + expect(signals).toEqual([]); + + subgroupAlive = false; + await observer.observeProcessTree(); + await expect(observer.waitForQuiescence(0)).resolves.toMatchObject({ + quiescent: true, + deadlineMet: true, + containmentSupported: true, + alivePidsAtDeadline: [], }); + expect(signals).toEqual([]); } finally { + await forceKillRetainedTestGroup(anchor); + controller.abort(); + observer.dispose(); + } + }); + + it("keeps a live unauthenticated descendant subgroup nonquiescent without granting it signal authority", async () => { + let rootPid = 0; + const subgroupPid = () => rootPid + 100; + const signals: Array = []; + const observer = new LocalManagedAgentProcessObserver({ + platform: "darwin", + readProcessTable: async () => + available([ + [ + rootPid, + { + parentPid: process.pid, + processGroupId: rootPid, + sessionId: rootPid, + startedAt: "root", + }, + ], + [ + subgroupPid(), + { + parentPid: rootPid, + processGroupId: subgroupPid(), + sessionId: subgroupPid(), + startedAt: "surviving-subgroup", + }, + ], + ]), + processGroupLiveness: () => "alive", + signalProcessGroup: (groupId, signal) => { + signals.push([groupId, signal]); + return "sent"; + }, + }); + const controller = new AbortController(); + const anchor = asChildProcess( + observer.spawn({ + ...activeNodeCommand(), + cwd: process.cwd(), + env: { ...process.env }, + signal: controller.signal, + }), + ); + rootPid = anchor.pid!; + try { + await observer.observeProcessTree(); + await expect(observer.waitForQuiescence(0)).resolves.toMatchObject({ + quiescent: false, + deadlineMet: false, + containmentSupported: false, + alivePidsAtDeadline: expect.arrayContaining([subgroupPid()]), + }); + expect(signals).toEqual([]); + } finally { + await forceKillRetainedTestGroup(anchor); + controller.abort(); + observer.dispose(); + } + }); + + it("keeps an unauthenticated subgroup permanently failed closed if its stable identity loses root ancestry", async () => { + let rootPid = 0; + let subgroupState: "descended" | "reparented" | "gone" = "descended"; + const subgroupPid = () => rootPid + 100; + const observer = new LocalManagedAgentProcessObserver({ + platform: "darwin", + readProcessTable: async () => { + const subgroup = + subgroupState === "gone" + ? [] + : [ + [ + subgroupPid(), + { + parentPid: + subgroupState === "descended" ? rootPid : process.pid, + processGroupId: subgroupPid(), + sessionId: subgroupPid(), + startedAt: "reparented-subgroup", + }, + ] as const, + ]; + return available([ + ...(subgroupState === "descended" + ? ([ + [ + rootPid, + { + parentPid: process.pid, + processGroupId: rootPid, + sessionId: rootPid, + startedAt: "root", + }, + ], + ] as const) + : []), + ...subgroup, + ]); + }, + processGroupLiveness: () => "gone", + signalProcessGroup: () => "sent", + }); + const controller = new AbortController(); + const anchor = asChildProcess( + observer.spawn({ + ...activeNodeCommand(), + cwd: process.cwd(), + env: { ...process.env }, + signal: controller.signal, + }), + ); + rootPid = anchor.pid!; + try { + await observer.observeProcessTree(); + subgroupState = "reparented"; + await observer.observeProcessTree(); + subgroupState = "gone"; + await observer.observeProcessTree(); + await expect(observer.waitForQuiescence(0)).resolves.toMatchObject({ + quiescent: false, + deadlineMet: false, + containmentSupported: false, + alivePidsAtDeadline: [], + }); + } finally { + await forceKillRetainedTestGroup(anchor); + controller.abort(); + observer.dispose(); + } + }); + + it("keeps a subgroup escape after an authorized root kill permanently failed closed", async () => { + let rootPid = 0; + let subgroupState: "root_group" | "reparented" | "gone" = "root_group"; + const subgroupPid = () => rootPid + 100; + const signals: Array = []; + const observer = new LocalManagedAgentProcessObserver({ + platform: "darwin", + readProcessTable: async () => { + if (subgroupState === "gone") return available([]); + return available([ + ...(subgroupState === "root_group" + ? ([ + [ + rootPid, + { + parentPid: process.pid, + processGroupId: rootPid, + sessionId: rootPid, + state: signals.some(([, signal]) => signal === "SIGSTOP") + ? "T" + : "S", + startedAt: "root", + }, + ], + ] as const) + : []), + [ + subgroupPid(), + { + parentPid: subgroupState === "root_group" ? rootPid : process.pid, + processGroupId: + subgroupState === "root_group" ? rootPid : subgroupPid(), + sessionId: + subgroupState === "root_group" ? rootPid : subgroupPid(), + state: + subgroupState === "root_group" && + signals.some(([, signal]) => signal === "SIGSTOP") + ? "T" + : "S", + startedAt: "survived-root-kill", + }, + ], + ]); + }, + processGroupLiveness: () => "alive", + signalProcessGroup: (groupId, signal) => { + signals.push([groupId, signal]); + if (signal === "SIGKILL") subgroupState = "reparented"; + return "sent"; + }, + }); + const controller = new AbortController(); + const anchor = asChildProcess( + observer.spawn({ + ...activeNodeCommand(), + cwd: process.cwd(), + env: { ...process.env }, + signal: controller.signal, + }), + ); + rootPid = anchor.pid!; + try { + await expect(observer.prepareCancellation()).resolves.toMatchObject({ + supported: true, + reason: "ready", + }); + controller.abort(); + expect(signals).toEqual([[rootPid, "SIGSTOP"]]); + await observer.observeProcessTree(); + expect(signals).toEqual([ + [rootPid, "SIGSTOP"], + [rootPid, "SIGKILL"], + ]); + await observer.observeProcessTree(); + subgroupState = "gone"; + await observer.observeProcessTree(); + await expect(observer.waitForQuiescence(0)).resolves.toMatchObject({ + quiescent: false, + deadlineMet: false, + containmentSupported: false, + alivePidsAtDeadline: [], + }); + } finally { + await forceKillRetainedTestGroup(anchor); + controller.abort(); + observer.dispose(); + } + }); + + it.skipIf(process.platform === "win32")( + "blocks L2 readiness when an authenticated tool run gains an unauthenticated descendant group", + async () => { + let injectUnknownDescendant = false; + let run: RegisteredDescendantToolRun | undefined; + let syntheticPid = 2_000_000_000; + const observer = new LocalManagedAgentProcessObserver({ + readProcessTable: async () => { + const observation = await readRealPosixProcessTable(); + if (!observation.available || !injectUnknownDescendant || !run) { + return observation; + } + const processes = new Map(observation.processes); + while (processes.has(syntheticPid)) syntheticPid -= 1; + processes.set(syntheticPid, { + parentPid: run.anchor.pid!, + processGroupId: syntheticPid, + sessionId: syntheticPid, + state: "S", + startedAt: "synthetic-unknown-l2-descendant", + }); + return { available: true, processes }; + }, + }); + try { + run = await startRegisteredDescendantToolRun( + observer, + "unknown-l2-descendant", + ); + injectUnknownDescendant = true; + await observer.observeProcessTree(); + await expect(observer.prepareCancellation()).resolves.toMatchObject({ + supported: false, + reason: "containment_escaped", + ownershipProven: false, + }); + } finally { + injectUnknownDescendant = false; + await cleanupRegisteredDescendantToolRun(run); + observer.dispose(); + } + }, + 15_000, + ); + + it.skipIf(process.platform === "win32")( + "never loses a reparented escaped tool grandchild or signals its new group", + async () => { + const fixture = await createManagedAgentFixture( + () => "reparented-tool-grandchild", + ); + fixtures.push(fixture); + const credentialFile = join(fixture.root, "tool-control.json"); + let rootPid = 0; + let escaped = false; + let toolParentPid = 0; + let toolChildPid = 0; + let toolGrandchildPid = 0; + let escapedGroupId = 0; + const signals: Array = []; + const observer = new LocalManagedAgentProcessObserver({ + readProcessTable: async () => { + await Promise.resolve(); + if (escaped) { + return available([ + [ + toolGrandchildPid, + { + parentPid: 1, + processGroupId: escapedGroupId, + sessionId: escapedGroupId, + startedAt: "tool-grandchild", + }, + ], + ]); + } + return available([ + [ + process.pid, + { + parentPid: process.ppid, + processGroupId: process.pid, + sessionId: process.pid, + startedAt: "host", + }, + ], + [ + rootPid, + { + parentPid: process.pid, + processGroupId: rootPid, + sessionId: rootPid, + startedAt: "root-100", + }, + ], + [ + toolParentPid, + { + parentPid: rootPid, + processGroupId: toolParentPid, + sessionId: toolParentPid, + startedAt: "tool-parent-200", + }, + ], + [ + toolChildPid, + { + parentPid: toolParentPid, + processGroupId: toolParentPid, + sessionId: toolParentPid, + startedAt: "tool-child-201", + }, + ], + [ + toolGrandchildPid, + { + parentPid: toolChildPid, + processGroupId: toolParentPid, + sessionId: toolParentPid, + startedAt: "tool-grandchild", + }, + ], + ]); + }, + processGroupLiveness: (groupId) => + escaped && groupId === escapedGroupId ? "alive" : "gone", + signalProcessGroup: (groupId, signal) => { + signals.push([groupId, signal]); + return "sent"; + }, + }); + const controller = new AbortController(); + observer.armToolProcessContainment(); + const anchor = asChildProcess( + observer.spawn({ + command: process.execPath, + args: [ + "--input-type=module", + "--eval", + EXPORT_TOOL_CONTROL_SCRIPT, + credentialFile, + ], + cwd: fixture.workspaceRoot, + env: { ...process.env }, + signal: controller.signal, + }), + ); + rootPid = anchor.pid!; + toolParentPid = rootPid + 10_000; + toolChildPid = toolParentPid + 1; + toolGrandchildPid = toolParentPid + 2; + escapedGroupId = toolParentPid + 3; + let registrations: readonly NetSocket[] = []; + try { + const credentials = await waitForToolControlCredentials(credentialFile); + registrations = await Promise.all([ + openToolRegistration(credentials, "parent", toolParentPid), + openToolRegistration(credentials, "child", toolChildPid), + ]); + await expect(observer.prepareCancellation()).resolves.toMatchObject({ + supported: true, + reason: "ready", + ownershipProven: true, + }); + + escaped = true; + const registrationClosures = registrations.map((socket) => + once(socket, "close"), + ); + for (const socket of registrations) socket.destroy(); + await Promise.all(registrationClosures); + await forceKillRetainedTestGroup(anchor); + + const teardown = await observer.emergencyCleanup(50); + + expect(teardown).toMatchObject({ + quiescent: false, + deadlineMet: false, + containmentSupported: false, + }); + expect(teardown.alivePidsAtDeadline).toEqual( + expect.arrayContaining([toolGrandchildPid, escapedGroupId]), + ); + expect(signals.some(([groupId]) => groupId === escapedGroupId)).toBe( + false, + ); + } finally { + for (const socket of registrations) socket.destroy(); + await forceKillRetainedTestGroup(anchor); + controller.abort(); + observer.dispose(); + } + }, + 10_000, + ); + + it("makes raw and forwarded aborts idempotent after ownership preparation", async () => { + let rootPid = 0; + const signals: Array = []; + const observer = new LocalManagedAgentProcessObserver({ + platform: "darwin", + readProcessTable: async () => + available([ + [ + rootPid, + { + parentPid: process.pid, + processGroupId: rootPid, + startedAt: "root", + }, + ], + [ + rootPid + 100, + { + parentPid: rootPid, + processGroupId: rootPid, + startedAt: "child", + }, + ], + ]), + signalProcessGroup: (groupId, signal) => { + signals.push([groupId, signal]); + return "sent"; + }, + }); + const rawController = new AbortController(); + const forwardedController = new AbortController(); + observer.bindAbortSignal(rawController.signal); + const child = asChildProcess( + observer.spawn({ + ...activeNodeCommand(), + cwd: process.cwd(), + env: { ...process.env }, + signal: forwardedController.signal, + }), + ); + rootPid = child.pid!; + try { + await observer.prepareCancellation(); + rawController.abort(); + forwardedController.abort(); + expect(signals).toEqual([[rootPid, "SIGSTOP"]]); + await observer.observeProcessTree(); + expect(signals).toEqual([ + [rootPid, "SIGSTOP"], + [rootPid, "SIGKILL"], + ]); + await observer.observeProcessTree(); + expect(signals).toHaveLength(2); + } finally { + await forceKillRetainedTestGroup(child); + observer.dispose(); + } + }); + + it("retries a transient failed SIGKILL while the trusted stopped root still anchors the group", async () => { + let rootPid = 0; + let killAttempts = 0; + const signals: Array = []; + const observer = new LocalManagedAgentProcessObserver({ + platform: "darwin", + readProcessTable: async () => + available([ + [ + rootPid, + { + parentPid: process.pid, + processGroupId: rootPid, + startedAt: "root", + }, + ], + ]), + signalProcessGroup: (groupId, signal) => { + signals.push([groupId, signal]); + if (signal === "SIGKILL" && killAttempts++ === 0) return "failure"; + return "sent"; + }, + }); + const rawController = new AbortController(); + const forwardedController = new AbortController(); + observer.bindAbortSignal(rawController.signal); + const child = asChildProcess( + observer.spawn({ + ...activeNodeCommand(), + cwd: process.cwd(), + env: { ...process.env }, + signal: forwardedController.signal, + }), + ); + rootPid = child.pid!; + try { + await observer.prepareCancellation(); + rawController.abort(); + await observer.emergencyCleanup(0); + + expect(signals).toEqual([ + [rootPid, "SIGSTOP"], + [rootPid, "SIGKILL"], + [rootPid, "SIGKILL"], + ]); + await expect(observer.waitForQuiescence(0)).resolves.toMatchObject({ + containmentSupported: true, + forceKillIssued: true, + quiescent: false, + }); + } finally { + await forceKillRetainedTestGroup(child); + observer.dispose(); + } + }); + + it("treats an unexpected group-liveness probe error as unknown, never gone", async () => { + let rootPid = 0; + const observer = new LocalManagedAgentProcessObserver({ + platform: "darwin", + readProcessTable: async () => + available([ + [ + rootPid, + { + parentPid: process.pid, + processGroupId: rootPid, + sessionId: 0, + state: "S", + startedAt: "reported-live-root", + }, + ], + ]), + processGroupLiveness: () => "unknown", + }); + const controller = new AbortController(); + const child = asChildProcess( + observer.spawn({ + command: process.execPath, + args: ["-e", "process.exit(0)"], + cwd: process.cwd(), + env: { ...process.env }, + signal: controller.signal, + }), + ); + rootPid = child.pid!; + await once(child, "exit"); + try { + await expect(observer.waitForQuiescence(0)).resolves.toMatchObject({ + quiescent: false, + deadlineMet: false, + containmentSupported: false, + }); + } finally { + controller.abort(); + observer.dispose(); + } + }); + + it("accepts complete process-table absence without probing a cached group", async () => { + let livenessProbes = 0; + const observer = new LocalManagedAgentProcessObserver({ + platform: "darwin", + readProcessTable: async () => available([]), + processGroupLiveness: () => { + livenessProbes += 1; + return "unknown"; + }, + }); + const controller = new AbortController(); + const child = asChildProcess( + observer.spawn({ + command: process.execPath, + args: ["-e", "process.exit(0)"], + cwd: process.cwd(), + env: { ...process.env }, + signal: controller.signal, + }), + ); + await once(child, "exit"); + try { + await observer.observeProcessTree(); + await expect(observer.waitForQuiescence(0)).resolves.toMatchObject({ + quiescent: true, + deadlineMet: true, + containmentSupported: true, + alivePidsAtDeadline: [], + }); + expect(livenessProbes).toBe(0); + } finally { + controller.abort(); + observer.dispose(); + } + }); + + it("reports quiescence after the caller budget as a missed deadline", async () => { + let now = 0; + let measureOverrun = false; + const observer = new LocalManagedAgentProcessObserver({ + platform: "darwin", + readProcessTable: async () => { + if (measureOverrun) now = 2; + return available([]); + }, + now: () => now, + }); + const controller = new AbortController(); + const child = asChildProcess( + observer.spawn({ + ...activeNodeCommand(), + cwd: process.cwd(), + env: { ...process.env }, + signal: controller.signal, + }), + ); + try { + await observer.observeProcessTree(); + now = 0; + measureOverrun = true; + + await expect(observer.waitForQuiescence(1)).resolves.toMatchObject({ + quiescent: true, + deadlineMet: false, + elapsedMs: 2, + processTableAvailable: true, + alivePidsAtDeadline: [], + }); + } finally { + if (typeof child.pid === "number") { + await forceKillRetainedTestGroup(child); + } controller.abort(); observer.dispose(); } diff --git a/packages/harness/src/experimental/managed-agent-spike/process-observer.ts b/packages/harness/src/experimental/managed-agent-spike/process-observer.ts index fb6970064..4284dc658 100644 --- a/packages/harness/src/experimental/managed-agent-spike/process-observer.ts +++ b/packages/harness/src/experimental/managed-agent-spike/process-observer.ts @@ -227,6 +227,8 @@ if (inner) { export interface ManagedAgentKernelProcessRecord { readonly parentPid: number; readonly processGroupId?: number; + /** POSIX session id, when the process table exposes it. */ + readonly sessionId?: number; /** POSIX process state used only to confirm an issued group stop. */ readonly state?: string; /** Kernel-reported creation time used for evidence, never POSIX authority. */ @@ -299,6 +301,12 @@ function sameProcess( return Boolean(left && right && left.startedAt === right.startedAt); } +function processIsZombie( + record: ManagedAgentKernelProcessRecord | undefined, +): boolean { + return record?.state?.startsWith("Z") ?? false; +} + function sameCapability(left: string, right: string): boolean { const leftBytes = Buffer.from(left, "utf8"); const rightBytes = Buffer.from(right, "utf8"); @@ -356,35 +364,53 @@ async function windowsProcessTable(): Promise { ); } -async function posixProcessTable(): Promise { - const { stdout } = await execFileAsync( - "/bin/ps", - ["-axo", "pid=,ppid=,pgid=,stat=,lstart="], - { - encoding: "utf8", - windowsHide: true, - maxBuffer: 4 * 1024 * 1024, - timeout: MANAGED_AGENT_PROCESS_HELPER_TIMEOUT_MS, - killSignal: "SIGKILL", - }, - ); +export function managedAgentPosixSessionColumn( + platform: NodeJS.Platform, +): "sess" | "sid" { + return platform === "darwin" ? "sess" : "sid"; +} + +export function parseManagedAgentPosixProcessTable( + stdout: string, +): ManagedAgentKernelProcessTable { const entries: Array = []; for (const line of stdout.split("\n")) { - const match = /^\s*(\d+)\s+(\d+)\s+(\d+)\s+(\S+)\s+(.+?)\s*$/.exec(line); + const match = /^\s*(\d+)\s+(\d+)\s+(\d+)\s+(\d+)\s+(\S+)\s+(.+?)\s*$/.exec( + line, + ); if (!match) continue; entries.push([ Number(match[1]), { parentPid: Number(match[2]), processGroupId: Number(match[3]), - state: match[4]!, - startedAt: match[5]!, + sessionId: Number(match[4]), + state: match[5]!, + startedAt: match[6]!, }, ]); } return new Map(entries); } +async function posixProcessTable( + platform: NodeJS.Platform, +): Promise { + const sessionColumn = managedAgentPosixSessionColumn(platform); + const { stdout } = await execFileAsync( + "/bin/ps", + ["-axo", `pid=,ppid=,pgid=,${sessionColumn}=,stat=,lstart=`], + { + encoding: "utf8", + windowsHide: true, + maxBuffer: 4 * 1024 * 1024, + timeout: MANAGED_AGENT_PROCESS_HELPER_TIMEOUT_MS, + killSignal: "SIGKILL", + }, + ); + return parseManagedAgentPosixProcessTable(stdout); +} + async function defaultReadProcessTable( platform: NodeJS.Platform, ): Promise { @@ -394,7 +420,7 @@ async function defaultReadProcessTable( processes: platform === "win32" ? await windowsProcessTable() - : await posixProcessTable(), + : await posixProcessTable(platform), }; } catch { return { available: false }; @@ -485,6 +511,14 @@ export class LocalManagedAgentProcessObserver implements ManagedAgentProcessObse readonly #delay: (milliseconds: number) => Promise; readonly #roots = new Map(); readonly #observedIdentities = new Map(); + // An unarmed SDK may create a short-lived subgroup that is still below the + // owned root. Its stable identities block readiness, quiescence, and root + // kill but never grant subgroup signal authority. Only positive death clears + // them; topology drift is handled as a permanent escape below. + readonly #pendingUnauthenticatedDescendants = new Map< + number, + ObservedIdentity + >(); readonly #observedPids = new Set(); readonly #sampler: NodeJS.Timeout; readonly #boundSignals = new WeakSet(); @@ -509,6 +543,7 @@ export class LocalManagedAgentProcessObserver implements ManagedAgentProcessObse #fallbackCleanupRequested = false; #lastTable: ManagedAgentKernelProcessTable | undefined; #processTableAvailable = false; + #processTableNeedsRefresh = false; #sampleTask: Promise | undefined; public constructor(options: LocalManagedAgentProcessObserverOptions = {}) { @@ -652,11 +687,46 @@ export class LocalManagedAgentProcessObserver implements ManagedAgentProcessObse this.#toolProcessContainmentArmed = true; if (this.#toolControlFailed) { for (const root of this.#roots.values()) { - root.containmentSupported = false; + this.#invalidateRootContainment(root); } } } + #invalidateRootContainment(root: OwnedRoot): void { + root.containmentSupported = false; + } + + #invalidateToolContainment(): void { + this.#toolProcessObservationInvalid = true; + } + + #hasPendingUnauthenticatedDescendants(rootPid: number): boolean { + return [...this.#pendingUnauthenticatedDescendants.values()].some( + (identity) => identity.rootPid === rootPid, + ); + } + + #expectedAfterAuthorizedGroupKill( + root: OwnedRoot, + observed: ManagedAgentKernelProcessRecord | undefined, + current: ManagedAgentKernelProcessRecord | undefined, + toolProcessGroupId: number | undefined, + ): boolean { + if (!sameProcess(observed, current)) return false; + if ( + observed?.processGroupId === root.pid && + current?.processGroupId === root.pid + ) { + return root.forceKillIssued; + } + return ( + typeof toolProcessGroupId === "number" && + observed?.processGroupId === toolProcessGroupId && + current?.processGroupId === toolProcessGroupId && + this.#toolProcessForceKillIssued + ); + } + public spawn(options: SpawnOptions): SpawnedProcess { const usePosixSupervisor = this.#platform === "darwin" || this.#platform === "linux"; @@ -710,23 +780,15 @@ export class LocalManagedAgentProcessObserver implements ManagedAgentProcessObse if (typeof child.pid === "number") { const pid = child.pid; if (usePosixSupervisor) { - let groupKillRequested = false; - Object.defineProperty(child, "killed", { - configurable: true, - enumerable: true, - get: () => groupKillRequested, - }); - child.kill = ((signal: NodeJS.Signals = "SIGTERM") => { - try { - process.kill(-pid, signal); - groupKillRequested = true; - return true; - } catch (error) { - if ((error as NodeJS.ErrnoException).code === "ESRCH") { - return false; - } - throw error; - } + child.kill = ((_signal: NodeJS.Signals = "SIGTERM") => { + // ProcessTransport calls kill() immediately before it forwards its + // private AbortSignal. Treat that first call as logical acceptance so + // the SDK will not retry through a cached PID, but preserve the live + // supervisor as the ancestry anchor. Only the subsequently forwarded + // signal may start sampled, identity-checked group cleanup. + if (!childActive(child) || child.killed) return false; + Reflect.set(child, "killed", true); + return true; }) as typeof child.kill; } this.#roots.set(pid, { @@ -800,15 +862,21 @@ export class LocalManagedAgentProcessObserver implements ManagedAgentProcessObse const groupMemberPids = typeof processGroupId === "number" ? [...table.entries()].flatMap(([pid, record]) => - record.processGroupId === processGroupId ? [pid] : [], + record.processGroupId === processGroupId && + !processIsZombie(record) + ? [pid] + : [], ) : []; if ( root && childActive(root.child) && table.get(root.pid)?.processGroupId === root.pid && + !processIsZombie(table.get(root.pid)) && parentIdentity && + !processIsZombie(parentIdentity) && childIdentity && + !processIsZombie(childIdentity) && typeof processGroupId === "number" && typeof hostProcessGroupId === "number" && processGroupId > 1 && @@ -818,6 +886,7 @@ export class LocalManagedAgentProcessObserver implements ManagedAgentProcessObse childIdentity.parentPid === parent.pid && childIdentity.processGroupId === processGroupId && groupLeaderIdentity?.processGroupId === processGroupId && + !processIsZombie(groupLeaderIdentity) && groupMemberPids.length > 0 && groupMemberPids.every((pid) => rootDescendants.has(pid)) ) { @@ -847,10 +916,12 @@ export class LocalManagedAgentProcessObserver implements ManagedAgentProcessObse if ( !registration.closed && current && + !processIsZombie(current) && + !this.#toolProcessForceKillIssued && (!sameProcess(registration.identity, current) || current.processGroupId !== processGroupId) ) { - this.#toolProcessObservationInvalid = true; + this.#invalidateToolContainment(); } } } @@ -871,6 +942,8 @@ export class LocalManagedAgentProcessObserver implements ManagedAgentProcessObse if ( !root || !childActive(root.child) || + !root.containmentSupported || + this.#processTableNeedsRefresh || this.#toolProcessObservationInvalid || typeof processGroupId !== "number" || !parent?.accepted || @@ -889,23 +962,39 @@ export class LocalManagedAgentProcessObserver implements ManagedAgentProcessObse const currentChild = table.get(child.pid); if ( !currentRoot || + processIsZombie(currentRoot) || !currentParent || + processIsZombie(currentParent) || !currentChild || + processIsZombie(currentChild) || currentRoot.processGroupId !== root.pid || (root.identity && !sameProcess(root.identity, currentRoot)) || + (root.identity && root.identity.sessionId !== currentRoot.sessionId) || (options.requireRootStopped && !currentRoot.state?.includes("T")) || !sameProcess(parent.identity, currentParent) || currentParent.processGroupId !== processGroupId || + parent.identity.sessionId !== currentParent.sessionId || !sameProcess(child.identity, currentChild) || currentChild.parentPid !== parent.pid || - currentChild.processGroupId !== processGroupId + currentChild.processGroupId !== processGroupId || + child.identity.sessionId !== currentChild.sessionId ) { return false; } const rootDescendants = descendantsOf(new Set([root.pid]), table); + const allowedGroups = new Set([root.pid, processGroupId]); + if ( + [...rootDescendants].some((pid) => { + const record = table.get(pid); + return !record || !allowedGroups.has(record.processGroupId ?? -1); + }) + ) { + return false; + } const groupMembers = [...table.entries()].filter( - ([, record]) => record.processGroupId === processGroupId, + ([, record]) => + record.processGroupId === processGroupId && !processIsZombie(record), ); return ( groupMembers.length > 0 && @@ -917,6 +1006,158 @@ export class LocalManagedAgentProcessObserver implements ManagedAgentProcessObse ); } + #observePosixOwnedProcesses(table: ManagedAgentKernelProcessTable): void { + for (const root of this.#roots.values()) { + const descendants = descendantsOf(new Set([root.pid]), table); + const toolProcessGroupId = + this.#toolProcessRootPid === root.pid + ? this.#toolProcessGroupId + : undefined; + const allowedGroups = new Set([root.pid]); + if (typeof toolProcessGroupId === "number") { + allowedGroups.add(toolProcessGroupId); + } + const validateAllowedGroups = + !this.#toolProcessContainmentArmed || + typeof toolProcessGroupId === "number"; + const currentlyOwned = new Set([root.pid, ...descendants]); + + for (const [pid, pending] of this.#pendingUnauthenticatedDescendants) { + if (pending.rootPid !== root.pid) continue; + const current = table.get(pid); + if (!sameProcess(pending.record, current) || processIsZombie(current)) { + this.#pendingUnauthenticatedDescendants.delete(pid); + } + } + + for (const [pid, record] of table) { + if (processIsZombie(record)) continue; + if ( + record.processGroupId !== root.pid && + record.processGroupId !== toolProcessGroupId + ) { + continue; + } + currentlyOwned.add(pid); + if (pid !== root.pid && !descendants.has(pid)) { + const observed = this.#observedIdentities.get(pid); + const expectedAfterAuthorizedKill = + this.#expectedAfterAuthorizedGroupKill( + root, + observed?.record, + record, + toolProcessGroupId, + ); + if (expectedAfterAuthorizedKill) continue; + if (record.processGroupId === toolProcessGroupId) { + this.#invalidateToolContainment(); + } else { + this.#invalidateRootContainment(root); + } + } + } + + for (const [pid, observed] of this.#observedIdentities) { + if (observed.rootPid !== root.pid) continue; + const current = table.get(pid); + // A successful complete process-table sample with no matching stable + // identity is positive evidence that the old process has exited. A + // recycled numeric PID never inherits the old observation. + // A kernel zombie is already dead and cannot execute, migrate, or + // authorize a signal. Its transient reparenting during reap is not a + // live containment escape. + if ( + !sameProcess(observed.record, current) || + processIsZombie(current) + ) { + continue; + } + this.#observedPids.add(pid); + if ( + current!.parentPid !== observed.record.parentPid || + current!.processGroupId !== observed.record.processGroupId || + current!.sessionId !== observed.record.sessionId || + (pid !== root.pid && !currentlyOwned.has(pid)) + ) { + const belongsToToolGroup = + typeof toolProcessGroupId === "number" && + (observed.record.processGroupId === toolProcessGroupId || + current!.processGroupId === toolProcessGroupId); + const expectedAfterAuthorizedKill = + this.#expectedAfterAuthorizedGroupKill( + root, + observed.record, + current, + toolProcessGroupId, + ); + if (expectedAfterAuthorizedKill) { + continue; + } + if (belongsToToolGroup) { + this.#invalidateToolContainment(); + } else { + this.#invalidateRootContainment(root); + } + } + } + + for (const pid of currentlyOwned) { + const current = table.get(pid); + if (!current || processIsZombie(current)) continue; + const isDescendant = descendants.has(pid); + const escapedOwnedAncestry = pid !== root.pid && !isDescendant; + const unauthenticatedDescendant = + validateAllowedGroups && + isDescendant && + !allowedGroups.has(current.processGroupId ?? -1); + if (escapedOwnedAncestry || unauthenticatedDescendant) { + const observed = this.#observedIdentities.get(pid); + const belongsToToolGroup = + typeof toolProcessGroupId === "number" && + (current.processGroupId === toolProcessGroupId || + observed?.record.processGroupId === toolProcessGroupId); + const pending = this.#pendingUnauthenticatedDescendants.get(pid); + if ( + unauthenticatedDescendant && + !this.#toolProcessContainmentArmed && + (!observed || + (pending?.rootPid === root.pid && + sameProcess(pending.record, current))) + ) { + if (!pending) { + this.#pendingUnauthenticatedDescendants.set(pid, { + rootPid: root.pid, + record: current, + }); + } + } else { + const expectedAfterAuthorizedKill = + this.#expectedAfterAuthorizedGroupKill( + root, + observed?.record, + current, + toolProcessGroupId, + ); + if (expectedAfterAuthorizedKill) continue; + if (belongsToToolGroup) { + this.#invalidateToolContainment(); + } else { + this.#invalidateRootContainment(root); + } + } + } + const observed = this.#observedIdentities.get(pid); + if (!observed || !sameProcess(observed.record, current)) { + this.#observedIdentities.set(pid, { + rootPid: root.pid, + record: current, + }); + } + this.#observedPids.add(pid); + } + } + } + public async observeProcessTree( timeoutMs = MANAGED_AGENT_PROCESS_HELPER_TIMEOUT_MS, ): Promise { @@ -934,41 +1175,32 @@ export class LocalManagedAgentProcessObserver implements ManagedAgentProcessObse const observation = await this.#boundedProcessTableRead(boundedTimeoutMs); if (!observation.available) { - this.#lastTable = undefined; - this.#processTableAvailable = false; + if ( + boundedTimeoutMs > 0 || + this.#processTableNeedsRefresh || + !this.#processTableAvailable + ) { + this.#lastTable = undefined; + this.#processTableAvailable = false; + } return false; } const table = observation.processes; this.#lastTable = table; this.#processTableAvailable = true; + this.#processTableNeedsRefresh = false; for (const root of this.#roots.values()) { if (this.#platform !== "win32") { - for (const [pid, observed] of this.#observedIdentities) { - if (observed.rootPid !== root.pid) continue; - const current = table.get(pid); - if ( - sameProcess(observed.record, current) && - current?.processGroupId !== root.pid - ) { - root.containmentSupported = false; - } - } const currentRoot = table.get(root.pid); if ( currentRoot && + !processIsZombie(currentRoot) && childActive(root.child) && + !root.forceKillIssued && currentRoot.processGroupId !== root.pid ) { - root.containmentSupported = false; - } - for (const [pid, record] of table) { - if (record.processGroupId !== root.pid) continue; - this.#observedIdentities.set(pid, { - rootPid: root.pid, - record, - }); - this.#observedPids.add(pid); + this.#invalidateRootContainment(root); } continue; } @@ -1002,6 +1234,14 @@ export class LocalManagedAgentProcessObserver implements ManagedAgentProcessObse } } this.#observeToolProcessContainment(table); + if (this.#platform !== "win32") { + this.#observePosixOwnedProcesses(table); + } + // Query.return() can remain pending while the SDK performs its own + // shutdown. Advance a requested fallback from each authoritative + // sample so the detached tool group is stopped/killed before the + // supervisor anchor is killed last, all within the same deadline. + this.#advanceFallbackCleanup(); return true; })().finally(() => { this.#sampleTask = undefined; @@ -1020,8 +1260,16 @@ export class LocalManagedAgentProcessObserver implements ManagedAgentProcessObse if (!available) { // A caller with a shorter absolute deadline must not reuse a stale table // while a longer background sample is still pending. - this.#lastTable = undefined; - this.#processTableAvailable = false; + if ( + boundedTimeoutMs > 0 || + this.#processTableNeedsRefresh || + !this.#processTableAvailable + ) { + this.#lastTable = undefined; + this.#processTableAvailable = false; + } else { + return true; + } } return available; } @@ -1037,7 +1285,9 @@ export class LocalManagedAgentProcessObserver implements ManagedAgentProcessObse processTableAvailable: this.#processTableAvailable, containmentSupported: [...this.#roots.values()].every( - ({ containmentSupported }) => containmentSupported, + ({ pid, containmentSupported }) => + containmentSupported && + !this.#hasPendingUnauthenticatedDescendants(pid), ) && (!this.#toolProcessContainmentArmed || (this.#toolProcessRegistrations.size === 2 && @@ -1048,7 +1298,7 @@ export class LocalManagedAgentProcessObserver implements ManagedAgentProcessObse if (this.#platform !== "darwin" && this.#platform !== "linux") { for (const root of this.#roots.values()) { - root.containmentSupported = false; + this.#invalidateRootContainment(root); } return unsupported("platform_unsupported"); } @@ -1060,14 +1310,20 @@ export class LocalManagedAgentProcessObserver implements ManagedAgentProcessObse if (!root.containmentSupported) { return unsupported("containment_escaped"); } + if (this.#hasPendingUnauthenticatedDescendants(root.pid)) { + return unsupported("containment_escaped"); + } if (!childActive(root.child)) { return unsupported("root_not_active"); } const currentRoot = this.#lastTable!.get(root.pid); if (!currentRoot || currentRoot.processGroupId !== root.pid) { - root.containmentSupported = false; + this.#invalidateRootContainment(root); return unsupported("root_not_group_leader"); } + if (processIsZombie(currentRoot)) { + return unsupported("root_not_active"); + } root.identity = currentRoot; if (this.#toolProcessContainmentArmed) { const parent = this.#toolProcessRegistrations.get("parent"); @@ -1109,16 +1365,56 @@ export class LocalManagedAgentProcessObserver implements ManagedAgentProcessObse }; } + #hasFreshRootGroupAuthority(root: OwnedRoot): boolean { + if (!root.containmentSupported || this.#processTableNeedsRefresh) { + return false; + } + // The observer-created supervisor group remains the bounded fallback when + // a helper read is unavailable. Once a complete table exists, however, it + // must not turn a dead/zombie or replaced numeric root into authority. + if (!this.#processTableAvailable) return true; + const current = this.#lastTable?.get(root.pid); + const baseline = + root.identity ?? this.#observedIdentities.get(root.pid)?.record; + if ( + !current || + processIsZombie(current) || + current.processGroupId !== root.pid + ) { + return false; + } + return baseline + ? sameProcess(baseline, current) && + current.parentPid === baseline.parentPid && + current.sessionId === baseline.sessionId + : true; + } + + #signalValidatedProcessGroup( + processGroupId: number, + signal: "SIGSTOP" | "SIGKILL", + ): ManagedAgentProcessSignalOutcome { + const outcome = this.#signalProcessGroup(processGroupId, signal); + if (outcome === "sent") this.#processTableNeedsRefresh = true; + return outcome; + } + #stopOwnedRootsSynchronously(): void { if (this.#platform !== "darwin" && this.#platform !== "linux") return; for (const root of this.#roots.values()) { - if (root.forceKillIssued || !childActive(root.child)) continue; + if ( + root.forceKillIssued || + !childActive(root.child) || + !this.#hasFreshRootGroupAuthority(root) + ) { + continue; + } if (!root.stopIssued) { - const stopOutcome = this.#signalProcessGroup(root.pid, "SIGSTOP"); + const stopOutcome = this.#signalValidatedProcessGroup( + root.pid, + "SIGSTOP", + ); root.stopIssued = stopOutcome === "sent"; - if (stopOutcome === "failure") { - root.containmentSupported = false; - } } } } @@ -1126,10 +1422,19 @@ export class LocalManagedAgentProcessObserver implements ManagedAgentProcessObse #killOwnedRootsSynchronously(): void { if (this.#platform !== "darwin" && this.#platform !== "linux") return; for (const root of this.#roots.values()) { - if (root.forceKillIssued || !childActive(root.child)) continue; - const killOutcome = this.#signalProcessGroup(root.pid, "SIGKILL"); + if ( + root.forceKillIssued || + !childActive(root.child) || + this.#hasPendingUnauthenticatedDescendants(root.pid) || + !this.#hasFreshRootGroupAuthority(root) + ) { + continue; + } + const killOutcome = this.#signalValidatedProcessGroup( + root.pid, + "SIGKILL", + ); root.forceKillIssued = killOutcome === "sent"; - if (killOutcome === "failure") root.containmentSupported = false; } } @@ -1162,6 +1467,15 @@ export class LocalManagedAgentProcessObserver implements ManagedAgentProcessObse const processGroupId = this.#toolProcessGroupId; if (typeof processGroupId !== "number") return; + const table = this.#lastTable!; + const liveToolGroupMembers = [...table.values()].some( + (record) => + record.processGroupId === processGroupId && !processIsZombie(record), + ); + if (this.#toolProcessForceKillIssued && !liveToolGroupMembers) { + this.#killOwnedRootsSynchronously(); + return; + } const groupLiveness = this.#processGroupLiveness(processGroupId); if (groupLiveness === "gone") { this.#killOwnedRootsSynchronously(); @@ -1169,7 +1483,6 @@ export class LocalManagedAgentProcessObserver implements ManagedAgentProcessObse } if (groupLiveness !== "alive") return; - const table = this.#lastTable!; if ( !this.#hasFreshToolAuthority(table, { requireRootStopped: true, @@ -1179,38 +1492,58 @@ export class LocalManagedAgentProcessObserver implements ManagedAgentProcessObse return; } if (!this.#toolProcessStopIssued) { - const stopOutcome = this.#signalProcessGroup(processGroupId, "SIGSTOP"); + const stopOutcome = this.#signalValidatedProcessGroup( + processGroupId, + "SIGSTOP", + ); this.#toolProcessStopIssued = stopOutcome === "sent"; if (stopOutcome === "gone") this.#killOwnedRootsSynchronously(); return; } if (!this.#toolProcessForceKillIssued) { - const killOutcome = this.#signalProcessGroup(processGroupId, "SIGKILL"); + const killOutcome = this.#signalValidatedProcessGroup( + processGroupId, + "SIGKILL", + ); this.#toolProcessForceKillIssued = killOutcome === "sent"; - if (killOutcome === "sent" || killOutcome === "gone") { + if (killOutcome === "gone") { this.#killOwnedRootsSynchronously(); } } } - async #currentObservation( + #currentObservation( startedAt: number, emergencyCleanupAttempted: boolean, - ): Promise { + ): ManagedAgentTeardownObservation { const roots = [...this.#roots.values()]; const alive = new Set(); for (const root of roots) { - if (childActive(root.child)) alive.add(root.pid); + const sampledRoot = this.#lastTable?.get(root.pid); + if ( + childActive(root.child) && + (!this.#processTableAvailable || + (sampledRoot && !processIsZombie(sampledRoot))) + ) { + alive.add(root.pid); + } if (!this.#processTableAvailable) continue; const table = this.#lastTable!; if (this.#platform !== "win32") { - for (const [pid, record] of table) { - if (record.processGroupId === root.pid) alive.add(pid); - } - const groupLiveness = this.#processGroupLiveness(root.pid); + const liveRootGroupPids = [...table.entries()].flatMap( + ([pid, record]) => + record.processGroupId === root.pid && !processIsZombie(record) + ? [pid] + : [], + ); + for (const pid of liveRootGroupPids) alive.add(pid); + const groupLiveness = + liveRootGroupPids.length > 0 + ? this.#processGroupLiveness(root.pid) + : "gone"; if (groupLiveness === "alive") alive.add(root.pid); if (groupLiveness === "unknown") { - root.containmentSupported = false; + this.#invalidateRootContainment(root); } } else { for (const [pid, observed] of this.#observedIdentities) { @@ -1232,20 +1565,47 @@ export class LocalManagedAgentProcessObserver implements ManagedAgentProcessObse } if (this.#processTableAvailable) { const table = this.#lastTable!; - if (typeof toolProcessGroupId === "number") { - for (const [pid, record] of table) { - if (record.processGroupId === toolProcessGroupId) { - alive.add(pid); + if (this.#platform !== "win32") { + for (const [pid, observed] of this.#observedIdentities) { + const current = table.get(pid); + if ( + !sameProcess(observed.record, current) || + processIsZombie(current) + ) { + continue; + } + alive.add(pid); + if ( + typeof current!.processGroupId === "number" && + (current!.parentPid !== observed.record.parentPid || + current!.processGroupId !== observed.record.processGroupId || + current!.sessionId !== observed.record.sessionId) + ) { + alive.add(current!.processGroupId); } } - const groupLiveness = this.#processGroupLiveness(toolProcessGroupId); + } + if (typeof toolProcessGroupId === "number") { + const liveToolGroupPids = [...table.entries()].flatMap( + ([pid, record]) => + record.processGroupId === toolProcessGroupId && + !processIsZombie(record) + ? [pid] + : [], + ); + for (const pid of liveToolGroupPids) alive.add(pid); + const groupLiveness = + liveToolGroupPids.length > 0 + ? this.#processGroupLiveness(toolProcessGroupId) + : "gone"; if (groupLiveness === "alive") alive.add(toolProcessGroupId); if (groupLiveness === "unknown") { - this.#toolProcessObservationInvalid = true; + this.#invalidateToolContainment(); } } for (const registration of toolRegistrations) { - if (table.has(registration.pid)) { + const current = table.get(registration.pid); + if (current && !processIsZombie(current)) { alive.add(registration.pid); } } @@ -1265,6 +1625,9 @@ export class LocalManagedAgentProcessObserver implements ManagedAgentProcessObse )); const containmentSupported = roots.every(({ containmentSupported: supported }) => supported) && + roots.every( + ({ pid }) => !this.#hasPendingUnauthenticatedDescendants(pid), + ) && (!this.#toolProcessContainmentArmed || (this.#toolControlAvailable && this.#toolProcessObservationComplete && @@ -1302,13 +1665,30 @@ export class LocalManagedAgentProcessObserver implements ManagedAgentProcessObse ): Promise { const startedAt = this.#now(); const boundedTimeoutMs = Math.max(0, timeoutMs); + if ( + boundedTimeoutMs === 0 && + this.#processTableAvailable && + !this.#processTableNeedsRefresh && + !this.#sampleTask + ) { + const cachedObservation = this.#currentObservation(startedAt, false); + if (cachedObservation.quiescent) { + // The fresh cached table already proved quiescence at call entry. Its + // evidence elapsed is therefore zero even if returning the Promise + // crosses a wall-clock millisecond boundary. + return { + ...cachedObservation, + deadlineMet: true, + elapsedMs: 0, + }; + } + } for (;;) { const elapsedBeforeSample = Math.max(0, this.#now() - startedAt); await this.observeProcessTree( Math.max(0, boundedTimeoutMs - elapsedBeforeSample), ); - this.#advanceFallbackCleanup(); - const observation = await this.#currentObservation(startedAt, false); + const observation = this.#currentObservation(startedAt, false); if (observation.quiescent) { return { ...observation, diff --git a/packages/harness/src/experimental/managed-agent-spike/runtime-sdk-loopback.test.ts b/packages/harness/src/experimental/managed-agent-spike/runtime-sdk-loopback.test.ts index c29529e5e..7376b64bd 100644 --- a/packages/harness/src/experimental/managed-agent-spike/runtime-sdk-loopback.test.ts +++ b/packages/harness/src/experimental/managed-agent-spike/runtime-sdk-loopback.test.ts @@ -1,10 +1,11 @@ -import { spawn } from "node:child_process"; +import { execFile, spawn } from "node:child_process"; import { once } from "node:events"; import { readFile } from "node:fs/promises"; import { createServer, type ServerResponse } from "node:http"; import { createRequire } from "node:module"; import type { AddressInfo } from "node:net"; import { dirname, join } from "node:path"; +import { promisify } from "node:util"; import { query as agentSdkQuery } from "@anthropic-ai/claude-agent-sdk"; import { expect, it } from "vitest"; @@ -16,11 +17,18 @@ import { waitForManagedAgentFixturePids, } from "./fixture.js"; import { MANAGED_AGENT_CONTRACT } from "./contract.js"; -import { LocalManagedAgentProcessObserver } from "./process-observer.js"; +import { + LocalManagedAgentProcessObserver, + managedAgentPosixSessionColumn, + parseManagedAgentPosixProcessTable, + type ManagedAgentKernelProcessTable, + type ManagedAgentProcessTableObservation, +} from "./process-observer.js"; import { qualifiedManagedAgentMcpToolName, runManagedAgentProbe, } from "./runtime.js"; +import type { ManagedAgentProcessObserver } from "./types.js"; const RUN_ID = "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa"; const EXECUTION_ID = "bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb"; @@ -32,6 +40,24 @@ const ALLOWED_BASH_COMMAND = "git status --short"; const DENIED_BASH_COMMAND = "touch denied-side-effect.txt"; const ECHO_NONCE_TOOL = qualifiedManagedAgentMcpToolName("echo_nonce"); const require = createRequire(import.meta.url); +const execFileAsync = promisify(execFile); + +async function readLoopbackProcessTable(): Promise { + try { + const sessionColumn = managedAgentPosixSessionColumn(process.platform); + const { stdout } = await execFileAsync( + "/bin/ps", + ["-axo", `pid=,ppid=,pgid=,${sessionColumn}=,stat=,lstart=`], + { encoding: "utf8", maxBuffer: 4 * 1024 * 1024, timeout: 1_000 }, + ); + return { + available: true, + processes: parseManagedAgentPosixProcessTable(stdout), + }; + } catch { + return { available: false }; + } +} function processExists(pid: number): boolean { try { @@ -245,6 +271,92 @@ function writeFinalResponse(response: ServerResponse, turn: number): void { it("enforces real-SDK built-in and in-process MCP calls with exact loopback correlation", async () => { const fixture = await createManagedAgentFixture(() => "loopback-nonce"); + const startedAt = Date.now(); + const stateSamples: Array<{ + readonly elapsedMs: number; + readonly processes?: ManagedAgentKernelProcessTable; + }> = []; + const groupSignals: Array<{ + readonly elapsedMs: number; + readonly groupId: number; + readonly signal: "SIGSTOP" | "SIGKILL"; + readonly outcome: "sent" | "gone" | "failure"; + }> = []; + const lifecycle: Array<{ + readonly elapsedMs: number; + readonly event: string; + }> = []; + const observer = new LocalManagedAgentProcessObserver({ + readProcessTable: async () => { + const observation = await readLoopbackProcessTable(); + stateSamples.push({ + elapsedMs: Date.now() - startedAt, + ...(observation.available ? { processes: observation.processes } : {}), + }); + return observation; + }, + signalProcessGroup: (groupId, signal) => { + let outcome: "sent" | "gone" | "failure"; + try { + process.kill(-groupId, signal); + outcome = "sent"; + } catch (error) { + outcome = + (error as NodeJS.ErrnoException).code === "ESRCH" + ? "gone" + : "failure"; + } + groupSignals.push({ + elapsedMs: Date.now() - startedAt, + groupId, + signal, + outcome, + }); + return outcome; + }, + }); + let supervisorPid: number | undefined; + const observedObserver: ManagedAgentProcessObserver = { + spawn: (options) => { + const child = observer.spawn(options); + const pid = Reflect.get(child, "pid"); + supervisorPid = typeof pid === "number" ? pid : undefined; + lifecycle.push({ + elapsedMs: Date.now() - startedAt, + event: "observer_spawned", + }); + return child; + }, + bindAbortSignal: (signal) => observer.bindAbortSignal(signal), + armToolProcessContainment: () => observer.armToolProcessContainment(), + prepareCancellation: () => observer.prepareCancellation(), + observeProcessTree: (timeoutMs) => observer.observeProcessTree(timeoutMs), + waitForQuiescence: async (timeoutMs) => { + lifecycle.push({ + elapsedMs: Date.now() - startedAt, + event: "wait_for_quiescence_started", + }); + const result = await observer.waitForQuiescence(timeoutMs); + lifecycle.push({ + elapsedMs: Date.now() - startedAt, + event: `wait_for_quiescence_settled:${result.quiescent}:${result.containmentSupported}`, + }); + return result; + }, + emergencyCleanup: async (timeoutMs) => { + lifecycle.push({ + elapsedMs: Date.now() - startedAt, + event: "host_emergency_cleanup_started", + }); + const result = await observer.emergencyCleanup(timeoutMs); + lifecycle.push({ + elapsedMs: Date.now() - startedAt, + event: `host_emergency_cleanup_settled:${result.quiescent}:${result.containmentSupported}`, + }); + return result; + }, + dispose: () => observer.dispose(), + }; const observations: LoopbackObservation[] = []; let helloCount = 0; let inferenceTurn = 0; @@ -339,8 +451,32 @@ it("enforces real-SDK built-in and in-process MCP calls with exact loopback corr }, { hermeticGatewayOrigin: `http://127.0.0.1:${address.port}`, - queryFactory: ({ prompt, options }) => - agentSdkQuery({ prompt, options }), + processObserver: observedObserver, + queryFactory: ({ prompt, options }) => { + const sdkQuery = agentSdkQuery({ prompt, options }); + return { + [Symbol.asyncIterator]: () => sdkQuery[Symbol.asyncIterator](), + close: () => { + lifecycle.push({ + elapsedMs: Date.now() - startedAt, + event: "sdk_close_called", + }); + sdkQuery.close(); + }, + return: async () => { + lifecycle.push({ + elapsedMs: Date.now() - startedAt, + event: "sdk_return_started", + }); + const returned = await sdkQuery.return(undefined); + lifecycle.push({ + elapsedMs: Date.now() - startedAt, + event: "sdk_return_settled", + }); + return returned; + }, + }; + }, uuid: () => { const id = ids.shift(); if (!id) throw new Error("unexpected UUID request"); @@ -366,7 +502,38 @@ it("enforces real-SDK built-in and in-process MCP calls with exact loopback corr expect( observations.map(({ mcpResultMatches }) => mcpResultMatches), ).toEqual([false, false, false, false, true]); - expect(result.terminal).toBe("success"); + const stateTransitions = stateSamples.reduce< + Array<{ + readonly elapsedMs: number; + readonly records: unknown; + }> + >((transitions, { elapsedMs, processes }) => { + const records = processes + ? [...processes.entries()] + .filter( + ([pid]) => + pid === supervisorPid || + result.teardown.observedPids.includes(pid), + ) + .map(([pid, record]) => ({ pid, ...record })) + : "unavailable"; + const previous = transitions.at(-1)?.records; + if (JSON.stringify(previous) !== JSON.stringify(records)) { + transitions.push({ elapsedMs, records }); + } + return transitions; + }, []); + expect( + result.terminal, + JSON.stringify({ + groupSignals, + lifecycle, + processTableSampleCount: stateSamples.length, + stateTransitions, + teardown: result.teardown, + terminationEvidence: result.terminationEvidence, + }), + ).toBe("success"); const requested = result.toolEvidence.filter( ({ status }) => status === "requested", @@ -447,6 +614,8 @@ it("enforces real-SDK built-in and in-process MCP calls with exact loopback corr expect(result.queryClosed).toBe(true); expect(result.teardown.quiescent).toBe(true); } finally { + await observer.emergencyCleanup(1_000); + observer.dispose(); server.closeAllConnections(); await new Promise((resolve) => server.close(() => resolve())); await fixture.cleanup(); @@ -462,7 +631,73 @@ it.skipIf( const fixture = await createManagedAgentFixture( () => "loopback-l2-cancellation", ); - const observer = new LocalManagedAgentProcessObserver(); + const startedAt = Date.now(); + const stateSamples: Array<{ + readonly elapsedMs: number; + readonly processes?: ManagedAgentKernelProcessTable; + }> = []; + const groupSignals: Array<{ + readonly elapsedMs: number; + readonly groupId: number; + readonly signal: "SIGSTOP" | "SIGKILL"; + readonly outcome: "sent" | "gone" | "failure"; + }> = []; + const observer = new LocalManagedAgentProcessObserver({ + readProcessTable: async () => { + const observation = await readLoopbackProcessTable(); + stateSamples.push({ + elapsedMs: Date.now() - startedAt, + ...(observation.available + ? { processes: observation.processes } + : {}), + }); + return observation; + }, + signalProcessGroup: (groupId, signal) => { + let outcome: "sent" | "gone" | "failure"; + try { + process.kill(-groupId, signal); + outcome = "sent"; + } catch (error) { + outcome = + (error as NodeJS.ErrnoException).code === "ESRCH" + ? "gone" + : "failure"; + } + groupSignals.push({ + elapsedMs: Date.now() - startedAt, + groupId, + signal, + outcome, + }); + return outcome; + }, + }); + const cleanupOrder: string[] = []; + let supervisorPid: number | undefined; + const observedObserver: ManagedAgentProcessObserver = { + spawn: (options) => { + options.signal.addEventListener( + "abort", + () => cleanupOrder.push("sdk_forwarded_signal"), + { once: true }, + ); + const child = observer.spawn(options); + const pid = Reflect.get(child, "pid"); + supervisorPid = typeof pid === "number" ? pid : undefined; + return child; + }, + bindAbortSignal: (signal) => observer.bindAbortSignal(signal), + armToolProcessContainment: () => observer.armToolProcessContainment(), + prepareCancellation: () => observer.prepareCancellation(), + observeProcessTree: (timeoutMs) => observer.observeProcessTree(timeoutMs), + waitForQuiescence: (timeoutMs) => observer.waitForQuiescence(timeoutMs), + emergencyCleanup: (timeoutMs) => { + cleanupOrder.push("host_emergency_cleanup"); + return observer.emergencyCleanup(timeoutMs); + }, + dispose: () => observer.dispose(), + }; const unrelated = spawn( process.execPath, ["-e", "setInterval(() => {}, 1000)"], @@ -470,6 +705,8 @@ it.skipIf( ); await once(unrelated, "spawn"); let fixturePids: readonly number[] = []; + let fixtureToolProcessGroupId: number | undefined; + let cancellationStartedAt: number | undefined; let inferenceTurn = 0; const server = createServer((request, response) => { if (request.method === "HEAD" && request.url === "/api/hello") { @@ -522,9 +759,23 @@ it.skipIf( }, { hermeticGatewayOrigin: `http://127.0.0.1:${address.port}`, - processObserver: observer, - queryFactory: ({ prompt, options }) => - agentSdkQuery({ prompt, options }), + processObserver: observedObserver, + queryFactory: ({ prompt, options }) => { + const sdkQuery = agentSdkQuery({ prompt, options }); + return { + [Symbol.asyncIterator]: () => sdkQuery[Symbol.asyncIterator](), + close: () => { + cleanupOrder.push("sdk_close_called"); + sdkQuery.close(); + }, + return: async () => { + cleanupOrder.push("sdk_return_started"); + const returned = await sdkQuery.return(undefined); + cleanupOrder.push("sdk_return_settled"); + return returned; + }, + }; + }, waitForCancellationSignal: async (signal) => { fixturePids = await waitForManagedAgentFixturePids( fixture, @@ -541,6 +792,27 @@ it.skipIf( expect( fixturePids.every((pid) => readiness.observedPids.includes(pid)), ).toBe(true); + let readinessProcessTable: + | ManagedAgentKernelProcessTable + | undefined; + for (let index = stateSamples.length - 1; index >= 0; index -= 1) { + const processes = stateSamples[index]?.processes; + if (!processes?.has(fixturePids[0]!)) continue; + readinessProcessTable = processes; + break; + } + fixtureToolProcessGroupId = readinessProcessTable?.get( + fixturePids[0]!, + )?.processGroupId; + expect(fixtureToolProcessGroupId).toBeTypeOf("number"); + expect( + fixturePids.every( + (pid) => + readinessProcessTable?.get(pid)?.processGroupId === + fixtureToolProcessGroupId, + ), + ).toBe(true); + cancellationStartedAt = Date.now(); }, uuid: () => { const id = ids.shift(); @@ -549,9 +821,45 @@ it.skipIf( }, }, ); + const cancellationElapsedMs = + Date.now() - (cancellationStartedAt ?? startedAt); + const stateTransitions = stateSamples.reduce< + Array<{ + readonly elapsedMs: number; + readonly records: unknown; + }> + >((transitions, { elapsedMs, processes }) => { + const records = processes + ? [...processes.entries()] + .filter( + ([pid]) => + pid === supervisorPid || + fixturePids.includes(pid) || + result.teardown.observedPids.includes(pid), + ) + .map(([pid, record]) => ({ pid, ...record })) + : "unavailable"; + const previous = transitions.at(-1)?.records; + if (JSON.stringify(previous) !== JSON.stringify(records)) { + transitions.push({ elapsedMs, records }); + } + return transitions; + }, []); expect(inferenceTurn).toBe(1); - expect(result.terminal).toBe("cancelled"); + expect( + result.terminal, + JSON.stringify({ + cleanupOrder, + elapsedMs: cancellationElapsedMs, + groupSignals, + queryClosed: result.queryClosed, + stateTransitions, + teardown: result.teardown, + terminationEvidence: result.terminationEvidence, + }), + ).toBe("cancelled"); + expect(cancellationElapsedMs).toBeLessThan(5_000); expect(result.cancellationRequested).toBe(true); expect(result.queryClosed).toBe(true); expect(result.teardown).toMatchObject({ @@ -568,6 +876,226 @@ it.skipIf( expect(fixturePids).toHaveLength(2); expect(fixturePids.every((pid) => !processExists(pid))).toBe(true); expect(processExists(unrelated.pid!)).toBe(true); + expect( + groupSignals.map(({ groupId, signal }) => [groupId, signal]), + ).toEqual([ + [supervisorPid, "SIGSTOP"], + [fixtureToolProcessGroupId, "SIGSTOP"], + [fixtureToolProcessGroupId, "SIGKILL"], + [supervisorPid, "SIGKILL"], + ]); + expect(cleanupOrder).toEqual( + expect.arrayContaining([ + "sdk_close_called", + "sdk_return_started", + "sdk_return_settled", + "host_emergency_cleanup", + ]), + ); + expect(cleanupOrder.indexOf("sdk_close_called")).toBeLessThan( + cleanupOrder.indexOf("sdk_return_started"), + ); + expect(cleanupOrder.indexOf("sdk_return_started")).toBeLessThan( + cleanupOrder.indexOf("sdk_return_settled"), + ); + expect(cleanupOrder.indexOf("sdk_return_settled")).toBeLessThan( + cleanupOrder.indexOf("host_emergency_cleanup"), + ); + const forwardedSignalIndex = cleanupOrder.indexOf("sdk_forwarded_signal"); + expect(forwardedSignalIndex).toBeGreaterThanOrEqual(0); + expect(forwardedSignalIndex).toBeLessThan( + cleanupOrder.indexOf("host_emergency_cleanup"), + ); + } finally { + await observer.emergencyCleanup(1_000); + observer.dispose(); + server.closeAllConnections(); + await new Promise((resolve) => server.close(() => resolve())); + for (const pid of fixturePids) { + if (!processExists(pid)) continue; + await forceKillTestProcess(pid); + } + if (typeof unrelated.pid === "number" && processExists(unrelated.pid)) { + unrelated.kill("SIGKILL"); + await waitForProcessDeath(unrelated.pid); + } + await fixture.cleanup(); + } + expect(fixturePids.every((pid) => !processExists(pid))).toBe(true); + }, + 20_000, +); + +it.skipIf( + process.platform === "win32" || + process.versions.node !== MANAGED_AGENT_CONTRACT.certificationNodeVersion, +)( + "fails closed through the runtime timeout fallback when the SDK signal is not forwarded", + async () => { + const fixture = await createManagedAgentFixture( + () => "loopback-l2-missing-forwarded-signal", + ); + const observer = new LocalManagedAgentProcessObserver(); + const neverForwardedController = new AbortController(); + const cleanupOrder: string[] = []; + const observedObserver: ManagedAgentProcessObserver = { + spawn: (options) => { + options.signal.addEventListener( + "abort", + () => cleanupOrder.push("sdk_forwarded_signal_unobserved"), + { once: true }, + ); + return observer.spawn({ + ...options, + signal: neverForwardedController.signal, + }); + }, + bindAbortSignal: (signal) => observer.bindAbortSignal(signal), + armToolProcessContainment: () => observer.armToolProcessContainment(), + prepareCancellation: () => observer.prepareCancellation(), + observeProcessTree: (timeoutMs) => observer.observeProcessTree(timeoutMs), + waitForQuiescence: (timeoutMs) => observer.waitForQuiescence(timeoutMs), + emergencyCleanup: (timeoutMs) => { + cleanupOrder.push("host_timeout_fallback"); + return observer.emergencyCleanup(timeoutMs); + }, + dispose: () => observer.dispose(), + }; + const unrelated = spawn( + process.execPath, + ["-e", "setInterval(() => {}, 1000)"], + { stdio: "ignore", windowsHide: true }, + ); + await once(unrelated, "spawn"); + let fixturePids: readonly number[] = []; + let cancellationStartedAt: number | undefined; + let inferenceTurn = 0; + const server = createServer((request, response) => { + if (request.method === "HEAD" && request.url === "/api/hello") { + response.writeHead(200).end(); + return; + } + if ( + request.method !== "POST" || + request.url?.split("?")[0] !== "/v1/messages" + ) { + response.writeHead(404).end(); + return; + } + request.resume(); + request.once("end", () => { + inferenceTurn += 1; + writeToolUseResponse(response, inferenceTurn, { + id: "toolu_loopback_l2_missing_forwarded_signal", + name: "Bash", + input: { command: fixture.l2BashCommand }, + }); + }); + }); + + await new Promise((resolve, reject) => { + server.once("error", reject); + server.listen(0, "127.0.0.1", resolve); + }); + try { + const address = server.address() as AddressInfo; + const ids = [RUN_ID, EXECUTION_ID]; + const result = await runManagedAgentProbe( + { + scenario: "L2", + workspaceRoot: fixture.workspaceRoot, + configRoot: fixture.configRoot, + target: "sonnet-5", + gatewayOrigin: `http://127.0.0.1:${address.port}`, + gatewayCredential: "sk-ant-api03-local-loopback-only", + prompt: fixture.prompt("L2"), + maxTurns: 4, + maxBudgetUsd: 0.25, + allowedBashCommands: [fixture.l2BashCommand], + pathRoleBindings: [], + expectedL1FinalBytes: [], + preservePaths: [ + FIXTURE_PATHS.dirtySentinel, + FIXTURE_PATHS.untrackedSentinel, + ], + }, + { + hermeticGatewayOrigin: `http://127.0.0.1:${address.port}`, + processObserver: observedObserver, + queryFactory: ({ prompt, options }) => { + const sdkQuery = agentSdkQuery({ prompt, options }); + return { + [Symbol.asyncIterator]: () => sdkQuery[Symbol.asyncIterator](), + close: () => { + cleanupOrder.push("sdk_close_called"); + sdkQuery.close(); + }, + return: async () => { + cleanupOrder.push("sdk_return_started"); + await sdkQuery.return(undefined); + cleanupOrder.push("sdk_return_underlying_settled"); + return new Promise>(() => undefined); + }, + }; + }, + waitForCancellationSignal: async (signal) => { + fixturePids = await waitForManagedAgentFixturePids( + fixture, + 10_000, + signal, + ); + await expect(observer.prepareCancellation()).resolves.toMatchObject( + { + supported: true, + reason: "ready", + containmentSupported: true, + ownershipProven: true, + }, + ); + cancellationStartedAt = Date.now(); + }, + uuid: () => { + const id = ids.shift(); + if (!id) throw new Error("unexpected UUID request"); + return id; + }, + }, + ); + const cancellationElapsedMs = + Date.now() - (cancellationStartedAt ?? Date.now()); + + expect(inferenceTurn).toBe(1); + expect(result.terminal).toBe("close_timeout"); + expect(result.cancellationRequested).toBe(true); + expect(result.queryClosed).toBe(false); + expect(result.teardown).toMatchObject({ + quiescent: true, + deadlineMet: true, + processTableAvailable: true, + containmentSupported: true, + ownershipProven: true, + forceKillIssued: true, + toolProcessObservationComplete: true, + toolProcessChannelsClosed: true, + alivePidsAtDeadline: [], + }); + expect(result.teardown.elapsedMs).toBeLessThanOrEqual(5_000); + expect(cancellationElapsedMs).toBeLessThanOrEqual(5_000); + expect(fixturePids).toHaveLength(2); + expect(fixturePids.every((pid) => !processExists(pid))).toBe(true); + expect(processExists(unrelated.pid!)).toBe(true); + expect(cleanupOrder).toEqual( + expect.arrayContaining([ + "sdk_close_called", + "sdk_return_started", + "sdk_return_underlying_settled", + "sdk_forwarded_signal_unobserved", + "host_timeout_fallback", + ]), + ); + expect( + cleanupOrder.indexOf("sdk_forwarded_signal_unobserved"), + ).toBeLessThan(cleanupOrder.indexOf("host_timeout_fallback")); } finally { await observer.emergencyCleanup(1_000); observer.dispose(); diff --git a/packages/harness/src/experimental/managed-agent-spike/runtime.test.ts b/packages/harness/src/experimental/managed-agent-spike/runtime.test.ts index a30ff3a20..292e2e6bc 100644 --- a/packages/harness/src/experimental/managed-agent-spike/runtime.test.ts +++ b/packages/harness/src/experimental/managed-agent-spike/runtime.test.ts @@ -967,6 +967,178 @@ describe("runManagedAgentProbe", () => { expect(result.terminationEvidence.beforePolicyOverride).toBe("query_error"); }, 10_000); + it("awaits async-generator cleanup after void close before host fallback", async () => { + const { config } = await probeConfig("L2"); + const observer = fakeObserver(); + let resolveCancellation!: () => void; + const cancellation = new Promise((resolve) => { + resolveCancellation = resolve; + }); + let resolveReturn!: () => void; + const returned = new Promise((resolve) => { + resolveReturn = resolve; + }); + let markToolArmed!: () => void; + const toolArmed = new Promise((resolve) => { + markToolArmed = resolve; + }); + let markCloseCalled!: () => void; + const closeCalled = new Promise((resolve) => { + markCloseCalled = resolve; + }); + const returnCleanup = vi.fn(async () => { + await returned; + return { done: true as const, value: undefined }; + }); + const close = vi.fn(() => markCloseCalled()); + let nextCall = 0; + + const resultPromise = runManagedAgentProbe(config, { + hermeticGatewayOrigin: config.gatewayOrigin, + processObserver: observer, + waitForCancellationSignal: async () => cancellation, + queryFactory: ({ options }) => ({ + [Symbol.asyncIterator]() { + return { + next: async (): Promise> => { + nextCall += 1; + if (nextCall === 1) { + return { + done: false, + value: { + type: "assistant", + message: { + id: "assistant_cleanup_order", + content: [ + { + type: "tool_use", + id: "toolu_cleanup_order", + name: "Bash", + input: { command: config.allowedBashCommands[0] }, + }, + ], + }, + }, + }; + } + await invokePreToolUse(options, { + toolName: "Bash", + toolInput: { command: config.allowedBashCommands[0] }, + toolUseId: "toolu_cleanup_order", + }); + markToolArmed(); + return new Promise>(() => undefined); + }, + return: returnCleanup, + }; + }, + close, + return: returnCleanup, + }), + }); + + await toolArmed; + resolveCancellation(); + await closeCalled; + await new Promise((resolve) => setImmediate(resolve)); + const returnStartedBeforeRelease = returnCleanup.mock.calls.length === 1; + const fallbackStartedBeforeRelease = + observer.emergencyCleanup.mock.calls.length > 0; + resolveReturn(); + const result = await resultPromise; + + expect(close).toHaveBeenCalledOnce(); + expect(returnStartedBeforeRelease).toBe(true); + expect(fallbackStartedBeforeRelease).toBe(false); + expect(observer.emergencyCleanup).toHaveBeenCalledOnce(); + expect(result.queryClosed).toBe(true); + expect(result.terminal).toBe("cancelled"); + expect(result.cancellationRequested).toBe(true); + }, 10_000); + + it.each([ + ["clean completion", false, "incomplete"], + ["SDK error-result completion", true, "sdk_result_error"], + ] as const)( + "retains armed L2 readiness and one deadline after %s", + async (_description, emitErrorResult, expectedTerminal) => { + const { config } = await probeConfig("L2"); + const observer = fakeObserver(); + let now = 1_000; + let readinessCompleted = false; + let readinessWasAborted = false; + let capturedOptions: Options | undefined; + + const result = await runManagedAgentProbe(config, { + hermeticGatewayOrigin: config.gatewayOrigin, + processObserver: observer, + now: () => now, + waitForCancellationSignal: (signal) => + new Promise((resolveReadiness, rejectReadiness) => { + const timer = setTimeout(() => { + now = 3_250; + readinessCompleted = true; + resolveReadiness(); + }, 25); + signal.addEventListener( + "abort", + () => { + if (!readinessCompleted) readinessWasAborted = true; + clearTimeout(timer); + rejectReadiness( + new Error("readiness aborted after early completion"), + ); + }, + { once: true }, + ); + }), + queryFactory: ({ options }) => ({ + async *[Symbol.asyncIterator]() { + capturedOptions = options; + yield { + type: "assistant", + message: { + id: "assistant_early_settlement", + content: [ + { + type: "tool_use", + id: "toolu_early_settlement", + name: "Bash", + input: { command: config.allowedBashCommands[0] }, + }, + ], + }, + }; + await invokePreToolUse(options, { + toolName: "Bash", + toolInput: { command: config.allowedBashCommands[0] }, + toolUseId: "toolu_early_settlement", + }); + if (emitErrorResult) { + yield { + type: "result", + subtype: "error_max_turns", + is_error: true, + num_turns: 1, + }; + } + }, + close: vi.fn(), + }), + }); + + expect(readinessCompleted).toBe(true); + expect(readinessWasAborted).toBe(false); + expect(capturedOptions?.abortController?.signal.aborted).toBe(true); + expect(observer.emergencyCleanup).toHaveBeenCalledWith(2_750); + expect(result.teardown.elapsedMs).toBe(2_250); + expect(result.queryClosed).toBe(true); + expect(result.cancellationRequested).toBe(false); + expect(result.terminal).toBe(expectedTerminal); + }, + 10_000, + ); + it("abandons a never-resolving iterator next immediately after raw cancellation", async () => { const { config } = await probeConfig("L2"); const observer = fakeObserver(); @@ -986,6 +1158,7 @@ describe("runManagedAgentProbe", () => { markNextStarted?.(); return new Promise>(() => undefined); }, + return: async () => ({ done: true, value: undefined }), }; }, close, @@ -1009,6 +1182,29 @@ describe("runManagedAgentProbe", () => { expect(observer.bindAbortSignal).not.toHaveBeenCalled(); }); + it("accepts an awaited close promise when no iterator return exists", async () => { + const { config } = await probeConfig(); + const close = vi.fn(async () => undefined); + const result = await runManagedAgentProbe(config, { + hermeticGatewayOrigin: config.gatewayOrigin, + processObserver: fakeObserver(), + queryFactory: () => ({ + [Symbol.asyncIterator]() { + return { + next: async () => ({ done: true as const, value: undefined }), + }; + }, + close, + }), + }); + + expect(close).toHaveBeenCalledOnce(); + expect(result.queryClosed).toBe(true); + expect(result.terminationEvidence.queryExecution).toBe( + "iteration_completed", + ); + }); + it("keeps a CLI-shaped process alive until bounded close and cleanup complete", async () => { const { config } = await probeConfig("L2"); const childProgram = String.raw` diff --git a/packages/harness/src/experimental/managed-agent-spike/runtime.ts b/packages/harness/src/experimental/managed-agent-spike/runtime.ts index ee46d6efd..1fa4c2b36 100644 --- a/packages/harness/src/experimental/managed-agent-spike/runtime.ts +++ b/packages/harness/src/experimental/managed-agent-spike/runtime.ts @@ -278,15 +278,45 @@ function buildManagedAgentPolicyDiagnostics( async function closeQueryBounded( query: ManagedAgentQuery, + iterator: AsyncIterator | undefined, timeoutMs = QUERY_CLOSE_TIMEOUT_MS, ): Promise { let timeout: NodeJS.Timeout | undefined; - const close = Promise.resolve() - .then(() => query.close()) - .then( - () => true, - () => false, - ); + let closeResult: void | Promise; + try { + closeResult = query.close(); + } catch { + return false; + } + const closeWasAwaitable = + closeResult !== undefined && + closeResult !== null && + typeof (closeResult as PromiseLike).then === "function"; + const closeSettled = Promise.resolve(closeResult).then( + () => true, + () => false, + ); + const cleanupSettled = + typeof query.return === "function" + ? Promise.resolve() + .then(() => query.return!()) + .then( + () => true, + () => false, + ) + : typeof iterator?.return === "function" + ? Promise.resolve() + .then(() => iterator.return!()) + .then( + () => true, + () => false, + ) + : closeWasAwaitable + ? closeSettled + : new Promise(() => undefined); + const close = Promise.all([closeSettled, cleanupSettled]).then((settled) => + settled.every(Boolean), + ); if (timeoutMs <= 0) { void close; return false; @@ -411,6 +441,7 @@ export async function runManagedAgentProbe( let cancellationRequestedAt: number | undefined; let abortStartedAt: number | undefined; let query: ManagedAgentQuery | undefined; + let iterator: AsyncIterator | undefined; let queryFailed = false; let queryClosed = false; let cancellationTriggerFailed = false; @@ -572,7 +603,7 @@ export async function runManagedAgentProbe( queryExecution = "construction_failed"; throw error; } - const iterator = query[Symbol.asyncIterator](); + iterator = query[Symbol.asyncIterator](); for (;;) { const step = await nextManagedAgentEvent( iterator, @@ -610,12 +641,12 @@ export async function runManagedAgentProbe( } finally { queryIterationSettled = true; const now = dependencies.now ?? Date.now; - const armedEarlyQueryTeardown = - queryFailed && + const armedEarlyQuerySettlement = toolProcessContainmentArmed && Boolean(cancellationTask) && - !abortController.signal.aborted; - if (armedEarlyQueryTeardown) { + !abortController.signal.aborted && + !cancellationSignalReady; + if (armedEarlyQuerySettlement) { abortStartedAt ??= now(); const readinessBudget = Math.max( 0, @@ -630,14 +661,17 @@ export async function runManagedAgentProbe( } } triggerController.abort(); - if (cancellationTask && !armedEarlyQueryTeardown) { + if (cancellationTask && !armedEarlyQuerySettlement) { await cancellationTask; } queryFailed ||= cancellationTriggerFailed; // Give the SDK its documented graceful-shutdown path before host // fallback containment. The observer binds only SpawnOptions.signal, // which the SDK forwards after stdin EOF and its bounded grace period. - if (queryFailed && !abortController.signal.aborted) { + if ( + (queryFailed || armedEarlyQuerySettlement) && + !abortController.signal.aborted + ) { abortStartedAt ??= (dependencies.now ?? Date.now)(); abortController.abort(); } @@ -651,7 +685,7 @@ export async function runManagedAgentProbe( (now() - abortStartedAt) - FORCE_CLEANUP_CONFIRMATION_RESERVE_MS, ); - queryClosed = await closeQueryBounded(query, closeBudgetMs); + queryClosed = await closeQueryBounded(query, iterator, closeBudgetMs); } if ((query && !queryClosed) || queryFailed) abortController.abort(); } diff --git a/packages/harness/src/experimental/managed-agent-spike/types.ts b/packages/harness/src/experimental/managed-agent-spike/types.ts index 4629b5eeb..cef7504bf 100644 --- a/packages/harness/src/experimental/managed-agent-spike/types.ts +++ b/packages/harness/src/experimental/managed-agent-spike/types.ts @@ -292,6 +292,8 @@ export interface ManagedAgentProbeResult { */ export interface ManagedAgentQuery extends AsyncIterable { close(): void | Promise; + /** Pinned SDK Query.return() awaits its fire-and-forget close cleanup. */ + return?(value?: void): Promise>; } export type ManagedAgentQueryFactory = (input: { From cb64b1d0687c636451feea15d5d83c86130f4980 Mon Sep 17 00:00:00 2001 From: Yash Date: Mon, 17 Aug 2026 04:55:40 -0700 Subject: [PATCH 17/24] fix(harness): enforce bounded agent teardown evidence --- .../managed-agent-spike/README.md | 35 +- .../managed-agent-spike/fixture.test.ts | 5 + .../managed-agent-spike/fixture.ts | 62 +- .../experimental/managed-agent-spike/index.ts | 2 + .../managed-agent-spike/probe-cli.ts | 2 +- .../process-observer.test.ts | 843 ++++++++++++++---- .../managed-agent-spike/process-observer.ts | 638 +++++++++++-- .../runtime-sdk-loopback.test.ts | 64 +- .../managed-agent-spike/runtime.test.ts | 131 ++- .../managed-agent-spike/runtime.ts | 111 ++- .../experimental/managed-agent-spike/types.ts | 27 +- 11 files changed, 1560 insertions(+), 360 deletions(-) diff --git a/packages/harness/src/experimental/managed-agent-spike/README.md b/packages/harness/src/experimental/managed-agent-spike/README.md index 01a9c3e83..d9a808ea4 100644 --- a/packages/harness/src/experimental/managed-agent-spike/README.md +++ b/packages/harness/src/experimental/managed-agent-spike/README.md @@ -150,20 +150,27 @@ the runtime immediately follows it with and awaits `Query.return()` under the same deadline. `queryClosed` means that awaitable cleanup settled; invoking `close()` alone is never completion evidence. Host emergency cleanup starts only after that cleanup settles, the forwarded signal has already requested the -fallback, or the bounded SDK-grace budget expires. The returned handle accepts -the first SDK `child.kill()` logically by setting `child.killed = true`, but it -intentionally sends no native signal. The SDK-forwarded abort signal requests -the sampled host fallback; only freshly validated host group cleanup sends -signals. The fallback first stops the observer-created SDK supervisor group. A -new process-table sample must then revalidate the active root identity, both -role identities, their relationship and shared group, every -current root/tool descendant's parent, group, session, and ancestry, and at -least one open lifetime channel. Only that fresh proof authorizes `SIGSTOP` to -the detached fixture group. A second fresh sample must show both the root and -every tool-group member stopped before `SIGKILL` is sent to the fixture group -and then the SDK supervisor group. Failed tool stop/kill attempts remain -retryable, but every retry requires another fresh proof. The five-second -absolute deadline bounds the entire sequence. +fallback, or the bounded SDK-grace budget expires. As a compatibility shim +certified only for the pinned Agent SDK 0.3.228, the returned handle accepts the +first SDK `child.kill()` logically by setting `child.killed = true`, but it +intentionally sends no native signal. The hermetic real-SDK loopback test is a +sequence sentinel for 0.3.228's exact close/return behavior: one logical kill, +the SDK-forwarded abort, a second rejected logical kill, return settlement, +then host fallback. An SDK upgrade must remove or recertify the shim before the +pin changes. The forwarded abort signal requests the sampled host fallback; +only freshly validated host group cleanup sends signals. The fallback first +stops the observer-created SDK supervisor group. A new process-table sample +must then revalidate the active root identity, both role identities, their +relationship and shared group, every current root/tool descendant's parent, +group, session, and ancestry, and at least one open lifetime channel. Only that +fresh proof authorizes `SIGSTOP` to the detached fixture group. A second fresh +sample must show both the root and every tool-group member stopped before +`SIGKILL` is sent to the fixture group. A third fresh sample must prove the +fixture group absent before the supervisor group receives `SIGKILL`, and a +fourth must prove that root group absent. Failed stop/kill attempts remain +retryable, but every attempt—including an ESRCH or helper failure—advances the +sample generation and requires another fresh proof. The five-second absolute +deadline bounds the entire sequence. If the root exits, a stable identity changes parent/group/session, a foreign member appears, ancestry is lost, both channels close prematurely, or a diff --git a/packages/harness/src/experimental/managed-agent-spike/fixture.test.ts b/packages/harness/src/experimental/managed-agent-spike/fixture.test.ts index 849b871bd..87628a184 100644 --- a/packages/harness/src/experimental/managed-agent-spike/fixture.test.ts +++ b/packages/harness/src/experimental/managed-agent-spike/fixture.test.ts @@ -46,6 +46,11 @@ describe("managed-agent disposable git fixture", () => { expect(fixture.prompt("L1")).toContain(FIXTURE_PATHS.untrackedSentinel); expect(fixture.prompt("L1")).not.toContain(fixture.nonce); expect(fixture.prompt("L2")).toContain(fixture.l2BashCommand); + expect(fixture.l2BashCommand).toContain("--host-cleanup-marker"); + expect(fixture.l2BashCommand).toContain(fixture.cooperativeExitMarker); + expect( + fixture.cooperativeExitMarker.startsWith(fixture.workspaceRoot), + ).toBe(false); expect(await verifyManagedAgentFixtureBytes(fixture)).toEqual([ { path: FIXTURE_PATHS.dirtySentinel, preserved: true }, { path: FIXTURE_PATHS.untrackedSentinel, preserved: true }, diff --git a/packages/harness/src/experimental/managed-agent-spike/fixture.ts b/packages/harness/src/experimental/managed-agent-spike/fixture.ts index 3f288e26d..11c384b9a 100644 --- a/packages/harness/src/experimental/managed-agent-spike/fixture.ts +++ b/packages/harness/src/experimental/managed-agent-spike/fixture.ts @@ -1,5 +1,6 @@ import { execFileSync } from "node:child_process"; import { createHash, randomUUID } from "node:crypto"; +import { existsSync } from "node:fs"; import { lstat, mkdir, @@ -53,6 +54,9 @@ export interface ManagedAgentFixture { readonly pathRoleBindings: readonly ManagedAgentPathRoleBinding[]; readonly expectedL1FinalBytes: readonly ManagedAgentL1ExpectedFileHash[]; readonly preservedBytes: Readonly>; + /** Host-only cooperative marker outside the model-writable workspace. */ + readonly cooperativeExitMarker: string; + requestCooperativeExit(): Promise; prompt(scenario: ManagedAgentProbeScenario): string; cleanup(): Promise; } @@ -80,12 +84,16 @@ function shellQuote(value: string): string { const LONG_RUNNING_SCRIPT = ` import { spawn } from "node:child_process"; -import { writeFileSync } from "node:fs"; +import { existsSync, unlinkSync, writeFileSync } from "node:fs"; import { createConnection } from "node:net"; import { resolve } from "node:path"; const pidFile = resolve(process.argv[2]); const requireControlRegistration = process.argv[3] === "--register-control"; +const cleanupMarkerIndex = process.argv.indexOf("--host-cleanup-marker"); +const cleanupMarker = cleanupMarkerIndex >= 0 + ? resolve(process.argv[cleanupMarkerIndex + 1]) + : undefined; const controlSocket = process.env[${JSON.stringify(MANAGED_AGENT_TOOL_CONTROL_SOCKET_ENV)}]; const controlCapability = process.env[${JSON.stringify(MANAGED_AGENT_TOOL_CONTROL_CAPABILITY_ENV)}]; if (requireControlRegistration && (!controlSocket || !controlCapability)) { @@ -100,6 +108,7 @@ const childProgram = [ 'delete process.env["${MANAGED_AGENT_TOOL_CONTROL_SOCKET_ENV}"];', 'delete process.env["${MANAGED_AGENT_TOOL_CONTROL_CAPABILITY_ENV}"];', 'process.on("SIGTERM", () => {});', + 'process.on("message", (message) => { if (message === "host-shutdown") process.exit(0); });', 'const publishReady = () => { if (process.send) process.send("ready"); };', 'const connectControl = () => {', ' if (!controlSocket || !controlCapability) { publishReady(); return; }', @@ -119,6 +128,10 @@ const childProgram = [ ' });', ' socket.on("data", (chunk) => {', ' response += chunk;', + ' if (response.includes(' + JSON.stringify('"shutdown":true') + ')) {', + ' socket.write(JSON.stringify({ shutdownAck: true }) + "\\\\n", () => process.exit(0));', + ' return;', + ' }', ' if (!response.includes("\\\\n")) return;', ' if (!response.includes(' + JSON.stringify('"registered":true') + ')) { socket.destroy(); return; }', ' registered = true;', @@ -169,6 +182,12 @@ const connectControl = () => { }); socket.on("data", (chunk) => { response += chunk; + if (response.includes('"shutdown":true')) { + socket.write(JSON.stringify({ shutdownAck: true }) + "\\n", () => + process.exit(0), + ); + return; + } if (!response.includes("\\n")) return; if (!response.includes('"registered":true')) { throw new Error("managed-agent tool registration rejected"); @@ -181,6 +200,17 @@ const connectControl = () => { socket.once("close", retry); }; if (requireControlRegistration) connectControl(); +if (cleanupMarker) { + process.on("exit", () => { + try { unlinkSync(cleanupMarker); } catch {} + }); + const cleanupPoll = setInterval(() => { + if (!existsSync(cleanupMarker)) return; + clearInterval(cleanupPoll); + if (child.connected) child.send("host-shutdown"); + child.once("exit", () => process.exit(0)); + }, 10); +} setInterval(() => {}, 1000); `.trimStart(); @@ -320,6 +350,10 @@ export async function createManagedAgentFixture( const cleanTargetReplacement = "managed target updated\n"; const createdTargetContents = "managed output created\n"; const outsideSentinel = join(outsideRoot, "outside-sentinel.txt"); + const cooperativeExitMarker = join( + root, + `.host-cleanup-${randomUUID().split("-").join("")}`, + ); await Promise.all([ writeFile( @@ -360,6 +394,8 @@ export async function createManagedAgentFixture( shellQuote(FIXTURE_PATHS.processScript), shellQuote(FIXTURE_PATHS.processPidFile), shellQuote("--register-control"), + shellQuote("--host-cleanup-marker"), + shellQuote(cooperativeExitMarker), ].join(" "); const pathRoleBindings = [ { path: FIXTURE_PATHS.cleanTarget, role: "clean_target" }, @@ -381,6 +417,27 @@ export async function createManagedAgentFixture( sha256: hash(createdTargetContents), }, ] as const satisfies readonly ManagedAgentL1ExpectedFileHash[]; + let cooperativeExitRequested = false; + const requestCooperativeExit = async (): Promise => { + if (!existsSync(root)) return; + if ( + cooperativeExitRequested || + !existsSync(join(workspaceRoot, FIXTURE_PATHS.processPidFile)) + ) { + return; + } + cooperativeExitRequested = true; + await writeFile(cooperativeExitMarker, "shutdown\n", { mode: 0o600 }); + const deadline = Date.now() + 1_000; + while (existsSync(cooperativeExitMarker) && Date.now() < deadline) { + await new Promise((resolveDelay) => setTimeout(resolveDelay, 10)); + } + if (existsSync(cooperativeExitMarker)) { + // Permit a later cleanup attempt if the fixture process had not begun + // polling yet. The marker is still removed with the disposable root. + cooperativeExitRequested = false; + } + }; return { root, @@ -398,6 +455,8 @@ export async function createManagedAgentFixture( [FIXTURE_PATHS.dirtySentinel]: Buffer.from(dirtyContents), [FIXTURE_PATHS.untrackedSentinel]: Buffer.from(untrackedContents), }, + cooperativeExitMarker, + requestCooperativeExit, prompt(scenario) { if (scenario === "L2") { return [ @@ -429,6 +488,7 @@ export async function createManagedAgentFixture( ].join("\n"); }, async cleanup() { + await requestCooperativeExit().catch(() => undefined); await rm(root, { recursive: true, force: true }); }, }; diff --git a/packages/harness/src/experimental/managed-agent-spike/index.ts b/packages/harness/src/experimental/managed-agent-spike/index.ts index 53efbf008..754f6f24c 100644 --- a/packages/harness/src/experimental/managed-agent-spike/index.ts +++ b/packages/harness/src/experimental/managed-agent-spike/index.ts @@ -53,6 +53,7 @@ export { } from "./permissions.js"; export { LocalManagedAgentProcessObserver, + MANAGED_AGENT_LOGICAL_KILL_SHIM_SDK_VERSION, createLocalManagedAgentProcessObserver, } from "./process-observer.js"; export { @@ -103,6 +104,7 @@ export type { ManagedAgentQueryExecutionOutcome, ManagedAgentQueryFactory, ManagedAgentSdkUsageEstimate, + ManagedAgentTeardownDeadline, ManagedAgentTeardownObservation, ManagedAgentTerminalClassification, ManagedAgentTerminationEvidence, diff --git a/packages/harness/src/experimental/managed-agent-spike/probe-cli.ts b/packages/harness/src/experimental/managed-agent-spike/probe-cli.ts index 9094357bd..4b3f03f1b 100644 --- a/packages/harness/src/experimental/managed-agent-spike/probe-cli.ts +++ b/packages/harness/src/experimental/managed-agent-spike/probe-cli.ts @@ -960,7 +960,7 @@ export async function executeManagedAgentProbeCli( }; return evaluateManagedAgentProbe(resultWithByteEvidence, fixturePids); } finally { - observer.dispose(); + await observer.dispose(); await fixture.cleanup(); } } diff --git a/packages/harness/src/experimental/managed-agent-spike/process-observer.test.ts b/packages/harness/src/experimental/managed-agent-spike/process-observer.test.ts index 372968835..87a123416 100644 --- a/packages/harness/src/experimental/managed-agent-spike/process-observer.test.ts +++ b/packages/harness/src/experimental/managed-agent-spike/process-observer.test.ts @@ -2,6 +2,7 @@ import { once } from "node:events"; import { ChildProcess, execFile, + execFileSync, spawn as spawnChild, type ChildProcessWithoutNullStreams, } from "node:child_process"; @@ -139,7 +140,7 @@ const DESCENDANT_TOOL_SCRIPT = String.raw` import { spawn } from "node:child_process"; import { writeFileSync } from "node:fs"; -const [toolScript, pidFile, credentialFile] = process.argv.slice(1); +const [toolScript, pidFile, credentialFile, cleanupMarker] = process.argv.slice(1); writeFileSync(credentialFile, JSON.stringify({ socketPath: process.env.SAPIOM_MANAGED_AGENT_TOOL_CONTROL_SOCKET, capability: process.env.SAPIOM_MANAGED_AGENT_TOOL_CONTROL_CAPABILITY, @@ -150,11 +151,13 @@ const tool = spawn( "--noprofile", "--norc", "-c", - 'exec "$1" "$2" "$3"', + 'exec "$1" "$2" "$3" "$4" "$5"', "managed-agent-tool", process.execPath, toolScript, pidFile, + "--host-cleanup-marker", + cleanupMarker, ], { detached: true, @@ -302,7 +305,101 @@ async function sendClosedToolRegistration( function asChildProcess( spawned: SpawnedProcess, ): ChildProcessWithoutNullStreams { - return spawned as ChildProcessWithoutNullStreams; + const child = spawned as ChildProcessWithoutNullStreams; + captureRetainedRootAuthority(child); + return child; +} + +interface RetainedRootAuthority { + readonly pid: number; + readonly record: ManagedAgentKernelProcessRecord; + readonly ancestry: readonly (readonly [ + number, + ManagedAgentKernelProcessRecord, + ])[]; +} + +const retainedRootAuthorities = new WeakMap< + ChildProcess, + RetainedRootAuthority +>(); + +function sameFullTestIdentity( + expected: ManagedAgentKernelProcessRecord, + current: ManagedAgentKernelProcessRecord | undefined, +): boolean { + return ( + expected.startedAt === current?.startedAt && + expected.parentPid === current.parentPid && + expected.processGroupId === current.processGroupId && + expected.sessionId === current.sessionId + ); +} + +function processAncestry( + pid: number, + table: ReadonlyMap, +): RetainedRootAuthority["ancestry"] { + const ancestry: Array = + []; + const seen = new Set(); + let currentPid = pid; + while (currentPid > 0 && !seen.has(currentPid)) { + seen.add(currentPid); + const record = table.get(currentPid); + if (!record) break; + ancestry.push([currentPid, record]); + currentPid = record.parentPid; + } + return ancestry; +} + +function captureRetainedRootAuthority(child: ChildProcess): void { + if ( + process.platform === "win32" || + typeof child.pid !== "number" || + retainedRootAuthorities.has(child) + ) { + return; + } + try { + const sessionColumn = managedAgentPosixSessionColumn(process.platform); + const stdout = execFileSync( + "/bin/ps", + ["-axo", `pid=,ppid=,pgid=,${sessionColumn}=,stat=,lstart=`], + { encoding: "utf8", maxBuffer: 4 * 1024 * 1024 }, + ); + const table = parseManagedAgentPosixProcessTable(stdout); + const record = table.get(child.pid); + if (!record) return; + retainedRootAuthorities.set(child, { + pid: child.pid, + record, + ancestry: processAncestry(child.pid, table), + }); + } catch { + // A missing acquisition snapshot permanently removes fallback authority. + } +} + +function retainedRootAuthorityMatches( + authority: RetainedRootAuthority, + table: ReadonlyMap, +): boolean { + const leader = table.get(authority.pid); + const currentAncestry = processAncestry(authority.pid, table); + return Boolean( + leader && + !leader.state?.startsWith("Z") && + leader.processGroupId === authority.pid && + sameFullTestIdentity(authority.record, leader) && + authority.ancestry.length === currentAncestry.length && + authority.ancestry.every( + ([pid, record], index) => + currentAncestry[index]?.[0] === pid && + sameFullTestIdentity(record, currentAncestry[index]?.[1]), + ), + ); } function processExists(pid: number): boolean { @@ -409,11 +506,7 @@ async function waitForChildExitBounded( async function forceKillExactTestProcess(child: ChildProcess): Promise { const pid = child.pid; if (typeof pid !== "number") return; - try { - process.kill(pid, "SIGKILL"); - } catch (error) { - if ((error as NodeJS.ErrnoException).code !== "ESRCH") throw error; - } + child.kill("SIGKILL"); await waitForTestProcessDeath( () => processExists(pid), `Unrelated process ${pid}`, @@ -436,44 +529,6 @@ async function captureExactTestProcessIdentities( return identities; } -async function forceKillExactTestProcessIdentities( - identities: ReadonlyMap, -): Promise { - if (identities.size === 0) return; - const observation = await readRealPosixProcessTable(); - if (!observation.available) { - throw new Error("Process table unavailable for exact test cleanup"); - } - for (const [pid, identity] of identities) { - const current = observation.processes.get(pid); - if ( - !current || - current.startedAt !== identity.startedAt || - current.state?.startsWith("Z") - ) { - continue; - } - try { - process.kill(pid, "SIGKILL"); - } catch (error) { - if ((error as NodeJS.ErrnoException).code !== "ESRCH") throw error; - } - } - const deadline = Date.now() + 1_000; - for (;;) { - const survivors = await liveExactTestProcessIdentities(identities); - if (survivors.length === 0) return; - if (Date.now() >= deadline) { - throw new Error( - `Exact test processes ${survivors - .map((pid) => pid) - .join(", ")} survived test cleanup`, - ); - } - await new Promise((resolveDelay) => setTimeout(resolveDelay, 10)); - } -} - async function liveExactTestProcessIdentities( identities: ReadonlyMap, ): Promise { @@ -483,15 +538,34 @@ async function liveExactTestProcessIdentities( } return [...identities].flatMap(([pid, identity]) => { const record = current.processes.get(pid); - return record?.startedAt === identity.startedAt && + return record && + sameFullTestIdentity(identity, record) && !record.state?.startsWith("Z") ? [pid] : []; }); } +async function waitForExactTestProcessIdentitiesToExit( + identities: ReadonlyMap, + timeoutMs = 1_000, +): Promise { + const deadline = Date.now() + timeoutMs; + let survivors = await liveExactTestProcessIdentities(identities); + while (survivors.length > 0 && Date.now() < deadline) { + await new Promise((resolveDelay) => setTimeout(resolveDelay, 10)); + survivors = await liveExactTestProcessIdentities(identities); + } + if (survivors.length > 0) { + throw new Error( + `Authenticated cooperative cleanup left ${survivors.length} tool process(es)`, + ); + } +} + async function forceKillRetainedTestGroup(root: ChildProcess): Promise { - const processGroupId = root.pid; + const authority = retainedRootAuthorities.get(root); + const processGroupId = authority?.pid; if ( typeof processGroupId !== "number" || root.exitCode !== null || @@ -504,28 +578,14 @@ async function forceKillRetainedTestGroup(root: ChildProcess): Promise { if (!observation.available) { throw new Error("Process table unavailable for retained group cleanup"); } - const leader = observation.processes.get(processGroupId); if ( - !leader || - leader.state?.startsWith("Z") || - leader.processGroupId !== processGroupId + !authority || + !retainedRootAuthorityMatches(authority, observation.processes) ) { throw new Error( `Refusing cached group cleanup for unverified root ${processGroupId}`, ); } - const identities = new Map( - [...observation.processes].filter( - ([, record]) => - record.processGroupId === processGroupId && - !record.state?.startsWith("Z"), - ), - ); - if (!identities.has(processGroupId)) { - throw new Error( - `Refusing group cleanup without live leader identity ${processGroupId}`, - ); - } if (root.exitCode !== null || root.signalCode !== null) return; // The retained, still-active ChildProcess plus this fresh kernel snapshot is @@ -537,7 +597,6 @@ async function forceKillRetainedTestGroup(root: ChildProcess): Promise { if ((error as NodeJS.ErrnoException).code !== "ESRCH") throw error; } await waitForChildExitBounded(root); - await forceKillExactTestProcessIdentities(identities); } async function proveRetainedGroupAuthority( @@ -556,7 +615,6 @@ async function proveRetainedGroupAuthority( return signalRealProcessGroup(processGroupId, signal); }, }); - const rawController = new AbortController(); const forwardedController = new AbortController(); const unrelated = spawnChild( process.execPath, @@ -564,7 +622,6 @@ async function proveRetainedGroupAuthority( { stdio: "ignore", windowsHide: true }, ); await once(unrelated, "spawn"); - observer.bindAbortSignal(rawController.signal); let anchor: ChildProcessWithoutNullStreams | undefined; let nonCooperativeChildPid: number | undefined; let exitMarker: string | undefined; @@ -659,7 +716,6 @@ async function proveRetainedGroupAuthority( `Escaped fixture child ${nonCooperativeChildPid}`, ); } - rawController.abort(); forwardedController.abort(); observer.dispose(); await forceKillExactTestProcess(unrelated); @@ -793,8 +849,16 @@ async function startRegisteredDescendantToolRun( } catch (setupError) { const cleanupErrors: unknown[] = []; try { + await observer.dispose(); if (toolIdentities) { - await forceKillExactTestProcessIdentities(toolIdentities); + const survivors = await liveExactTestProcessIdentities(toolIdentities); + if (survivors.length > 0) { + cleanupErrors.push( + new Error( + `Authenticated cooperative cleanup left ${survivors.length} tool process(es)`, + ), + ); + } } else if (toolPids || typeof toolProcessGroupId === "number") { cleanupErrors.push( new Error( @@ -811,7 +875,7 @@ async function startRegisteredDescendantToolRun( cleanupErrors.push(error); } finally { forwardedController.abort(); - observer.dispose(); + await observer.dispose(); } if (cleanupErrors.length > 0) { throw setupAndCleanupFailure(setupError, cleanupErrors); @@ -826,7 +890,8 @@ async function cleanupRegisteredDescendantToolRun( if (!run) return; const cleanupErrors: unknown[] = []; try { - await forceKillExactTestProcessIdentities(run.toolIdentities); + await run.observer.dispose(); + await waitForExactTestProcessIdentitiesToExit(run.toolIdentities); } catch (error) { cleanupErrors.push(error); } @@ -839,7 +904,7 @@ async function cleanupRegisteredDescendantToolRun( cleanupErrors.push(error); } finally { run.forwardedController.abort(); - run.observer.dispose(); + await run.observer.dispose(); } if (cleanupErrors.length > 0) { throw registeredDescendantCleanupFailure(cleanupErrors); @@ -847,6 +912,39 @@ async function cleanupRegisteredDescendantToolRun( } describe("LocalManagedAgentProcessObserver", () => { + it.each([ + ["parent", { parentPid: 2, processGroupId: 41, sessionId: 41 }], + ["process group", { parentPid: 1, processGroupId: 99, sessionId: 41 }], + ["session", { parentPid: 1, processGroupId: 41, sessionId: 99 }], + ] as const)( + "refuses retained-root fallback when same-start identity changes %s topology", + (_description, changedTopology) => { + const baseline = { + parentPid: 1, + processGroupId: 41, + sessionId: 41, + state: "S", + startedAt: "same-second-start", + } satisfies ManagedAgentKernelProcessRecord; + const authority: RetainedRootAuthority = { + pid: 41, + record: baseline, + ancestry: [[41, baseline]], + }; + const current = new Map([ + [ + 41, + { + ...baseline, + ...changedTopology, + }, + ] as const, + ]); + + expect(retainedRootAuthorityMatches(authority, current)).toBe(false); + }, + ); + it.each([ ["darwin", "sess"], ["linux", "sid"], @@ -1057,8 +1155,12 @@ describe("LocalManagedAgentProcessObserver", () => { }); } finally { nativeKillSpy.mockRestore(); - observer.dispose(); - if (anchor.exitCode === null && anchor.signalCode === null) { + await observer.dispose(); + if ( + anchor.connected && + anchor.exitCode === null && + anchor.signalCode === null + ) { anchor.disconnect(); await waitForChildExitBounded(anchor); } @@ -1074,7 +1176,6 @@ describe("LocalManagedAgentProcessObserver", () => { const fixture = await createManagedAgentFixture(() => "process-observer"); fixtures.push(fixture); const observer = new LocalManagedAgentProcessObserver(); - const rawController = new AbortController(); const forwardedController = new AbortController(); const unrelated = spawnChild( process.execPath, @@ -1084,7 +1185,6 @@ describe("LocalManagedAgentProcessObserver", () => { await once(unrelated, "spawn"); let root: ChildProcessWithoutNullStreams | undefined; let ownedProcessGroupId: number | undefined; - observer.bindAbortSignal(rawController.signal); try { root = asChildProcess( observer.spawn({ @@ -1110,7 +1210,7 @@ describe("LocalManagedAgentProcessObserver", () => { expect(readiness.observedPids).not.toContain(unrelated.pid); const startedAt = Date.now(); - rawController.abort(); + forwardedController.abort(); const teardown = await observer.emergencyCleanup(1_000); expect(teardown).toMatchObject({ @@ -1126,7 +1226,6 @@ describe("LocalManagedAgentProcessObserver", () => { expect(Date.now() - startedAt).toBeLessThan(1_000); expect(processExists(unrelated.pid!)).toBe(true); } finally { - rawController.abort(); forwardedController.abort(); if (root && typeof ownedProcessGroupId === "number") { // Test-harness safety must not depend on the observer behavior under @@ -1166,14 +1265,8 @@ describe("LocalManagedAgentProcessObserver", () => { const ownedProcessGroupId = anchor.pid; expect(ownedProcessGroupId).toBeTypeOf("number"); let observerDisposed = false; - let fixtureIdentities: ReadonlyMap< - number, - ManagedAgentKernelProcessRecord - > = new Map(); try { const fixturePids = await waitForManagedAgentFixturePids(fixture); - fixtureIdentities = - await captureExactTestProcessIdentities(fixturePids); await expect( prepareCancellationAfterTransientReadFailure(observer), ).resolves.toMatchObject({ @@ -1186,9 +1279,8 @@ describe("LocalManagedAgentProcessObserver", () => { // observer sampling before the kernel delivers the group SIGKILL so a // transient, already-signalled reparent cannot make the assertion // scheduler-dependent. - observer.dispose(); + await observer.dispose(); observerDisposed = true; - anchor.disconnect(); await waitForChildExitBounded(anchor); await Promise.all( fixturePids.map((pid) => @@ -1208,15 +1300,57 @@ describe("LocalManagedAgentProcessObserver", () => { if (anchor.exitCode === null && anchor.signalCode === null) { await forceKillRetainedTestGroup(anchor); } - await forceKillExactTestProcessIdentities(fixtureIdentities); controller.abort(); - if (!observerDisposed) observer.dispose(); + if (!observerDisposed) await observer.dispose(); await forceKillExactTestProcess(unrelated); } }, 10_000, ); + it.skipIf(process.platform === "win32")( + "cooperatively shuts down both authenticated fixture processes without a numeric fixture signal", + async () => { + const signals: Array = []; + const observer = new LocalManagedAgentProcessObserver({ + signalProcessGroup: (groupId, signal) => { + signals.push([groupId, signal]); + return "failure"; + }, + }); + let run: RegisteredDescendantToolRun | undefined; + try { + run = await startRegisteredDescendantToolRun( + observer, + "cooperative-cleanup", + ); + + await observer.dispose(); + + await Promise.all( + run.toolPids.map((pid) => + waitForTestProcessDeath( + () => processExists(pid), + `Cooperative fixture process ${pid}`, + ), + ), + ); + expect(signals).toEqual([]); + } finally { + await observer.dispose(); + run?.forwardedController.abort(); + if ( + run && + run.anchor.exitCode === null && + run.anchor.signalCode === null + ) { + await forceKillRetainedTestGroup(run.anchor); + } + } + }, + 10_000, + ); + it("fails preparation closed after a fast root exits and never signals its former numeric group", async () => { const signals: Array = []; const observer = new LocalManagedAgentProcessObserver({ @@ -1226,9 +1360,7 @@ describe("LocalManagedAgentProcessObserver", () => { return "sent"; }, }); - const rawController = new AbortController(); const forwardedController = new AbortController(); - observer.bindAbortSignal(rawController.signal); const child = asChildProcess( observer.spawn({ command: process.execPath, @@ -1246,7 +1378,6 @@ describe("LocalManagedAgentProcessObserver", () => { supported: false, reason: "root_not_active", }); - rawController.abort(); forwardedController.abort(); await observer.emergencyCleanup(0); expect(signals).toEqual([]); @@ -1284,10 +1415,6 @@ describe("LocalManagedAgentProcessObserver", () => { await once(unrelated, "spawn"); let anchor: ChildProcessWithoutNullStreams | undefined; let fixturePids: readonly number[] = []; - let fixtureIdentities: ReadonlyMap< - number, - ManagedAgentKernelProcessRecord - > = new Map(); try { observer.armToolProcessContainment(); anchor = asChildProcess( @@ -1307,8 +1434,6 @@ describe("LocalManagedAgentProcessObserver", () => { }), ); fixturePids = await waitForManagedAgentFixturePids(fixture); - fixtureIdentities = - await captureExactTestProcessIdentities(fixturePids); await expect( prepareCancellationAfterTransientReadFailure(observer), ).resolves.toMatchObject({ @@ -1333,11 +1458,10 @@ describe("LocalManagedAgentProcessObserver", () => { expect(processExists(unrelated.pid!)).toBe(true); } finally { forwardedController.abort(); - await forceKillExactTestProcessIdentities(fixtureIdentities); + await observer.dispose(); if (anchor) { await forceKillRetainedTestGroup(anchor); } - observer.dispose(); await forceKillExactTestProcess(unrelated); } }, @@ -1425,12 +1549,10 @@ describe("LocalManagedAgentProcessObserver", () => { expect(setupEvidence!.anchor.signalCode).toBe("SIGKILL"); } finally { if (setupEvidence) { - await forceKillExactTestProcessIdentities( - setupEvidence.toolIdentities, - ); + await observer.dispose(); await forceKillRetainedTestGroup(setupEvidence.anchor); } - observer.dispose(); + await observer.dispose(); } }, 15_000, @@ -1482,7 +1604,7 @@ describe("LocalManagedAgentProcessObserver", () => { expect(teardown).toMatchObject({ quiescent: false, deadlineMet: false, - forceKillIssued: true, + forceKillIssued: false, }); expect( signals.filter(([groupId]) => groupId === toolProcessGroupId), @@ -1547,7 +1669,6 @@ describe("LocalManagedAgentProcessObserver", () => { ); toolProcessGroupId = run.toolProcessGroupId; registeredPids = run.toolPids; - await forceKillExactTestProcessIdentities(run.toolIdentities); simulatePidReuse = true; const teardown = await observer.emergencyCleanup(250); @@ -1555,7 +1676,7 @@ describe("LocalManagedAgentProcessObserver", () => { expect(teardown).toMatchObject({ quiescent: false, deadlineMet: false, - forceKillIssued: true, + forceKillIssued: false, }); expect( signals.filter(([groupId]) => groupId === toolProcessGroupId), @@ -1685,10 +1806,6 @@ describe("LocalManagedAgentProcessObserver", () => { const credentialFile = join(fixture.root, "tool-control.json"); let anchor: ChildProcessWithoutNullStreams | undefined; let detachedTool: ChildProcess | undefined; - let detachedToolIdentities: ReadonlyMap< - number, - ManagedAgentKernelProcessRecord - > = new Map(); let registrations: readonly NetSocket[] = []; try { observer.armToolProcessContainment(); @@ -1709,7 +1826,12 @@ describe("LocalManagedAgentProcessObserver", () => { const credentials = await waitForToolControlCredentials(credentialFile); detachedTool = spawnChild( process.execPath, - [FIXTURE_PATHS.processScript, FIXTURE_PATHS.processPidFile], + [ + FIXTURE_PATHS.processScript, + FIXTURE_PATHS.processPidFile, + "--host-cleanup-marker", + fixture.cooperativeExitMarker, + ], { cwd: fixture.workspaceRoot, detached: true, @@ -1720,10 +1842,8 @@ describe("LocalManagedAgentProcessObserver", () => { ); const [toolParentPid, toolChildPid] = await waitForManagedAgentFixturePids(fixture); - detachedToolIdentities = await captureExactTestProcessIdentities([ - toolParentPid, - toolChildPid, - ]); + expect(toolParentPid).toBe(detachedTool.pid); + expect(toolChildPid).toBeTypeOf("number"); registrations = await Promise.all([ startToolRegistration(credentials, "parent", toolParentPid), startToolRegistration(credentials, "child", toolChildPid), @@ -1742,12 +1862,13 @@ describe("LocalManagedAgentProcessObserver", () => { } finally { for (const registration of registrations) registration.destroy(); forwardedController.abort(); - await forceKillExactTestProcessIdentities(detachedToolIdentities); + await fixture.requestCooperativeExit(); if (detachedTool) await waitForChildExitBounded(detachedTool); + await observer.dispose(); if (anchor) { await forceKillRetainedTestGroup(anchor); } - observer.dispose(); + await observer.dispose(); } }, 15_000, @@ -1764,10 +1885,7 @@ describe("LocalManagedAgentProcessObserver", () => { const forwardedController = new AbortController(); const credentialFile = join(fixture.root, "tool-control.json"); let anchor: ChildProcessWithoutNullStreams | undefined; - let detachedToolIdentities: ReadonlyMap< - number, - ManagedAgentKernelProcessRecord - > = new Map(); + let toolPids: readonly number[] = []; let parentRegistration: NetSocket | undefined; let childRegistration: NetSocket | undefined; try { @@ -1782,6 +1900,7 @@ describe("LocalManagedAgentProcessObserver", () => { join(fixture.workspaceRoot, FIXTURE_PATHS.processScript), join(fixture.workspaceRoot, FIXTURE_PATHS.processPidFile), credentialFile, + fixture.cooperativeExitMarker, ], cwd: fixture.workspaceRoot, env: { ...process.env }, @@ -1791,10 +1910,7 @@ describe("LocalManagedAgentProcessObserver", () => { const credentials = await waitForToolControlCredentials(credentialFile); const [toolParentPid, toolChildPid] = await waitForManagedAgentFixturePids(fixture); - detachedToolIdentities = await captureExactTestProcessIdentities([ - toolParentPid, - toolChildPid, - ]); + toolPids = [toolParentPid, toolChildPid]; await sendClosedToolRegistration(credentials, "parent", toolParentPid); [parentRegistration, childRegistration] = await Promise.all([ @@ -1809,7 +1925,15 @@ describe("LocalManagedAgentProcessObserver", () => { containmentSupported: true, }); - await forceKillExactTestProcessIdentities(detachedToolIdentities); + await fixture.requestCooperativeExit(); + await Promise.all( + toolPids.map((pid) => + waitForTestProcessDeath( + () => processExists(pid), + `Marker-authenticated fixture process ${pid}`, + ), + ), + ); forwardedController.abort(); const openChannelObservation = await observer.emergencyCleanup(1_000); await waitForChildExitBounded(anchor); @@ -1822,16 +1946,17 @@ describe("LocalManagedAgentProcessObserver", () => { childRegistration.destroy(); const finalObservation = await observer.waitForQuiescence(3_000); expect(finalObservation).toMatchObject({ - quiescent: true, - deadlineMet: true, + quiescent: false, + deadlineMet: false, }); } finally { parentRegistration?.destroy(); childRegistration?.destroy(); forwardedController.abort(); - await forceKillExactTestProcessIdentities(detachedToolIdentities); + await fixture.requestCooperativeExit(); + await observer.dispose(); if (anchor) await forceKillRetainedTestGroup(anchor); - observer.dispose(); + await observer.dispose(); } }, 15_000, @@ -1915,7 +2040,7 @@ describe("LocalManagedAgentProcessObserver", () => { expect(Date.now() - shortConfirmationStartedAt).toBeLessThan(150); controller.abort(); - expect(signals).toEqual([[child.pid!, "SIGSTOP"]]); + expect(signals).toEqual([]); } finally { await forceKillRetainedTestGroup(child); controller.abort(); @@ -2015,7 +2140,7 @@ describe("LocalManagedAgentProcessObserver", () => { signals.push([groupId, signal]); return "sent"; }, - now: () => now, + monotonicNow: () => now, delay: async (milliseconds) => { now += Math.max(1, milliseconds); }, @@ -2091,7 +2216,7 @@ describe("LocalManagedAgentProcessObserver", () => { signals.push([groupId, signal]); return "sent"; }, - now: () => now, + monotonicNow: () => now, delay: async (milliseconds) => { now += Math.max(1, milliseconds); }, @@ -2194,7 +2319,7 @@ describe("LocalManagedAgentProcessObserver", () => { signals.push([groupId, signal]); return "sent"; }, - now: () => now, + monotonicNow: () => now, delay: async (milliseconds) => { now += Math.max(1, milliseconds); }, @@ -2223,11 +2348,6 @@ describe("LocalManagedAgentProcessObserver", () => { stage = "exiting"; await observer.observeProcessTree(); - await expect(observer.waitForQuiescence(0)).resolves.toMatchObject({ - quiescent: false, - containmentSupported: true, - alivePidsAtDeadline: expect.arrayContaining([rootPid + 100]), - }); stage = "gone"; await observer.observeProcessTree(); @@ -2295,6 +2415,7 @@ describe("LocalManagedAgentProcessObserver", () => { it("allows normal quiescence only after a still-descended unauthenticated subgroup is positively dead", async () => { let rootPid = 0; + let rootAlive = true; let subgroupAlive = true; const subgroupPid = () => rootPid + 100; const signals: Array = []; @@ -2302,9 +2423,9 @@ describe("LocalManagedAgentProcessObserver", () => { platform: "darwin", readProcessTable: async () => { await Promise.resolve(); - return available( - subgroupAlive - ? [ + return available([ + ...(rootAlive + ? ([ [ rootPid, { @@ -2314,6 +2435,10 @@ describe("LocalManagedAgentProcessObserver", () => { startedAt: "root", }, ], + ] as const) + : []), + ...(subgroupAlive + ? ([ [ subgroupPid(), { @@ -2323,11 +2448,18 @@ describe("LocalManagedAgentProcessObserver", () => { startedAt: "short-lived-subgroup", }, ], - ] - : [], - ); + ] as const) + : []), + ]); }, - processGroupLiveness: () => (subgroupAlive ? "alive" : "gone"), + processGroupLiveness: (processGroupId) => + processGroupId === rootPid + ? rootAlive + ? "alive" + : "gone" + : subgroupAlive + ? "alive" + : "gone", signalProcessGroup: (groupId, signal) => { signals.push([groupId, signal]); return "sent"; @@ -2345,20 +2477,18 @@ describe("LocalManagedAgentProcessObserver", () => { rootPid = anchor.pid!; try { await observer.observeProcessTree(); - await expect(observer.waitForQuiescence(0)).resolves.toMatchObject({ - quiescent: false, - containmentSupported: false, - alivePidsAtDeadline: expect.arrayContaining([subgroupPid()]), - }); await expect(observer.prepareCancellation()).resolves.toMatchObject({ supported: false, reason: "containment_escaped", ownershipProven: false, + observedPids: expect.arrayContaining([subgroupPid()]), }); expect(signals).toEqual([]); subgroupAlive = false; await observer.observeProcessTree(); + rootAlive = false; + await observer.observeProcessTree(); await expect(observer.waitForQuiescence(0)).resolves.toMatchObject({ quiescent: true, deadlineMet: true, @@ -2503,6 +2633,105 @@ describe("LocalManagedAgentProcessObserver", () => { } }); + it("retains a same-PGID subgroup child when its leader exits and the survivor reparents between samples", async () => { + let rootPid = 0; + const subgroupLeaderPid = () => rootPid + 100; + const subgroupChildPid = () => rootPid + 101; + let stage: "complete" | "leader_gone" | "all_gone" = "complete"; + const signals: Array = []; + const observer = new LocalManagedAgentProcessObserver({ + platform: "darwin", + readProcessTable: async () => + available([ + [ + rootPid, + { + parentPid: process.pid, + processGroupId: rootPid, + sessionId: rootPid, + state: "S", + startedAt: "pending-root", + }, + ], + ...(stage === "complete" + ? ([ + [ + subgroupLeaderPid(), + { + parentPid: rootPid, + processGroupId: subgroupLeaderPid(), + sessionId: subgroupLeaderPid(), + state: "S", + startedAt: "pending-leader", + }, + ], + ] as const) + : []), + ...(stage !== "all_gone" + ? ([ + [ + subgroupChildPid(), + { + parentPid: + stage === "complete" ? subgroupLeaderPid() : process.pid, + processGroupId: subgroupLeaderPid(), + sessionId: subgroupLeaderPid(), + state: "S", + startedAt: "pending-child", + }, + ], + ] as const) + : []), + ]), + signalProcessGroup: (groupId, signal) => { + signals.push([groupId, signal]); + return "sent"; + }, + }); + const forwardedController = new AbortController(); + const anchor = asChildProcess( + observer.spawn({ + ...activeNodeCommand(), + cwd: process.cwd(), + env: { ...process.env }, + signal: forwardedController.signal, + }), + ); + rootPid = anchor.pid!; + try { + await expect(observer.prepareCancellation()).resolves.toMatchObject({ + supported: false, + reason: "containment_escaped", + observedPids: expect.arrayContaining([ + subgroupLeaderPid(), + subgroupChildPid(), + ]), + }); + + stage = "leader_gone"; + await observer.observeProcessTree(); + await expect(observer.prepareCancellation()).resolves.toMatchObject({ + supported: false, + reason: "containment_escaped", + observedPids: expect.arrayContaining([subgroupChildPid()]), + }); + forwardedController.abort(); + expect(signals).toEqual([]); + + stage = "all_gone"; + await observer.observeProcessTree(); + await expect(observer.prepareCancellation()).resolves.toMatchObject({ + supported: false, + reason: "containment_escaped", + }); + expect(signals).toEqual([]); + } finally { + await forceKillRetainedTestGroup(anchor); + forwardedController.abort(); + observer.dispose(); + } + }); + it("keeps a subgroup escape after an authorized root kill permanently failed closed", async () => { let rootPid = 0; let subgroupState: "root_group" | "reparented" | "gone" = "root_group"; @@ -2788,7 +3017,7 @@ describe("LocalManagedAgentProcessObserver", () => { 10_000, ); - it("makes raw and forwarded aborts idempotent after ownership preparation", async () => { + it("makes repeated SDK-forwarded abort delivery idempotent after ownership preparation", async () => { let rootPid = 0; const signals: Array = []; const observer = new LocalManagedAgentProcessObserver({ @@ -2800,6 +3029,9 @@ describe("LocalManagedAgentProcessObserver", () => { { parentPid: process.pid, processGroupId: rootPid, + state: signals.some(([, signal]) => signal === "SIGSTOP") + ? "T" + : "S", startedAt: "root", }, ], @@ -2808,6 +3040,9 @@ describe("LocalManagedAgentProcessObserver", () => { { parentPid: rootPid, processGroupId: rootPid, + state: signals.some(([, signal]) => signal === "SIGSTOP") + ? "T" + : "S", startedAt: "child", }, ], @@ -2817,9 +3052,7 @@ describe("LocalManagedAgentProcessObserver", () => { return "sent"; }, }); - const rawController = new AbortController(); const forwardedController = new AbortController(); - observer.bindAbortSignal(rawController.signal); const child = asChildProcess( observer.spawn({ ...activeNodeCommand(), @@ -2831,7 +3064,7 @@ describe("LocalManagedAgentProcessObserver", () => { rootPid = child.pid!; try { await observer.prepareCancellation(); - rawController.abort(); + forwardedController.abort(); forwardedController.abort(); expect(signals).toEqual([[rootPid, "SIGSTOP"]]); await observer.observeProcessTree(); @@ -2847,32 +3080,40 @@ describe("LocalManagedAgentProcessObserver", () => { } }); - it("retries a transient failed SIGKILL while the trusted stopped root still anchors the group", async () => { + it("retries transient root stop and kill failures only after fresh authority samples", async () => { let rootPid = 0; + let processTableReads = 0; + let stopAttempts = 0; let killAttempts = 0; const signals: Array = []; + const signalReadCounts: number[] = []; const observer = new LocalManagedAgentProcessObserver({ platform: "darwin", - readProcessTable: async () => - available([ + readProcessTable: async () => { + processTableReads += 1; + return available([ [ rootPid, { parentPid: process.pid, processGroupId: rootPid, + state: signals.some(([, signal]) => signal === "SIGSTOP") + ? "T" + : "S", startedAt: "root", }, ], - ]), + ]); + }, signalProcessGroup: (groupId, signal) => { signals.push([groupId, signal]); + signalReadCounts.push(processTableReads); + if (signal === "SIGSTOP" && stopAttempts++ === 0) return "failure"; if (signal === "SIGKILL" && killAttempts++ === 0) return "failure"; return "sent"; }, }); - const rawController = new AbortController(); const forwardedController = new AbortController(); - observer.bindAbortSignal(rawController.signal); const child = asChildProcess( observer.spawn({ ...activeNodeCommand(), @@ -2884,14 +3125,20 @@ describe("LocalManagedAgentProcessObserver", () => { rootPid = child.pid!; try { await observer.prepareCancellation(); - rawController.abort(); - await observer.emergencyCleanup(0); + await observer.emergencyCleanup(100); expect(signals).toEqual([ + [rootPid, "SIGSTOP"], [rootPid, "SIGSTOP"], [rootPid, "SIGKILL"], [rootPid, "SIGKILL"], ]); + expect( + signalReadCounts.every( + (readCount, index) => + index === 0 || readCount > signalReadCounts[index - 1]!, + ), + ).toBe(true); await expect(observer.waitForQuiescence(0)).resolves.toMatchObject({ containmentSupported: true, forceKillIssued: true, @@ -2991,7 +3238,7 @@ describe("LocalManagedAgentProcessObserver", () => { if (measureOverrun) now = 2; return available([]); }, - now: () => now, + monotonicNow: () => now, }); const controller = new AbortController(); const child = asChildProcess( @@ -3065,4 +3312,272 @@ describe("LocalManagedAgentProcessObserver", () => { observer.dispose(); } }); + + it("does not let a process-table read started before a signal authorize the next signal", async () => { + let rootPid = 0; + const reads: Array< + (observation: ManagedAgentProcessTableObservation) => void + > = []; + const signals: Array = []; + const observer = new LocalManagedAgentProcessObserver({ + platform: "darwin", + readProcessTable: () => + new Promise((resolveRead) => { + reads.push(resolveRead); + }), + processGroupLiveness: () => "alive", + signalProcessGroup: (groupId, signal) => { + signals.push([groupId, signal]); + return "sent"; + }, + }); + const forwardedController = new AbortController(); + const anchor = asChildProcess( + observer.spawn({ + ...activeNodeCommand(), + cwd: process.cwd(), + env: { ...process.env }, + signal: forwardedController.signal, + }), + ); + rootPid = anchor.pid!; + const rootTable = () => + available([ + [ + rootPid, + { + parentPid: process.pid, + processGroupId: rootPid, + sessionId: rootPid, + state: signals.some(([, signal]) => signal === "SIGSTOP") + ? "T" + : "S", + startedAt: "epoch-root", + }, + ], + ]); + + try { + await vi.waitFor(() => expect(reads).toHaveLength(1)); + const initialSample = observer.observeProcessTree(); + reads.shift()!(rootTable()); + await initialSample; + + const readinessTask = observer.prepareCancellation(); + await vi.waitFor(() => expect(reads).toHaveLength(1)); + reads.shift()!(rootTable()); + await expect(readinessTask).resolves.toMatchObject({ + supported: true, + reason: "ready", + }); + + const preSignalSample = observer.observeProcessTree(); + await vi.waitFor(() => expect(reads).toHaveLength(1)); + forwardedController.abort(); + expect(signals).toEqual([[rootPid, "SIGSTOP"]]); + + reads.shift()!(rootTable()); + await preSignalSample; + expect(signals).toEqual([[rootPid, "SIGSTOP"]]); + + const postSignalSample = observer.observeProcessTree(); + await vi.waitFor(() => expect(reads).toHaveLength(1)); + reads.shift()!(rootTable()); + await postSignalSample; + expect(signals).toEqual([ + [rootPid, "SIGSTOP"], + [rootPid, "SIGKILL"], + ]); + } finally { + await forceKillRetainedTestGroup(anchor); + forwardedController.abort(); + observer.dispose(); + } + }); + + it.each(["deadline", "dispose"] as const)( + "seals held process-table reads after %s so late completion cannot mutate or signal", + async (sealKind) => { + let rootPid = 0; + let monotonicTime = 0; + let resolveRead!: ( + observation: ManagedAgentProcessTableObservation, + ) => void; + const signals: Array = []; + const observer = new LocalManagedAgentProcessObserver({ + platform: "darwin", + monotonicNow: () => monotonicTime, + readProcessTable: () => + new Promise((resolve) => { + resolveRead = resolve; + }), + signalProcessGroup: (groupId, signal) => { + signals.push([groupId, signal]); + return "sent"; + }, + }); + const forwardedController = new AbortController(); + const anchor = asChildProcess( + observer.spawn({ + ...activeNodeCommand(), + cwd: process.cwd(), + env: { ...process.env }, + signal: forwardedController.signal, + }), + ); + rootPid = anchor.pid!; + const deadline = Object.freeze({ startedAtMs: 0, deadlineAtMs: 10 }); + try { + await vi.waitFor(() => expect(resolveRead).toBeTypeOf("function")); + if (sealKind === "deadline") { + monotonicTime = 11; + await observer.waitForQuiescence(deadline); + } else { + await observer.dispose(); + } + const signalsAtSeal = [...signals]; + resolveRead( + available([ + [ + rootPid, + { + parentPid: process.pid, + processGroupId: rootPid, + sessionId: rootPid, + state: "T", + startedAt: "late-root", + }, + ], + ]), + ); + await new Promise((resolve) => setImmediate(resolve)); + forwardedController.abort(); + await new Promise((resolve) => setImmediate(resolve)); + + expect(signals).toEqual(signalsAtSeal); + expect(await observer.observeProcessTree(deadline)).toBe(false); + } finally { + await forceKillRetainedTestGroup(anchor); + forwardedController.abort(); + await observer.dispose(); + } + }, + ); + + it("discards an in-flight background sample that completes after a newly adopted deadline", async () => { + let monotonicTime = 0; + let resolveRead!: ( + observation: ManagedAgentProcessTableObservation, + ) => void; + const signals: Array = []; + const observer = new LocalManagedAgentProcessObserver({ + platform: "darwin", + monotonicNow: () => monotonicTime, + readProcessTable: () => + new Promise((resolve) => { + resolveRead = resolve; + }), + processGroupLiveness: () => "gone", + signalProcessGroup: (groupId, signal) => { + signals.push([groupId, signal]); + return "sent"; + }, + }); + const forwardedController = new AbortController(); + const anchor = asChildProcess( + observer.spawn({ + ...activeNodeCommand(), + cwd: process.cwd(), + env: { ...process.env }, + signal: forwardedController.signal, + }), + ); + const deadline = Object.freeze({ startedAtMs: 0, deadlineAtMs: 10 }); + try { + await vi.waitFor(() => expect(resolveRead).toBeTypeOf("function")); + const observationTask = observer.waitForQuiescence(deadline); + monotonicTime = 11; + resolveRead(available([])); + + await expect(observationTask).resolves.toMatchObject({ + quiescent: false, + deadlineMet: false, + processTableAvailable: false, + }); + expect(signals).toEqual([]); + expect(await observer.observeProcessTree(deadline)).toBe(false); + } finally { + await forceKillRetainedTestGroup(anchor); + forwardedController.abort(); + await observer.dispose(); + } + }); + + it.skipIf(process.platform === "win32")( + "rejects tool-registration data delivered after the adopted deadline", + async () => { + const fixture = await createManagedAgentFixture( + () => "late-tool-registration", + ); + fixtures.push(fixture); + let monotonicTime = 0; + const credentialFile = join(fixture.root, "late-control.json"); + const observer = new LocalManagedAgentProcessObserver({ + monotonicNow: () => monotonicTime, + readProcessTable: () => new Promise(() => undefined), + }); + observer.armToolProcessContainment(); + const forwardedController = new AbortController(); + const anchor = asChildProcess( + observer.spawn({ + command: process.execPath, + args: [ + "--input-type=module", + "--eval", + EXPORT_TOOL_CONTROL_SCRIPT, + credentialFile, + ], + cwd: fixture.workspaceRoot, + env: { ...process.env }, + signal: forwardedController.signal, + }), + ); + let socket: NetSocket | undefined; + const deadline = Object.freeze({ startedAtMs: 0, deadlineAtMs: 10 }); + try { + const credentials = await waitForToolControlCredentials(credentialFile); + socket = createConnection(credentials.socketPath); + socket.on("error", () => undefined); + await once(socket, "connect"); + const closed = once(socket, "close").then(() => true); + const observationTask = observer.waitForQuiescence(deadline); + monotonicTime = 11; + socket.write( + `${JSON.stringify({ + capability: credentials.capability, + role: "parent", + pid: process.pid, + })}\n`, + ); + + await expect( + Promise.race([ + closed, + new Promise((resolveTimeout) => + setTimeout(() => resolveTimeout(false), 50), + ), + ]), + ).resolves.toBe(true); + await expect(observationTask).resolves.toMatchObject({ + quiescent: false, + deadlineMet: false, + }); + } finally { + socket?.destroy(); + forwardedController.abort(); + await observer.dispose(); + await forceKillRetainedTestGroup(anchor); + } + }, + ); }); diff --git a/packages/harness/src/experimental/managed-agent-spike/process-observer.ts b/packages/harness/src/experimental/managed-agent-spike/process-observer.ts index 4284dc658..efc6ee5dd 100644 --- a/packages/harness/src/experimental/managed-agent-spike/process-observer.ts +++ b/packages/harness/src/experimental/managed-agent-spike/process-observer.ts @@ -12,6 +12,7 @@ import { } from "node:net"; import { tmpdir } from "node:os"; import { join } from "node:path"; +import { performance } from "node:perf_hooks"; import { promisify } from "node:util"; import type { @@ -19,9 +20,11 @@ import type { SpawnOptions, } from "@anthropic-ai/claude-agent-sdk"; +import { MANAGED_AGENT_CONTRACT } from "./contract.js"; import type { ManagedAgentCancellationReadiness, ManagedAgentProcessObserver, + ManagedAgentTeardownDeadline, ManagedAgentTeardownObservation, } from "./types.js"; @@ -36,6 +39,8 @@ export const MANAGED_AGENT_TOOL_CONTROL_SOCKET_ENV = export const MANAGED_AGENT_TOOL_CONTROL_CAPABILITY_ENV = "SAPIOM_MANAGED_AGENT_TOOL_CONTROL_CAPABILITY"; const TOOL_REGISTRATION_MAX_BYTES = 1_024; +const DISPOSE_DRAIN_TIMEOUT_MS = 500; +export const MANAGED_AGENT_LOGICAL_KILL_SHIM_SDK_VERSION = "0.3.228" as const; /** * The POSIX supervisor is the observer-owned process-group leader. The real @@ -51,6 +56,7 @@ import { spawn } from "node:child_process"; const PAYLOAD_ENV = "SAPIOM_MANAGED_AGENT_SUPERVISOR_PAYLOAD"; const HELPER_TIMEOUT_MS = 200; const POLL_INTERVAL_MS = 25; +const EMPTY_GROUP_EXIT_GRACE_MS = 750; const MAX_PROCESS_TABLE_BYTES = 4 * 1024 * 1024; function fail(message) { @@ -164,6 +170,7 @@ function readOtherGroupMembers() { let innerClosed = false; let innerExitCode = 1; let membershipCheckRunning = false; +let emptyGroupObservedAt; let pollTimer; function scheduleMembershipCheck(delayMs = 0) { @@ -178,6 +185,14 @@ async function checkMembership() { const members = await readOtherGroupMembers(); membershipCheckRunning = false; if (members && members.length === 0) { + const now = Date.now(); + emptyGroupObservedAt ??= now; + const remainingGrace = + EMPTY_GROUP_EXIT_GRACE_MS - (now - emptyGroupObservedAt); + if (remainingGrace > 0) { + scheduleMembershipCheck(Math.min(POLL_INTERVAL_MS, remainingGrace)); + return; + } process.stdin.unpipe(); process.stdin.destroy(); if (process.connected) { @@ -187,6 +202,7 @@ async function checkMembership() { process.exitCode = innerExitCode; return; } + emptyGroupObservedAt = undefined; scheduleMembershipCheck(POLL_INTERVAL_MS); } @@ -260,7 +276,7 @@ export interface LocalManagedAgentProcessObserverOptions { processGroupId: number, signal: "SIGSTOP" | "SIGKILL", ) => ManagedAgentProcessSignalOutcome; - readonly now?: () => number; + readonly monotonicNow?: () => number; readonly delay?: (milliseconds: number) => Promise; } @@ -290,6 +306,25 @@ interface ObservedIdentity { readonly record: ManagedAgentKernelProcessRecord; } +interface PendingUnauthenticatedSubgroup { + readonly key: string; + readonly rootPid: number; + readonly rootIdentity: ManagedAgentKernelProcessRecord; + readonly processGroupId: number; + readonly sessionId: number; + readonly members: Map< + string, + { readonly pid: number; readonly record: ManagedAgentKernelProcessRecord } + >; +} + +interface ProcessSampleTask { + readonly token: symbol; + readonly generation: number; + readonly lifecycleEpoch: number; + readonly promise: Promise; +} + function defaultDelay(milliseconds: number): Promise { return new Promise((resolveDelay) => setTimeout(resolveDelay, milliseconds)); } @@ -507,17 +542,17 @@ export class LocalManagedAgentProcessObserver implements ManagedAgentProcessObse processGroupId: number, signal: "SIGSTOP" | "SIGKILL", ) => ManagedAgentProcessSignalOutcome; - readonly #now: () => number; + readonly #monotonicNow: () => number; readonly #delay: (milliseconds: number) => Promise; readonly #roots = new Map(); readonly #observedIdentities = new Map(); - // An unarmed SDK may create a short-lived subgroup that is still below the - // owned root. Its stable identities block readiness, quiescence, and root - // kill but never grant subgroup signal authority. Only positive death clears - // them; topology drift is handled as a permanent escape below. - readonly #pendingUnauthenticatedDescendants = new Map< - number, - ObservedIdentity + // An unarmed SDK may create a short-lived subgroup below the owned root. + // Track the subgroup identity and every member generation, not merely its + // leader: leader exit/reparenting must never make a surviving member vanish. + // These records only block readiness/root kill and never grant signal power. + readonly #pendingUnauthenticatedSubgroups = new Map< + string, + PendingUnauthenticatedSubgroup >(); readonly #observedPids = new Set(); readonly #sampler: NodeJS.Timeout; @@ -544,7 +579,13 @@ export class LocalManagedAgentProcessObserver implements ManagedAgentProcessObse #lastTable: ManagedAgentKernelProcessTable | undefined; #processTableAvailable = false; #processTableNeedsRefresh = false; - #sampleTask: Promise | undefined; + #sampleGeneration = 0; + #lifecycleEpoch = 0; + #sealed = false; + #hostDisposing = false; + #teardownDeadline: ManagedAgentTeardownDeadline | undefined; + #sampleTask: ProcessSampleTask | undefined; + #disposeTask: Promise | undefined; public constructor(options: LocalManagedAgentProcessObserverOptions = {}) { this.#platform = options.platform ?? process.platform; @@ -555,7 +596,7 @@ export class LocalManagedAgentProcessObserver implements ManagedAgentProcessObse options.processGroupLiveness ?? defaultProcessGroupLiveness; this.#signalProcessGroup = options.signalProcessGroup ?? defaultSignalProcessGroup; - this.#now = options.now ?? Date.now; + this.#monotonicNow = options.monotonicNow ?? (() => performance.now()); this.#delay = options.delay ?? defaultDelay; if (this.#platform === "darwin" || this.#platform === "linux") { this.#startToolControlServer(); @@ -567,6 +608,64 @@ export class LocalManagedAgentProcessObserver implements ManagedAgentProcessObse this.#sampler.unref(); } + #adoptDeadline( + deadline: ManagedAgentTeardownDeadline, + ): ManagedAgentTeardownDeadline { + if ( + !Number.isFinite(deadline.startedAtMs) || + !Number.isFinite(deadline.deadlineAtMs) || + deadline.deadlineAtMs < deadline.startedAtMs + ) { + throw new Error("managed-agent teardown deadline is invalid"); + } + if (!this.#teardownDeadline) { + this.#teardownDeadline = Object.freeze({ ...deadline }); + } else if ( + deadline.startedAtMs !== this.#teardownDeadline.startedAtMs || + deadline.deadlineAtMs !== this.#teardownDeadline.deadlineAtMs + ) { + // A later caller may not reset or extend the one teardown deadline. + throw new Error("managed-agent teardown deadline changed after adoption"); + } + return this.#teardownDeadline; + } + + #remainingMs(deadline: ManagedAgentTeardownDeadline): number { + return Math.max(0, deadline.deadlineAtMs - this.#monotonicNow()); + } + + #deadlineExpiredAndSeal(): boolean { + if ( + !this.#teardownDeadline || + this.#monotonicNow() < this.#teardownDeadline.deadlineAtMs + ) { + return false; + } + this.#seal(); + return true; + } + + #normalizeDeadline( + input: ManagedAgentTeardownDeadline | number, + ): ManagedAgentTeardownDeadline { + if (typeof input !== "number") return this.#adoptDeadline(input); + if (this.#teardownDeadline) return this.#teardownDeadline; + const startedAtMs = this.#monotonicNow(); + return this.#adoptDeadline( + Object.freeze({ + startedAtMs, + deadlineAtMs: startedAtMs + Math.max(0, input), + }), + ); + } + + #seal(): void { + if (this.#sealed) return; + this.#sealed = true; + this.#lifecycleEpoch += 1; + clearInterval(this.#sampler); + } + #startToolControlServer(): void { try { const directory = mkdtempSync( @@ -596,11 +695,24 @@ export class LocalManagedAgentProcessObserver implements ManagedAgentProcessObse } #receiveToolRegistration(socket: NetSocket): void { + if (this.#sealed || this.#deadlineExpiredAndSeal()) { + socket.destroy(); + return; + } + const lifecycleEpoch = this.#lifecycleEpoch; this.#toolControlSockets.add(socket); socket.on("error", () => undefined); let registration: ToolProcessRegistration | undefined; socket.once("close", () => { this.#toolControlSockets.delete(socket); + if ( + this.#sealed || + this.#deadlineExpiredAndSeal() || + this.#hostDisposing || + lifecycleEpoch !== this.#lifecycleEpoch + ) { + return; + } if (!registration) return; const current = this.#toolProcessRegistrations.get(registration.role); if (current !== registration) return; @@ -621,7 +733,15 @@ export class LocalManagedAgentProcessObserver implements ManagedAgentProcessObse socket.destroy(); }; socket.on("data", (chunk: Buffer) => { - if (handled) return; + if ( + handled || + this.#sealed || + this.#deadlineExpiredAndSeal() || + lifecycleEpoch !== this.#lifecycleEpoch + ) { + if (this.#sealed) socket.destroy(); + return; + } body += chunk.toString("utf8"); if (Buffer.byteLength(body, "utf8") > TOOL_REGISTRATION_MAX_BYTES) { reject(); @@ -667,12 +787,13 @@ export class LocalManagedAgentProcessObserver implements ManagedAgentProcessObse }); } - public bindAbortSignal(signal: AbortSignal): void { + #bindAbortSignal(signal: AbortSignal): void { if (this.#boundSignals.has(signal)) return; this.#boundSignals.add(signal); signal.addEventListener( "abort", () => { + if (this.#sealed || this.#deadlineExpiredAndSeal()) return; this.#requestFallbackCleanupSynchronously(); }, { once: true }, @@ -683,6 +804,7 @@ export class LocalManagedAgentProcessObserver implements ManagedAgentProcessObse } public armToolProcessContainment(): void { + if (this.#sealed || this.#deadlineExpiredAndSeal()) return; if (this.#toolProcessContainmentArmed) return; this.#toolProcessContainmentArmed = true; if (this.#toolControlFailed) { @@ -701,8 +823,41 @@ export class LocalManagedAgentProcessObserver implements ManagedAgentProcessObse } #hasPendingUnauthenticatedDescendants(rootPid: number): boolean { - return [...this.#pendingUnauthenticatedDescendants.values()].some( - (identity) => identity.rootPid === rootPid, + return [...this.#pendingUnauthenticatedSubgroups.values()].some( + (subgroup) => subgroup.rootPid === rootPid, + ); + } + + #subgroupKey( + rootPid: number, + rootIdentity: ManagedAgentKernelProcessRecord, + processGroupId: number, + sessionId: number, + ): string { + return JSON.stringify([ + rootPid, + rootIdentity.startedAt, + rootIdentity.parentPid, + rootIdentity.processGroupId, + rootIdentity.sessionId, + processGroupId, + sessionId, + ]); + } + + #memberKey(pid: number, record: ManagedAgentKernelProcessRecord): string { + return JSON.stringify([pid, record.startedAt]); + } + + #sameIdentityAndTopology( + expected: ManagedAgentKernelProcessRecord, + current: ManagedAgentKernelProcessRecord | undefined, + ): boolean { + return ( + sameProcess(expected, current) && + expected.parentPid === current!.parentPid && + expected.processGroupId === current!.processGroupId && + expected.sessionId === current!.sessionId ); } @@ -728,6 +883,9 @@ export class LocalManagedAgentProcessObserver implements ManagedAgentProcessObse } public spawn(options: SpawnOptions): SpawnedProcess { + if (this.#sealed || this.#hostDisposing || this.#deadlineExpiredAndSeal()) { + throw new Error("managed-agent process observer is closed"); + } const usePosixSupervisor = this.#platform === "darwin" || this.#platform === "linux"; const child = ( @@ -780,12 +938,23 @@ export class LocalManagedAgentProcessObserver implements ManagedAgentProcessObse if (typeof child.pid === "number") { const pid = child.pid; if (usePosixSupervisor) { + if ( + MANAGED_AGENT_CONTRACT.agentSdkVersion !== + MANAGED_AGENT_LOGICAL_KILL_SHIM_SDK_VERSION + ) { + throw new Error( + `The managed-agent logical kill shim is certified only for Agent SDK ${MANAGED_AGENT_LOGICAL_KILL_SHIM_SDK_VERSION}`, + ); + } child.kill = ((_signal: NodeJS.Signals = "SIGTERM") => { - // ProcessTransport calls kill() immediately before it forwards its - // private AbortSignal. Treat that first call as logical acceptance so - // the SDK will not retry through a cached PID, but preserve the live - // supervisor as the ancestry anchor. Only the subsequently forwarded - // signal may start sampled, identity-checked group cleanup. + // Agent SDK 0.3.228's ProcessTransport calls kill() immediately before + // it forwards its private AbortSignal. This compatibility shim must + // be removed or recertified when that SDK pin changes. Treat the call + // as logical acceptance so the SDK will not retry through a cached + // PID, but preserve the live supervisor as the ancestry anchor. Only + // the subsequently forwarded signal may start sampled, + // identity-checked group cleanup. The real-SDK loopback test is the + // sequence sentinel for this exact kill-then-abort behavior. if (!childActive(child) || child.killed) return false; Reflect.set(child, "killed", true); return true; @@ -803,7 +972,7 @@ export class LocalManagedAgentProcessObserver implements ManagedAgentProcessObse // The SDK's forwarded SpawnOptions.signal arrives only after its own // graceful close. Keep it as an idempotent fallback; runtime deliberately // does not bind the raw Options.abortController to host process signals. - this.bindAbortSignal(options.signal); + this.#bindAbortSignal(options.signal); void this.observeProcessTree(); } return child; @@ -1006,9 +1175,143 @@ export class LocalManagedAgentProcessObserver implements ManagedAgentProcessObse ); } + #rememberPendingSubgroupMembers( + subgroup: PendingUnauthenticatedSubgroup, + root: OwnedRoot, + table: ManagedAgentKernelProcessTable, + rootDescendants: ReadonlySet, + ): void { + for (const [pid, record] of table) { + if ( + processIsZombie(record) || + record.processGroupId !== subgroup.processGroupId || + record.sessionId !== subgroup.sessionId + ) { + continue; + } + const key = this.#memberKey(pid, record); + const existing = subgroup.members.get(key); + if (!existing) subgroup.members.set(key, { pid, record }); + this.#observedPids.add(pid); + if (!rootDescendants.has(pid)) { + this.#invalidateRootContainment(root); + } + if (existing && !this.#sameIdentityAndTopology(existing.record, record)) { + this.#invalidateRootContainment(root); + } + } + } + + #observePendingUnauthenticatedSubgroups( + root: OwnedRoot, + table: ManagedAgentKernelProcessTable, + rootDescendants: ReadonlySet, + ): Set { + const liveMemberPids = new Set(); + for (const [key, subgroup] of this.#pendingUnauthenticatedSubgroups) { + if (subgroup.rootPid !== root.pid) continue; + const currentRoot = table.get(root.pid); + if (!this.#sameIdentityAndTopology(subgroup.rootIdentity, currentRoot)) { + // The root identity is part of the pending subgroup's immutable + // provenance. Losing it while the subgroup is unresolved is sticky. + this.#invalidateRootContainment(root); + } + this.#rememberPendingSubgroupMembers( + subgroup, + root, + table, + rootDescendants, + ); + + let rememberedMemberAlive = false; + for (const member of subgroup.members.values()) { + const current = table.get(member.pid); + if (!sameProcess(member.record, current) || processIsZombie(current)) { + continue; + } + rememberedMemberAlive = true; + liveMemberPids.add(member.pid); + if ( + !this.#sameIdentityAndTopology(member.record, current) || + !rootDescendants.has(member.pid) + ) { + this.#invalidateRootContainment(root); + } + } + const currentGroupMembers = [...table.entries()].filter( + ([, record]) => + !processIsZombie(record) && + record.processGroupId === subgroup.processGroupId && + record.sessionId === subgroup.sessionId, + ); + for (const [pid] of currentGroupMembers) liveMemberPids.add(pid); + if (!rememberedMemberAlive && currentGroupMembers.length === 0) { + // Only a complete, authoritative sample with every remembered member + // and every replacement group member absent can clear the blocker. + this.#pendingUnauthenticatedSubgroups.delete(key); + } + } + return liveMemberPids; + } + + #registerPendingUnauthenticatedSubgroup( + root: OwnedRoot, + table: ManagedAgentKernelProcessTable, + rootDescendants: ReadonlySet, + record: ManagedAgentKernelProcessRecord, + ): void { + const rootIdentity = table.get(root.pid); + const processGroupId = record.processGroupId; + const sessionId = record.sessionId; + if ( + !rootIdentity || + processIsZombie(rootIdentity) || + typeof processGroupId !== "number" || + typeof sessionId !== "number" + ) { + this.#invalidateRootContainment(root); + return; + } + const key = this.#subgroupKey( + root.pid, + rootIdentity, + processGroupId, + sessionId, + ); + let subgroup = this.#pendingUnauthenticatedSubgroups.get(key); + if (!subgroup) { + subgroup = { + key, + rootPid: root.pid, + rootIdentity, + processGroupId, + sessionId, + members: new Map(), + }; + this.#pendingUnauthenticatedSubgroups.set(key, subgroup); + } + this.#rememberPendingSubgroupMembers( + subgroup, + root, + table, + rootDescendants, + ); + } + #observePosixOwnedProcesses(table: ManagedAgentKernelProcessTable): void { for (const root of this.#roots.values()) { - const descendants = descendantsOf(new Set([root.pid]), table); + const rootDescendants = descendantsOf(new Set([root.pid]), table); + const pendingMemberPids = this.#observePendingUnauthenticatedSubgroups( + root, + table, + rootDescendants, + ); + // Continue observing children below every pending member even if its + // original leader exits between complete samples. + const descendants = descendantsOf( + new Set([root.pid, ...pendingMemberPids]), + table, + ); const toolProcessGroupId = this.#toolProcessRootPid === root.pid ? this.#toolProcessGroupId @@ -1022,14 +1325,6 @@ export class LocalManagedAgentProcessObserver implements ManagedAgentProcessObse typeof toolProcessGroupId === "number"; const currentlyOwned = new Set([root.pid, ...descendants]); - for (const [pid, pending] of this.#pendingUnauthenticatedDescendants) { - if (pending.rootPid !== root.pid) continue; - const current = table.get(pid); - if (!sameProcess(pending.record, current) || processIsZombie(current)) { - this.#pendingUnauthenticatedDescendants.delete(pid); - } - } - for (const [pid, record] of table) { if (processIsZombie(record)) continue; if ( @@ -1116,20 +1411,17 @@ export class LocalManagedAgentProcessObserver implements ManagedAgentProcessObse typeof toolProcessGroupId === "number" && (current.processGroupId === toolProcessGroupId || observed?.record.processGroupId === toolProcessGroupId); - const pending = this.#pendingUnauthenticatedDescendants.get(pid); if ( unauthenticatedDescendant && !this.#toolProcessContainmentArmed && - (!observed || - (pending?.rootPid === root.pid && - sameProcess(pending.record, current))) + (!observed || sameProcess(observed.record, current)) ) { - if (!pending) { - this.#pendingUnauthenticatedDescendants.set(pid, { - rootPid: root.pid, - record: current, - }); - } + this.#registerPendingUnauthenticatedSubgroup( + root, + table, + rootDescendants, + current, + ); } else { const expectedAfterAuthorizedKill = this.#expectedAfterAuthorizedGroupKill( @@ -1159,8 +1451,16 @@ export class LocalManagedAgentProcessObserver implements ManagedAgentProcessObse } public async observeProcessTree( - timeoutMs = MANAGED_AGENT_PROCESS_HELPER_TIMEOUT_MS, + deadline?: ManagedAgentTeardownDeadline, ): Promise { + const activeDeadline = deadline + ? this.#adoptDeadline(deadline) + : this.#teardownDeadline; + if (this.#sealed) return false; + if (activeDeadline && this.#remainingMs(activeDeadline) <= 0) { + this.#seal(); + return false; + } if (this.#roots.size === 0 && !this.#toolProcessContainmentArmed) { this.#lastTable = new Map(); this.#processTableAvailable = true; @@ -1168,12 +1468,41 @@ export class LocalManagedAgentProcessObserver implements ManagedAgentProcessObse } const boundedTimeoutMs = Math.max( 0, - Math.min(MANAGED_AGENT_PROCESS_HELPER_TIMEOUT_MS, timeoutMs), + Math.min( + MANAGED_AGENT_PROCESS_HELPER_TIMEOUT_MS, + activeDeadline + ? this.#remainingMs(activeDeadline) + : MANAGED_AGENT_PROCESS_HELPER_TIMEOUT_MS, + ), ); - if (!this.#sampleTask) { - this.#sampleTask = (async () => { + const generation = this.#sampleGeneration; + const lifecycleEpoch = this.#lifecycleEpoch; + const reusableSample = + this.#sampleTask?.generation === generation && + this.#sampleTask.lifecycleEpoch === lifecycleEpoch + ? this.#sampleTask + : undefined; + if (!reusableSample) { + const sampleToken = Symbol("managed-agent-process-sample"); + const promise = (async () => { const observation = await this.#boundedProcessTableRead(boundedTimeoutMs); + // A deadline can be adopted while a background sample is already in + // flight. Consult the current observer deadline at completion so that + // such a sample cannot install evidence after the newly adopted bound. + const completionDeadline = this.#teardownDeadline ?? activeDeadline; + const completedBeforeDeadline = completionDeadline + ? this.#monotonicNow() < completionDeadline.deadlineAtMs + : true; + if ( + this.#sealed || + lifecycleEpoch !== this.#lifecycleEpoch || + generation !== this.#sampleGeneration || + !completedBeforeDeadline + ) { + if (!completedBeforeDeadline) this.#seal(); + return false; + } if (!observation.available) { if ( boundedTimeoutMs > 0 || @@ -1244,11 +1573,20 @@ export class LocalManagedAgentProcessObserver implements ManagedAgentProcessObse this.#advanceFallbackCleanup(); return true; })().finally(() => { - this.#sampleTask = undefined; + if (this.#sampleTask?.token === sampleToken) { + this.#sampleTask = undefined; + } }); + const sampleState: ProcessSampleTask = { + token: sampleToken, + generation, + lifecycleEpoch, + promise, + }; + this.#sampleTask = sampleState; } - const sample = this.#sampleTask; + const sample = (reusableSample ?? this.#sampleTask)!.promise; let timeout: NodeJS.Timeout | undefined; const available = await Promise.race([ sample, @@ -1257,7 +1595,7 @@ export class LocalManagedAgentProcessObserver implements ManagedAgentProcessObse }), ]); if (timeout) clearTimeout(timeout); - if (!available) { + if (!available && !this.#sealed) { // A caller with a shorter absolute deadline must not reuse a stale table // while a longer background sample is still pending. if ( @@ -1365,21 +1703,26 @@ export class LocalManagedAgentProcessObserver implements ManagedAgentProcessObse }; } - #hasFreshRootGroupAuthority(root: OwnedRoot): boolean { + #hasFreshRootGroupAuthority( + root: OwnedRoot, + requireStopped = false, + ): boolean { if (!root.containmentSupported || this.#processTableNeedsRefresh) { return false; } - // The observer-created supervisor group remains the bounded fallback when - // a helper read is unavailable. Once a complete table exists, however, it - // must not turn a dead/zombie or replaced numeric root into authority. - if (!this.#processTableAvailable) return true; + // The observer-created group alone is not signal authority. A missing + // helper sample must fail closed because a cached numeric PGID can outlive + // its original leader and be reused before the ChildProcess exit event is + // delivered. + if (!this.#processTableAvailable) return false; const current = this.#lastTable?.get(root.pid); const baseline = root.identity ?? this.#observedIdentities.get(root.pid)?.record; if ( !current || processIsZombie(current) || - current.processGroupId !== root.pid + current.processGroupId !== root.pid || + (requireStopped && !current.state?.includes("T")) ) { return false; } @@ -1394,12 +1737,18 @@ export class LocalManagedAgentProcessObserver implements ManagedAgentProcessObse processGroupId: number, signal: "SIGSTOP" | "SIGKILL", ): ManagedAgentProcessSignalOutcome { + if (this.#sealed || this.#deadlineExpiredAndSeal()) return "failure"; const outcome = this.#signalProcessGroup(processGroupId, signal); - if (outcome === "sent") this.#processTableNeedsRefresh = true; + // Every attempt invalidates every sample that started before it, including + // ESRCH and helper failures. Only a complete read started in this new + // generation may prove the attempted target gone or authorize a next step. + this.#sampleGeneration += 1; + this.#processTableNeedsRefresh = true; return outcome; } #stopOwnedRootsSynchronously(): void { + if (this.#sealed) return; if (this.#platform !== "darwin" && this.#platform !== "linux") return; for (const root of this.#roots.values()) { if ( @@ -1420,13 +1769,15 @@ export class LocalManagedAgentProcessObserver implements ManagedAgentProcessObse } #killOwnedRootsSynchronously(): void { + if (this.#sealed) return; if (this.#platform !== "darwin" && this.#platform !== "linux") return; for (const root of this.#roots.values()) { if ( root.forceKillIssued || !childActive(root.child) || this.#hasPendingUnauthenticatedDescendants(root.pid) || - !this.#hasFreshRootGroupAuthority(root) + !root.stopIssued || + !this.#hasFreshRootGroupAuthority(root, true) ) { continue; } @@ -1439,24 +1790,25 @@ export class LocalManagedAgentProcessObserver implements ManagedAgentProcessObse } #requestFallbackCleanupSynchronously(): void { + if (this.#sealed || this.#deadlineExpiredAndSeal()) return; this.#fallbackCleanupRequested = true; this.#stopOwnedRootsSynchronously(); - if ( - !this.#toolProcessContainmentArmed || - !this.#toolProcessObservationComplete - ) { - this.#killOwnedRootsSynchronously(); - } } #advanceFallbackCleanup(): void { if ( + this.#sealed || + this.#deadlineExpiredAndSeal() || !this.#fallbackCleanupRequested || this.#platform === "win32" || !this.#processTableAvailable ) { return; } + // A failed root STOP attempt invalidates its authorizing sample just like + // every other signal outcome. Retry it only here, after a new complete + // generation has restored fresh root authority. + this.#stopOwnedRootsSynchronously(); if ( !this.#toolProcessContainmentArmed || !this.#toolProcessObservationComplete @@ -1638,7 +1990,7 @@ export class LocalManagedAgentProcessObserver implements ManagedAgentProcessObse toolProcessObservationComplete; const forceKillIssued = roots.length > 0 && roots.every(({ forceKillIssued }) => forceKillIssued); - const elapsedMs = Math.max(0, this.#now() - startedAt); + const elapsedMs = Math.max(0, this.#monotonicNow() - startedAt); const quiescent = processTableAvailable && containmentSupported && @@ -1661,12 +2013,16 @@ export class LocalManagedAgentProcessObserver implements ManagedAgentProcessObse } public async waitForQuiescence( - timeoutMs: number, + deadlineInput: ManagedAgentTeardownDeadline | number, ): Promise { - const startedAt = this.#now(); - const boundedTimeoutMs = Math.max(0, timeoutMs); + const adoptedDeadline = this.#normalizeDeadline(deadlineInput); + const startedAt = adoptedDeadline.startedAtMs; + const boundedTimeoutMs = Math.max( + 0, + adoptedDeadline.deadlineAtMs - adoptedDeadline.startedAtMs, + ); if ( - boundedTimeoutMs === 0 && + this.#remainingMs(adoptedDeadline) === 0 && this.#processTableAvailable && !this.#processTableNeedsRefresh && !this.#sampleTask @@ -1676,43 +2032,54 @@ export class LocalManagedAgentProcessObserver implements ManagedAgentProcessObse // The fresh cached table already proved quiescence at call entry. Its // evidence elapsed is therefore zero even if returning the Promise // crosses a wall-clock millisecond boundary. - return { + const result = { ...cachedObservation, deadlineMet: true, elapsedMs: 0, }; + this.#seal(); + return result; } } for (;;) { - const elapsedBeforeSample = Math.max(0, this.#now() - startedAt); - await this.observeProcessTree( - Math.max(0, boundedTimeoutMs - elapsedBeforeSample), - ); + if (this.#remainingMs(adoptedDeadline) <= 0 || this.#sealed) { + const observation = this.#currentObservation(startedAt, false); + this.#seal(); + return { ...observation, deadlineMet: false }; + } + await this.observeProcessTree(adoptedDeadline); const observation = this.#currentObservation(startedAt, false); if (observation.quiescent) { + const deadlineMet = + this.#monotonicNow() <= adoptedDeadline.deadlineAtMs; + if (!deadlineMet) this.#seal(); return { ...observation, - deadlineMet: observation.elapsedMs <= boundedTimeoutMs, + deadlineMet, }; } - if (observation.elapsedMs >= boundedTimeoutMs) { + if ( + observation.elapsedMs >= boundedTimeoutMs || + this.#remainingMs(adoptedDeadline) <= 0 + ) { + this.#seal(); return { ...observation, deadlineMet: false }; } await this.#delay( - Math.min(QUIESCENCE_POLL_MS, boundedTimeoutMs - observation.elapsedMs), + Math.min(QUIESCENCE_POLL_MS, this.#remainingMs(adoptedDeadline)), ); } } public async emergencyCleanup( - timeoutMs: number, + deadlineInput: ManagedAgentTeardownDeadline | number, ): Promise { - const startedAt = this.#now(); - const boundedTimeoutMs = Math.max(0, timeoutMs); + const adoptedDeadline = this.#normalizeDeadline(deadlineInput); + const startedAt = adoptedDeadline.startedAtMs; this.#requestFallbackCleanupSynchronously(); - const confirmation = await this.waitForQuiescence(boundedTimeoutMs); + const confirmation = await this.waitForQuiescence(adoptedDeadline); if (!confirmation.quiescent) this.#killOwnedRootsSynchronously(); - const elapsedMs = Math.max(0, this.#now() - startedAt); + const elapsedMs = Math.max(0, this.#monotonicNow() - startedAt); const roots = [...this.#roots.values()]; const forceKillIssued = roots.length > 0 && roots.every((root) => root.forceKillIssued); @@ -1720,16 +2087,115 @@ export class LocalManagedAgentProcessObserver implements ManagedAgentProcessObse ...confirmation, forceKillIssued, elapsedMs, - deadlineMet: confirmation.quiescent && elapsedMs <= boundedTimeoutMs, + deadlineMet: + confirmation.quiescent && + this.#monotonicNow() <= adoptedDeadline.deadlineAtMs, emergencyCleanupAttempted: true, }; } - public dispose(): void { - clearInterval(this.#sampler); + public dispose(): Promise { + this.#disposeTask ??= this.#disposeInternal(); + return this.#disposeTask; + } + + async #disposeInternal(): Promise { + this.#hostDisposing = true; + this.#seal(); + + // The two exact fixture processes authenticated these retained channels + // with an observer-created capability that was never written into the + // workspace. Ask them to exit cooperatively and require an acknowledgement; + // workspace PID-file contents are diagnostic only and never signal input. + const acknowledgementTasks = [ + ...this.#toolProcessRegistrations.values(), + ].flatMap((registration) => { + const socket = registration.socket; + if (socket.destroyed || registration.closed) return []; + return [ + new Promise((resolveAcknowledgement) => { + let settled = false; + let body = ""; + const finish = (acknowledged: boolean): void => { + if (settled) return; + settled = true; + socket.off("data", onData); + socket.off("close", onClose); + resolveAcknowledgement(acknowledged); + }; + const onData = (chunk: Buffer | string): void => { + body += chunk.toString(); + if (body.includes('"shutdownAck":true')) finish(true); + }; + const onClose = (): void => finish(false); + socket.on("data", onData); + socket.once("close", onClose); + try { + socket.write('{"shutdown":true}\n'); + } catch { + finish(false); + } + }), + ]; + }); + if (acknowledgementTasks.length > 0) { + let timeout: NodeJS.Timeout | undefined; + await Promise.race([ + Promise.all(acknowledgementTasks), + new Promise((resolveTimeout) => { + timeout = setTimeout(resolveTimeout, DISPOSE_DRAIN_TIMEOUT_MS); + }), + ]); + if (timeout) clearTimeout(timeout); + } + + // Closing the retained IPC handle lets the supervisor kill its own exact + // process group without the test harness supplying any numeric PID/PGID. + const rootExitTasks = [...this.#roots.values()].flatMap(({ child }) => { + if (!childActive(child)) return []; + if (child.connected) child.disconnect(); + return [ + new Promise((resolveExit) => { + if (!childActive(child)) { + resolveExit(); + return; + } + child.once("close", () => resolveExit()); + }), + ]; + }); + if (rootExitTasks.length > 0) { + let timeout: NodeJS.Timeout | undefined; + await Promise.race([ + Promise.all(rootExitTasks), + new Promise((resolveTimeout) => { + timeout = setTimeout(resolveTimeout, DISPOSE_DRAIN_TIMEOUT_MS); + }), + ]); + if (timeout) clearTimeout(timeout); + } + for (const socket of this.#toolControlSockets) socket.destroy(); this.#toolControlSockets.clear(); - this.#toolControlServer?.close(); + const serverClose = new Promise((resolveClose) => { + const server = this.#toolControlServer; + if (!server?.listening) { + resolveClose(); + return; + } + server.close(() => resolveClose()); + }); + let serverCloseTimeout: NodeJS.Timeout | undefined; + await Promise.race([ + serverClose, + new Promise((resolveTimeout) => { + serverCloseTimeout = setTimeout( + resolveTimeout, + DISPOSE_DRAIN_TIMEOUT_MS, + ); + }), + ]); + if (serverCloseTimeout) clearTimeout(serverCloseTimeout); if (this.#toolControlDirectory) { try { rmSync(this.#toolControlDirectory, { recursive: true, force: true }); @@ -1740,6 +2206,8 @@ export class LocalManagedAgentProcessObserver implements ManagedAgentProcessObse } } -export function createLocalManagedAgentProcessObserver(): ManagedAgentProcessObserver { - return new LocalManagedAgentProcessObserver(); +export function createLocalManagedAgentProcessObserver( + options: LocalManagedAgentProcessObserverOptions = {}, +): ManagedAgentProcessObserver { + return new LocalManagedAgentProcessObserver(options); } diff --git a/packages/harness/src/experimental/managed-agent-spike/runtime-sdk-loopback.test.ts b/packages/harness/src/experimental/managed-agent-spike/runtime-sdk-loopback.test.ts index 7376b64bd..8cdf3f78d 100644 --- a/packages/harness/src/experimental/managed-agent-spike/runtime-sdk-loopback.test.ts +++ b/packages/harness/src/experimental/managed-agent-spike/runtime-sdk-loopback.test.ts @@ -80,15 +80,6 @@ async function waitForProcessDeath( throw new Error(`Test process ${pid} survived cleanup`); } -async function forceKillTestProcess(pid: number): Promise { - try { - process.kill(pid, "SIGKILL"); - } catch (error) { - if ((error as NodeJS.ErrnoException).code !== "ESRCH") throw error; - } - await waitForProcessDeath(pid); -} - interface LoopbackObservation { readonly headerNames: readonly string[]; readonly evalSourceMatches: boolean; @@ -327,7 +318,6 @@ it("enforces real-SDK built-in and in-process MCP calls with exact loopback corr }); return child; }, - bindAbortSignal: (signal) => observer.bindAbortSignal(signal), armToolProcessContainment: () => observer.armToolProcessContainment(), prepareCancellation: () => observer.prepareCancellation(), observeProcessTree: (timeoutMs) => observer.observeProcessTree(timeoutMs), @@ -683,11 +673,15 @@ it.skipIf( { once: true }, ); const child = observer.spawn(options); + const logicalKill = child.kill.bind(child); + Reflect.set(child, "kill", (signal: NodeJS.Signals = "SIGTERM") => { + cleanupOrder.push("sdk_logical_kill"); + return logicalKill(signal); + }); const pid = Reflect.get(child, "pid"); supervisorPid = typeof pid === "number" ? pid : undefined; return child; }, - bindAbortSignal: (signal) => observer.bindAbortSignal(signal), armToolProcessContainment: () => observer.armToolProcessContainment(), prepareCancellation: () => observer.prepareCancellation(), observeProcessTree: (timeoutMs) => observer.observeProcessTree(timeoutMs), @@ -902,19 +896,31 @@ it.skipIf( cleanupOrder.indexOf("host_emergency_cleanup"), ); const forwardedSignalIndex = cleanupOrder.indexOf("sdk_forwarded_signal"); + const logicalKillIndexes = cleanupOrder.flatMap((step, index) => + step === "sdk_logical_kill" ? [index] : [], + ); + expect(cleanupOrder).toEqual([ + "sdk_close_called", + "sdk_return_started", + "sdk_logical_kill", + "sdk_forwarded_signal", + "sdk_logical_kill", + "sdk_return_settled", + "host_emergency_cleanup", + ]); + expect(logicalKillIndexes).toEqual([2, 4]); expect(forwardedSignalIndex).toBeGreaterThanOrEqual(0); + expect(logicalKillIndexes[0]).toBeLessThan(forwardedSignalIndex); + expect(logicalKillIndexes[1]).toBeGreaterThan(forwardedSignalIndex); expect(forwardedSignalIndex).toBeLessThan( cleanupOrder.indexOf("host_emergency_cleanup"), ); } finally { await observer.emergencyCleanup(1_000); - observer.dispose(); + await observer.dispose(); server.closeAllConnections(); await new Promise((resolve) => server.close(() => resolve())); - for (const pid of fixturePids) { - if (!processExists(pid)) continue; - await forceKillTestProcess(pid); - } + await Promise.all(fixturePids.map((pid) => waitForProcessDeath(pid))); if (typeof unrelated.pid === "number" && processExists(unrelated.pid)) { unrelated.kill("SIGKILL"); await waitForProcessDeath(unrelated.pid); @@ -950,7 +956,6 @@ it.skipIf( signal: neverForwardedController.signal, }); }, - bindAbortSignal: (signal) => observer.bindAbortSignal(signal), armToolProcessContainment: () => observer.armToolProcessContainment(), prepareCancellation: () => observer.prepareCancellation(), observeProcessTree: (timeoutMs) => observer.observeProcessTree(timeoutMs), @@ -1098,13 +1103,10 @@ it.skipIf( ).toBeLessThan(cleanupOrder.indexOf("host_timeout_fallback")); } finally { await observer.emergencyCleanup(1_000); - observer.dispose(); + await observer.dispose(); server.closeAllConnections(); await new Promise((resolve) => server.close(() => resolve())); - for (const pid of fixturePids) { - if (!processExists(pid)) continue; - await forceKillTestProcess(pid); - } + await Promise.all(fixturePids.map((pid) => waitForProcessDeath(pid))); if (typeof unrelated.pid === "number" && processExists(unrelated.pid)) { unrelated.kill("SIGKILL"); await waitForProcessDeath(unrelated.pid); @@ -1120,7 +1122,7 @@ it.skipIf( process.platform === "win32" || process.versions.node !== MANAGED_AGENT_CONTRACT.certificationNodeVersion, )( - "records a teardown timeout when readiness failure leaves the Bash fixture alive", + "keeps readiness-failure evidence fail-closed after cooperative disposal", async () => { const fixture = await createManagedAgentFixture( () => "loopback-l2-early-error", @@ -1224,16 +1226,20 @@ it.skipIf( result.teardown.alivePidsAtDeadline.includes(pid), ), ).toBe(true); - expect(fixturePids.every((pid) => processExists(pid))).toBe(true); + await Promise.all(fixturePids.map((pid) => waitForProcessDeath(pid))); + expect(fixturePids.every((pid) => !processExists(pid))).toBe(true); + expect(result.terminal).toBe("teardown_timeout"); + expect( + fixturePids.every((pid) => + result.teardown.alivePidsAtDeadline.includes(pid), + ), + ).toBe(true); } finally { await observer.emergencyCleanup(1_000); - observer.dispose(); + await observer.dispose(); server.closeAllConnections(); await new Promise((resolve) => server.close(() => resolve())); - for (const pid of fixturePids) { - if (!processExists(pid)) continue; - await forceKillTestProcess(pid); - } + await Promise.all(fixturePids.map((pid) => waitForProcessDeath(pid))); await fixture.cleanup(); } expect(recordedTerminal).toBe("teardown_timeout"); diff --git a/packages/harness/src/experimental/managed-agent-spike/runtime.test.ts b/packages/harness/src/experimental/managed-agent-spike/runtime.test.ts index 292e2e6bc..aa9a3beb7 100644 --- a/packages/harness/src/experimental/managed-agent-spike/runtime.test.ts +++ b/packages/harness/src/experimental/managed-agent-spike/runtime.test.ts @@ -69,7 +69,6 @@ function fakeObserver( spawn: vi.fn(() => { throw new Error("fake query must not spawn"); }), - bindAbortSignal: vi.fn(), armToolProcessContainment: vi.fn(), prepareCancellation: vi.fn(async () => ({ supported: true, @@ -358,6 +357,29 @@ describe("runManagedAgentProbe", () => { } }); + it("returns recursively immutable evidence after observer finalization", async () => { + const { config } = await probeConfig(); + const result = await runManagedAgentProbe(config, { + hermeticGatewayOrigin: config.gatewayOrigin, + processObserver: fakeObserver(), + queryFactory: () => + queryFromEvents([ + { + type: "system", + subtype: "init", + session_id: SUCCESS_SESSION_ID, + }, + { type: "result", subtype: "success", is_error: false }, + ]), + }); + + expect(Object.isFrozen(result)).toBe(true); + expect(Object.isFrozen(result.teardown)).toBe(true); + expect(Object.isFrozen(result.events)).toBe(true); + expect(Object.isFrozen(result.events[0])).toBe(true); + expect(Object.isFrozen(result.correlation)).toBe(true); + }); + it("does not follow pre-existing config child symlinks and fails invalid roots before query construction", async () => { const { config, fixture } = await probeConfig(); const externalConfig = join(fixture.root, "external-config"); @@ -496,7 +518,6 @@ describe("runManagedAgentProbe", () => { expect(JSON.stringify(result)).not.toContain( "synthetic private iteration failure", ); - expect(observer.bindAbortSignal).not.toHaveBeenCalled(); expect(shutdownOrder).toEqual(["sdk_query_close", "host_fallback"]); }); @@ -920,7 +941,7 @@ describe("runManagedAgentProbe", () => { const result = await runManagedAgentProbe(config, { hermeticGatewayOrigin: config.gatewayOrigin, processObserver: observer, - now: () => now, + monotonicNow: () => now, waitForCancellationSignal: (signal) => new Promise((resolveReadiness, rejectReadiness) => { const timer = setTimeout(() => { @@ -961,7 +982,10 @@ describe("runManagedAgentProbe", () => { expect(readinessCompleted).toBe(true); expect(readinessWasAborted).toBe(false); expect(observer.armToolProcessContainment).toHaveBeenCalledOnce(); - expect(observer.emergencyCleanup).toHaveBeenCalledWith(2_750); + expect(observer.emergencyCleanup).toHaveBeenCalledWith({ + startedAtMs: 1_000, + deadlineAtMs: 6_000, + }); expect(result.teardown.elapsedMs).toBe(2_250); expect(result.cancellationRequested).toBe(false); expect(result.terminationEvidence.beforePolicyOverride).toBe("query_error"); @@ -1057,13 +1081,36 @@ describe("runManagedAgentProbe", () => { }, 10_000); it.each([ - ["clean completion", false, "incomplete"], - ["SDK error-result completion", true, "sdk_result_error"], + ["clean completion", false, "incomplete", undefined], + ["SDK error-result completion", true, "sdk_result_error", undefined], + [ + "clean completion with a live process", + false, + "teardown_timeout", + "liveness", + ], + [ + "SDK error-result completion with an open tool channel", + true, + "teardown_timeout", + "channel", + ], ] as const)( "retains armed L2 readiness and one deadline after %s", - async (_description, emitErrorResult, expectedTerminal) => { + async (_description, emitErrorResult, expectedTerminal, failClosedOn) => { const { config } = await probeConfig("L2"); - const observer = fakeObserver(); + const observer = fakeObserver( + failClosedOn + ? { + ...quiescentTeardown(), + quiescent: false, + deadlineMet: false, + toolProcessChannelsClosed: failClosedOn !== "channel", + observedPids: failClosedOn === "liveness" ? [7_001] : [], + alivePidsAtDeadline: failClosedOn === "liveness" ? [7_001] : [], + } + : quiescentTeardown(), + ); let now = 1_000; let readinessCompleted = false; let readinessWasAborted = false; @@ -1072,7 +1119,7 @@ describe("runManagedAgentProbe", () => { const result = await runManagedAgentProbe(config, { hermeticGatewayOrigin: config.gatewayOrigin, processObserver: observer, - now: () => now, + monotonicNow: () => now, waitForCancellationSignal: (signal) => new Promise((resolveReadiness, rejectReadiness) => { const timer = setTimeout(() => { @@ -1130,11 +1177,23 @@ describe("runManagedAgentProbe", () => { expect(readinessCompleted).toBe(true); expect(readinessWasAborted).toBe(false); expect(capturedOptions?.abortController?.signal.aborted).toBe(true); - expect(observer.emergencyCleanup).toHaveBeenCalledWith(2_750); + expect(observer.emergencyCleanup).toHaveBeenCalledWith({ + startedAtMs: 1_000, + deadlineAtMs: 6_000, + }); expect(result.teardown.elapsedMs).toBe(2_250); expect(result.queryClosed).toBe(true); expect(result.cancellationRequested).toBe(false); expect(result.terminal).toBe(expectedTerminal); + expect(result.terminationEvidence.beforePolicyOverride).toBe( + expectedTerminal, + ); + if (failClosedOn === "liveness") { + expect(result.teardown.alivePidsAtDeadline).toEqual([7_001]); + } + if (failClosedOn === "channel") { + expect(result.teardown.toolProcessChannelsClosed).toBe(false); + } }, 10_000, ); @@ -1179,7 +1238,6 @@ describe("runManagedAgentProbe", () => { expect(result.terminal).toBe("cancelled"); expect(result.terminationEvidence.queryExecution).toBe("iteration_aborted"); expect(close).toHaveBeenCalledOnce(); - expect(observer.bindAbortSignal).not.toHaveBeenCalled(); }); it("accepts an awaited close promise when no iterator return exists", async () => { @@ -1229,7 +1287,6 @@ const teardown = { }; const observer = { spawn() { throw new Error("fake query must not spawn"); }, - bindAbortSignal() {}, armToolProcessContainment() {}, async prepareCancellation() { return { @@ -1305,7 +1362,7 @@ process.stdout.write(JSON.stringify({ const result = await runManagedAgentProbe(config, { hermeticGatewayOrigin: config.gatewayOrigin, processObserver: observer, - now: () => now, + monotonicNow: () => now, waitForCancellationSignal: async () => undefined, queryFactory: () => ({ [Symbol.asyncIterator]() { @@ -1317,7 +1374,10 @@ process.stdout.write(JSON.stringify({ }), }); - expect(observer.emergencyCleanup).toHaveBeenCalledWith(2_750); + expect(observer.emergencyCleanup).toHaveBeenCalledWith({ + startedAtMs: 1_000, + deadlineAtMs: 6_000, + }); expect(result.teardown).toMatchObject({ quiescent: true, deadlineMet: true, @@ -1326,6 +1386,49 @@ process.stdout.write(JSON.stringify({ expect(result.terminal).toBe("cancelled"); }); + it("never extends the teardown budget or reports deadline success when wall time rolls back", async () => { + const wallClock = vi.spyOn(Date, "now").mockReturnValue(10_000); + try { + const { config } = await probeConfig("L2"); + let monotonicTime = 10_000; + const observer = fakeObserver(); + const result = await runManagedAgentProbe(config, { + hermeticGatewayOrigin: config.gatewayOrigin, + processObserver: observer, + monotonicNow: () => monotonicTime, + waitForCancellationSignal: async () => undefined, + queryFactory: () => ({ + [Symbol.asyncIterator]() { + return { + next: () => new Promise>(() => undefined), + return: () => + new Promise>(() => undefined), + }; + }, + close: () => { + wallClock.mockReturnValue(-100_000); + monotonicTime = 15_001; + }, + return: () => + new Promise>(() => undefined), + }), + }); + const deadline = observer.emergencyCleanup.mock.calls[0]?.[0] as + | { readonly startedAtMs: number; readonly deadlineAtMs: number } + | undefined; + + expect(deadline).toEqual({ + startedAtMs: 10_000, + deadlineAtMs: 15_000, + }); + expect(deadline!.deadlineAtMs - deadline!.startedAtMs).toBe(5_000); + expect(result.teardown.deadlineMet).toBe(false); + expect(result.teardown.elapsedMs).toBeGreaterThanOrEqual(5_000); + } finally { + wallClock.mockRestore(); + } + }); + it("records teardown failure before attempting emergency cleanup", async () => { const { config } = await probeConfig(); const observer = fakeObserver({ diff --git a/packages/harness/src/experimental/managed-agent-spike/runtime.ts b/packages/harness/src/experimental/managed-agent-spike/runtime.ts index 1fa4c2b36..261264de9 100644 --- a/packages/harness/src/experimental/managed-agent-spike/runtime.ts +++ b/packages/harness/src/experimental/managed-agent-spike/runtime.ts @@ -1,4 +1,5 @@ import { randomUUID } from "node:crypto"; +import { performance } from "node:perf_hooks"; import { createSdkMcpServer, @@ -40,6 +41,7 @@ import type { ManagedAgentQuery, ManagedAgentQueryExecutionOutcome, ManagedAgentTeardownObservation, + ManagedAgentTeardownDeadline, ManagedAgentTerminalClassification, ManagedAgentToolEvidence, } from "./types.js"; @@ -279,7 +281,8 @@ function buildManagedAgentPolicyDiagnostics( async function closeQueryBounded( query: ManagedAgentQuery, iterator: AsyncIterator | undefined, - timeoutMs = QUERY_CLOSE_TIMEOUT_MS, + deadline: ManagedAgentTeardownDeadline, + monotonicNow: () => number, ): Promise { let timeout: NodeJS.Timeout | undefined; let closeResult: void | Promise; @@ -317,6 +320,12 @@ async function closeQueryBounded( const close = Promise.all([closeSettled, cleanupSettled]).then((settled) => settled.every(Boolean), ); + const timeoutMs = Math.max( + 0, + deadline.deadlineAtMs - + monotonicNow() - + FORCE_CLEANUP_CONFIRMATION_RESERVE_MS, + ); if (timeoutMs <= 0) { void close; return false; @@ -340,8 +349,10 @@ async function closeQueryBounded( async function waitForTaskBounded( task: Promise, - timeoutMs: number, + deadline: ManagedAgentTeardownDeadline, + monotonicNow: () => number, ): Promise { + const timeoutMs = Math.max(0, deadline.deadlineAtMs - monotonicNow()); if (timeoutMs <= 0) return false; let timeout: NodeJS.Timeout | undefined; try { @@ -406,6 +417,17 @@ function classifyTerminal(input: { return "incomplete"; } +function deepFreezeEvidence(value: T, seen = new WeakSet()): T { + if (typeof value !== "object" || value === null || seen.has(value)) { + return value; + } + seen.add(value); + for (const nested of Object.values(value)) { + deepFreezeEvidence(nested, seen); + } + return Object.freeze(value); +} + export async function runManagedAgentProbe( config: ManagedAgentProbeConfig, dependencies: ManagedAgentProbeDependencies = {}, @@ -434,12 +456,22 @@ export async function runManagedAgentProbe( const mcpRuntime = createManagedAgentMcpRuntime(config.expectedMcpNonce); const abortController = new AbortController(); const triggerController = new AbortController(); + const monotonicNow = dependencies.monotonicNow ?? (() => performance.now()); const before = await captureManagedAgentWorkspaceSnapshot( validated.canonicalWorkspaceRoot, ); let cancellationRequested = false; - let cancellationRequestedAt: number | undefined; - let abortStartedAt: number | undefined; + let teardownDeadline: ManagedAgentTeardownDeadline | undefined; + const ensureTeardownDeadline = (): ManagedAgentTeardownDeadline => { + if (!teardownDeadline) { + const startedAtMs = monotonicNow(); + teardownDeadline = Object.freeze({ + startedAtMs, + deadlineAtMs: startedAtMs + MANAGED_AGENT_TEARDOWN_TIMEOUT_MS, + }); + } + return teardownDeadline; + }; let query: ManagedAgentQuery | undefined; let iterator: AsyncIterator | undefined; let queryFailed = false; @@ -464,7 +496,8 @@ export async function runManagedAgentProbe( executionId, }); const processObserver = - dependencies.processObserver ?? createLocalManagedAgentProcessObserver(); + dependencies.processObserver ?? + createLocalManagedAgentProcessObserver({ monotonicNow }); const policyBoundary = createManagedAgentPolicyBoundary({ canonicalWorkspaceRoot: validated.canonicalWorkspaceRoot, allowedBuiltinTools: @@ -559,6 +592,7 @@ export async function runManagedAgentProbe( policyPreflightFailed = true; recorder.recordLifecycle("policy_preflight_failed"); triggerController.abort(); + ensureTeardownDeadline(); abortController.abort(); } const cancellationTask = @@ -570,16 +604,15 @@ export async function runManagedAgentProbe( cancellationSignalReady = true; if (queryIterationSettled) return; cancellationRequested = true; - cancellationRequestedAt = (dependencies.now ?? Date.now)(); - abortStartedAt = cancellationRequestedAt; recorder.recordLifecycle("cancellation_requested"); + ensureTeardownDeadline(); abortController.abort(); }) .catch(() => { if (!triggerController.signal.aborted) { cancellationTriggerFailed = true; if (!queryIterationSettled) { - abortStartedAt = (dependencies.now ?? Date.now)(); + ensureTeardownDeadline(); abortController.abort(); } } @@ -640,21 +673,17 @@ export async function runManagedAgentProbe( if (!abortController.signal.aborted) queryFailed = true; } finally { queryIterationSettled = true; - const now = dependencies.now ?? Date.now; const armedEarlyQuerySettlement = toolProcessContainmentArmed && Boolean(cancellationTask) && !abortController.signal.aborted && !cancellationSignalReady; if (armedEarlyQuerySettlement) { - abortStartedAt ??= now(); - const readinessBudget = Math.max( - 0, - MANAGED_AGENT_TEARDOWN_TIMEOUT_MS - (now() - abortStartedAt), - ); + const deadline = ensureTeardownDeadline(); const taskSettled = await waitForTaskBounded( cancellationTask!, - readinessBudget, + deadline, + monotonicNow, ); if (!taskSettled || !cancellationSignalReady) { cancellationTriggerFailed = true; @@ -672,43 +701,37 @@ export async function runManagedAgentProbe( (queryFailed || armedEarlyQuerySettlement) && !abortController.signal.aborted ) { - abortStartedAt ??= (dependencies.now ?? Date.now)(); + ensureTeardownDeadline(); abortController.abort(); } if (query) { - const closeBudgetMs = - abortStartedAt === undefined - ? QUERY_CLOSE_TIMEOUT_MS - : Math.max( - 0, - MANAGED_AGENT_TEARDOWN_TIMEOUT_MS - - (now() - abortStartedAt) - - FORCE_CLEANUP_CONFIRMATION_RESERVE_MS, - ); - queryClosed = await closeQueryBounded(query, iterator, closeBudgetMs); + // Establish the one deadline before close() or iterator.return() can + // begin. Natural completion is teardown-relevant too. + const deadline = ensureTeardownDeadline(); + queryClosed = await closeQueryBounded( + query, + iterator, + deadline, + monotonicNow, + ); + } + if ((query && !queryClosed) || queryFailed) { + ensureTeardownDeadline(); + abortController.abort(); } - if ((query && !queryClosed) || queryFailed) abortController.abort(); } } - const now = dependencies.now ?? Date.now; - const teardownStartedAt = abortStartedAt ?? now(); - const remainingBudget = (): number => - Math.max( - 0, - MANAGED_AGENT_TEARDOWN_TIMEOUT_MS - (now() - teardownStartedAt), - ); + const deadline = ensureTeardownDeadline(); if (abortController.signal.aborted) { - teardown = await processObserver.emergencyCleanup(remainingBudget()); + teardown = await processObserver.emergencyCleanup(deadline); } else { - teardown = await processObserver.waitForQuiescence( - Math.max(0, remainingBudget() - FORCE_CLEANUP_CONFIRMATION_RESERVE_MS), - ); + teardown = await processObserver.waitForQuiescence(deadline); if (!teardown.quiescent) { - teardown = await processObserver.emergencyCleanup(remainingBudget()); + teardown = await processObserver.emergencyCleanup(deadline); } } - const totalElapsedMs = now() - teardownStartedAt; + const totalElapsedMs = Math.max(0, monotonicNow() - deadline.startedAtMs); teardown = { ...teardown, elapsedMs: totalElapsedMs, @@ -716,7 +739,7 @@ export async function runManagedAgentProbe( teardown.quiescent && teardown.processTableAvailable && teardown.containmentSupported && - totalElapsedMs <= MANAGED_AGENT_TEARDOWN_TIMEOUT_MS, + monotonicNow() <= deadline.deadlineAtMs, }; const beforePolicyOverride = classifyTerminal({ teardown, @@ -762,7 +785,7 @@ export async function runManagedAgentProbe( ...(eventNormalizationFailure ? { eventNormalizationFailure } : {}), }; } finally { - processObserver.dispose(); + await processObserver.dispose(); } const after = await captureManagedAgentWorkspaceSnapshot( @@ -773,7 +796,7 @@ export async function runManagedAgentProbe( recorder.permissionEvidence, guardRejections, ); - return { + return deepFreezeEvidence({ contractVersion: 1, runId, scenario: config.scenario, @@ -821,5 +844,5 @@ export async function runManagedAgentProbe( } : {}), ...(recorder.usage ? { sdkUsage: recorder.usage } : {}), - }; + }); } diff --git a/packages/harness/src/experimental/managed-agent-spike/types.ts b/packages/harness/src/experimental/managed-agent-spike/types.ts index cef7504bf..7b555ddee 100644 --- a/packages/harness/src/experimental/managed-agent-spike/types.ts +++ b/packages/harness/src/experimental/managed-agent-spike/types.ts @@ -303,8 +303,6 @@ export type ManagedAgentQueryFactory = (input: { export interface ManagedAgentProcessObserver { spawn(options: SpawnOptions): SpawnedProcess; - /** Bind only the SDK-forwarded post-grace SpawnOptions signal. */ - bindAbortSignal(signal: AbortSignal): void; /** * Arm the two host-authenticated lifetime observations used only by the * exact E0.4 L2 fixture. Tool-reported identities never grant authority by @@ -314,13 +312,25 @@ export interface ManagedAgentProcessObserver { /** Prove the narrow POSIX observation model before allowing L2 to cancel. */ prepareCancellation(): Promise; /** Sample only members owned by the host-observed process anchors. */ - observeProcessTree(timeoutMs?: number): Promise; + observeProcessTree(deadline?: ManagedAgentTeardownDeadline): Promise; waitForQuiescence( - timeoutMs: number, + deadline: ManagedAgentTeardownDeadline, ): Promise; - /** Idempotently run the anchored fallback and confirm within this budget. */ - emergencyCleanup(timeoutMs: number): Promise; - dispose(): void; + /** Idempotently run the anchored fallback and confirm before this deadline. */ + emergencyCleanup( + deadline: ManagedAgentTeardownDeadline, + ): Promise; + dispose(): void | Promise; +} + +/** + * One immutable monotonic deadline shared by SDK close/return and host process + * containment. It is created once at the first teardown-relevant event and is + * never extended from wall-clock time or a later cleanup phase. + */ +export interface ManagedAgentTeardownDeadline { + readonly startedAtMs: number; + readonly deadlineAtMs: number; } export type ManagedAgentCancellationReadinessReason = @@ -352,7 +362,8 @@ export interface ManagedAgentProbeDependencies { readonly hermeticGatewayOrigin?: string; readonly processObserver?: ManagedAgentProcessObserver; readonly uuid?: () => string; - readonly now?: () => number; + /** Injectable monotonic clock; wall time is never cancellation authority. */ + readonly monotonicNow?: () => number; readonly waitForCancellationSignal?: (signal: AbortSignal) => Promise; readonly policySettingsGuard?: (input: { readonly cwd: string; From 92cfd7a58b3e1c8ac1d56f443f471b0c0fb89ed7 Mon Sep 17 00:00:00 2001 From: Yash Date: Mon, 17 Aug 2026 05:31:36 -0700 Subject: [PATCH 18/24] fix(harness): close managed-agent teardown gaps --- .../managed-agent-spike/README.md | 29 ++- .../managed-agent-spike/events.test.ts | 55 +++- .../managed-agent-spike/events.ts | 33 ++- .../managed-agent-spike/fixture.test.ts | 28 ++- .../managed-agent-spike/fixture.ts | 36 +-- .../managed-agent-spike/probe-cli.test.ts | 26 ++ .../managed-agent-spike/probe-cli.ts | 8 +- .../process-observer.test.ts | 237 +++++++++++++++--- .../managed-agent-spike/process-observer.ts | 90 ++++--- .../runtime-sdk-loopback.test.ts | 17 +- .../managed-agent-spike/runtime.test.ts | 42 ++++ .../managed-agent-spike/runtime.ts | 31 ++- .../experimental/managed-agent-spike/types.ts | 20 ++ 13 files changed, 534 insertions(+), 118 deletions(-) diff --git a/packages/harness/src/experimental/managed-agent-spike/README.md b/packages/harness/src/experimental/managed-agent-spike/README.md index d9a808ea4..e52a89633 100644 --- a/packages/harness/src/experimental/managed-agent-spike/README.md +++ b/packages/harness/src/experimental/managed-agent-spike/README.md @@ -59,6 +59,14 @@ in memory; raw or hashed IDs are not emitted. SDK `result.num_turns` is retained separately as bounded informational evidence and is not used as the BigQuery call-count key. +The durable result also records content-free, non-authoritative SDK model +evidence. A completed L1 run must observe the selected alias in both the SDK +init event and the sole `result.modelUsage` key. A cancelled L2 run may have no +result event, so it requires the matching init model and then relies on gateway +reconciliation. These checks prove what the SDK reported, not what the gateway +served: BigQuery provider/model, fallback, token, and cost rows remain the +authoritative exact-deployment evidence for every live inference turn. + The hermetic pinned-SDK loopback exercises Read, allowed and denied Bash, and a real in-process `echo_nonce` MCP turn. It requires one primary `PreToolUse` decision for each request and separately verifies the MCP handler invocation @@ -142,8 +150,12 @@ permanently fails containment closed. Once L2 tool containment is armed, only the authenticated supervisor and fixture groups are permitted; an additional descendant group rejects readiness. -The runtime gives the Agent SDK its documented abort and bounded query-close -path first. It does not bind the raw per-run `Options.abortController` to host +The runtime creates one immutable monotonic deadline and makes the observer +adopt that same object before the first abort, `Query.close()`, or +`Query.return()`. An SDK-forwarded signal observed before adoption is remembered +but grants no signal authority until the bounded deadline exists. The runtime +then gives the Agent SDK its documented abort and bounded query-close path +first. It does not bind the raw per-run `Options.abortController` to host signals. Only the SDK-forwarded post-grace `SpawnOptions.signal` can trigger the fallback. In SDK 0.3.228, `Query.close()` starts cleanup but returns `void`, so the runtime immediately follows it with and awaits `Query.return()` under the @@ -172,6 +184,13 @@ retryable, but every attempt—including an ESRCH or helper failure—advances t sample generation and requires another fresh proof. The five-second absolute deadline bounds the entire sequence. +Deadline expiry seals all evidence collection and ordinary fallback authority. +Disposal has one narrower leak-prevention rule: if this observer successfully +stopped an owned group before sealing, that stopped kernel group cannot execute, +fork, exit, or have its PGID recycled, so disposal may issue its final `SIGKILL` +without a new sample. This never changes a failed deadline result and never +applies to a group that was not stopped while authority was fresh. + If the root exits, a stable identity changes parent/group/session, a foreign member appears, ancestry is lost, both channels close prematurely, or a process-table read is unavailable, the observer never signals the detached @@ -210,6 +229,12 @@ universal Bash containment, other command shapes, Windows Job Objects, and production recovery belong to later epics, and this probe does not claim those guarantees. +The disposable fixture uses a host-owned lifetime lease outside the writable +workspace. The lease exists before launch; `shutdown` contents or a missing +lease both make the fixture parent stop its child and exit. Cleanup can therefore +remove the temporary root without turning a startup race into a permanently +running process. + ## Pre-v2 live evidence The exact-trace-v1 campaign completed both Sonnet 5 L1 repetitions and the diff --git a/packages/harness/src/experimental/managed-agent-spike/events.test.ts b/packages/harness/src/experimental/managed-agent-spike/events.test.ts index 6711a1b7c..7842a7d2e 100644 --- a/packages/harness/src/experimental/managed-agent-spike/events.test.ts +++ b/packages/harness/src/experimental/managed-agent-spike/events.test.ts @@ -11,7 +11,7 @@ const SESSION_ID = "11111111-1111-4111-8111-111111111111"; describe("ManagedAgentEventRecorder", () => { it("retains structural evidence while redacting message and tool content", () => { - const recorder = new ManagedAgentEventRecorder("run-1"); + const recorder = new ManagedAgentEventRecorder("run-1", "expected-model"); recorder.observeSdkEvent({ type: "system", subtype: "init", @@ -62,6 +62,12 @@ describe("ManagedAgentEventRecorder", () => { }, total_cost_usd: 0.001, num_turns: 7, + modelUsage: { + "unexpected-model": { + inputTokens: 7, + outputTokens: 3, + }, + }, }); expect(recorder.recordTerminal("success")).toBe(true); expect(recorder.recordTerminal("query_error")).toBe(false); @@ -77,6 +83,14 @@ describe("ManagedAgentEventRecorder", () => { }); expect(recorder.inferenceTurns).toBe(1); expect(recorder.sdkNumTurns).toBe(7); + expect(recorder.modelEvidence).toEqual({ + authority: "sdk_non_authoritative", + initModelObserved: true, + initModelMatchesExpectedAlias: false, + resultModelUsageObserved: true, + resultModelUsageMatchesExpectedAlias: false, + resultModelCount: 1, + }); expect(recorder.toolEvidence).toEqual([ { toolUseId: normalizeManagedAgentToolUseId("tool-1"), @@ -107,7 +121,7 @@ describe("ManagedAgentEventRecorder", () => { }); it("redacts attacker-controlled session, tool, and permission identifiers from all evidence", () => { - const recorder = new ManagedAgentEventRecorder("run-2"); + const recorder = new ManagedAgentEventRecorder("run-2", "expected-model"); const sessionSecret = "session-secret-credential"; const toolIdSecret = "tool-id-secret-credential"; const permissionIdSecret = "permission-id-secret-credential"; @@ -192,6 +206,31 @@ describe("ManagedAgentEventRecorder", () => { } }); + it("requires both SDK init and result usage to report only the selected alias", () => { + const expectedModel = "claude-sonnet-5-anthropic-anthropic-eval"; + const recorder = new ManagedAgentEventRecorder("run-model", expectedModel); + recorder.observeSdkEvent({ + type: "system", + subtype: "init", + model: expectedModel, + }); + recorder.observeSdkEvent({ + type: "result", + subtype: "success", + is_error: false, + modelUsage: { [expectedModel]: { inputTokens: 1, outputTokens: 1 } }, + }); + + expect(recorder.modelEvidence).toEqual({ + authority: "sdk_non_authoritative", + initModelObserved: true, + initModelMatchesExpectedAlias: true, + resultModelUsageObserved: true, + resultModelUsageMatchesExpectedAlias: true, + resultModelCount: 1, + }); + }); + it("rejects missing, empty, and overlong tool-use identifiers instead of normalizing sentinels", () => { const invalidIds = [ undefined, @@ -204,7 +243,10 @@ describe("ManagedAgentEventRecorder", () => { ManagedAgentEventError, ); - const requested = new ManagedAgentEventRecorder("invalid-requested"); + const requested = new ManagedAgentEventRecorder( + "invalid-requested", + "expected-model", + ); expect(() => requested.observeSdkEvent({ type: "assistant", @@ -223,7 +265,10 @@ describe("ManagedAgentEventRecorder", () => { ).toThrow(ManagedAgentEventError); expect(requested.toolEvidence).toEqual([]); - const completed = new ManagedAgentEventRecorder("invalid-completed"); + const completed = new ManagedAgentEventRecorder( + "invalid-completed", + "expected-model", + ); expect(() => completed.observeSdkEvent({ type: "user", @@ -243,7 +288,7 @@ describe("ManagedAgentEventRecorder", () => { }); it("counts distinct hashed assistant ids and keeps bounded SDK turns separate", () => { - const recorder = new ManagedAgentEventRecorder("run-3"); + const recorder = new ManagedAgentEventRecorder("run-3", "expected-model"); for (const messageId of [ "private-message-a", "private-message-a", diff --git a/packages/harness/src/experimental/managed-agent-spike/events.ts b/packages/harness/src/experimental/managed-agent-spike/events.ts index f33da4eaf..222732895 100644 --- a/packages/harness/src/experimental/managed-agent-spike/events.ts +++ b/packages/harness/src/experimental/managed-agent-spike/events.ts @@ -5,6 +5,7 @@ import type { ManagedAgentEventNormalizationFailureReason, ManagedAgentPermissionEvidence, ManagedAgentProbeEvent, + ManagedAgentSdkModelEvidence, ManagedAgentSdkUsageEstimate, ManagedAgentTerminalClassification, ManagedAgentToolEvidence, @@ -155,16 +156,23 @@ export class ManagedAgentEventRecorder { readonly #permissionEvidence: ManagedAgentPermissionEvidence[] = []; readonly #inferenceMessageIds = new Set(); readonly #runId: string; + readonly #expectedModelAlias: string; #terminalRecorded = false; #sessionId: string | undefined; #usage: ManagedAgentSdkUsageEstimate | undefined; #sdkNumTurns: number | undefined; + #initModelObserved = false; + #initModelMatchesExpectedAlias = false; + #resultModelUsageObserved = false; + #resultModelUsageMatchesExpectedAlias = false; + #resultModelCount = 0; #sdkResult: | { readonly isError: boolean; readonly subtype?: string } | undefined; - public constructor(runId: string) { + public constructor(runId: string, expectedModelAlias: string) { this.#runId = runId; + this.#expectedModelAlias = expectedModelAlias; } public get events(): readonly ManagedAgentProbeEvent[] { @@ -187,6 +195,18 @@ export class ManagedAgentEventRecorder { return this.#usage; } + public get modelEvidence(): ManagedAgentSdkModelEvidence { + return { + authority: "sdk_non_authoritative", + initModelObserved: this.#initModelObserved, + initModelMatchesExpectedAlias: this.#initModelMatchesExpectedAlias, + resultModelUsageObserved: this.#resultModelUsageObserved, + resultModelUsageMatchesExpectedAlias: + this.#resultModelUsageMatchesExpectedAlias, + resultModelCount: this.#resultModelCount, + }; + } + public get inferenceTurns(): number { return this.#inferenceMessageIds.size; } @@ -243,6 +263,10 @@ export class ManagedAgentEventRecorder { if (sessionId && !this.#sessionId) this.#sessionId = sessionId; if (type === "system" && subtype === "init") { + const initModel = optionalString(event.model); + this.#initModelObserved = initModel !== undefined; + this.#initModelMatchesExpectedAlias = + initModel === this.#expectedModelAlias; this.#append({ type: "lifecycle", subtype: "sdk_init", sessionId }); return; } @@ -325,6 +349,13 @@ export class ManagedAgentEventRecorder { const isError = event.is_error === true || subtype !== "success"; this.#sdkResult = { isError, ...(subtype ? { subtype } : {}) }; this.#usage = sdkUsage(event); + const modelUsage = asRecord(event.modelUsage); + const resultModels = modelUsage ? Object.keys(modelUsage) : []; + this.#resultModelCount = resultModels.length; + this.#resultModelUsageObserved = resultModels.length > 0; + this.#resultModelUsageMatchesExpectedAlias = + resultModels.length === 1 && + resultModels[0] === this.#expectedModelAlias; this.#append({ type: "sdk_result", subtype, diff --git a/packages/harness/src/experimental/managed-agent-spike/fixture.test.ts b/packages/harness/src/experimental/managed-agent-spike/fixture.test.ts index 87628a184..081a458b4 100644 --- a/packages/harness/src/experimental/managed-agent-spike/fixture.test.ts +++ b/packages/harness/src/experimental/managed-agent-spike/fixture.test.ts @@ -1,5 +1,6 @@ -import { execFileSync } from "node:child_process"; -import { writeFile } from "node:fs/promises"; +import { execFileSync, spawn } from "node:child_process"; +import { once } from "node:events"; +import { rm, writeFile } from "node:fs/promises"; import { join } from "node:path"; import { afterEach, describe, expect, it } from "vitest"; @@ -57,6 +58,29 @@ describe("managed-agent disposable git fixture", () => { ]); }); + it("treats a deleted host lifetime lease as shutdown during startup", async () => { + const fixture = await createManagedAgentFixture(() => "lease-shutdown"); + fixtures.push(fixture); + await rm(fixture.cooperativeExitMarker, { force: true }); + const processScript = join( + fixture.workspaceRoot, + FIXTURE_PATHS.processScript, + ); + const child = spawn( + process.execPath, + [ + processScript, + join(fixture.workspaceRoot, FIXTURE_PATHS.processPidFile), + "--host-cleanup-marker", + fixture.cooperativeExitMarker, + ], + { stdio: "ignore", windowsHide: true }, + ); + + const [exitCode] = await once(child, "exit"); + expect(exitCode).toBe(0); + }); + it("renders L1 as eleven exact ordered calls without resolving the escape link", async () => { const fixture = await createManagedAgentFixture(() => "prompt-contract"); fixtures.push(fixture); diff --git a/packages/harness/src/experimental/managed-agent-spike/fixture.ts b/packages/harness/src/experimental/managed-agent-spike/fixture.ts index 11c384b9a..c674ca54d 100644 --- a/packages/harness/src/experimental/managed-agent-spike/fixture.ts +++ b/packages/harness/src/experimental/managed-agent-spike/fixture.ts @@ -15,6 +15,7 @@ import { } from "node:fs/promises"; import { tmpdir } from "node:os"; import { basename, join, relative, resolve } from "node:path"; +import { performance } from "node:perf_hooks"; import { MANAGED_AGENT_L1_CERTIFICATION_CONTRACT } from "./contract.js"; import { @@ -84,7 +85,7 @@ function shellQuote(value: string): string { const LONG_RUNNING_SCRIPT = ` import { spawn } from "node:child_process"; -import { existsSync, unlinkSync, writeFileSync } from "node:fs"; +import { existsSync, readFileSync, unlinkSync, writeFileSync } from "node:fs"; import { createConnection } from "node:net"; import { resolve } from "node:path"; @@ -205,7 +206,16 @@ if (cleanupMarker) { try { unlinkSync(cleanupMarker); } catch {} }); const cleanupPoll = setInterval(() => { - if (!existsSync(cleanupMarker)) return; + // The host creates a lifetime lease before launch. A missing lease means + // the disposable fixture root was removed during a startup race and must + // therefore be treated as shutdown, never as permission to keep running. + const shutdownRequested = + !existsSync(cleanupMarker) || + (() => { + try { return readFileSync(cleanupMarker, "utf8") === "shutdown\\n"; } + catch { return true; } + })(); + if (!shutdownRequested) return; clearInterval(cleanupPoll); if (child.connected) child.send("host-shutdown"); child.once("exit", () => process.exit(0)); @@ -367,6 +377,9 @@ export async function createManagedAgentFixture( { mode: 0o600 }, ), writeFile(outsideSentinel, outsideContents), + // This host-owned lease lives outside the model-writable workspace. The + // fixture treats deletion as shutdown too, closing the setup/cleanup race. + writeFile(cooperativeExitMarker, "run\n", { mode: 0o600 }), ]); await symlink(outsideSentinel, join(workspaceRoot, FIXTURE_PATHS.escapeLink)); @@ -420,23 +433,14 @@ export async function createManagedAgentFixture( let cooperativeExitRequested = false; const requestCooperativeExit = async (): Promise => { if (!existsSync(root)) return; - if ( - cooperativeExitRequested || - !existsSync(join(workspaceRoot, FIXTURE_PATHS.processPidFile)) - ) { - return; + if (!cooperativeExitRequested) { + cooperativeExitRequested = true; + await writeFile(cooperativeExitMarker, "shutdown\n", { mode: 0o600 }); } - cooperativeExitRequested = true; - await writeFile(cooperativeExitMarker, "shutdown\n", { mode: 0o600 }); - const deadline = Date.now() + 1_000; - while (existsSync(cooperativeExitMarker) && Date.now() < deadline) { + const deadline = performance.now() + 1_000; + while (existsSync(cooperativeExitMarker) && performance.now() < deadline) { await new Promise((resolveDelay) => setTimeout(resolveDelay, 10)); } - if (existsSync(cooperativeExitMarker)) { - // Permit a later cleanup attempt if the fixture process had not begun - // polling yet. The marker is still removed with the disposable root. - cooperativeExitRequested = false; - } }; return { diff --git a/packages/harness/src/experimental/managed-agent-spike/probe-cli.test.ts b/packages/harness/src/experimental/managed-agent-spike/probe-cli.test.ts index e0d6f547a..1ac733ead 100644 --- a/packages/harness/src/experimental/managed-agent-spike/probe-cli.test.ts +++ b/packages/harness/src/experimental/managed-agent-spike/probe-cli.test.ts @@ -111,6 +111,14 @@ function passingL1Result(): ManagedAgentProbeResult { scenario: "L1", target: "sonnet-5", modelAlias: "claude-sonnet-5-anthropic-anthropic-eval", + sdkModelEvidence: { + authority: "sdk_non_authoritative", + initModelObserved: true, + initModelMatchesExpectedAlias: true, + resultModelUsageObserved: true, + resultModelUsageMatchesExpectedAlias: true, + resultModelCount: 1, + }, sdkSessionId: "11111111-1111-4111-8111-111111111111", inferenceTurns: 8, sdkNumTurns: 8, @@ -632,6 +640,24 @@ describe("managed-agent probe CLI", () => { ).toEqual({ id: "builtin_tools_succeeded", passed: false }); }); + it("fails exact-model certification when SDK-observed model evidence is missing or mixed", () => { + const passing = passingL1Result(); + const report = evaluateManagedAgentProbe({ + ...passing, + sdkModelEvidence: { + ...passing.sdkModelEvidence, + resultModelUsageMatchesExpectedAlias: false, + resultModelCount: 2, + }, + }); + + expect(report.checks).toContainEqual({ + id: "exact_model_alias", + passed: false, + }); + expect(report.outcome).toBe("fail"); + }); + it.each([ ["clean_target", "e"], ["dirty_sentinel", "f"], diff --git a/packages/harness/src/experimental/managed-agent-spike/probe-cli.ts b/packages/harness/src/experimental/managed-agent-spike/probe-cli.ts index 4b3f03f1b..90a991fe7 100644 --- a/packages/harness/src/experimental/managed-agent-spike/probe-cli.ts +++ b/packages/harness/src/experimental/managed-agent-spike/probe-cli.ts @@ -687,7 +687,13 @@ export function evaluateManagedAgentProbe( id: "exact_model_alias", passed: result.modelAlias === - resolveManagedAgentModelTarget(result.target).alias, + resolveManagedAgentModelTarget(result.target).alias && + result.sdkModelEvidence.initModelObserved && + result.sdkModelEvidence.initModelMatchesExpectedAlias && + (result.sdkModelEvidence.resultModelUsageObserved + ? result.sdkModelEvidence.resultModelUsageMatchesExpectedAlias && + result.sdkModelEvidence.resultModelCount === 1 + : result.scenario === "L2" && result.terminal === "cancelled"), }, { id: "sdk_session_observed", passed: Boolean(result.sdkSessionId) }, { id: "query_closed", passed: result.queryClosed }, diff --git a/packages/harness/src/experimental/managed-agent-spike/process-observer.test.ts b/packages/harness/src/experimental/managed-agent-spike/process-observer.test.ts index 87a123416..0675ee416 100644 --- a/packages/harness/src/experimental/managed-agent-spike/process-observer.test.ts +++ b/packages/harness/src/experimental/managed-agent-spike/process-observer.test.ts @@ -9,6 +9,7 @@ import { import { readFile, writeFile } from "node:fs/promises"; import { createConnection, type Socket as NetSocket } from "node:net"; import { join } from "node:path"; +import { performance } from "node:perf_hooks"; import { promisify } from "node:util"; import type { SpawnedProcess } from "@anthropic-ai/claude-agent-sdk"; @@ -33,6 +34,13 @@ import { const fixtures: ManagedAgentFixture[] = []; const execFileAsync = promisify(execFile); +function deadlineAfter(timeoutMs: number, startedAtMs = performance.now()) { + return Object.freeze({ + startedAtMs, + deadlineAtMs: startedAtMs + timeoutMs, + }); +} + afterEach(async () => { await Promise.all(fixtures.splice(0).map((fixture) => fixture.cleanup())); }); @@ -685,7 +693,9 @@ async function proveRetainedGroupAuthority( expect(escapedReadiness.observedPids).not.toContain(unrelated.pid); expect(processExists(nonCooperativeChildPid)).toBe(true); expect(processExists(unrelated.pid!)).toBe(true); - await expect(observer.waitForQuiescence(0)).resolves.toMatchObject({ + await expect( + observer.waitForQuiescence(deadlineAfter(1)), + ).resolves.toMatchObject({ quiescent: false, deadlineMet: false, containmentSupported: false, @@ -1148,7 +1158,9 @@ describe("LocalManagedAgentProcessObserver", () => { expect(anchor.kill("SIGTERM")).toBe(false); expect(nativeKillSpy).not.toHaveBeenCalled(); expect(processExists(processGroupId)).toBe(true); - await expect(observer.waitForQuiescence(0)).resolves.toMatchObject({ + await expect( + observer.waitForQuiescence(deadlineAfter(1)), + ).resolves.toMatchObject({ quiescent: false, containmentSupported: true, forceKillIssued: false, @@ -1211,7 +1223,7 @@ describe("LocalManagedAgentProcessObserver", () => { const startedAt = Date.now(); forwardedController.abort(); - const teardown = await observer.emergencyCleanup(1_000); + const teardown = await observer.emergencyCleanup(deadlineAfter(1_000)); expect(teardown).toMatchObject({ quiescent: true, @@ -1379,7 +1391,7 @@ describe("LocalManagedAgentProcessObserver", () => { reason: "root_not_active", }); forwardedController.abort(); - await observer.emergencyCleanup(0); + await observer.emergencyCleanup(deadlineAfter(1)); expect(signals).toEqual([]); } finally { observer.dispose(); @@ -1442,7 +1454,7 @@ describe("LocalManagedAgentProcessObserver", () => { ownershipProven: true, }); - const teardown = await observer.emergencyCleanup(2_000); + const teardown = await observer.emergencyCleanup(deadlineAfter(2_000)); expect(teardown).toMatchObject({ quiescent: true, @@ -1489,6 +1501,8 @@ describe("LocalManagedAgentProcessObserver", () => { rootProcessGroupId = run.anchor.pid!; toolProcessGroupId = run.toolProcessGroupId; + const teardownDeadline = deadlineAfter(3_000); + observer.beginTeardown(teardownDeadline); run.forwardedController.abort(); const deadline = Date.now() + 2_000; while ( @@ -1508,7 +1522,9 @@ describe("LocalManagedAgentProcessObserver", () => { [toolProcessGroupId, "SIGKILL"], [rootProcessGroupId, "SIGKILL"], ]); - await expect(observer.waitForQuiescence(1_000)).resolves.toMatchObject({ + await expect( + observer.waitForQuiescence(teardownDeadline), + ).resolves.toMatchObject({ quiescent: true, deadlineMet: true, containmentSupported: true, @@ -1599,7 +1615,7 @@ describe("LocalManagedAgentProcessObserver", () => { toolProcessGroupId = run.toolProcessGroupId; injectForeignMember = true; - const teardown = await observer.emergencyCleanup(250); + const teardown = await observer.emergencyCleanup(deadlineAfter(250)); expect(teardown).toMatchObject({ quiescent: false, @@ -1671,7 +1687,7 @@ describe("LocalManagedAgentProcessObserver", () => { registeredPids = run.toolPids; simulatePidReuse = true; - const teardown = await observer.emergencyCleanup(250); + const teardown = await observer.emergencyCleanup(deadlineAfter(250)); expect(teardown).toMatchObject({ quiescent: false, @@ -1724,7 +1740,7 @@ describe("LocalManagedAgentProcessObserver", () => { ); toolProcessGroupId = run.toolProcessGroupId; - const teardown = await observer.emergencyCleanup(3_000); + const teardown = await observer.emergencyCleanup(deadlineAfter(3_000)); expect(teardown).toMatchObject({ quiescent: true, @@ -1775,7 +1791,7 @@ describe("LocalManagedAgentProcessObserver", () => { await forceKillRetainedTestGroup(run.anchor); expect(processGroupExists(toolProcessGroupId)).toBe(true); - const teardown = await observer.emergencyCleanup(250); + const teardown = await observer.emergencyCleanup(deadlineAfter(250)); expect(teardown).toMatchObject({ quiescent: false, @@ -1855,7 +1871,7 @@ describe("LocalManagedAgentProcessObserver", () => { reason: "tool_process_not_registered", }); - const teardown = await observer.emergencyCleanup(100); + const teardown = await observer.emergencyCleanup(deadlineAfter(100)); expect(teardown.quiescent).toBe(false); expect(processGroupExists(detachedTool.pid!)).toBe(true); @@ -1934,8 +1950,11 @@ describe("LocalManagedAgentProcessObserver", () => { ), ), ); + const teardownDeadline = deadlineAfter(1_000); + observer.beginTeardown(teardownDeadline); forwardedController.abort(); - const openChannelObservation = await observer.emergencyCleanup(1_000); + const openChannelObservation = + await observer.emergencyCleanup(teardownDeadline); await waitForChildExitBounded(anchor); expect(openChannelObservation).toMatchObject({ quiescent: false, @@ -1944,7 +1963,8 @@ describe("LocalManagedAgentProcessObserver", () => { parentRegistration.destroy(); childRegistration.destroy(); - const finalObservation = await observer.waitForQuiescence(3_000); + const finalObservation = + await observer.waitForQuiescence(teardownDeadline); expect(finalObservation).toMatchObject({ quiescent: false, deadlineMet: false, @@ -1981,12 +2001,17 @@ describe("LocalManagedAgentProcessObserver", () => { if (anchor.exitCode === null && anchor.signalCode === null) { await once(anchor, "exit"); } - await expect(observer.waitForQuiescence(50)).resolves.toMatchObject({ + const teardownDeadline = deadlineAfter(50); + await expect( + observer.waitForQuiescence(teardownDeadline), + ).resolves.toMatchObject({ quiescent: false, deadlineMet: false, containmentSupported: false, }); - await expect(observer.emergencyCleanup(50)).resolves.toMatchObject({ + await expect( + observer.emergencyCleanup(teardownDeadline), + ).resolves.toMatchObject({ quiescent: false, deadlineMet: false, containmentSupported: false, @@ -2032,7 +2057,9 @@ describe("LocalManagedAgentProcessObserver", () => { }); expect(Date.now() - startedAt).toBeLessThan(1_000); const shortConfirmationStartedAt = Date.now(); - await expect(observer.waitForQuiescence(50)).resolves.toMatchObject({ + await expect( + observer.waitForQuiescence(deadlineAfter(50)), + ).resolves.toMatchObject({ quiescent: false, deadlineMet: false, processTableAvailable: false, @@ -2048,6 +2075,117 @@ describe("LocalManagedAgentProcessObserver", () => { } }); + it("remembers an SDK abort but grants no fallback signal before deadline adoption", async () => { + let rootPid = 0; + const signals: Array = []; + const observer = new LocalManagedAgentProcessObserver({ + platform: "darwin", + readProcessTable: async () => + available([ + [ + rootPid, + { + parentPid: process.pid, + processGroupId: rootPid, + sessionId: rootPid, + state: signals.some(([, signal]) => signal === "SIGSTOP") + ? "T" + : "S", + startedAt: "abort-before-deadline", + }, + ], + ]), + signalProcessGroup: (groupId, signal) => { + signals.push([groupId, signal]); + return "sent"; + }, + }); + const controller = new AbortController(); + const child = asChildProcess( + observer.spawn({ + ...activeNodeCommand(), + cwd: process.cwd(), + env: { ...process.env }, + signal: controller.signal, + }), + ); + rootPid = child.pid!; + try { + await expect(observer.prepareCancellation()).resolves.toMatchObject({ + supported: true, + reason: "ready", + }); + + controller.abort(); + expect(signals).toEqual([]); + + observer.beginTeardown(deadlineAfter(100)); + expect(signals).toEqual([[rootPid, "SIGSTOP"]]); + } finally { + await forceKillRetainedTestGroup(child); + await observer.dispose(); + } + }); + + it.skipIf(process.platform === "win32")( + "kills a stopped owned group during disposal when post-stop observation misses the deadline", + async () => { + let hangAfterStop = false; + const signals: Array = []; + const observer = new LocalManagedAgentProcessObserver({ + readProcessTable: () => + hangAfterStop + ? new Promise(() => undefined) + : readRealPosixProcessTable(), + signalProcessGroup: (groupId, signal) => { + signals.push([groupId, signal]); + const outcome = signalRealProcessGroup(groupId, signal); + if (signal === "SIGSTOP" && outcome === "sent") { + hangAfterStop = true; + } + return outcome; + }, + }); + const controller = new AbortController(); + const anchor = asChildProcess( + observer.spawn({ + ...activeNodeCommand(), + cwd: process.cwd(), + env: { ...process.env }, + signal: controller.signal, + }), + ); + try { + await expect( + prepareCancellationAfterTransientReadFailure(observer), + ).resolves.toMatchObject({ supported: true, reason: "ready" }); + const teardownDeadline = deadlineAfter(75); + observer.beginTeardown(teardownDeadline); + controller.abort(); + + const teardown = await observer.emergencyCleanup(teardownDeadline); + expect(teardown).toMatchObject({ + quiescent: false, + deadlineMet: false, + forceKillIssued: false, + }); + expect(signals.map(([, signal]) => signal)).toEqual(["SIGSTOP"]); + + await observer.dispose(); + await waitForChildExitBounded(anchor); + expect(signals.map(([, signal]) => signal)).toEqual([ + "SIGSTOP", + "SIGKILL", + ]); + expect(processGroupExists(anchor.pid!)).toBe(false); + } finally { + await forceKillRetainedTestGroup(anchor); + await observer.dispose(); + } + }, + 10_000, + ); + it("marks an observed POSIX group escape unsupported without authorizing an individual signal", async () => { let rootPid = 0; let escaped = false; @@ -2093,7 +2231,9 @@ describe("LocalManagedAgentProcessObserver", () => { }); escaped = true; await observer.observeProcessTree(); - await expect(observer.waitForQuiescence(0)).resolves.toMatchObject({ + await expect( + observer.waitForQuiescence(deadlineAfter(1)), + ).resolves.toMatchObject({ quiescent: false, containmentSupported: false, }); @@ -2163,7 +2303,9 @@ describe("LocalManagedAgentProcessObserver", () => { zombie = true; await observer.observeProcessTree(); - const observation = await observer.waitForQuiescence(1); + const observation = await observer.waitForQuiescence( + deadlineAfter(1, now), + ); expect(observation).toMatchObject({ quiescent: false, containmentSupported: true, @@ -2239,7 +2381,8 @@ describe("LocalManagedAgentProcessObserver", () => { escaped = true; await observer.observeProcessTree(); - const observation = await observer.waitForQuiescence(1); + const teardownDeadline = deadlineAfter(1, now); + const observation = await observer.waitForQuiescence(teardownDeadline); expect(observation).toMatchObject({ quiescent: false, containmentSupported: false, @@ -2252,7 +2395,9 @@ describe("LocalManagedAgentProcessObserver", () => { controller.abort(); gone = true; await observer.observeProcessTree(); - await expect(observer.waitForQuiescence(1)).resolves.toMatchObject({ + await expect( + observer.waitForQuiescence(teardownDeadline), + ).resolves.toMatchObject({ quiescent: false, containmentSupported: false, }); @@ -2339,6 +2484,8 @@ describe("LocalManagedAgentProcessObserver", () => { supported: true, reason: "ready", }); + const teardownDeadline = deadlineAfter(100, now); + observer.beginTeardown(teardownDeadline); controller.abort(); await observer.observeProcessTree(); expect(signals).toEqual([ @@ -2351,7 +2498,9 @@ describe("LocalManagedAgentProcessObserver", () => { stage = "gone"; await observer.observeProcessTree(); - await expect(observer.waitForQuiescence(0)).resolves.toMatchObject({ + await expect( + observer.waitForQuiescence(teardownDeadline), + ).resolves.toMatchObject({ quiescent: true, deadlineMet: true, containmentSupported: true, @@ -2489,7 +2638,9 @@ describe("LocalManagedAgentProcessObserver", () => { await observer.observeProcessTree(); rootAlive = false; await observer.observeProcessTree(); - await expect(observer.waitForQuiescence(0)).resolves.toMatchObject({ + await expect( + observer.waitForQuiescence(deadlineAfter(1)), + ).resolves.toMatchObject({ quiescent: true, deadlineMet: true, containmentSupported: true, @@ -2548,7 +2699,9 @@ describe("LocalManagedAgentProcessObserver", () => { rootPid = anchor.pid!; try { await observer.observeProcessTree(); - await expect(observer.waitForQuiescence(0)).resolves.toMatchObject({ + await expect( + observer.waitForQuiescence(deadlineAfter(1)), + ).resolves.toMatchObject({ quiescent: false, deadlineMet: false, containmentSupported: false, @@ -2620,7 +2773,9 @@ describe("LocalManagedAgentProcessObserver", () => { await observer.observeProcessTree(); subgroupState = "gone"; await observer.observeProcessTree(); - await expect(observer.waitForQuiescence(0)).resolves.toMatchObject({ + await expect( + observer.waitForQuiescence(deadlineAfter(1)), + ).resolves.toMatchObject({ quiescent: false, deadlineMet: false, containmentSupported: false, @@ -2798,6 +2953,8 @@ describe("LocalManagedAgentProcessObserver", () => { supported: true, reason: "ready", }); + const teardownDeadline = deadlineAfter(100); + observer.beginTeardown(teardownDeadline); controller.abort(); expect(signals).toEqual([[rootPid, "SIGSTOP"]]); await observer.observeProcessTree(); @@ -2808,7 +2965,9 @@ describe("LocalManagedAgentProcessObserver", () => { await observer.observeProcessTree(); subgroupState = "gone"; await observer.observeProcessTree(); - await expect(observer.waitForQuiescence(0)).resolves.toMatchObject({ + await expect( + observer.waitForQuiescence(teardownDeadline), + ).resolves.toMatchObject({ quiescent: false, deadlineMet: false, containmentSupported: false, @@ -2994,7 +3153,7 @@ describe("LocalManagedAgentProcessObserver", () => { await Promise.all(registrationClosures); await forceKillRetainedTestGroup(anchor); - const teardown = await observer.emergencyCleanup(50); + const teardown = await observer.emergencyCleanup(deadlineAfter(50)); expect(teardown).toMatchObject({ quiescent: false, @@ -3064,6 +3223,7 @@ describe("LocalManagedAgentProcessObserver", () => { rootPid = child.pid!; try { await observer.prepareCancellation(); + observer.beginTeardown(deadlineAfter(100)); forwardedController.abort(); forwardedController.abort(); expect(signals).toEqual([[rootPid, "SIGSTOP"]]); @@ -3125,7 +3285,8 @@ describe("LocalManagedAgentProcessObserver", () => { rootPid = child.pid!; try { await observer.prepareCancellation(); - await observer.emergencyCleanup(100); + const teardownDeadline = deadlineAfter(100); + await observer.emergencyCleanup(teardownDeadline); expect(signals).toEqual([ [rootPid, "SIGSTOP"], @@ -3139,7 +3300,9 @@ describe("LocalManagedAgentProcessObserver", () => { index === 0 || readCount > signalReadCounts[index - 1]!, ), ).toBe(true); - await expect(observer.waitForQuiescence(0)).resolves.toMatchObject({ + await expect( + observer.waitForQuiescence(teardownDeadline), + ).resolves.toMatchObject({ containmentSupported: true, forceKillIssued: true, quiescent: false, @@ -3182,7 +3345,9 @@ describe("LocalManagedAgentProcessObserver", () => { rootPid = child.pid!; await once(child, "exit"); try { - await expect(observer.waitForQuiescence(0)).resolves.toMatchObject({ + await expect( + observer.waitForQuiescence(deadlineAfter(1)), + ).resolves.toMatchObject({ quiescent: false, deadlineMet: false, containmentSupported: false, @@ -3216,7 +3381,9 @@ describe("LocalManagedAgentProcessObserver", () => { await once(child, "exit"); try { await observer.observeProcessTree(); - await expect(observer.waitForQuiescence(0)).resolves.toMatchObject({ + await expect( + observer.waitForQuiescence(deadlineAfter(1)), + ).resolves.toMatchObject({ quiescent: true, deadlineMet: true, containmentSupported: true, @@ -3254,7 +3421,9 @@ describe("LocalManagedAgentProcessObserver", () => { now = 0; measureOverrun = true; - await expect(observer.waitForQuiescence(1)).resolves.toMatchObject({ + await expect( + observer.waitForQuiescence(deadlineAfter(1, 0)), + ).resolves.toMatchObject({ quiescent: true, deadlineMet: false, elapsedMs: 2, @@ -3302,8 +3471,9 @@ describe("LocalManagedAgentProcessObserver", () => { forgedPids, ); await observer.observeProcessTree(); - const teardown = await observer.waitForQuiescence(0); - await observer.emergencyCleanup(0); + const teardownDeadline = deadlineAfter(1); + const teardown = await observer.waitForQuiescence(teardownDeadline); + await observer.emergencyCleanup(teardownDeadline); expect(teardown.observedPids).not.toContain(forgedPids[0]); expect(teardown.observedPids).not.toContain(forgedPids[1]); @@ -3373,6 +3543,7 @@ describe("LocalManagedAgentProcessObserver", () => { const preSignalSample = observer.observeProcessTree(); await vi.waitFor(() => expect(reads).toHaveLength(1)); + observer.beginTeardown(deadlineAfter(1_000)); forwardedController.abort(); expect(signals).toEqual([[rootPid, "SIGSTOP"]]); diff --git a/packages/harness/src/experimental/managed-agent-spike/process-observer.ts b/packages/harness/src/experimental/managed-agent-spike/process-observer.ts index efc6ee5dd..b9dd67f75 100644 --- a/packages/harness/src/experimental/managed-agent-spike/process-observer.ts +++ b/packages/harness/src/experimental/managed-agent-spike/process-observer.ts @@ -52,6 +52,7 @@ export const MANAGED_AGENT_LOGICAL_KILL_SHIM_SDK_VERSION = "0.3.228" as const; */ const MANAGED_AGENT_POSIX_SUPERVISOR_SOURCE = String.raw` import { spawn } from "node:child_process"; +import { performance } from "node:perf_hooks"; const PAYLOAD_ENV = "SAPIOM_MANAGED_AGENT_SUPERVISOR_PAYLOAD"; const HELPER_TIMEOUT_MS = 200; @@ -185,7 +186,7 @@ async function checkMembership() { const members = await readOtherGroupMembers(); membershipCheckRunning = false; if (members && members.length === 0) { - const now = Date.now(); + const now = performance.now(); emptyGroupObservedAt ??= now; const remainingGrace = EMPTY_GROUP_EXIT_GRACE_MS - (now - emptyGroupObservedAt); @@ -583,6 +584,7 @@ export class LocalManagedAgentProcessObserver implements ManagedAgentProcessObse #lifecycleEpoch = 0; #sealed = false; #hostDisposing = false; + #abortObserved = false; #teardownDeadline: ManagedAgentTeardownDeadline | undefined; #sampleTask: ProcessSampleTask | undefined; #disposeTask: Promise | undefined; @@ -630,6 +632,11 @@ export class LocalManagedAgentProcessObserver implements ManagedAgentProcessObse return this.#teardownDeadline; } + public beginTeardown(deadline: ManagedAgentTeardownDeadline): void { + this.#adoptDeadline(deadline); + if (this.#abortObserved) this.#requestFallbackCleanupSynchronously(); + } + #remainingMs(deadline: ManagedAgentTeardownDeadline): number { return Math.max(0, deadline.deadlineAtMs - this.#monotonicNow()); } @@ -645,20 +652,6 @@ export class LocalManagedAgentProcessObserver implements ManagedAgentProcessObse return true; } - #normalizeDeadline( - input: ManagedAgentTeardownDeadline | number, - ): ManagedAgentTeardownDeadline { - if (typeof input !== "number") return this.#adoptDeadline(input); - if (this.#teardownDeadline) return this.#teardownDeadline; - const startedAtMs = this.#monotonicNow(); - return this.#adoptDeadline( - Object.freeze({ - startedAtMs, - deadlineAtMs: startedAtMs + Math.max(0, input), - }), - ); - } - #seal(): void { if (this.#sealed) return; this.#sealed = true; @@ -793,13 +786,19 @@ export class LocalManagedAgentProcessObserver implements ManagedAgentProcessObse signal.addEventListener( "abort", () => { + this.#abortObserved = true; + // The SDK may forward its private signal before runtime enters its + // teardown path. Remember it, but never signal from an unbounded + // window; beginTeardown() will synchronously replay the request. + if (!this.#teardownDeadline) return; if (this.#sealed || this.#deadlineExpiredAndSeal()) return; this.#requestFallbackCleanupSynchronously(); }, { once: true }, ); if (signal.aborted) { - this.#requestFallbackCleanupSynchronously(); + this.#abortObserved = true; + if (this.#teardownDeadline) this.#requestFallbackCleanupSynchronously(); } } @@ -1795,6 +1794,36 @@ export class LocalManagedAgentProcessObserver implements ManagedAgentProcessObse this.#stopOwnedRootsSynchronously(); } + /** + * Disposal may run after the evidence deadline has irreversibly sealed the + * observer. A group for which this observer successfully issued SIGSTOP + * cannot execute, exit, fork, or have its PGID recycled while stopped, so + * that frozen kernel anchor remains safe for one final SIGKILL without a + * new process-table sample. This is leak prevention only: it never changes + * the already-finalized teardown evidence or grants authority for a group + * that was not stopped before sealing. + */ + #forceKillStoppedGroupsForDisposalSynchronously(): void { + if (this.#platform !== "darwin" && this.#platform !== "linux") return; + + const toolProcessGroupId = this.#toolProcessGroupId; + if ( + this.#toolProcessStopIssued && + !this.#toolProcessForceKillIssued && + typeof toolProcessGroupId === "number" + ) { + const outcome = this.#signalProcessGroup(toolProcessGroupId, "SIGKILL"); + this.#toolProcessForceKillIssued = outcome === "sent"; + } + + for (const root of this.#roots.values()) { + if (root.stopIssued && !root.forceKillIssued && childActive(root.child)) { + const outcome = this.#signalProcessGroup(root.pid, "SIGKILL"); + root.forceKillIssued = outcome === "sent"; + } + } + } + #advanceFallbackCleanup(): void { if ( this.#sealed || @@ -2013,34 +2042,14 @@ export class LocalManagedAgentProcessObserver implements ManagedAgentProcessObse } public async waitForQuiescence( - deadlineInput: ManagedAgentTeardownDeadline | number, + deadline: ManagedAgentTeardownDeadline, ): Promise { - const adoptedDeadline = this.#normalizeDeadline(deadlineInput); + const adoptedDeadline = this.#adoptDeadline(deadline); const startedAt = adoptedDeadline.startedAtMs; const boundedTimeoutMs = Math.max( 0, adoptedDeadline.deadlineAtMs - adoptedDeadline.startedAtMs, ); - if ( - this.#remainingMs(adoptedDeadline) === 0 && - this.#processTableAvailable && - !this.#processTableNeedsRefresh && - !this.#sampleTask - ) { - const cachedObservation = this.#currentObservation(startedAt, false); - if (cachedObservation.quiescent) { - // The fresh cached table already proved quiescence at call entry. Its - // evidence elapsed is therefore zero even if returning the Promise - // crosses a wall-clock millisecond boundary. - const result = { - ...cachedObservation, - deadlineMet: true, - elapsedMs: 0, - }; - this.#seal(); - return result; - } - } for (;;) { if (this.#remainingMs(adoptedDeadline) <= 0 || this.#sealed) { const observation = this.#currentObservation(startedAt, false); @@ -2072,9 +2081,9 @@ export class LocalManagedAgentProcessObserver implements ManagedAgentProcessObse } public async emergencyCleanup( - deadlineInput: ManagedAgentTeardownDeadline | number, + deadline: ManagedAgentTeardownDeadline, ): Promise { - const adoptedDeadline = this.#normalizeDeadline(deadlineInput); + const adoptedDeadline = this.#adoptDeadline(deadline); const startedAt = adoptedDeadline.startedAtMs; this.#requestFallbackCleanupSynchronously(); const confirmation = await this.waitForQuiescence(adoptedDeadline); @@ -2102,6 +2111,7 @@ export class LocalManagedAgentProcessObserver implements ManagedAgentProcessObse async #disposeInternal(): Promise { this.#hostDisposing = true; this.#seal(); + this.#forceKillStoppedGroupsForDisposalSynchronously(); // The two exact fixture processes authenticated these retained channels // with an observer-created capability that was never written into the diff --git a/packages/harness/src/experimental/managed-agent-spike/runtime-sdk-loopback.test.ts b/packages/harness/src/experimental/managed-agent-spike/runtime-sdk-loopback.test.ts index 8cdf3f78d..924c2acdb 100644 --- a/packages/harness/src/experimental/managed-agent-spike/runtime-sdk-loopback.test.ts +++ b/packages/harness/src/experimental/managed-agent-spike/runtime-sdk-loopback.test.ts @@ -318,6 +318,7 @@ it("enforces real-SDK built-in and in-process MCP calls with exact loopback corr }); return child; }, + beginTeardown: (deadline) => observer.beginTeardown(deadline), armToolProcessContainment: () => observer.armToolProcessContainment(), prepareCancellation: () => observer.prepareCancellation(), observeProcessTree: (timeoutMs) => observer.observeProcessTree(timeoutMs), @@ -524,6 +525,14 @@ it("enforces real-SDK built-in and in-process MCP calls with exact loopback corr terminationEvidence: result.terminationEvidence, }), ).toBe("success"); + expect(result.sdkModelEvidence).toEqual({ + authority: "sdk_non_authoritative", + initModelObserved: true, + initModelMatchesExpectedAlias: true, + resultModelUsageObserved: true, + resultModelUsageMatchesExpectedAlias: true, + resultModelCount: 1, + }); const requested = result.toolEvidence.filter( ({ status }) => status === "requested", @@ -604,8 +613,7 @@ it("enforces real-SDK built-in and in-process MCP calls with exact loopback corr expect(result.queryClosed).toBe(true); expect(result.teardown.quiescent).toBe(true); } finally { - await observer.emergencyCleanup(1_000); - observer.dispose(); + await observer.dispose(); server.closeAllConnections(); await new Promise((resolve) => server.close(() => resolve())); await fixture.cleanup(); @@ -682,6 +690,7 @@ it.skipIf( supervisorPid = typeof pid === "number" ? pid : undefined; return child; }, + beginTeardown: (deadline) => observer.beginTeardown(deadline), armToolProcessContainment: () => observer.armToolProcessContainment(), prepareCancellation: () => observer.prepareCancellation(), observeProcessTree: (timeoutMs) => observer.observeProcessTree(timeoutMs), @@ -916,7 +925,6 @@ it.skipIf( cleanupOrder.indexOf("host_emergency_cleanup"), ); } finally { - await observer.emergencyCleanup(1_000); await observer.dispose(); server.closeAllConnections(); await new Promise((resolve) => server.close(() => resolve())); @@ -956,6 +964,7 @@ it.skipIf( signal: neverForwardedController.signal, }); }, + beginTeardown: (deadline) => observer.beginTeardown(deadline), armToolProcessContainment: () => observer.armToolProcessContainment(), prepareCancellation: () => observer.prepareCancellation(), observeProcessTree: (timeoutMs) => observer.observeProcessTree(timeoutMs), @@ -1102,7 +1111,6 @@ it.skipIf( cleanupOrder.indexOf("sdk_forwarded_signal_unobserved"), ).toBeLessThan(cleanupOrder.indexOf("host_timeout_fallback")); } finally { - await observer.emergencyCleanup(1_000); await observer.dispose(); server.closeAllConnections(); await new Promise((resolve) => server.close(() => resolve())); @@ -1235,7 +1243,6 @@ it.skipIf( ), ).toBe(true); } finally { - await observer.emergencyCleanup(1_000); await observer.dispose(); server.closeAllConnections(); await new Promise((resolve) => server.close(() => resolve())); diff --git a/packages/harness/src/experimental/managed-agent-spike/runtime.test.ts b/packages/harness/src/experimental/managed-agent-spike/runtime.test.ts index aa9a3beb7..d78b77a1d 100644 --- a/packages/harness/src/experimental/managed-agent-spike/runtime.test.ts +++ b/packages/harness/src/experimental/managed-agent-spike/runtime.test.ts @@ -61,6 +61,7 @@ function quiescentTeardown(): ManagedAgentTeardownObservation { function fakeObserver( teardown: ManagedAgentTeardownObservation = quiescentTeardown(), ): ManagedAgentProcessObserver & { + beginTeardown: ReturnType; waitForQuiescence: ReturnType; emergencyCleanup: ReturnType; dispose: ReturnType; @@ -69,6 +70,7 @@ function fakeObserver( spawn: vi.fn(() => { throw new Error("fake query must not spawn"); }), + beginTeardown: vi.fn(), armToolProcessContainment: vi.fn(), prepareCancellation: vi.fn(async () => ({ supported: true, @@ -1287,6 +1289,7 @@ const teardown = { }; const observer = { spawn() { throw new Error("fake query must not spawn"); }, + beginTeardown() {}, armToolProcessContainment() {}, async prepareCancellation() { return { @@ -1386,6 +1389,45 @@ process.stdout.write(JSON.stringify({ expect(result.terminal).toBe("cancelled"); }); + it("adopts the exact teardown deadline before close and iterator return", async () => { + const { config } = await probeConfig(); + const observer = fakeObserver(); + const order: string[] = []; + observer.beginTeardown.mockImplementation(() => { + order.push("observer_deadline_adopted"); + }); + const close = vi.fn(() => { + order.push("query_close"); + }); + const queryReturn = vi.fn(async () => { + order.push("query_return"); + return { done: true as const, value: undefined }; + }); + + await runManagedAgentProbe(config, { + hermeticGatewayOrigin: config.gatewayOrigin, + processObserver: observer, + queryFactory: () => ({ + [Symbol.asyncIterator]() { + return { + next: async () => ({ done: true as const, value: undefined }), + }; + }, + close, + return: queryReturn, + }), + }); + + expect(order).toEqual([ + "observer_deadline_adopted", + "query_close", + "query_return", + ]); + expect(observer.beginTeardown).toHaveBeenCalledOnce(); + const adoptedDeadline = observer.beginTeardown.mock.calls[0]?.[0]; + expect(observer.waitForQuiescence.mock.calls[0]?.[0]).toBe(adoptedDeadline); + }); + it("never extends the teardown budget or reports deadline success when wall time rolls back", async () => { const wallClock = vi.spyOn(Date, "now").mockReturnValue(10_000); try { diff --git a/packages/harness/src/experimental/managed-agent-spike/runtime.ts b/packages/harness/src/experimental/managed-agent-spike/runtime.ts index 261264de9..94f6692d8 100644 --- a/packages/harness/src/experimental/managed-agent-spike/runtime.ts +++ b/packages/harness/src/experimental/managed-agent-spike/runtime.ts @@ -452,7 +452,7 @@ export async function runManagedAgentProbe( config.target, executionId, ); - const recorder = new ManagedAgentEventRecorder(runId); + const recorder = new ManagedAgentEventRecorder(runId, validated.model.alias); const mcpRuntime = createManagedAgentMcpRuntime(config.expectedMcpNonce); const abortController = new AbortController(); const triggerController = new AbortController(); @@ -460,6 +460,18 @@ export async function runManagedAgentProbe( const before = await captureManagedAgentWorkspaceSnapshot( validated.canonicalWorkspaceRoot, ); + const childEnvironment = buildManagedAgentChildEnvironment({ + ambient: process.env, + configRoot: validated.canonicalConfigRoot, + gatewayOrigin: validated.gatewayOrigin, + gatewayCredential: config.gatewayCredential, + modelAlias: validated.model.alias, + evalSource, + executionId, + }); + const processObserver = + dependencies.processObserver ?? + createLocalManagedAgentProcessObserver({ monotonicNow }); let cancellationRequested = false; let teardownDeadline: ManagedAgentTeardownDeadline | undefined; const ensureTeardownDeadline = (): ManagedAgentTeardownDeadline => { @@ -469,6 +481,10 @@ export async function runManagedAgentProbe( startedAtMs, deadlineAtMs: startedAtMs + MANAGED_AGENT_TEARDOWN_TIMEOUT_MS, }); + // The observer must see the exact same deadline before the first abort, + // SDK close, or iterator return. This also arms a previously observed + // SDK-forwarded abort without granting it an unbounded cleanup window. + processObserver.beginTeardown(teardownDeadline); } return teardownDeadline; }; @@ -486,18 +502,6 @@ export async function runManagedAgentProbe( let queryExecution: ManagedAgentQueryExecutionOutcome = "not_started"; const guardRejections: ManagedAgentPreToolUseGuardRejection[] = []; - const childEnvironment = buildManagedAgentChildEnvironment({ - ambient: process.env, - configRoot: validated.canonicalConfigRoot, - gatewayOrigin: validated.gatewayOrigin, - gatewayCredential: config.gatewayCredential, - modelAlias: validated.model.alias, - evalSource, - executionId, - }); - const processObserver = - dependencies.processObserver ?? - createLocalManagedAgentProcessObserver({ monotonicNow }); const policyBoundary = createManagedAgentPolicyBoundary({ canonicalWorkspaceRoot: validated.canonicalWorkspaceRoot, allowedBuiltinTools: @@ -802,6 +806,7 @@ export async function runManagedAgentProbe( scenario: config.scenario, target: config.target, modelAlias: validated.model.alias, + sdkModelEvidence: recorder.modelEvidence, ...(recorder.sessionId ? { sdkSessionId: recorder.sessionId } : {}), inferenceTurns: recorder.inferenceTurns, ...(recorder.sdkNumTurns === undefined diff --git a/packages/harness/src/experimental/managed-agent-spike/types.ts b/packages/harness/src/experimental/managed-agent-spike/types.ts index 7b555ddee..24ad90f6c 100644 --- a/packages/harness/src/experimental/managed-agent-spike/types.ts +++ b/packages/harness/src/experimental/managed-agent-spike/types.ts @@ -136,6 +136,20 @@ export interface ManagedAgentSdkUsageEstimate { readonly estimatedCostUsd?: number; } +/** + * Content-free SDK evidence that the configured alias was also reported by + * the running SDK. Gateway reconciliation remains authoritative for the + * upstream provider/model and fallback state. + */ +export interface ManagedAgentSdkModelEvidence { + readonly authority: "sdk_non_authoritative"; + readonly initModelObserved: boolean; + readonly initModelMatchesExpectedAlias: boolean; + readonly resultModelUsageObserved: boolean; + readonly resultModelUsageMatchesExpectedAlias: boolean; + readonly resultModelCount: number; +} + export interface ManagedAgentWorkspaceChange { readonly path: string; readonly change: "created" | "modified" | "deleted"; @@ -250,6 +264,7 @@ export interface ManagedAgentProbeResult { readonly scenario: ManagedAgentProbeScenario; readonly target: ManagedAgentModelTargetId; readonly modelAlias: string; + readonly sdkModelEvidence: ManagedAgentSdkModelEvidence; readonly sdkSessionId?: string; /** Distinct, hashed assistant message IDs; authoritative for BQ call count. */ readonly inferenceTurns: number; @@ -303,6 +318,11 @@ export type ManagedAgentQueryFactory = (input: { export interface ManagedAgentProcessObserver { spawn(options: SpawnOptions): SpawnedProcess; + /** + * Adopt the runtime's one immutable monotonic teardown deadline before any + * abort, Query.close(), or Query.return() operation may begin. + */ + beginTeardown(deadline: ManagedAgentTeardownDeadline): void; /** * Arm the two host-authenticated lifetime observations used only by the * exact E0.4 L2 fixture. Tool-reported identities never grant authority by From 5fb1b8883e0aeedca3ffc08c7e8832ef532aaaf4 Mon Sep 17 00:00:00 2001 From: Yash Date: Mon, 17 Aug 2026 06:09:42 -0700 Subject: [PATCH 19/24] fix(harness): close managed-agent review gaps --- .github/workflows/harness.yml | 26 +++ .../managed-agent-spike/README.md | 69 +++---- .../managed-agent-spike/fixture.test.ts | 71 +++++++- .../managed-agent-spike/fixture.ts | 26 ++- .../managed-agent-spike/permissions.test.ts | 32 ++++ .../managed-agent-spike/permissions.ts | 17 ++ .../managed-agent-spike/probe-cli.test.ts | 23 +-- .../managed-agent-spike/probe-cli.ts | 11 +- .../process-observer.test.ts | 128 +++++++------ .../managed-agent-spike/process-observer.ts | 168 +++++------------- .../runtime-sdk-loopback.test.ts | 53 ++++-- .../managed-agent-spike/runtime.test.ts | 23 +++ .../managed-agent-spike/runtime.ts | 9 + 13 files changed, 418 insertions(+), 238 deletions(-) diff --git a/.github/workflows/harness.yml b/.github/workflows/harness.yml index d6a10c4ce..71b1f86a9 100644 --- a/.github/workflows/harness.yml +++ b/.github/workflows/harness.yml @@ -56,6 +56,32 @@ concurrency: cancel-in-progress: true jobs: + # Real SDK transport sentinel. This exercises the pinned bundled runtime and + # must not silently skip when the repository's floating Node matrix moves. + managed-agent-sdk-sentinel: + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + with: + persist-credentials: false + + - uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v4.4.0 + + - name: Use certification Node.js + uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 + with: + node-version: "22.23.2" + cache: "pnpm" + + - name: Install dependencies + run: pnpm install --frozen-lockfile + + - name: Run real Agent SDK loopback sentinels + run: >- + pnpm --filter @sapiom/harness exec vitest run + src/experimental/managed-agent-spike/runtime-sdk-loopback.test.ts + # Playwright mock-mode tier — harness web/e2e specs (VITE_MOCK=1, chromium only). # The live/real-pty tiers (e2e:live, sim) are opt-in local only; nothing here # invokes them — they require real agent binaries and credentials not in CI. diff --git a/packages/harness/src/experimental/managed-agent-spike/README.md b/packages/harness/src/experimental/managed-agent-spike/README.md index e52a89633..ac8a432c2 100644 --- a/packages/harness/src/experimental/managed-agent-spike/README.md +++ b/packages/harness/src/experimental/managed-agent-spike/README.md @@ -9,8 +9,10 @@ Codex flows. Every model-requested Read, Edit, Write, Bash, and in-process MCP call is gated by one programmatic `PreToolUse` hook registered without a matcher. The hook runs before the SDK's permission evaluation, applies canonical-path containment, -exact Bash equality, and an MCP allowlist, and returns a complete fresh input -object only when allowing the call. Unknown tools fail closed. +exact Bash equality, an exact Bash input shape of `{ command: string }`, and an +MCP allowlist, and returns a complete fresh input object only when allowing the +call. Extra SDK fields such as background execution or timeout controls fail +closed even when the command string itself matches. Unknown tools fail closed. The hook also requires a non-empty, bounded `tool_use_id` from the event. When the SDK supplies the optional callback ID, it must be independently bounded and @@ -31,11 +33,13 @@ probe's isolated HOME and `CLAUDE_CONFIG_DIR` calls SDK `resolveSettings()`. query. Policy helpers fail closed because SDK 0.3.228 does not execute them in `resolveSettings()` and therefore cannot prove parity with query startup. -The subprocess requires a Node executable. E0.4 uses the current Node host; -Electron-as-Node and packaged executable resolution are deliberately deferred -to E0.7. The runtime also exposes only a narrow async-iterator/close query -interface. It does not expose or call SDK `Query.mcpCall()`, whose trusted -control channel bypasses permission checks. +The subprocess requires a Node executable. Every direct gateway invocation, +including calls through the exported programmatic runtime, requires exact Node +22.23.2. Hermetic tests may use another Node only through the explicit injected +gateway/query seam. Electron-as-Node and packaged executable resolution are +deliberately deferred to E0.7. The runtime also exposes only a narrow +async-iterator/close query interface. It does not expose or call SDK +`Query.mcpCall()`, whose trusted control channel bypasses permission checks. ## Correlation and turn evidence @@ -66,6 +70,10 @@ result event, so it requires the matching init model and then relies on gateway reconciliation. These checks prove what the SDK reported, not what the gateway served: BigQuery provider/model, fallback, token, and cost rows remain the authoritative exact-deployment evidence for every live inference turn. +Accordingly, the local report uses `outcome: "local_pass"`, labels the check +`sdk_model_alias_observed`, and always emits +`deploymentProvenance: "requires_gateway_reconciliation"`. It never calls a +locally passing run deployment-certified. The hermetic pinned-SDK loopback exercises Read, allowed and denied Bash, and a real in-process `echo_nonce` MCP turn. It requires one primary `PreToolUse` @@ -170,26 +178,25 @@ sequence sentinel for 0.3.228's exact close/return behavior: one logical kill, the SDK-forwarded abort, a second rejected logical kill, return settlement, then host fallback. An SDK upgrade must remove or recertify the shim before the pin changes. The forwarded abort signal requests the sampled host fallback; -only freshly validated host group cleanup sends signals. The fallback first -stops the observer-created SDK supervisor group. A new process-table sample -must then revalidate the active root identity, both role identities, their -relationship and shared group, every current root/tool descendant's parent, -group, session, and ancestry, and at least one open lifetime channel. Only that -fresh proof authorizes `SIGSTOP` to the detached fixture group. A second fresh -sample must show both the root and every tool-group member stopped before -`SIGKILL` is sent to the fixture group. A third fresh sample must prove the -fixture group absent before the supervisor group receives `SIGKILL`, and a -fourth must prove that root group absent. Failed stop/kill attempts remain -retryable, but every attempt—including an ESRCH or helper failure—advances the -sample generation and requires another fresh proof. The five-second absolute -deadline bounds the entire sequence. - -Deadline expiry seals all evidence collection and ordinary fallback authority. -Disposal has one narrower leak-prevention rule: if this observer successfully -stopped an owned group before sealing, that stopped kernel group cannot execute, -fork, exit, or have its PGID recycled, so disposal may issue its final `SIGKILL` -without a new sample. This never changes a failed deadline result and never -applies to a group that was not stopped while authority was fresh. +only freshly validated host group cleanup sends signals. The request invalidates +every sample started before teardown. A complete post-request sample must +revalidate the active root identity, both role identities, their relationship +and shared group, every current root/tool descendant's parent, group, session, +and ancestry, and at least one open lifetime channel. Only that fresh proof +authorizes one direct `SIGKILL` to the detached fixture group. The signal +invalidates its authorizing sample. A second complete sample must prove the +fixture group absent before a freshly revalidated supervisor group receives +`SIGKILL`, and a third must prove that root group absent. Failed kill attempts +remain retryable, but every attempt—including an ESRCH or helper failure—moves +to a new sample generation and requires another fresh proof. The five-second +absolute deadline bounds the entire sequence. + +Deadline expiry or a successful quiescence observation seals all evidence, +closes the spawn gate, and permanently revokes numeric signal authority. +Disposal never signals a cached PID or PGID, even if child exit delivery lags +or the number is reused. It instead asks authenticated fixture sockets to shut +down and disconnects the retained, observer-created supervisor IPC handle; the +still-running supervisor can kill only its own exact process group. If the root exits, a stable identity changes parent/group/session, a foreign member appears, ancestry is lost, both channels close prematurely, or a @@ -198,8 +205,8 @@ group. This includes an inner SDK command exit that reparents a surviving descendant: an unchanged old PGID does not retain authority after ancestry is lost. A successful complete table that no longer contains the stable identity is positive exit evidence; otherwise an escaped same-identity PID and its new -group remain in final liveness accounting. The observer may still stop or kill -its own live SDK supervisor group, but the run remains a fail-closed +group remain in final liveness accounting. The observer may still kill its own +freshly revalidated live SDK supervisor group, but the run remains a fail-closed `teardown_timeout` while any tool process or lifetime channel remains. This also prevents numeric PID/PGID reuse from converting cached evidence into authority. `forceKillIssued` describes only owned SDK supervisor roots and is not required @@ -233,7 +240,9 @@ The disposable fixture uses a host-owned lifetime lease outside the writable workspace. The lease exists before launch; `shutdown` contents or a missing lease both make the fixture parent stop its child and exit. Cleanup can therefore remove the temporary root without turning a startup race into a permanently -running process. +The child also exits on IPC disconnect, and a failed readiness-file publication +shuts it down before the parent exits; a hermetic regression removes the real +fixture root during delayed readiness and verifies that neither process remains. ## Pre-v2 live evidence diff --git a/packages/harness/src/experimental/managed-agent-spike/fixture.test.ts b/packages/harness/src/experimental/managed-agent-spike/fixture.test.ts index 081a458b4..0baf5700e 100644 --- a/packages/harness/src/experimental/managed-agent-spike/fixture.test.ts +++ b/packages/harness/src/experimental/managed-agent-spike/fixture.test.ts @@ -1,6 +1,7 @@ import { execFileSync, spawn } from "node:child_process"; import { once } from "node:events"; -import { rm, writeFile } from "node:fs/promises"; +import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; import { join } from "node:path"; import { afterEach, describe, expect, it } from "vitest"; @@ -18,6 +19,31 @@ import { const fixtures: ManagedAgentFixture[] = []; +function processExists(pid: number): boolean { + try { + process.kill(pid, 0); + return true; + } catch (error) { + return (error as NodeJS.ErrnoException).code === "EPERM"; + } +} + +async function waitForDirectChildPid(parentPid: number): Promise { + const deadline = Date.now() + 2_000; + while (Date.now() < deadline) { + const output = execFileSync("/bin/ps", ["-axo", "pid=,ppid="], { + encoding: "utf8", + windowsHide: true, + }); + for (const line of output.split("\n")) { + const match = /^\s*(\d+)\s+(\d+)\s*$/.exec(line); + if (match && Number(match[2]) === parentPid) return Number(match[1]); + } + await new Promise((resolveDelay) => setTimeout(resolveDelay, 10)); + } + throw new Error("fixture child did not start"); +} + afterEach(async () => { await Promise.all(fixtures.splice(0).map((fixture) => fixture.cleanup())); }); @@ -81,6 +107,49 @@ describe("managed-agent disposable git fixture", () => { expect(exitCode).toBe(0); }); + it.skipIf(process.platform === "win32")( + "does not orphan the child when the fixture root disappears before delayed readiness", + async () => { + const fixture = await createManagedAgentFixture( + () => "deleted-root-readiness", + ); + fixtures.push(fixture); + const externalLeaseRoot = await mkdtemp( + join(tmpdir(), "managed-agent-fixture-lease-"), + ); + const externalLease = join(externalLeaseRoot, "lease"); + await writeFile(externalLease, "run\n", { mode: 0o600 }); + const child = spawn( + process.execPath, + [ + join(fixture.workspaceRoot, FIXTURE_PATHS.processScript), + join(fixture.workspaceRoot, FIXTURE_PATHS.processPidFile), + "--host-cleanup-marker", + externalLease, + "--host-readiness-delay-ms", + "500", + ], + { stdio: "ignore", windowsHide: true }, + ); + await once(child, "spawn"); + const descendantPid = await waitForDirectChildPid(child.pid!); + + try { + const exitTask = once(child, "exit"); + await rm(fixture.root, { recursive: true, force: true }); + const [exitCode] = await exitTask; + expect(exitCode).toBe(1); + await new Promise((resolveDelay) => setTimeout(resolveDelay, 25)); + expect(processExists(descendantPid)).toBe(false); + } finally { + if (processExists(descendantPid)) { + process.kill(descendantPid, "SIGKILL"); + } + await rm(externalLeaseRoot, { recursive: true, force: true }); + } + }, + ); + it("renders L1 as eleven exact ordered calls without resolving the escape link", async () => { const fixture = await createManagedAgentFixture(() => "prompt-contract"); fixtures.push(fixture); diff --git a/packages/harness/src/experimental/managed-agent-spike/fixture.ts b/packages/harness/src/experimental/managed-agent-spike/fixture.ts index c674ca54d..1a443f9da 100644 --- a/packages/harness/src/experimental/managed-agent-spike/fixture.ts +++ b/packages/harness/src/experimental/managed-agent-spike/fixture.ts @@ -95,6 +95,14 @@ const cleanupMarkerIndex = process.argv.indexOf("--host-cleanup-marker"); const cleanupMarker = cleanupMarkerIndex >= 0 ? resolve(process.argv[cleanupMarkerIndex + 1]) : undefined; +const readinessDelayIndex = process.argv.indexOf("--host-readiness-delay-ms"); +const parsedReadinessDelay = readinessDelayIndex >= 0 + ? Number(process.argv[readinessDelayIndex + 1]) + : 0; +const readinessDelayMs = Number.isSafeInteger(parsedReadinessDelay) && + parsedReadinessDelay >= 0 && parsedReadinessDelay <= 5_000 + ? parsedReadinessDelay + : 0; const controlSocket = process.env[${JSON.stringify(MANAGED_AGENT_TOOL_CONTROL_SOCKET_ENV)}]; const controlCapability = process.env[${JSON.stringify(MANAGED_AGENT_TOOL_CONTROL_CAPABILITY_ENV)}]; if (requireControlRegistration && (!controlSocket || !controlCapability)) { @@ -108,9 +116,17 @@ const childProgram = [ 'const controlCapability = process.env["${MANAGED_AGENT_TOOL_CONTROL_CAPABILITY_ENV}"];', 'delete process.env["${MANAGED_AGENT_TOOL_CONTROL_SOCKET_ENV}"];', 'delete process.env["${MANAGED_AGENT_TOOL_CONTROL_CAPABILITY_ENV}"];', + 'const readinessDelayMs = ' + JSON.stringify(readinessDelayMs) + ';', 'process.on("SIGTERM", () => {});', 'process.on("message", (message) => { if (message === "host-shutdown") process.exit(0); });', - 'const publishReady = () => { if (process.send) process.send("ready"); };', + 'process.on("disconnect", () => process.exit(0));', + 'let readyPublished = false;', + 'const publishReady = () => {', + ' if (readyPublished) return;', + ' readyPublished = true;', + ' const sendReady = () => { if (process.send) process.send("ready"); };', + ' if (readinessDelayMs > 0) setTimeout(sendReady, readinessDelayMs); else sendReady();', + '};', 'const connectControl = () => {', ' if (!controlSocket || !controlCapability) { publishReady(); return; }', ' const socket = createConnection(controlSocket);', @@ -159,7 +175,13 @@ let childReady = false; let controlReady = !requireControlRegistration; const publishReadiness = () => { if (!childReady || !controlReady) return; - writeFileSync(pidFile, JSON.stringify({ parentPid: process.pid, childPid: child.pid })); + try { + writeFileSync(pidFile, JSON.stringify({ parentPid: process.pid, childPid: child.pid })); + } catch { + child.once("exit", () => process.exit(1)); + if (child.connected) child.send("host-shutdown"); + else child.kill("SIGKILL"); + } }; child.once("message", () => { childReady = true; diff --git a/packages/harness/src/experimental/managed-agent-spike/permissions.test.ts b/packages/harness/src/experimental/managed-agent-spike/permissions.test.ts index 5988e7576..59d7b4be6 100644 --- a/packages/harness/src/experimental/managed-agent-spike/permissions.test.ts +++ b/packages/harness/src/experimental/managed-agent-spike/permissions.test.ts @@ -304,6 +304,25 @@ describe("managed-agent universal policy boundary", () => { ).resolves.toMatchObject({ hookSpecificOutput: { permissionDecision: "deny" }, }); + await expect( + invoke("Bash", { + command: "git status --short", + run_in_background: true, + }), + ).resolves.toMatchObject({ + hookSpecificOutput: { + permissionDecision: "deny", + permissionDecisionReason: expect.stringContaining("invalid_input"), + }, + }); + await expect( + invoke("Bash", { command: "git status --short", timeout: 10 }), + ).resolves.toMatchObject({ + hookSpecificOutput: { + permissionDecision: "deny", + permissionDecisionReason: expect.stringContaining("invalid_input"), + }, + }); const readInput = { file_path: "inside.txt", preserve: "metadata" }; await expect(invoke("Read", readInput)).resolves.toMatchObject({ hookSpecificOutput: { @@ -354,6 +373,8 @@ describe("managed-agent universal policy boundary", () => { ).toEqual([ ["allow", "exact_bash_command", "pre_tool_use", "bash:exact_command"], ["deny", "bash_command_not_allowed", "pre_tool_use", "bash:unregistered"], + ["deny", "invalid_input", "pre_tool_use", "bash:unregistered"], + ["deny", "invalid_input", "pre_tool_use", "bash:unregistered"], ["allow", "fixture_path", "pre_tool_use", "read:clean_target"], ["allow", "fixture_path", "pre_tool_use", "write:managed_output"], [ @@ -611,6 +632,17 @@ describe("managed-agent universal policy boundary", () => { ).resolves.toMatchObject({ behavior: "allow" }); expect(evidence).toHaveLength(2); expect(evidence[1]?.source).toBe("can_use_tool_fallback"); + + await expect( + boundary.canUseToolFallback( + "Bash", + { command: "git status --short", run_in_background: true }, + { signal, toolUseID: "fallback-extra", requestId: "request-3" }, + ), + ).resolves.toMatchObject({ + behavior: "deny", + message: expect.stringContaining("invalid_input"), + }); }); it("fails closed when aborted before or during asynchronous path validation", async () => { diff --git a/packages/harness/src/experimental/managed-agent-spike/permissions.ts b/packages/harness/src/experimental/managed-agent-spike/permissions.ts index b4ace4859..27535a0c8 100644 --- a/packages/harness/src/experimental/managed-agent-spike/permissions.ts +++ b/packages/harness/src/experimental/managed-agent-spike/permissions.ts @@ -232,6 +232,19 @@ function asRecord(value: unknown): Record | undefined { : undefined; } +function hasExactKeys( + value: Record, + expectedKeys: readonly string[], +): boolean { + const actualKeys = Object.keys(value); + return ( + actualKeys.length === expectedKeys.length && + expectedKeys.every((key) => + Object.prototype.hasOwnProperty.call(value, key), + ) + ); +} + function denied( reason: ManagedAgentPermissionReason, operationId: ManagedAgentOperationId = "unknown", @@ -263,6 +276,7 @@ function classifyManagedAgentOperation( const input = asRecord(rawInput); if (toolName === "Bash") { return input && + hasExactKeys(input, ["command"]) && typeof input.command === "string" && allowedCommands.has(input.command) ? "bash:exact_command" @@ -318,6 +332,9 @@ async function evaluateManagedAgentPolicy( }; } if (toolName === "Bash") { + if (!hasExactKeys(input, ["command"])) { + return denied("invalid_input", operationId); + } const command = typeof input.command === "string" ? input.command : undefined; if (!command) return denied("invalid_input", operationId); diff --git a/packages/harness/src/experimental/managed-agent-spike/probe-cli.test.ts b/packages/harness/src/experimental/managed-agent-spike/probe-cli.test.ts index 1ac733ead..80afcaa3b 100644 --- a/packages/harness/src/experimental/managed-agent-spike/probe-cli.test.ts +++ b/packages/harness/src/experimental/managed-agent-spike/probe-cli.test.ts @@ -652,7 +652,7 @@ describe("managed-agent probe CLI", () => { }); expect(report.checks).toContainEqual({ - id: "exact_model_alias", + id: "sdk_model_alias_observed", passed: false, }); expect(report.outcome).toBe("fail"); @@ -674,7 +674,10 @@ describe("managed-agent probe CLI", () => { ); const report = evaluateManagedAgentProbe(result); - expect(report.outcome).toBe("pass"); + expect(report.outcome).toBe("local_pass"); + expect(report.deploymentProvenance).toBe( + "requires_gateway_reconciliation", + ); expect(report).toMatchObject({ l1Certification: { contractVersion: 2, @@ -689,7 +692,7 @@ describe("managed-agent probe CLI", () => { it("records zero optional Reads as nonblocking efficiency evidence", () => { expect(evaluateManagedAgentProbe(passingL1Result())).toMatchObject({ - outcome: "pass", + outcome: "local_pass", l1Certification: { evaluatorVersion: "managed-agent-l1-evaluator-v2", optionalReadCount: 0, @@ -711,7 +714,7 @@ describe("managed-agent probe CLI", () => { expect( evaluateManagedAgentProbe(maximallyBatchedL1Result(optionalRole)), ).toMatchObject({ - outcome: "pass", + outcome: "local_pass", checks: expect.arrayContaining([ { id: "exact_l1_tool_trace", passed: true }, ]), @@ -873,10 +876,10 @@ describe("managed-agent probe CLI", () => { withPermissionsBeforeOwnRequests(passingL1Result()); expect(evaluateManagedAgentProbe(requestBeforePermission).outcome).toBe( - "pass", + "local_pass", ); expect(evaluateManagedAgentProbe(permissionBeforeRequest).outcome).toBe( - "pass", + "local_pass", ); }); @@ -1068,7 +1071,7 @@ describe("managed-agent probe CLI", () => { workspaceChanges: [...passing.workspaceChanges].reverse(), }); - expect(report.outcome).toBe("pass"); + expect(report.outcome).toBe("local_pass"); expect(report.checks).toContainEqual({ id: "exact_workspace_delta", passed: true, @@ -1093,7 +1096,7 @@ describe("managed-agent probe CLI", () => { it("accepts exactly one permitted Bash request for L2 and rejects any extra tool call", () => { const passing = passingL2Result(); expect(evaluateManagedAgentProbe(passing, [12_345, 12_346])).toMatchObject({ - outcome: "pass", + outcome: "local_pass", checks: expect.arrayContaining([ { id: "exact_l2_bash_only_trace", passed: true }, { id: "l2_containment_prepared", passed: true }, @@ -1141,7 +1144,7 @@ describe("managed-agent probe CLI", () => { teardown: { ...passing.teardown, forceKillIssued: false }, }; expect(evaluateManagedAgentProbe(graceful, [12_345, 12_346]).outcome).toBe( - "pass", + "local_pass", ); for (const [field, checkId] of [ @@ -1614,7 +1617,7 @@ describe("managed-agent probe CLI", () => { it("requires positive permission evidence and distinct lexical and symlink denials", () => { const passing = passingL1Result(); - expect(evaluateManagedAgentProbe(passing).outcome).toBe("pass"); + expect(evaluateManagedAgentProbe(passing).outcome).toBe("local_pass"); const falsePass: ManagedAgentProbeResult = { ...passing, diff --git a/packages/harness/src/experimental/managed-agent-spike/probe-cli.ts b/packages/harness/src/experimental/managed-agent-spike/probe-cli.ts index 90a991fe7..f815ca96b 100644 --- a/packages/harness/src/experimental/managed-agent-spike/probe-cli.ts +++ b/packages/harness/src/experimental/managed-agent-spike/probe-cli.ts @@ -42,7 +42,9 @@ export interface ManagedAgentProbeCheck { } export interface ManagedAgentProbeReport { - readonly outcome: "pass" | "fail"; + /** Local protocol/host result; never authoritative deployment certification. */ + readonly outcome: "local_pass" | "fail"; + readonly deploymentProvenance: "requires_gateway_reconciliation"; readonly checks: readonly ManagedAgentProbeCheck[]; readonly result: ManagedAgentProbeResult; readonly l1Certification?: { @@ -684,7 +686,7 @@ export function evaluateManagedAgentProbe( ); const checks: ManagedAgentProbeCheck[] = [ { - id: "exact_model_alias", + id: "sdk_model_alias_observed", passed: result.modelAlias === resolveManagedAgentModelTarget(result.target).alias && @@ -854,7 +856,8 @@ export function evaluateManagedAgentProbe( } const report: ManagedAgentProbeReport = { - outcome: checks.every(({ passed }) => passed) ? "pass" : "fail", + outcome: checks.every(({ passed }) => passed) ? "local_pass" : "fail", + deploymentProvenance: "requires_gateway_reconciliation", checks, result, }; @@ -979,7 +982,7 @@ async function main(): Promise { return; } process.stdout.write(`${JSON.stringify(report, null, 2)}\n`); - if (report.outcome !== "pass") process.exitCode = 1; + if (report.outcome === "fail") process.exitCode = 1; } catch (error) { const message = error instanceof Error ? error.message : "Unknown probe failure"; diff --git a/packages/harness/src/experimental/managed-agent-spike/process-observer.test.ts b/packages/harness/src/experimental/managed-agent-spike/process-observer.test.ts index 0675ee416..4619927b0 100644 --- a/packages/harness/src/experimental/managed-agent-spike/process-observer.test.ts +++ b/packages/harness/src/experimental/managed-agent-spike/process-observer.test.ts @@ -1183,7 +1183,7 @@ describe("LocalManagedAgentProcessObserver", () => { ); it.skipIf(process.platform === "win32")( - "force-stops and kills the exact non-cooperative fixture group, then confirms death inside one deadline", + "kills the exact non-cooperative fixture group and confirms death inside one deadline", async () => { const fixture = await createManagedAgentFixture(() => "process-observer"); fixtures.push(fixture); @@ -1411,7 +1411,7 @@ describe("LocalManagedAgentProcessObserver", () => { ); it.skipIf(process.platform === "win32")( - "stops and kills a freshly revalidated detached tool group wholly descended from the owned root", + "kills a freshly revalidated detached tool group wholly descended from the owned root", async () => { const fixture = await createManagedAgentFixture( () => "anchored-descendant-tool", @@ -1517,8 +1517,6 @@ describe("LocalManagedAgentProcessObserver", () => { } expect(signals).toEqual([ - [rootProcessGroupId, "SIGSTOP"], - [toolProcessGroupId, "SIGSTOP"], [toolProcessGroupId, "SIGKILL"], [rootProcessGroupId, "SIGKILL"], ]); @@ -1707,11 +1705,10 @@ describe("LocalManagedAgentProcessObserver", () => { ); it.skipIf(process.platform === "win32")( - "retries detached tool stop and kill failures only after fresh authority checks", + "retries detached tool kill failures only after fresh authority checks", async () => { let processTableReads = 0; let toolProcessGroupId: number | undefined; - let stopAttempts = 0; let killAttempts = 0; const toolSignals: Array<{ readonly signal: "SIGSTOP" | "SIGKILL"; @@ -1727,7 +1724,6 @@ describe("LocalManagedAgentProcessObserver", () => { return signalRealProcessGroup(groupId, signal); } toolSignals.push({ signal, processTableReads }); - if (signal === "SIGSTOP" && stopAttempts++ === 0) return "failure"; if (signal === "SIGKILL" && killAttempts++ === 0) return "failure"; return signalRealProcessGroup(groupId, signal); }, @@ -1749,8 +1745,6 @@ describe("LocalManagedAgentProcessObserver", () => { alivePidsAtDeadline: [], }); expect(toolSignals.map(({ signal }) => signal)).toEqual([ - "SIGSTOP", - "SIGSTOP", "SIGKILL", "SIGKILL", ]); @@ -2120,30 +2114,77 @@ describe("LocalManagedAgentProcessObserver", () => { expect(signals).toEqual([]); observer.beginTeardown(deadlineAfter(100)); - expect(signals).toEqual([[rootPid, "SIGSTOP"]]); + await vi.waitFor(() => expect(signals).toEqual([[rootPid, "SIGKILL"]])); } finally { await forceKillRetainedTestGroup(child); await observer.dispose(); } }); + it("closes the spawn gate as soon as teardown adopts its immutable deadline", async () => { + const observer = new LocalManagedAgentProcessObserver({ + platform: "darwin", + readProcessTable: async () => available([]), + }); + try { + observer.beginTeardown(deadlineAfter(1_000)); + expect(() => + observer.spawn({ + ...activeNodeCommand(), + cwd: process.cwd(), + env: { ...process.env }, + signal: new AbortController().signal, + }), + ).toThrow("managed-agent process observer is closed"); + } finally { + await observer.dispose(); + } + }); + + it("seals a successful quiescence observation against delayed SDK spawns", async () => { + const observer = new LocalManagedAgentProcessObserver({ + platform: "darwin", + readProcessTable: async () => available([]), + }); + try { + await expect( + observer.waitForQuiescence(deadlineAfter(1_000)), + ).resolves.toMatchObject({ quiescent: true, deadlineMet: true }); + expect(() => + observer.spawn({ + ...activeNodeCommand(), + cwd: process.cwd(), + env: { ...process.env }, + signal: new AbortController().signal, + }), + ).toThrow("managed-agent process observer is closed"); + } finally { + await observer.dispose(); + } + }); + it.skipIf(process.platform === "win32")( - "kills a stopped owned group during disposal when post-stop observation misses the deadline", + "never signals a cached group during disposal after a post-kill observation misses the deadline", async () => { - let hangAfterStop = false; - const signals: Array = []; + let hangAfterKill = false; + let cachedNumericIdentityUnsafe = false; + const signals: Array = []; const observer = new LocalManagedAgentProcessObserver({ readProcessTable: () => - hangAfterStop + hangAfterKill ? new Promise(() => undefined) : readRealPosixProcessTable(), signalProcessGroup: (groupId, signal) => { - signals.push([groupId, signal]); - const outcome = signalRealProcessGroup(groupId, signal); - if (signal === "SIGSTOP" && outcome === "sent") { - hangAfterStop = true; + if (cachedNumericIdentityUnsafe) { + throw new Error("attempted to signal a potentially reused PGID"); } - return outcome; + signals.push([groupId, signal]); + // Once the kernel accepted SIGKILL, the original group can be reaped + // and its number reused before Node delivers ChildProcess close. + // Simulate that uncertainty by making every later numeric use fatal. + hangAfterKill = true; + cachedNumericIdentityUnsafe = true; + return "sent"; }, }); const controller = new AbortController(); @@ -2167,16 +2208,14 @@ describe("LocalManagedAgentProcessObserver", () => { expect(teardown).toMatchObject({ quiescent: false, deadlineMet: false, - forceKillIssued: false, + forceKillIssued: true, }); - expect(signals.map(([, signal]) => signal)).toEqual(["SIGSTOP"]); + expect(cachedNumericIdentityUnsafe).toBe(true); + expect(signals).toEqual([[anchor.pid!, "SIGKILL"]]); await observer.dispose(); await waitForChildExitBounded(anchor); - expect(signals.map(([, signal]) => signal)).toEqual([ - "SIGSTOP", - "SIGKILL", - ]); + expect(signals).toEqual([[anchor.pid!, "SIGKILL"]]); expect(processGroupExists(anchor.pid!)).toBe(false); } finally { await forceKillRetainedTestGroup(anchor); @@ -2488,10 +2527,7 @@ describe("LocalManagedAgentProcessObserver", () => { observer.beginTeardown(teardownDeadline); controller.abort(); await observer.observeProcessTree(); - expect(signals).toEqual([ - [rootPid, "SIGSTOP"], - [rootPid, "SIGKILL"], - ]); + expect(signals).toEqual([[rootPid, "SIGKILL"]]); stage = "exiting"; await observer.observeProcessTree(); @@ -2956,12 +2992,8 @@ describe("LocalManagedAgentProcessObserver", () => { const teardownDeadline = deadlineAfter(100); observer.beginTeardown(teardownDeadline); controller.abort(); - expect(signals).toEqual([[rootPid, "SIGSTOP"]]); await observer.observeProcessTree(); - expect(signals).toEqual([ - [rootPid, "SIGSTOP"], - [rootPid, "SIGKILL"], - ]); + expect(signals).toEqual([[rootPid, "SIGKILL"]]); await observer.observeProcessTree(); subgroupState = "gone"; await observer.observeProcessTree(); @@ -3226,24 +3258,19 @@ describe("LocalManagedAgentProcessObserver", () => { observer.beginTeardown(deadlineAfter(100)); forwardedController.abort(); forwardedController.abort(); - expect(signals).toEqual([[rootPid, "SIGSTOP"]]); await observer.observeProcessTree(); - expect(signals).toEqual([ - [rootPid, "SIGSTOP"], - [rootPid, "SIGKILL"], - ]); + expect(signals).toEqual([[rootPid, "SIGKILL"]]); await observer.observeProcessTree(); - expect(signals).toHaveLength(2); + expect(signals).toHaveLength(1); } finally { await forceKillRetainedTestGroup(child); observer.dispose(); } }); - it("retries transient root stop and kill failures only after fresh authority samples", async () => { + it("retries transient root kill failures only after fresh authority samples", async () => { let rootPid = 0; let processTableReads = 0; - let stopAttempts = 0; let killAttempts = 0; const signals: Array = []; const signalReadCounts: number[] = []; @@ -3268,7 +3295,6 @@ describe("LocalManagedAgentProcessObserver", () => { signalProcessGroup: (groupId, signal) => { signals.push([groupId, signal]); signalReadCounts.push(processTableReads); - if (signal === "SIGSTOP" && stopAttempts++ === 0) return "failure"; if (signal === "SIGKILL" && killAttempts++ === 0) return "failure"; return "sent"; }, @@ -3289,8 +3315,6 @@ describe("LocalManagedAgentProcessObserver", () => { await observer.emergencyCleanup(teardownDeadline); expect(signals).toEqual([ - [rootPid, "SIGSTOP"], - [rootPid, "SIGSTOP"], [rootPid, "SIGKILL"], [rootPid, "SIGKILL"], ]); @@ -3545,20 +3569,22 @@ describe("LocalManagedAgentProcessObserver", () => { await vi.waitFor(() => expect(reads).toHaveLength(1)); observer.beginTeardown(deadlineAfter(1_000)); forwardedController.abort(); - expect(signals).toEqual([[rootPid, "SIGSTOP"]]); + await vi.waitFor(() => expect(reads).toHaveLength(2)); + expect(signals).toEqual([]); + // Completing the read that started before teardown cannot install its + // evidence or authorize a signal in the new lifecycle generation. reads.shift()!(rootTable()); await preSignalSample; - expect(signals).toEqual([[rootPid, "SIGSTOP"]]); + expect(signals).toEqual([]); + // Only the complete read that started after teardown can authorize the + // one direct group kill. const postSignalSample = observer.observeProcessTree(); await vi.waitFor(() => expect(reads).toHaveLength(1)); reads.shift()!(rootTable()); await postSignalSample; - expect(signals).toEqual([ - [rootPid, "SIGSTOP"], - [rootPid, "SIGKILL"], - ]); + expect(signals).toEqual([[rootPid, "SIGKILL"]]); } finally { await forceKillRetainedTestGroup(anchor); forwardedController.abort(); diff --git a/packages/harness/src/experimental/managed-agent-spike/process-observer.ts b/packages/harness/src/experimental/managed-agent-spike/process-observer.ts index b9dd67f75..330a52a59 100644 --- a/packages/harness/src/experimental/managed-agent-spike/process-observer.ts +++ b/packages/harness/src/experimental/managed-agent-spike/process-observer.ts @@ -246,7 +246,7 @@ export interface ManagedAgentKernelProcessRecord { readonly processGroupId?: number; /** POSIX session id, when the process table exposes it. */ readonly sessionId?: number; - /** POSIX process state used only to confirm an issued group stop. */ + /** POSIX process state used to treat zombies as already dead. */ readonly state?: string; /** Kernel-reported creation time used for evidence, never POSIX authority. */ readonly startedAt: string; @@ -275,7 +275,7 @@ export interface LocalManagedAgentProcessObserverOptions { ) => ManagedAgentProcessGroupLiveness; readonly signalProcessGroup?: ( processGroupId: number, - signal: "SIGSTOP" | "SIGKILL", + signal: "SIGKILL", ) => ManagedAgentProcessSignalOutcome; readonly monotonicNow?: () => number; readonly delay?: (milliseconds: number) => Promise; @@ -287,7 +287,6 @@ interface OwnedRoot { identity?: ManagedAgentKernelProcessRecord; containmentSupported: boolean; ownershipProven: boolean; - stopIssued: boolean; forceKillIssued: boolean; } @@ -479,7 +478,7 @@ function defaultProcessGroupLiveness( function defaultSignalProcessGroup( processGroupId: number, - signal: "SIGSTOP" | "SIGKILL", + signal: "SIGKILL", ): ManagedAgentProcessSignalOutcome { try { process.kill(-processGroupId, signal); @@ -527,11 +526,11 @@ function descendantsOf( * * This is not universal built-in Bash containment or a process-tree killer. * Windows, an unavailable process table, missing lifetime channels, or - * identity/ancestry drift fail certification closed. The fallback stops the - * owned root first, revalidates all authority, then stops and revalidates the - * exact fixture group before killing it. POSIX `lstart` remains one component - * of fresh identity evidence, not standalone authority. Workspace PID-file - * contents never enter this class. + * identity/ancestry drift fail certification closed. The fallback freshly + * validates and kills the exact fixture group, proves it absent with a new + * sample, then freshly validates and kills the owned supervisor root. POSIX + * `lstart` remains one component of fresh identity evidence, not standalone + * authority. Workspace PID-file contents never enter this class. */ export class LocalManagedAgentProcessObserver implements ManagedAgentProcessObserver { readonly #platform: NodeJS.Platform; @@ -541,7 +540,7 @@ export class LocalManagedAgentProcessObserver implements ManagedAgentProcessObse ) => ManagedAgentProcessGroupLiveness; readonly #signalProcessGroup: ( processGroupId: number, - signal: "SIGSTOP" | "SIGKILL", + signal: "SIGKILL", ) => ManagedAgentProcessSignalOutcome; readonly #monotonicNow: () => number; readonly #delay: (milliseconds: number) => Promise; @@ -574,7 +573,6 @@ export class LocalManagedAgentProcessObserver implements ManagedAgentProcessObse #toolProcessRootPid: number | undefined; #toolProcessObservationComplete = false; #toolProcessObservationInvalid = false; - #toolProcessStopIssued = false; #toolProcessForceKillIssued = false; #fallbackCleanupRequested = false; #lastTable: ManagedAgentKernelProcessTable | undefined; @@ -882,7 +880,12 @@ export class LocalManagedAgentProcessObserver implements ManagedAgentProcessObse } public spawn(options: SpawnOptions): SpawnedProcess { - if (this.#sealed || this.#hostDisposing || this.#deadlineExpiredAndSeal()) { + if ( + this.#sealed || + this.#hostDisposing || + this.#teardownDeadline || + this.#deadlineExpiredAndSeal() + ) { throw new Error("managed-agent process observer is closed"); } const usePosixSupervisor = @@ -964,7 +967,6 @@ export class LocalManagedAgentProcessObserver implements ManagedAgentProcessObse child, containmentSupported: true, ownershipProven: false, - stopIssued: false, forceKillIssued: false, }); this.#observedPids.add(pid); @@ -1094,13 +1096,7 @@ export class LocalManagedAgentProcessObserver implements ManagedAgentProcessObse } } - #hasFreshToolAuthority( - table: ManagedAgentKernelProcessTable, - options: { - readonly requireRootStopped: boolean; - readonly requireToolStopped: boolean; - }, - ): boolean { + #hasFreshToolAuthority(table: ManagedAgentKernelProcessTable): boolean { const rootPid = this.#toolProcessRootPid; const processGroupId = this.#toolProcessGroupId; const root = @@ -1138,7 +1134,6 @@ export class LocalManagedAgentProcessObserver implements ManagedAgentProcessObse currentRoot.processGroupId !== root.pid || (root.identity && !sameProcess(root.identity, currentRoot)) || (root.identity && root.identity.sessionId !== currentRoot.sessionId) || - (options.requireRootStopped && !currentRoot.state?.includes("T")) || !sameProcess(parent.identity, currentParent) || currentParent.processGroupId !== processGroupId || parent.identity.sessionId !== currentParent.sessionId || @@ -1166,11 +1161,7 @@ export class LocalManagedAgentProcessObserver implements ManagedAgentProcessObse ); return ( groupMembers.length > 0 && - groupMembers.every( - ([pid, record]) => - rootDescendants.has(pid) && - (!options.requireToolStopped || record.state?.includes("T")), - ) + groupMembers.every(([pid]) => rootDescendants.has(pid)) ); } @@ -1567,8 +1558,9 @@ export class LocalManagedAgentProcessObserver implements ManagedAgentProcessObse } // Query.return() can remain pending while the SDK performs its own // shutdown. Advance a requested fallback from each authoritative - // sample so the detached tool group is stopped/killed before the - // supervisor anchor is killed last, all within the same deadline. + // sample so the detached tool group is killed and then confirmed gone + // before the supervisor anchor is killed last, all within the same + // deadline. this.#advanceFallbackCleanup(); return true; })().finally(() => { @@ -1680,12 +1672,7 @@ export class LocalManagedAgentProcessObserver implements ManagedAgentProcessObse if (this.#toolProcessObservationInvalid) { return unsupported("tool_process_identity_invalid"); } - if ( - !this.#hasFreshToolAuthority(this.#lastTable!, { - requireRootStopped: false, - requireToolStopped: false, - }) - ) { + if (!this.#hasFreshToolAuthority(this.#lastTable!)) { return unsupported("tool_process_identity_invalid"); } this.#toolProcessObservationComplete = true; @@ -1702,10 +1689,7 @@ export class LocalManagedAgentProcessObserver implements ManagedAgentProcessObse }; } - #hasFreshRootGroupAuthority( - root: OwnedRoot, - requireStopped = false, - ): boolean { + #hasFreshRootGroupAuthority(root: OwnedRoot): boolean { if (!root.containmentSupported || this.#processTableNeedsRefresh) { return false; } @@ -1720,8 +1704,7 @@ export class LocalManagedAgentProcessObserver implements ManagedAgentProcessObse if ( !current || processIsZombie(current) || - current.processGroupId !== root.pid || - (requireStopped && !current.state?.includes("T")) + current.processGroupId !== root.pid ) { return false; } @@ -1734,7 +1717,7 @@ export class LocalManagedAgentProcessObserver implements ManagedAgentProcessObse #signalValidatedProcessGroup( processGroupId: number, - signal: "SIGSTOP" | "SIGKILL", + signal: "SIGKILL", ): ManagedAgentProcessSignalOutcome { if (this.#sealed || this.#deadlineExpiredAndSeal()) return "failure"; const outcome = this.#signalProcessGroup(processGroupId, signal); @@ -1746,27 +1729,6 @@ export class LocalManagedAgentProcessObserver implements ManagedAgentProcessObse return outcome; } - #stopOwnedRootsSynchronously(): void { - if (this.#sealed) return; - if (this.#platform !== "darwin" && this.#platform !== "linux") return; - for (const root of this.#roots.values()) { - if ( - root.forceKillIssued || - !childActive(root.child) || - !this.#hasFreshRootGroupAuthority(root) - ) { - continue; - } - if (!root.stopIssued) { - const stopOutcome = this.#signalValidatedProcessGroup( - root.pid, - "SIGSTOP", - ); - root.stopIssued = stopOutcome === "sent"; - } - } - } - #killOwnedRootsSynchronously(): void { if (this.#sealed) return; if (this.#platform !== "darwin" && this.#platform !== "linux") return; @@ -1775,8 +1737,7 @@ export class LocalManagedAgentProcessObserver implements ManagedAgentProcessObse root.forceKillIssued || !childActive(root.child) || this.#hasPendingUnauthenticatedDescendants(root.pid) || - !root.stopIssued || - !this.#hasFreshRootGroupAuthority(root, true) + !this.#hasFreshRootGroupAuthority(root) ) { continue; } @@ -1790,38 +1751,15 @@ export class LocalManagedAgentProcessObserver implements ManagedAgentProcessObse #requestFallbackCleanupSynchronously(): void { if (this.#sealed || this.#deadlineExpiredAndSeal()) return; - this.#fallbackCleanupRequested = true; - this.#stopOwnedRootsSynchronously(); - } - - /** - * Disposal may run after the evidence deadline has irreversibly sealed the - * observer. A group for which this observer successfully issued SIGSTOP - * cannot execute, exit, fork, or have its PGID recycled while stopped, so - * that frozen kernel anchor remains safe for one final SIGKILL without a - * new process-table sample. This is leak prevention only: it never changes - * the already-finalized teardown evidence or grants authority for a group - * that was not stopped before sealing. - */ - #forceKillStoppedGroupsForDisposalSynchronously(): void { - if (this.#platform !== "darwin" && this.#platform !== "linux") return; - - const toolProcessGroupId = this.#toolProcessGroupId; - if ( - this.#toolProcessStopIssued && - !this.#toolProcessForceKillIssued && - typeof toolProcessGroupId === "number" - ) { - const outcome = this.#signalProcessGroup(toolProcessGroupId, "SIGKILL"); - this.#toolProcessForceKillIssued = outcome === "sent"; - } - - for (const root of this.#roots.values()) { - if (root.stopIssued && !root.forceKillIssued && childActive(root.child)) { - const outcome = this.#signalProcessGroup(root.pid, "SIGKILL"); - root.forceKillIssued = outcome === "sent"; - } + if (!this.#fallbackCleanupRequested) { + this.#fallbackCleanupRequested = true; + // The cleanup request itself is a lifecycle boundary. Discard any + // earlier sample so the first numeric signal can only be authorized by + // a complete process-table read that began after teardown started. + this.#sampleGeneration += 1; + this.#processTableNeedsRefresh = true; } + void this.observeProcessTree(this.#teardownDeadline); } #advanceFallbackCleanup(): void { @@ -1834,17 +1772,14 @@ export class LocalManagedAgentProcessObserver implements ManagedAgentProcessObse ) { return; } - // A failed root STOP attempt invalidates its authorizing sample just like - // every other signal outcome. Retry it only here, after a new complete - // generation has restored fresh root authority. - this.#stopOwnedRootsSynchronously(); - if ( - !this.#toolProcessContainmentArmed || - !this.#toolProcessObservationComplete - ) { + if (!this.#toolProcessContainmentArmed) { this.#killOwnedRootsSynchronously(); return; } + // Once tool containment is armed, fail closed until the authenticated + // parent/child identities and their separate group are complete. Killing + // only the supervisor group could otherwise strand an unknown tool group. + if (!this.#toolProcessObservationComplete) return; const processGroupId = this.#toolProcessGroupId; if (typeof processGroupId !== "number") return; @@ -1853,10 +1788,7 @@ export class LocalManagedAgentProcessObserver implements ManagedAgentProcessObse (record) => record.processGroupId === processGroupId && !processIsZombie(record), ); - if (this.#toolProcessForceKillIssued && !liveToolGroupMembers) { - this.#killOwnedRootsSynchronously(); - return; - } + if (liveToolGroupMembers && this.#toolProcessForceKillIssued) return; const groupLiveness = this.#processGroupLiveness(processGroupId); if (groupLiveness === "gone") { this.#killOwnedRootsSynchronously(); @@ -1864,21 +1796,7 @@ export class LocalManagedAgentProcessObserver implements ManagedAgentProcessObse } if (groupLiveness !== "alive") return; - if ( - !this.#hasFreshToolAuthority(table, { - requireRootStopped: true, - requireToolStopped: this.#toolProcessStopIssued, - }) - ) { - return; - } - if (!this.#toolProcessStopIssued) { - const stopOutcome = this.#signalValidatedProcessGroup( - processGroupId, - "SIGSTOP", - ); - this.#toolProcessStopIssued = stopOutcome === "sent"; - if (stopOutcome === "gone") this.#killOwnedRootsSynchronously(); + if (!this.#hasFreshToolAuthority(table)) { return; } if (!this.#toolProcessForceKillIssued) { @@ -2061,7 +1979,9 @@ export class LocalManagedAgentProcessObserver implements ManagedAgentProcessObse if (observation.quiescent) { const deadlineMet = this.#monotonicNow() <= adoptedDeadline.deadlineAtMs; - if (!deadlineMet) this.#seal(); + // Seal atomically with the successful observation so no delayed SDK + // spawn can appear after quiescence has been certified. + this.#seal(); return { ...observation, deadlineMet, @@ -2087,7 +2007,6 @@ export class LocalManagedAgentProcessObserver implements ManagedAgentProcessObse const startedAt = adoptedDeadline.startedAtMs; this.#requestFallbackCleanupSynchronously(); const confirmation = await this.waitForQuiescence(adoptedDeadline); - if (!confirmation.quiescent) this.#killOwnedRootsSynchronously(); const elapsedMs = Math.max(0, this.#monotonicNow() - startedAt); const roots = [...this.#roots.values()]; const forceKillIssued = @@ -2111,7 +2030,6 @@ export class LocalManagedAgentProcessObserver implements ManagedAgentProcessObse async #disposeInternal(): Promise { this.#hostDisposing = true; this.#seal(); - this.#forceKillStoppedGroupsForDisposalSynchronously(); // The two exact fixture processes authenticated these retained channels // with an observer-created capability that was never written into the diff --git a/packages/harness/src/experimental/managed-agent-spike/runtime-sdk-loopback.test.ts b/packages/harness/src/experimental/managed-agent-spike/runtime-sdk-loopback.test.ts index 924c2acdb..cee3f7d9f 100644 --- a/packages/harness/src/experimental/managed-agent-spike/runtime-sdk-loopback.test.ts +++ b/packages/harness/src/experimental/managed-agent-spike/runtime-sdk-loopback.test.ts @@ -260,6 +260,19 @@ function writeFinalResponse(response: ServerResponse, turn: number): void { response.end(); } +function writeHangingStream(response: ServerResponse, turn: number): void { + response.writeHead(200, { + "cache-control": "no-cache", + "content-type": "text/event-stream", + "request-id": `req_loopback_${turn}`, + }); + // The first fake-model turn launches Bash. Claude Code may immediately + // request another turn after its Bash implementation backgrounds a long + // command. Keep that synthetic continuation open until cancellation instead + // of manufacturing duplicate tool calls that a real model never requested. + response.write(": awaiting managed-agent cancellation\n\n"); +} + it("enforces real-SDK built-in and in-process MCP calls with exact loopback correlation", async () => { const fixture = await createManagedAgentFixture(() => "loopback-nonce"); const startedAt = Date.now(); @@ -726,11 +739,15 @@ it.skipIf( request.resume(); request.once("end", () => { inferenceTurn += 1; - writeToolUseResponse(response, inferenceTurn, { - id: "toolu_loopback_l2_bash", - name: "Bash", - input: { command: fixture.l2BashCommand }, - }); + if (inferenceTurn === 1) { + writeToolUseResponse(response, inferenceTurn, { + id: "toolu_loopback_l2_bash", + name: "Bash", + input: { command: fixture.l2BashCommand }, + }); + } else { + writeHangingStream(response, inferenceTurn); + } }); }); @@ -849,7 +866,9 @@ it.skipIf( return transitions; }, []); - expect(inferenceTurn).toBe(1); + expect(inferenceTurn).toBeGreaterThanOrEqual(1); + expect(inferenceTurn).toBeLessThanOrEqual(2); + expect(result.inferenceTurns).toBe(1); expect( result.terminal, JSON.stringify({ @@ -882,8 +901,6 @@ it.skipIf( expect( groupSignals.map(({ groupId, signal }) => [groupId, signal]), ).toEqual([ - [supervisorPid, "SIGSTOP"], - [fixtureToolProcessGroupId, "SIGSTOP"], [fixtureToolProcessGroupId, "SIGKILL"], [supervisorPid, "SIGKILL"], ]); @@ -999,11 +1016,15 @@ it.skipIf( request.resume(); request.once("end", () => { inferenceTurn += 1; - writeToolUseResponse(response, inferenceTurn, { - id: "toolu_loopback_l2_missing_forwarded_signal", - name: "Bash", - input: { command: fixture.l2BashCommand }, - }); + if (inferenceTurn === 1) { + writeToolUseResponse(response, inferenceTurn, { + id: "toolu_loopback_l2_missing_forwarded_signal", + name: "Bash", + input: { command: fixture.l2BashCommand }, + }); + } else { + writeHangingStream(response, inferenceTurn); + } }); }); @@ -1078,7 +1099,9 @@ it.skipIf( const cancellationElapsedMs = Date.now() - (cancellationStartedAt ?? Date.now()); - expect(inferenceTurn).toBe(1); + expect(inferenceTurn).toBeGreaterThanOrEqual(1); + expect(inferenceTurn).toBeLessThanOrEqual(2); + expect(result.inferenceTurns).toBe(1); expect(result.terminal).toBe("close_timeout"); expect(result.cancellationRequested).toBe(true); expect(result.queryClosed).toBe(false); @@ -1224,7 +1247,7 @@ it.skipIf( deadlineMet: false, containmentSupported: false, ownershipProven: false, - forceKillIssued: true, + forceKillIssued: false, toolProcessObservationComplete: false, toolProcessChannelsClosed: false, }); diff --git a/packages/harness/src/experimental/managed-agent-spike/runtime.test.ts b/packages/harness/src/experimental/managed-agent-spike/runtime.test.ts index d78b77a1d..86621afd6 100644 --- a/packages/harness/src/experimental/managed-agent-spike/runtime.test.ts +++ b/packages/harness/src/experimental/managed-agent-spike/runtime.test.ts @@ -8,6 +8,7 @@ import { afterEach, describe, expect, it, vi } from "vitest"; import type { Options } from "@anthropic-ai/claude-agent-sdk"; import { + MANAGED_AGENT_CONTRACT, MANAGED_AGENT_MODEL_ENVIRONMENT_VARIABLES, resolveManagedAgentModelTarget, } from "./contract.js"; @@ -1553,6 +1554,28 @@ process.stdout.write(JSON.stringify({ expect(queryFactory).not.toHaveBeenCalled(); }); + it.skipIf( + process.versions.node === MANAGED_AGENT_CONTRACT.certificationNodeVersion, + )( + "enforces the exact Node pin inside the exported direct-gateway runtime", + async () => { + const { config } = await probeConfig(); + const queryFactory = vi.fn(() => queryFromEvents([])); + await expect( + runManagedAgentProbe( + { + ...config, + gatewayOrigin: MANAGED_AGENT_CONTRACT.directGatewayOrigin, + }, + { processObserver: fakeObserver(), queryFactory }, + ), + ).rejects.toThrow( + `Direct managed-agent probes require Node ${MANAGED_AGENT_CONTRACT.certificationNodeVersion}`, + ); + expect(queryFactory).not.toHaveBeenCalled(); + }, + ); + it("rejects the hermetic origin seam without an injected query factory", async () => { const { config } = await probeConfig(); await expect( diff --git a/packages/harness/src/experimental/managed-agent-spike/runtime.ts b/packages/harness/src/experimental/managed-agent-spike/runtime.ts index 94f6692d8..fb3de320a 100644 --- a/packages/harness/src/experimental/managed-agent-spike/runtime.ts +++ b/packages/harness/src/experimental/managed-agent-spike/runtime.ts @@ -11,6 +11,7 @@ import { import { z } from "zod"; import { + MANAGED_AGENT_CONTRACT, MANAGED_AGENT_L1_CERTIFICATION_CONTRACT, validateManagedAgentProbeConfig, } from "./contract.js"; @@ -440,6 +441,14 @@ export async function runManagedAgentProbe( ? { hermeticGatewayOrigin: dependencies.hermeticGatewayOrigin } : {}), }); + if ( + !dependencies.hermeticGatewayOrigin && + process.versions.node !== MANAGED_AGENT_CONTRACT.certificationNodeVersion + ) { + throw new Error( + `Direct managed-agent probes require Node ${MANAGED_AGENT_CONTRACT.certificationNodeVersion}; current runtime is ${process.versions.node}`, + ); + } if (config.scenario === "L2" && !dependencies.waitForCancellationSignal) { throw new Error("L2 requires an explicit cancellation signal dependency"); } From 5380bea445dd6fcf34edc4ce88700f598b4456c7 Mon Sep 17 00:00:00 2001 From: Yash Date: Mon, 17 Aug 2026 06:44:56 -0700 Subject: [PATCH 20/24] fix(harness): make agent teardown process-bound Refs SAP-2632 --- .../managed-agent-spike/README.md | 51 +- .../managed-agent-spike/fixture.test.ts | 16 +- .../managed-agent-spike/fixture.ts | 10 +- .../experimental/managed-agent-spike/index.ts | 1 - .../process-observer.test.ts | 512 ++++++------------ .../managed-agent-spike/process-observer.ts | 235 +++++--- .../runtime-sdk-loopback.test.ts | 108 ++-- .../experimental/managed-agent-spike/types.ts | 2 +- 8 files changed, 431 insertions(+), 504 deletions(-) diff --git a/packages/harness/src/experimental/managed-agent-spike/README.md b/packages/harness/src/experimental/managed-agent-spike/README.md index ac8a432c2..925ecb128 100644 --- a/packages/harness/src/experimental/managed-agent-spike/README.md +++ b/packages/harness/src/experimental/managed-agent-spike/README.md @@ -170,26 +170,30 @@ the runtime immediately follows it with and awaits `Query.return()` under the same deadline. `queryClosed` means that awaitable cleanup settled; invoking `close()` alone is never completion evidence. Host emergency cleanup starts only after that cleanup settles, the forwarded signal has already requested the -fallback, or the bounded SDK-grace budget expires. As a compatibility shim -certified only for the pinned Agent SDK 0.3.228, the returned handle accepts the -first SDK `child.kill()` logically by setting `child.killed = true`, but it -intentionally sends no native signal. The hermetic real-SDK loopback test is a -sequence sentinel for 0.3.228's exact close/return behavior: one logical kill, -the SDK-forwarded abort, a second rejected logical kill, return settlement, -then host fallback. An SDK upgrade must remove or recertify the shim before the -pin changes. The forwarded abort signal requests the sampled host fallback; -only freshly validated host group cleanup sends signals. The request invalidates +fallback, or the bounded SDK-grace budget expires. The returned process handle +keeps the native `ChildProcess.kill()` contract. SDK `SIGTERM` calls are really +sent to the observer-owned supervisor, whose explicit TERM/INT/HUP handlers keep +the ancestry anchor alive for the bounded fallback. The hermetic real-SDK +loopback test is the sequence sentinel for 0.3.228's close/return, native-kill, +forwarded-abort, and fallback behavior. + +The forwarded abort signal requests the sampled host fallback and invalidates every sample started before teardown. A complete post-request sample must revalidate the active root identity, both role identities, their relationship and shared group, every current root/tool descendant's parent, group, session, -and ancestry, and at least one open lifetime channel. Only that fresh proof -authorizes one direct `SIGKILL` to the detached fixture group. The signal -invalidates its authorizing sample. A second complete sample must prove the -fixture group absent before a freshly revalidated supervisor group receives -`SIGKILL`, and a third must prove that root group absent. Failed kill attempts -remain retryable, but every attempt—including an ESRCH or helper failure—moves -to a new sample generation and requires another fresh proof. The five-second -absolute deadline bounds the entire sequence. +and ancestry, and at least one open lifetime channel. This evidence never +authorizes a host-side numeric signal. Instead, the observer writes a forced- +termination request to a still-open authenticated fixture socket. That exact +process instance calls `kill(0, SIGKILL)` and therefore terminates only its own +current group. The request invalidates its authorizing sample. A second complete +sample must prove the fixture group absent before the observer disconnects the +retained supervisor IPC channel; that exact supervisor instance then terminates +its own current group. A third complete sample proves the root group absent. +Failed channel requests remain retryable, but every attempt moves to a new +sample generation and requires another fresh proof. The five-second absolute +deadline bounds the entire sequence. A PID or PGID can disappear and be reused +between any sample and request without redirecting termination, because neither +channel is addressed by that number. Deadline expiry or a successful quiescence observation seals all evidence, closes the spawn gate, and permanently revokes numeric signal authority. @@ -200,15 +204,16 @@ still-running supervisor can kill only its own exact process group. If the root exits, a stable identity changes parent/group/session, a foreign member appears, ancestry is lost, both channels close prematurely, or a -process-table read is unavailable, the observer never signals the detached -group. This includes an inner SDK command exit that reparents a surviving +process-table read is unavailable, the observer never requests detached-group +termination. This includes an inner SDK command exit that reparents a surviving descendant: an unchanged old PGID does not retain authority after ancestry is lost. A successful complete table that no longer contains the stable identity is positive exit evidence; otherwise an escaped same-identity PID and its new -group remain in final liveness accounting. The observer may still kill its own -freshly revalidated live SDK supervisor group, but the run remains a fail-closed -`teardown_timeout` while any tool process or lifetime channel remains. This also -prevents numeric PID/PGID reuse from converting cached evidence into authority. +group remain in final liveness accounting. The observer may still ask its exact +live SDK supervisor over retained IPC to terminate itself, but the run remains +a fail-closed `teardown_timeout` while any tool process or lifetime channel +remains. This also prevents numeric PID/PGID reuse from converting cached +evidence into authority. `forceKillIssued` describes only owned SDK supervisor roots and is not required when SDK graceful shutdown succeeds. diff --git a/packages/harness/src/experimental/managed-agent-spike/fixture.test.ts b/packages/harness/src/experimental/managed-agent-spike/fixture.test.ts index 0baf5700e..ee5ad6322 100644 --- a/packages/harness/src/experimental/managed-agent-spike/fixture.test.ts +++ b/packages/harness/src/experimental/managed-agent-spike/fixture.test.ts @@ -28,6 +28,18 @@ function processExists(pid: number): boolean { } } +async function waitForProcessExit(pid: number, timeoutMs = 2_000) { + const deadline = Date.now() + timeoutMs; + while (processExists(pid) && Date.now() < deadline) { + await new Promise((resolveDelay) => setTimeout(resolveDelay, 10)); + } + if (processExists(pid)) { + throw new Error( + `Fixture descendant ${pid} survived its retained lifetime-lease shutdown`, + ); + } +} + async function waitForDirectChildPid(parentPid: number): Promise { const deadline = Date.now() + 2_000; while (Date.now() < deadline) { @@ -142,10 +154,8 @@ describe("managed-agent disposable git fixture", () => { await new Promise((resolveDelay) => setTimeout(resolveDelay, 25)); expect(processExists(descendantPid)).toBe(false); } finally { - if (processExists(descendantPid)) { - process.kill(descendantPid, "SIGKILL"); - } await rm(externalLeaseRoot, { recursive: true, force: true }); + await waitForProcessExit(descendantPid); } }, ); diff --git a/packages/harness/src/experimental/managed-agent-spike/fixture.ts b/packages/harness/src/experimental/managed-agent-spike/fixture.ts index 1a443f9da..7435fdd36 100644 --- a/packages/harness/src/experimental/managed-agent-spike/fixture.ts +++ b/packages/harness/src/experimental/managed-agent-spike/fixture.ts @@ -145,6 +145,10 @@ const childProgram = [ ' });', ' socket.on("data", (chunk) => {', ' response += chunk;', + ' if (response.includes(' + JSON.stringify('"forceKill":true') + ')) {', + ' try { process.kill(0, "SIGKILL"); } catch { process.exit(1); }', + ' return;', + ' }', ' if (response.includes(' + JSON.stringify('"shutdown":true') + ')) {', ' socket.write(JSON.stringify({ shutdownAck: true }) + "\\\\n", () => process.exit(0));', ' return;', @@ -180,7 +184,7 @@ const publishReadiness = () => { } catch { child.once("exit", () => process.exit(1)); if (child.connected) child.send("host-shutdown"); - else child.kill("SIGKILL"); + else process.exit(1); } }; child.once("message", () => { @@ -205,6 +209,10 @@ const connectControl = () => { }); socket.on("data", (chunk) => { response += chunk; + if (response.includes('"forceKill":true')) { + try { process.kill(0, "SIGKILL"); } catch { process.exit(1); } + return; + } if (response.includes('"shutdown":true')) { socket.write(JSON.stringify({ shutdownAck: true }) + "\\n", () => process.exit(0), diff --git a/packages/harness/src/experimental/managed-agent-spike/index.ts b/packages/harness/src/experimental/managed-agent-spike/index.ts index 754f6f24c..09da2f954 100644 --- a/packages/harness/src/experimental/managed-agent-spike/index.ts +++ b/packages/harness/src/experimental/managed-agent-spike/index.ts @@ -53,7 +53,6 @@ export { } from "./permissions.js"; export { LocalManagedAgentProcessObserver, - MANAGED_AGENT_LOGICAL_KILL_SHIM_SDK_VERSION, createLocalManagedAgentProcessObserver, } from "./process-observer.js"; export { diff --git a/packages/harness/src/experimental/managed-agent-spike/process-observer.test.ts b/packages/harness/src/experimental/managed-agent-spike/process-observer.test.ts index 4619927b0..b49f4c847 100644 --- a/packages/harness/src/experimental/managed-agent-spike/process-observer.test.ts +++ b/packages/harness/src/experimental/managed-agent-spike/process-observer.test.ts @@ -2,7 +2,6 @@ import { once } from "node:events"; import { ChildProcess, execFile, - execFileSync, spawn as spawnChild, type ChildProcessWithoutNullStreams, } from "node:child_process"; @@ -105,6 +104,20 @@ function activeNodeCommand(): { command: string; args: string[] } { }; } +function spawnCooperativeTestProcess(): ChildProcess { + return spawnChild( + process.execPath, + [ + "-e", + 'process.on("disconnect", () => process.exit(0)); setInterval(() => {}, 1000)', + ], + { + stdio: ["ignore", "ignore", "ignore", "ipc"], + windowsHide: true, + }, + ); +} + const FAST_EXIT_ROOT_SCRIPT = String.raw` import { spawn } from "node:child_process"; import { existsSync, writeFileSync } from "node:fs"; @@ -313,25 +326,9 @@ async function sendClosedToolRegistration( function asChildProcess( spawned: SpawnedProcess, ): ChildProcessWithoutNullStreams { - const child = spawned as ChildProcessWithoutNullStreams; - captureRetainedRootAuthority(child); - return child; -} - -interface RetainedRootAuthority { - readonly pid: number; - readonly record: ManagedAgentKernelProcessRecord; - readonly ancestry: readonly (readonly [ - number, - ManagedAgentKernelProcessRecord, - ])[]; + return spawned as ChildProcessWithoutNullStreams; } -const retainedRootAuthorities = new WeakMap< - ChildProcess, - RetainedRootAuthority ->(); - function sameFullTestIdentity( expected: ManagedAgentKernelProcessRecord, current: ManagedAgentKernelProcessRecord | undefined, @@ -344,72 +341,6 @@ function sameFullTestIdentity( ); } -function processAncestry( - pid: number, - table: ReadonlyMap, -): RetainedRootAuthority["ancestry"] { - const ancestry: Array = - []; - const seen = new Set(); - let currentPid = pid; - while (currentPid > 0 && !seen.has(currentPid)) { - seen.add(currentPid); - const record = table.get(currentPid); - if (!record) break; - ancestry.push([currentPid, record]); - currentPid = record.parentPid; - } - return ancestry; -} - -function captureRetainedRootAuthority(child: ChildProcess): void { - if ( - process.platform === "win32" || - typeof child.pid !== "number" || - retainedRootAuthorities.has(child) - ) { - return; - } - try { - const sessionColumn = managedAgentPosixSessionColumn(process.platform); - const stdout = execFileSync( - "/bin/ps", - ["-axo", `pid=,ppid=,pgid=,${sessionColumn}=,stat=,lstart=`], - { encoding: "utf8", maxBuffer: 4 * 1024 * 1024 }, - ); - const table = parseManagedAgentPosixProcessTable(stdout); - const record = table.get(child.pid); - if (!record) return; - retainedRootAuthorities.set(child, { - pid: child.pid, - record, - ancestry: processAncestry(child.pid, table), - }); - } catch { - // A missing acquisition snapshot permanently removes fallback authority. - } -} - -function retainedRootAuthorityMatches( - authority: RetainedRootAuthority, - table: ReadonlyMap, -): boolean { - const leader = table.get(authority.pid); - const currentAncestry = processAncestry(authority.pid, table); - return Boolean( - leader && - !leader.state?.startsWith("Z") && - leader.processGroupId === authority.pid && - sameFullTestIdentity(authority.record, leader) && - authority.ancestry.length === currentAncestry.length && - authority.ancestry.every( - ([pid, record], index) => - currentAncestry[index]?.[0] === pid && - sameFullTestIdentity(record, currentAncestry[index]?.[1]), - ), - ); -} - function processExists(pid: number): boolean { try { process.kill(pid, 0); @@ -428,20 +359,6 @@ function processGroupExists(processGroupId: number): boolean { } } -function signalRealProcessGroup( - processGroupId: number, - signal: "SIGSTOP" | "SIGKILL", -): "sent" | "gone" | "failure" { - try { - process.kill(-processGroupId, signal); - return "sent"; - } catch (error) { - return (error as NodeJS.ErrnoException).code === "ESRCH" - ? "gone" - : "failure"; - } -} - function realProcessGroupLiveness( processGroupId: number, ): "alive" | "gone" | "unknown" { @@ -511,10 +428,17 @@ async function waitForChildExitBounded( ]); } -async function forceKillExactTestProcess(child: ChildProcess): Promise { +async function stopExactTestProcess(child: ChildProcess): Promise { const pid = child.pid; if (typeof pid !== "number") return; - child.kill("SIGKILL"); + if (child.exitCode === null && child.signalCode === null) { + if (!child.connected) { + throw new Error( + `Refusing cleanup for test process ${pid} without its retained IPC channel`, + ); + } + child.disconnect(); + } await waitForTestProcessDeath( () => processExists(pid), `Unrelated process ${pid}`, @@ -571,39 +495,16 @@ async function waitForExactTestProcessIdentitiesToExit( } } -async function forceKillRetainedTestGroup(root: ChildProcess): Promise { - const authority = retainedRootAuthorities.get(root); - const processGroupId = authority?.pid; - if ( - typeof processGroupId !== "number" || - root.exitCode !== null || - root.signalCode !== null - ) { - return; - } - - const observation = await readRealPosixProcessTable(); - if (!observation.available) { - throw new Error("Process table unavailable for retained group cleanup"); - } - if ( - !authority || - !retainedRootAuthorityMatches(authority, observation.processes) - ) { +async function stopRetainedTestGroup(root: ChildProcess): Promise { + if (root.exitCode !== null || root.signalCode !== null) return; + if (!root.connected) { throw new Error( - `Refusing cached group cleanup for unverified root ${processGroupId}`, + "Refusing supervisor cleanup without its retained process-bound IPC channel", ); } - if (root.exitCode !== null || root.signalCode !== null) return; - - // The retained, still-active ChildProcess plus this fresh kernel snapshot is - // the complete authority for the one group signal below. Never probe or - // signal this negative PGID again after the leader can have exited. - try { - process.kill(-processGroupId, "SIGKILL"); - } catch (error) { - if ((error as NodeJS.ErrnoException).code !== "ESRCH") throw error; - } + // The supervisor's disconnect handler kills its own current group. No + // cached numeric PID or PGID crosses this test-cleanup boundary. + root.disconnect(); await waitForChildExitBounded(root); } @@ -614,21 +515,15 @@ async function proveRetainedGroupAuthority( () => `fast-root-exit-${exitTiming}`, ); fixtures.push(fixture); - const productionGroupSignals: Array< - readonly [number, "SIGSTOP" | "SIGKILL"] - > = []; + const productionGroupSignals: Array = []; const observer = new LocalManagedAgentProcessObserver({ - signalProcessGroup: (processGroupId, signal) => { + testOnlyRequestTermination: (processGroupId, signal) => { productionGroupSignals.push([processGroupId, signal]); - return signalRealProcessGroup(processGroupId, signal); + return "failure"; }, }); const forwardedController = new AbortController(); - const unrelated = spawnChild( - process.execPath, - ["-e", "setInterval(() => {}, 1000)"], - { stdio: "ignore", windowsHide: true }, - ); + const unrelated = spawnCooperativeTestProcess(); await once(unrelated, "spawn"); let anchor: ChildProcessWithoutNullStreams | undefined; let nonCooperativeChildPid: number | undefined; @@ -717,7 +612,7 @@ async function proveRetainedGroupAuthority( } if (anchor && anchor.exitCode === null && anchor.signalCode === null) { if (anchor.connected) anchor.disconnect(); - else await forceKillRetainedTestGroup(anchor); + else await stopRetainedTestGroup(anchor); await waitForChildExitBounded(anchor); } if (typeof nonCooperativeChildPid === "number") { @@ -728,7 +623,7 @@ async function proveRetainedGroupAuthority( } forwardedController.abort(); observer.dispose(); - await forceKillExactTestProcess(unrelated); + await stopExactTestProcess(unrelated); } } @@ -880,7 +775,7 @@ async function startRegisteredDescendantToolRun( cleanupErrors.push(error); } try { - await forceKillRetainedTestGroup(anchor); + await stopRetainedTestGroup(anchor); } catch (error) { cleanupErrors.push(error); } finally { @@ -909,7 +804,7 @@ async function cleanupRegisteredDescendantToolRun( if (run.anchor.exitCode === null && run.anchor.signalCode === null) { await waitForChildExitBounded(run.anchor, 100).catch(() => undefined); } - await forceKillRetainedTestGroup(run.anchor); + await stopRetainedTestGroup(run.anchor); } catch (error) { cleanupErrors.push(error); } finally { @@ -922,39 +817,6 @@ async function cleanupRegisteredDescendantToolRun( } describe("LocalManagedAgentProcessObserver", () => { - it.each([ - ["parent", { parentPid: 2, processGroupId: 41, sessionId: 41 }], - ["process group", { parentPid: 1, processGroupId: 99, sessionId: 41 }], - ["session", { parentPid: 1, processGroupId: 41, sessionId: 99 }], - ] as const)( - "refuses retained-root fallback when same-start identity changes %s topology", - (_description, changedTopology) => { - const baseline = { - parentPid: 1, - processGroupId: 41, - sessionId: 41, - state: "S", - startedAt: "same-second-start", - } satisfies ManagedAgentKernelProcessRecord; - const authority: RetainedRootAuthority = { - pid: 41, - record: baseline, - ancestry: [[41, baseline]], - }; - const current = new Map([ - [ - 41, - { - ...baseline, - ...changedTopology, - }, - ] as const, - ]); - - expect(retainedRootAuthorityMatches(authority, current)).toBe(false); - }, - ); - it.each([ ["darwin", "sess"], ["linux", "sid"], @@ -1007,7 +869,7 @@ describe("LocalManagedAgentProcessObserver", () => { await once(child, "exit"); const killSpy = vi.spyOn(process, "kill"); try { - await forceKillRetainedTestGroup(child); + await stopRetainedTestGroup(child); expect( killSpy.mock.calls.some( ([pid]) => typeof pid === "number" && pid < 0, @@ -1052,7 +914,7 @@ describe("LocalManagedAgentProcessObserver", () => { expect(signalCode).toBeNull(); } finally { if (typeof anchor.pid === "number") { - await forceKillRetainedTestGroup(anchor); + await stopRetainedTestGroup(anchor); } controller.abort(); observer.dispose(); @@ -1089,7 +951,7 @@ describe("LocalManagedAgentProcessObserver", () => { expect(forwardedStderrBytes).toBe(1024 * 1024); } finally { if (typeof anchor.pid === "number") { - await forceKillRetainedTestGroup(anchor); + await stopRetainedTestGroup(anchor); } controller.abort(); observer.dispose(); @@ -1128,7 +990,7 @@ describe("LocalManagedAgentProcessObserver", () => { ); it.skipIf(process.platform === "win32")( - "records an SDK kill logically without signaling the live supervisor anchor", + "honors SDK SIGTERM calls while the supervisor keeps its owned group anchored", async () => { const nativeKillSpy = vi.spyOn(ChildProcess.prototype, "kill"); const observer = new LocalManagedAgentProcessObserver(); @@ -1155,8 +1017,10 @@ describe("LocalManagedAgentProcessObserver", () => { expect(anchor.killed).toBe(false); expect(anchor.kill("SIGTERM")).toBe(true); expect(anchor.killed).toBe(true); - expect(anchor.kill("SIGTERM")).toBe(false); - expect(nativeKillSpy).not.toHaveBeenCalled(); + expect(anchor.kill("SIGTERM")).toBe(true); + expect(nativeKillSpy).toHaveBeenCalledTimes(2); + expect(nativeKillSpy).toHaveBeenNthCalledWith(1, "SIGTERM"); + expect(nativeKillSpy).toHaveBeenNthCalledWith(2, "SIGTERM"); expect(processExists(processGroupId)).toBe(true); await expect( observer.waitForQuiescence(deadlineAfter(1)), @@ -1189,11 +1053,7 @@ describe("LocalManagedAgentProcessObserver", () => { fixtures.push(fixture); const observer = new LocalManagedAgentProcessObserver(); const forwardedController = new AbortController(); - const unrelated = spawnChild( - process.execPath, - ["-e", "setInterval(() => {}, 1000)"], - { stdio: "ignore", windowsHide: true }, - ); + const unrelated = spawnCooperativeTestProcess(); await once(unrelated, "spawn"); let root: ChildProcessWithoutNullStreams | undefined; let ownedProcessGroupId: number | undefined; @@ -1243,10 +1103,10 @@ describe("LocalManagedAgentProcessObserver", () => { // Test-harness safety must not depend on the observer behavior under // test. Exact test-owned PGID authority is retained until death is // independently confirmed, including when an assertion fails. - await forceKillRetainedTestGroup(root); + await stopRetainedTestGroup(root); } observer.dispose(); - await forceKillExactTestProcess(unrelated); + await stopExactTestProcess(unrelated); } }, 15_000, @@ -1259,11 +1119,7 @@ describe("LocalManagedAgentProcessObserver", () => { fixtures.push(fixture); const observer = new LocalManagedAgentProcessObserver(); const controller = new AbortController(); - const unrelated = spawnChild( - process.execPath, - ["-e", "setInterval(() => {}, 1000)"], - { stdio: "ignore", windowsHide: true }, - ); + const unrelated = spawnCooperativeTestProcess(); await once(unrelated, "spawn"); const anchor = asChildProcess( observer.spawn({ @@ -1310,11 +1166,11 @@ describe("LocalManagedAgentProcessObserver", () => { await waitForChildExitBounded(anchor, 250).catch(() => undefined); } if (anchor.exitCode === null && anchor.signalCode === null) { - await forceKillRetainedTestGroup(anchor); + await stopRetainedTestGroup(anchor); } controller.abort(); if (!observerDisposed) await observer.dispose(); - await forceKillExactTestProcess(unrelated); + await stopExactTestProcess(unrelated); } }, 10_000, @@ -1325,7 +1181,7 @@ describe("LocalManagedAgentProcessObserver", () => { async () => { const signals: Array = []; const observer = new LocalManagedAgentProcessObserver({ - signalProcessGroup: (groupId, signal) => { + testOnlyRequestTermination: (groupId, signal) => { signals.push([groupId, signal]); return "failure"; }, @@ -1356,7 +1212,7 @@ describe("LocalManagedAgentProcessObserver", () => { run.anchor.exitCode === null && run.anchor.signalCode === null ) { - await forceKillRetainedTestGroup(run.anchor); + await stopRetainedTestGroup(run.anchor); } } }, @@ -1367,7 +1223,7 @@ describe("LocalManagedAgentProcessObserver", () => { const signals: Array = []; const observer = new LocalManagedAgentProcessObserver({ platform: "darwin", - signalProcessGroup: (groupId, signal) => { + testOnlyRequestTermination: (groupId, signal) => { signals.push([groupId, signal]); return "sent"; }, @@ -1419,11 +1275,7 @@ describe("LocalManagedAgentProcessObserver", () => { fixtures.push(fixture); const observer = new LocalManagedAgentProcessObserver(); const forwardedController = new AbortController(); - const unrelated = spawnChild( - process.execPath, - ["-e", "setInterval(() => {}, 1000)"], - { stdio: "ignore", windowsHide: true }, - ); + const unrelated = spawnCooperativeTestProcess(); await once(unrelated, "spawn"); let anchor: ChildProcessWithoutNullStreams | undefined; let fixturePids: readonly number[] = []; @@ -1472,24 +1324,26 @@ describe("LocalManagedAgentProcessObserver", () => { forwardedController.abort(); await observer.dispose(); if (anchor) { - await forceKillRetainedTestGroup(anchor); + await stopRetainedTestGroup(anchor); } - await forceKillExactTestProcess(unrelated); + await stopExactTestProcess(unrelated); } }, 15_000, ); it.skipIf(process.platform === "win32")( - "advances forwarded-signal fallback on fresh samples and kills the supervisor anchor last", + "uses process-bound channels in tool-then-supervisor order without host SIGKILL", async () => { let rootProcessGroupId: number | undefined; let toolProcessGroupId: number | undefined; - const signals: Array = []; + const terminationRequests: Array = []; + const hostKillSpy = vi.spyOn(process, "kill"); const observer = new LocalManagedAgentProcessObserver({ - signalProcessGroup: (groupId, signal) => { - signals.push([groupId, signal]); - return signalRealProcessGroup(groupId, signal); + onTerminationRequest: ({ processGroupId }, outcome) => { + if (outcome === "sent") { + terminationRequests.push([processGroupId, "SIGKILL"]); + } }, }); let run: RegisteredDescendantToolRun | undefined; @@ -1506,7 +1360,7 @@ describe("LocalManagedAgentProcessObserver", () => { run.forwardedController.abort(); const deadline = Date.now() + 2_000; while ( - !signals.some( + !terminationRequests.some( ([groupId, signal]) => groupId === rootProcessGroupId && signal === "SIGKILL", ) && @@ -1516,10 +1370,13 @@ describe("LocalManagedAgentProcessObserver", () => { await new Promise((resolveDelay) => setTimeout(resolveDelay, 10)); } - expect(signals).toEqual([ + expect(terminationRequests).toEqual([ [toolProcessGroupId, "SIGKILL"], [rootProcessGroupId, "SIGKILL"], ]); + expect( + hostKillSpy.mock.calls.some(([, signal]) => signal === "SIGKILL"), + ).toBe(false); await expect( observer.waitForQuiescence(teardownDeadline), ).resolves.toMatchObject({ @@ -1531,6 +1388,7 @@ describe("LocalManagedAgentProcessObserver", () => { alivePidsAtDeadline: [], }); } finally { + hostKillSpy.mockRestore(); await cleanupRegisteredDescendantToolRun(run); observer.dispose(); } @@ -1564,7 +1422,7 @@ describe("LocalManagedAgentProcessObserver", () => { } finally { if (setupEvidence) { await observer.dispose(); - await forceKillRetainedTestGroup(setupEvidence.anchor); + await stopRetainedTestGroup(setupEvidence.anchor); } await observer.dispose(); } @@ -1577,7 +1435,7 @@ describe("LocalManagedAgentProcessObserver", () => { async () => { let injectForeignMember = false; let toolProcessGroupId: number | undefined; - const signals: Array = []; + const signals: Array = []; const observer = new LocalManagedAgentProcessObserver({ readProcessTable: async () => { const observation = await readRealPosixProcessTable(); @@ -1599,9 +1457,9 @@ describe("LocalManagedAgentProcessObserver", () => { }); return { available: true, processes }; }, - signalProcessGroup: (groupId, signal) => { + testOnlyRequestTermination: (groupId, signal) => { signals.push([groupId, signal]); - return signalRealProcessGroup(groupId, signal); + return "failure"; }, }); let run: RegisteredDescendantToolRun | undefined; @@ -1638,7 +1496,7 @@ describe("LocalManagedAgentProcessObserver", () => { let simulatePidReuse = false; let toolProcessGroupId: number | undefined; let registeredPids: readonly [number, number] | undefined; - const signals: Array = []; + const signals: Array = []; const observer = new LocalManagedAgentProcessObserver({ readProcessTable: async () => { const observation = await readRealPosixProcessTable(); @@ -1670,9 +1528,9 @@ describe("LocalManagedAgentProcessObserver", () => { simulatePidReuse && groupId === toolProcessGroupId ? "alive" : realProcessGroupLiveness(groupId), - signalProcessGroup: (groupId, signal) => { + testOnlyRequestTermination: (groupId, signal) => { signals.push([groupId, signal]); - return signalRealProcessGroup(groupId, signal); + return "failure"; }, }); let run: RegisteredDescendantToolRun | undefined; @@ -1711,7 +1569,7 @@ describe("LocalManagedAgentProcessObserver", () => { let toolProcessGroupId: number | undefined; let killAttempts = 0; const toolSignals: Array<{ - readonly signal: "SIGSTOP" | "SIGKILL"; + readonly signal: "SIGKILL"; readonly processTableReads: number; }> = []; const observer = new LocalManagedAgentProcessObserver({ @@ -1719,13 +1577,14 @@ describe("LocalManagedAgentProcessObserver", () => { processTableReads += 1; return readRealPosixProcessTable(); }, - signalProcessGroup: (groupId, signal) => { - if (groupId !== toolProcessGroupId) { - return signalRealProcessGroup(groupId, signal); + testOnlyBeforeTerminationRequest: ({ target }) => { + if (target === "tool" && killAttempts++ === 0) return "failure"; + return undefined; + }, + onTerminationRequest: ({ processGroupId, target }) => { + if (target === "tool" && processGroupId === toolProcessGroupId) { + toolSignals.push({ signal: "SIGKILL", processTableReads }); } - toolSignals.push({ signal, processTableReads }); - if (signal === "SIGKILL" && killAttempts++ === 0) return "failure"; - return signalRealProcessGroup(groupId, signal); }, }); let run: RegisteredDescendantToolRun | undefined; @@ -1768,11 +1627,11 @@ describe("LocalManagedAgentProcessObserver", () => { "never signals the detached tool group after the owned root exits and ancestry is lost", async () => { let toolProcessGroupId: number | undefined; - const signals: Array = []; + const signals: Array = []; const observer = new LocalManagedAgentProcessObserver({ - signalProcessGroup: (groupId, signal) => { + testOnlyRequestTermination: (groupId, signal) => { signals.push([groupId, signal]); - return signalRealProcessGroup(groupId, signal); + return "failure"; }, }); let run: RegisteredDescendantToolRun | undefined; @@ -1782,7 +1641,7 @@ describe("LocalManagedAgentProcessObserver", () => { "root-exit-loses-tool-ancestry", ); toolProcessGroupId = run.toolProcessGroupId; - await forceKillRetainedTestGroup(run.anchor); + await stopRetainedTestGroup(run.anchor); expect(processGroupExists(toolProcessGroupId)).toBe(true); const teardown = await observer.emergencyCleanup(deadlineAfter(250)); @@ -1876,7 +1735,7 @@ describe("LocalManagedAgentProcessObserver", () => { if (detachedTool) await waitForChildExitBounded(detachedTool); await observer.dispose(); if (anchor) { - await forceKillRetainedTestGroup(anchor); + await stopRetainedTestGroup(anchor); } await observer.dispose(); } @@ -1969,7 +1828,7 @@ describe("LocalManagedAgentProcessObserver", () => { forwardedController.abort(); await fixture.requestCooperativeExit(); await observer.dispose(); - if (anchor) await forceKillRetainedTestGroup(anchor); + if (anchor) await stopRetainedTestGroup(anchor); await observer.dispose(); } }, @@ -2012,7 +1871,7 @@ describe("LocalManagedAgentProcessObserver", () => { }); } finally { controller.abort(); - await forceKillRetainedTestGroup(anchor); + await stopRetainedTestGroup(anchor); observer.dispose(); } }, @@ -2024,14 +1883,9 @@ describe("LocalManagedAgentProcessObserver", () => { const observer = new LocalManagedAgentProcessObserver({ platform: "darwin", readProcessTable: () => new Promise(() => undefined), - signalProcessGroup: (groupId, signal) => { + testOnlyRequestTermination: (groupId, signal) => { signals.push([groupId, signal]); - try { - process.kill(-groupId, signal); - } catch { - // The test-owned group may already have exited between signals. - } - return "sent"; + return "failure"; }, }); const controller = new AbortController(); @@ -2063,7 +1917,7 @@ describe("LocalManagedAgentProcessObserver", () => { controller.abort(); expect(signals).toEqual([]); } finally { - await forceKillRetainedTestGroup(child); + await stopRetainedTestGroup(child); controller.abort(); observer.dispose(); } @@ -2071,7 +1925,7 @@ describe("LocalManagedAgentProcessObserver", () => { it("remembers an SDK abort but grants no fallback signal before deadline adoption", async () => { let rootPid = 0; - const signals: Array = []; + const signals: Array = []; const observer = new LocalManagedAgentProcessObserver({ platform: "darwin", readProcessTable: async () => @@ -2082,14 +1936,12 @@ describe("LocalManagedAgentProcessObserver", () => { parentPid: process.pid, processGroupId: rootPid, sessionId: rootPid, - state: signals.some(([, signal]) => signal === "SIGSTOP") - ? "T" - : "S", + state: "S", startedAt: "abort-before-deadline", }, ], ]), - signalProcessGroup: (groupId, signal) => { + testOnlyRequestTermination: (groupId, signal) => { signals.push([groupId, signal]); return "sent"; }, @@ -2116,7 +1968,7 @@ describe("LocalManagedAgentProcessObserver", () => { observer.beginTeardown(deadlineAfter(100)); await vi.waitFor(() => expect(signals).toEqual([[rootPid, "SIGKILL"]])); } finally { - await forceKillRetainedTestGroup(child); + await stopRetainedTestGroup(child); await observer.dispose(); } }); @@ -2164,27 +2016,23 @@ describe("LocalManagedAgentProcessObserver", () => { }); it.skipIf(process.platform === "win32")( - "never signals a cached group during disposal after a post-kill observation misses the deadline", + "never uses a host numeric signal when channel confirmation misses the deadline", async () => { - let hangAfterKill = false; - let cachedNumericIdentityUnsafe = false; - const signals: Array = []; + let hangAfterRequest = false; + const terminationRequests: Array = []; + const hostKillSpy = vi.spyOn(process, "kill"); const observer = new LocalManagedAgentProcessObserver({ readProcessTable: () => - hangAfterKill + hangAfterRequest ? new Promise(() => undefined) : readRealPosixProcessTable(), - signalProcessGroup: (groupId, signal) => { - if (cachedNumericIdentityUnsafe) { - throw new Error("attempted to signal a potentially reused PGID"); + onTerminationRequest: ({ processGroupId }, outcome) => { + if (outcome === "sent") { + terminationRequests.push([processGroupId, "SIGKILL"]); + // From this point onward, the sampled numeric PGID could be reused. + // No later host operation may signal it. + hangAfterRequest = true; } - signals.push([groupId, signal]); - // Once the kernel accepted SIGKILL, the original group can be reaped - // and its number reused before Node delivers ChildProcess close. - // Simulate that uncertainty by making every later numeric use fatal. - hangAfterKill = true; - cachedNumericIdentityUnsafe = true; - return "sent"; }, }); const controller = new AbortController(); @@ -2210,15 +2058,19 @@ describe("LocalManagedAgentProcessObserver", () => { deadlineMet: false, forceKillIssued: true, }); - expect(cachedNumericIdentityUnsafe).toBe(true); - expect(signals).toEqual([[anchor.pid!, "SIGKILL"]]); + expect(hangAfterRequest).toBe(true); + expect(terminationRequests).toEqual([[anchor.pid!, "SIGKILL"]]); + expect( + hostKillSpy.mock.calls.some(([, signal]) => signal === "SIGKILL"), + ).toBe(false); await observer.dispose(); await waitForChildExitBounded(anchor); - expect(signals).toEqual([[anchor.pid!, "SIGKILL"]]); + expect(terminationRequests).toEqual([[anchor.pid!, "SIGKILL"]]); expect(processGroupExists(anchor.pid!)).toBe(false); } finally { - await forceKillRetainedTestGroup(anchor); + hostKillSpy.mockRestore(); + await stopRetainedTestGroup(anchor); await observer.dispose(); } }, @@ -2248,7 +2100,7 @@ describe("LocalManagedAgentProcessObserver", () => { const observer = new LocalManagedAgentProcessObserver({ platform: "darwin", readProcessTable: table, - signalProcessGroup: (groupId, signal) => { + testOnlyRequestTermination: (groupId, signal) => { signals.push([groupId, signal]); return "sent"; }, @@ -2278,7 +2130,7 @@ describe("LocalManagedAgentProcessObserver", () => { }); expect(signals).toEqual([]); } finally { - await forceKillRetainedTestGroup(child); + await stopRetainedTestGroup(child); controller.abort(); observer.dispose(); } @@ -2315,7 +2167,7 @@ describe("LocalManagedAgentProcessObserver", () => { ], ]), processGroupLiveness: () => "gone", - signalProcessGroup: (groupId, signal) => { + testOnlyRequestTermination: (groupId, signal) => { signals.push([groupId, signal]); return "sent"; }, @@ -2353,7 +2205,7 @@ describe("LocalManagedAgentProcessObserver", () => { expect(observation.alivePidsAtDeadline).not.toContain(rootPid + 200); expect(signals).toEqual([]); } finally { - await forceKillRetainedTestGroup(anchor); + await stopRetainedTestGroup(anchor); controller.abort(); observer.dispose(); } @@ -2393,7 +2245,7 @@ describe("LocalManagedAgentProcessObserver", () => { ], ]), processGroupLiveness: () => "gone", - signalProcessGroup: (groupId, signal) => { + testOnlyRequestTermination: (groupId, signal) => { signals.push([groupId, signal]); return "sent"; }, @@ -2442,7 +2294,7 @@ describe("LocalManagedAgentProcessObserver", () => { }); expect(signals).toEqual([]); } finally { - await forceKillRetainedTestGroup(anchor); + await stopRetainedTestGroup(anchor); controller.abort(); observer.dispose(); } @@ -2452,7 +2304,7 @@ describe("LocalManagedAgentProcessObserver", () => { let rootPid = 0; let stage: "owned" | "exiting" | "gone" = "owned"; let now = 0; - const signals: Array = []; + const signals: Array = []; const observer = new LocalManagedAgentProcessObserver({ platform: "darwin", readProcessTable: async () => { @@ -2478,9 +2330,7 @@ describe("LocalManagedAgentProcessObserver", () => { parentPid: process.pid, processGroupId: rootPid, sessionId: 0, - state: signals.some(([, signal]) => signal === "SIGSTOP") - ? "Ts" - : "Ss", + state: "Ss", startedAt: "root", }, ], @@ -2490,16 +2340,14 @@ describe("LocalManagedAgentProcessObserver", () => { parentPid: rootPid, processGroupId: rootPid, sessionId: 0, - state: signals.some(([, signal]) => signal === "SIGSTOP") - ? "T" - : "S", + state: "S", startedAt: "child", }, ], ]); }, processGroupLiveness: () => (stage === "gone" ? "gone" : "alive"), - signalProcessGroup: (groupId, signal) => { + testOnlyRequestTermination: (groupId, signal) => { signals.push([groupId, signal]); return "sent"; }, @@ -2543,7 +2391,7 @@ describe("LocalManagedAgentProcessObserver", () => { alivePidsAtDeadline: [], }); } finally { - await forceKillRetainedTestGroup(anchor); + await stopRetainedTestGroup(anchor); controller.abort(); observer.dispose(); } @@ -2568,7 +2416,7 @@ describe("LocalManagedAgentProcessObserver", () => { ], ]), processGroupLiveness: () => "alive", - signalProcessGroup: (groupId, signal) => { + testOnlyRequestTermination: (groupId, signal) => { signals.push([groupId, signal]); return "sent"; }, @@ -2592,7 +2440,7 @@ describe("LocalManagedAgentProcessObserver", () => { controller.abort(); expect(signals).toEqual([]); } finally { - await forceKillRetainedTestGroup(anchor); + await stopRetainedTestGroup(anchor); controller.abort(); observer.dispose(); } @@ -2645,7 +2493,7 @@ describe("LocalManagedAgentProcessObserver", () => { : subgroupAlive ? "alive" : "gone", - signalProcessGroup: (groupId, signal) => { + testOnlyRequestTermination: (groupId, signal) => { signals.push([groupId, signal]); return "sent"; }, @@ -2684,7 +2532,7 @@ describe("LocalManagedAgentProcessObserver", () => { }); expect(signals).toEqual([]); } finally { - await forceKillRetainedTestGroup(anchor); + await stopRetainedTestGroup(anchor); controller.abort(); observer.dispose(); } @@ -2718,7 +2566,7 @@ describe("LocalManagedAgentProcessObserver", () => { ], ]), processGroupLiveness: () => "alive", - signalProcessGroup: (groupId, signal) => { + testOnlyRequestTermination: (groupId, signal) => { signals.push([groupId, signal]); return "sent"; }, @@ -2745,7 +2593,7 @@ describe("LocalManagedAgentProcessObserver", () => { }); expect(signals).toEqual([]); } finally { - await forceKillRetainedTestGroup(anchor); + await stopRetainedTestGroup(anchor); controller.abort(); observer.dispose(); } @@ -2791,7 +2639,7 @@ describe("LocalManagedAgentProcessObserver", () => { ]); }, processGroupLiveness: () => "gone", - signalProcessGroup: () => "sent", + testOnlyRequestTermination: () => "sent", }); const controller = new AbortController(); const anchor = asChildProcess( @@ -2818,7 +2666,7 @@ describe("LocalManagedAgentProcessObserver", () => { alivePidsAtDeadline: [], }); } finally { - await forceKillRetainedTestGroup(anchor); + await stopRetainedTestGroup(anchor); controller.abort(); observer.dispose(); } @@ -2874,7 +2722,7 @@ describe("LocalManagedAgentProcessObserver", () => { ] as const) : []), ]), - signalProcessGroup: (groupId, signal) => { + testOnlyRequestTermination: (groupId, signal) => { signals.push([groupId, signal]); return "sent"; }, @@ -2917,7 +2765,7 @@ describe("LocalManagedAgentProcessObserver", () => { }); expect(signals).toEqual([]); } finally { - await forceKillRetainedTestGroup(anchor); + await stopRetainedTestGroup(anchor); forwardedController.abort(); observer.dispose(); } @@ -2941,9 +2789,7 @@ describe("LocalManagedAgentProcessObserver", () => { parentPid: process.pid, processGroupId: rootPid, sessionId: rootPid, - state: signals.some(([, signal]) => signal === "SIGSTOP") - ? "T" - : "S", + state: "S", startedAt: "root", }, ], @@ -2957,20 +2803,16 @@ describe("LocalManagedAgentProcessObserver", () => { subgroupState === "root_group" ? rootPid : subgroupPid(), sessionId: subgroupState === "root_group" ? rootPid : subgroupPid(), - state: - subgroupState === "root_group" && - signals.some(([, signal]) => signal === "SIGSTOP") - ? "T" - : "S", + state: "S", startedAt: "survived-root-kill", }, ], ]); }, processGroupLiveness: () => "alive", - signalProcessGroup: (groupId, signal) => { + testOnlyRequestTermination: (groupId, signal) => { signals.push([groupId, signal]); - if (signal === "SIGKILL") subgroupState = "reparented"; + subgroupState = "reparented"; return "sent"; }, }); @@ -3006,7 +2848,7 @@ describe("LocalManagedAgentProcessObserver", () => { alivePidsAtDeadline: [], }); } finally { - await forceKillRetainedTestGroup(anchor); + await stopRetainedTestGroup(anchor); controller.abort(); observer.dispose(); } @@ -3071,7 +2913,7 @@ describe("LocalManagedAgentProcessObserver", () => { let toolChildPid = 0; let toolGrandchildPid = 0; let escapedGroupId = 0; - const signals: Array = []; + const signals: Array = []; const observer = new LocalManagedAgentProcessObserver({ readProcessTable: async () => { await Promise.resolve(); @@ -3138,7 +2980,7 @@ describe("LocalManagedAgentProcessObserver", () => { }, processGroupLiveness: (groupId) => escaped && groupId === escapedGroupId ? "alive" : "gone", - signalProcessGroup: (groupId, signal) => { + testOnlyRequestTermination: (groupId, signal) => { signals.push([groupId, signal]); return "sent"; }, @@ -3183,7 +3025,7 @@ describe("LocalManagedAgentProcessObserver", () => { ); for (const socket of registrations) socket.destroy(); await Promise.all(registrationClosures); - await forceKillRetainedTestGroup(anchor); + await stopRetainedTestGroup(anchor); const teardown = await observer.emergencyCleanup(deadlineAfter(50)); @@ -3200,7 +3042,7 @@ describe("LocalManagedAgentProcessObserver", () => { ); } finally { for (const socket of registrations) socket.destroy(); - await forceKillRetainedTestGroup(anchor); + await stopRetainedTestGroup(anchor); controller.abort(); observer.dispose(); } @@ -3220,9 +3062,7 @@ describe("LocalManagedAgentProcessObserver", () => { { parentPid: process.pid, processGroupId: rootPid, - state: signals.some(([, signal]) => signal === "SIGSTOP") - ? "T" - : "S", + state: "S", startedAt: "root", }, ], @@ -3231,14 +3071,12 @@ describe("LocalManagedAgentProcessObserver", () => { { parentPid: rootPid, processGroupId: rootPid, - state: signals.some(([, signal]) => signal === "SIGSTOP") - ? "T" - : "S", + state: "S", startedAt: "child", }, ], ]), - signalProcessGroup: (groupId, signal) => { + testOnlyRequestTermination: (groupId, signal) => { signals.push([groupId, signal]); return "sent"; }, @@ -3263,7 +3101,7 @@ describe("LocalManagedAgentProcessObserver", () => { await observer.observeProcessTree(); expect(signals).toHaveLength(1); } finally { - await forceKillRetainedTestGroup(child); + await stopRetainedTestGroup(child); observer.dispose(); } }); @@ -3284,18 +3122,16 @@ describe("LocalManagedAgentProcessObserver", () => { { parentPid: process.pid, processGroupId: rootPid, - state: signals.some(([, signal]) => signal === "SIGSTOP") - ? "T" - : "S", + state: "S", startedAt: "root", }, ], ]); }, - signalProcessGroup: (groupId, signal) => { + testOnlyRequestTermination: (groupId, signal) => { signals.push([groupId, signal]); signalReadCounts.push(processTableReads); - if (signal === "SIGKILL" && killAttempts++ === 0) return "failure"; + if (killAttempts++ === 0) return "failure"; return "sent"; }, }); @@ -3332,7 +3168,7 @@ describe("LocalManagedAgentProcessObserver", () => { quiescent: false, }); } finally { - await forceKillRetainedTestGroup(child); + await stopRetainedTestGroup(child); observer.dispose(); } }); @@ -3456,7 +3292,7 @@ describe("LocalManagedAgentProcessObserver", () => { }); } finally { if (typeof child.pid === "number") { - await forceKillRetainedTestGroup(child); + await stopRetainedTestGroup(child); } controller.abort(); observer.dispose(); @@ -3485,7 +3321,7 @@ describe("LocalManagedAgentProcessObserver", () => { ); const signals: Array = []; const observer = new LocalManagedAgentProcessObserver({ - signalProcessGroup: (groupId, signal) => { + testOnlyRequestTermination: (groupId, signal) => { signals.push([groupId, signal]); return "sent"; }, @@ -3512,7 +3348,7 @@ describe("LocalManagedAgentProcessObserver", () => { const reads: Array< (observation: ManagedAgentProcessTableObservation) => void > = []; - const signals: Array = []; + const signals: Array = []; const observer = new LocalManagedAgentProcessObserver({ platform: "darwin", readProcessTable: () => @@ -3520,7 +3356,7 @@ describe("LocalManagedAgentProcessObserver", () => { reads.push(resolveRead); }), processGroupLiveness: () => "alive", - signalProcessGroup: (groupId, signal) => { + testOnlyRequestTermination: (groupId, signal) => { signals.push([groupId, signal]); return "sent"; }, @@ -3543,9 +3379,7 @@ describe("LocalManagedAgentProcessObserver", () => { parentPid: process.pid, processGroupId: rootPid, sessionId: rootPid, - state: signals.some(([, signal]) => signal === "SIGSTOP") - ? "T" - : "S", + state: "S", startedAt: "epoch-root", }, ], @@ -3573,20 +3407,20 @@ describe("LocalManagedAgentProcessObserver", () => { expect(signals).toEqual([]); // Completing the read that started before teardown cannot install its - // evidence or authorize a signal in the new lifecycle generation. + // evidence or authorize a request in the new lifecycle generation. reads.shift()!(rootTable()); await preSignalSample; expect(signals).toEqual([]); // Only the complete read that started after teardown can authorize the - // one direct group kill. + // one process-bound termination request. const postSignalSample = observer.observeProcessTree(); await vi.waitFor(() => expect(reads).toHaveLength(1)); reads.shift()!(rootTable()); await postSignalSample; expect(signals).toEqual([[rootPid, "SIGKILL"]]); } finally { - await forceKillRetainedTestGroup(anchor); + await stopRetainedTestGroup(anchor); forwardedController.abort(); observer.dispose(); } @@ -3600,7 +3434,7 @@ describe("LocalManagedAgentProcessObserver", () => { let resolveRead!: ( observation: ManagedAgentProcessTableObservation, ) => void; - const signals: Array = []; + const signals: Array = []; const observer = new LocalManagedAgentProcessObserver({ platform: "darwin", monotonicNow: () => monotonicTime, @@ -3608,7 +3442,7 @@ describe("LocalManagedAgentProcessObserver", () => { new Promise((resolve) => { resolveRead = resolve; }), - signalProcessGroup: (groupId, signal) => { + testOnlyRequestTermination: (groupId, signal) => { signals.push([groupId, signal]); return "sent"; }, @@ -3654,7 +3488,7 @@ describe("LocalManagedAgentProcessObserver", () => { expect(signals).toEqual(signalsAtSeal); expect(await observer.observeProcessTree(deadline)).toBe(false); } finally { - await forceKillRetainedTestGroup(anchor); + await stopRetainedTestGroup(anchor); forwardedController.abort(); await observer.dispose(); } @@ -3666,7 +3500,7 @@ describe("LocalManagedAgentProcessObserver", () => { let resolveRead!: ( observation: ManagedAgentProcessTableObservation, ) => void; - const signals: Array = []; + const signals: Array = []; const observer = new LocalManagedAgentProcessObserver({ platform: "darwin", monotonicNow: () => monotonicTime, @@ -3675,7 +3509,7 @@ describe("LocalManagedAgentProcessObserver", () => { resolveRead = resolve; }), processGroupLiveness: () => "gone", - signalProcessGroup: (groupId, signal) => { + testOnlyRequestTermination: (groupId, signal) => { signals.push([groupId, signal]); return "sent"; }, @@ -3704,7 +3538,7 @@ describe("LocalManagedAgentProcessObserver", () => { expect(signals).toEqual([]); expect(await observer.observeProcessTree(deadline)).toBe(false); } finally { - await forceKillRetainedTestGroup(anchor); + await stopRetainedTestGroup(anchor); forwardedController.abort(); await observer.dispose(); } @@ -3773,7 +3607,7 @@ describe("LocalManagedAgentProcessObserver", () => { socket?.destroy(); forwardedController.abort(); await observer.dispose(); - await forceKillRetainedTestGroup(anchor); + await stopRetainedTestGroup(anchor); } }, ); diff --git a/packages/harness/src/experimental/managed-agent-spike/process-observer.ts b/packages/harness/src/experimental/managed-agent-spike/process-observer.ts index 330a52a59..a990edd7f 100644 --- a/packages/harness/src/experimental/managed-agent-spike/process-observer.ts +++ b/packages/harness/src/experimental/managed-agent-spike/process-observer.ts @@ -20,7 +20,6 @@ import type { SpawnOptions, } from "@anthropic-ai/claude-agent-sdk"; -import { MANAGED_AGENT_CONTRACT } from "./contract.js"; import type { ManagedAgentCancellationReadiness, ManagedAgentProcessObserver, @@ -40,7 +39,6 @@ export const MANAGED_AGENT_TOOL_CONTROL_CAPABILITY_ENV = "SAPIOM_MANAGED_AGENT_TOOL_CONTROL_CAPABILITY"; const TOOL_REGISTRATION_MAX_BYTES = 1_024; const DISPOSE_DRAIN_TIMEOUT_MS = 500; -export const MANAGED_AGENT_LOGICAL_KILL_SHIM_SDK_VERSION = "0.3.228" as const; /** * The POSIX supervisor is the observer-owned process-group leader. The real @@ -91,7 +89,7 @@ for (const signal of ["SIGHUP", "SIGINT", "SIGTERM"]) { function killOwnedGroup() { try { - process.kill(-process.pid, "SIGKILL"); + process.kill(0, "SIGKILL"); } catch { process.exit(1); } @@ -124,7 +122,6 @@ function readOtherGroupMembers() { resolveMembers(members); }; const timeout = setTimeout(() => { - try { helper.kill("SIGKILL"); } catch {} finish(undefined); }, HELPER_TIMEOUT_MS); helper.stdout.on("data", (chunk) => { @@ -132,7 +129,7 @@ function readOtherGroupMembers() { output += chunk.toString("utf8"); if (Buffer.byteLength(output) > MAX_PROCESS_TABLE_BYTES) { overflowed = true; - try { helper.kill("SIGKILL"); } catch {} + helper.stdout.destroy(); } }); helper.once("error", () => finish(undefined)); @@ -265,7 +262,13 @@ export type ManagedAgentProcessTableObservation = | { readonly available: false }; export type ManagedAgentProcessGroupLiveness = "alive" | "gone" | "unknown"; -export type ManagedAgentProcessSignalOutcome = "sent" | "gone" | "failure"; +export type ManagedAgentTerminationRequestOutcome = "sent" | "gone" | "failure"; + +export interface ManagedAgentTerminationRequest { + readonly target: "root" | "tool"; + /** Diagnostic identity only. The production request path never signals it. */ + readonly processGroupId: number; +} export interface LocalManagedAgentProcessObserverOptions { readonly platform?: NodeJS.Platform; @@ -273,10 +276,25 @@ export interface LocalManagedAgentProcessObserverOptions { readonly processGroupLiveness?: ( processGroupId: number, ) => ManagedAgentProcessGroupLiveness; - readonly signalProcessGroup?: ( + /** + * Deterministic unit-test seam. Production callers must leave this unset: + * the default path requests termination only over retained process-bound + * channels and never turns a sampled numeric PGID into signal authority. + */ + readonly testOnlyRequestTermination?: ( processGroupId: number, signal: "SIGKILL", - ) => ManagedAgentProcessSignalOutcome; + target: ManagedAgentTerminationRequest["target"], + ) => ManagedAgentTerminationRequestOutcome; + /** Return an outcome to veto one request; return undefined to use the channel. */ + readonly testOnlyBeforeTerminationRequest?: ( + request: ManagedAgentTerminationRequest, + ) => ManagedAgentTerminationRequestOutcome | undefined; + /** Read-only test telemetry emitted after a channel request is attempted. */ + readonly onTerminationRequest?: ( + request: ManagedAgentTerminationRequest, + outcome: ManagedAgentTerminationRequestOutcome, + ) => void; readonly monotonicNow?: () => number; readonly delay?: (milliseconds: number) => Promise; } @@ -476,19 +494,6 @@ function defaultProcessGroupLiveness( } } -function defaultSignalProcessGroup( - processGroupId: number, - signal: "SIGKILL", -): ManagedAgentProcessSignalOutcome { - try { - process.kill(-processGroupId, signal); - return "sent"; - } catch (error) { - const code = (error as NodeJS.ErrnoException).code; - return code === "ESRCH" ? "gone" : "failure"; - } -} - function childActive(child: ChildProcessWithoutNullStreams): boolean { return child.exitCode === null && child.signalCode === null; } @@ -527,10 +532,11 @@ function descendantsOf( * This is not universal built-in Bash containment or a process-tree killer. * Windows, an unavailable process table, missing lifetime channels, or * identity/ancestry drift fail certification closed. The fallback freshly - * validates and kills the exact fixture group, proves it absent with a new - * sample, then freshly validates and kills the owned supervisor root. POSIX - * `lstart` remains one component of fresh identity evidence, not standalone - * authority. Workspace PID-file contents never enter this class. + * validates the exact fixture group, asks a still-open authenticated member to + * terminate its own current group, proves it absent with a new sample, then + * asks the owned supervisor over retained IPC to terminate its own group. + * POSIX `lstart` remains evidence only: a sampled numeric PID/PGID is never + * host signal authority. Workspace PID-file contents never enter this class. */ export class LocalManagedAgentProcessObserver implements ManagedAgentProcessObserver { readonly #platform: NodeJS.Platform; @@ -538,10 +544,24 @@ export class LocalManagedAgentProcessObserver implements ManagedAgentProcessObse readonly #processGroupLiveness: ( processGroupId: number, ) => ManagedAgentProcessGroupLiveness; - readonly #signalProcessGroup: ( - processGroupId: number, - signal: "SIGKILL", - ) => ManagedAgentProcessSignalOutcome; + readonly #testOnlyRequestTermination: + | (( + processGroupId: number, + signal: "SIGKILL", + target: ManagedAgentTerminationRequest["target"], + ) => ManagedAgentTerminationRequestOutcome) + | undefined; + readonly #testOnlyBeforeTerminationRequest: + | (( + request: ManagedAgentTerminationRequest, + ) => ManagedAgentTerminationRequestOutcome | undefined) + | undefined; + readonly #onTerminationRequest: + | (( + request: ManagedAgentTerminationRequest, + outcome: ManagedAgentTerminationRequestOutcome, + ) => void) + | undefined; readonly #monotonicNow: () => number; readonly #delay: (milliseconds: number) => Promise; readonly #roots = new Map(); @@ -594,8 +614,10 @@ export class LocalManagedAgentProcessObserver implements ManagedAgentProcessObse (() => defaultReadProcessTable(this.#platform)); this.#processGroupLiveness = options.processGroupLiveness ?? defaultProcessGroupLiveness; - this.#signalProcessGroup = - options.signalProcessGroup ?? defaultSignalProcessGroup; + this.#testOnlyRequestTermination = options.testOnlyRequestTermination; + this.#testOnlyBeforeTerminationRequest = + options.testOnlyBeforeTerminationRequest; + this.#onTerminationRequest = options.onTerminationRequest; this.#monotonicNow = options.monotonicNow ?? (() => performance.now()); this.#delay = options.delay ?? defaultDelay; if (this.#platform === "darwin" || this.#platform === "linux") { @@ -939,29 +961,6 @@ export class LocalManagedAgentProcessObserver implements ManagedAgentProcessObse child.stderr.on("error", () => undefined); if (typeof child.pid === "number") { const pid = child.pid; - if (usePosixSupervisor) { - if ( - MANAGED_AGENT_CONTRACT.agentSdkVersion !== - MANAGED_AGENT_LOGICAL_KILL_SHIM_SDK_VERSION - ) { - throw new Error( - `The managed-agent logical kill shim is certified only for Agent SDK ${MANAGED_AGENT_LOGICAL_KILL_SHIM_SDK_VERSION}`, - ); - } - child.kill = ((_signal: NodeJS.Signals = "SIGTERM") => { - // Agent SDK 0.3.228's ProcessTransport calls kill() immediately before - // it forwards its private AbortSignal. This compatibility shim must - // be removed or recertified when that SDK pin changes. Treat the call - // as logical acceptance so the SDK will not retry through a cached - // PID, but preserve the live supervisor as the ancestry anchor. Only - // the subsequently forwarded signal may start sampled, - // identity-checked group cleanup. The real-SDK loopback test is the - // sequence sentinel for this exact kill-then-abort behavior. - if (!childActive(child) || child.killed) return false; - Reflect.set(child, "killed", true); - return true; - }) as typeof child.kill; - } this.#roots.set(pid, { pid, child, @@ -1715,21 +1714,103 @@ export class LocalManagedAgentProcessObserver implements ManagedAgentProcessObse : true; } - #signalValidatedProcessGroup( + #recordTerminationRequest( + request: ManagedAgentTerminationRequest, + outcome: ManagedAgentTerminationRequestOutcome, + ): ManagedAgentTerminationRequestOutcome { + this.#onTerminationRequest?.(request, outcome); + return outcome; + } + + #requestToolGroupTermination( processGroupId: number, - signal: "SIGKILL", - ): ManagedAgentProcessSignalOutcome { + ): ManagedAgentTerminationRequestOutcome { if (this.#sealed || this.#deadlineExpiredAndSeal()) return "failure"; - const outcome = this.#signalProcessGroup(processGroupId, signal); - // Every attempt invalidates every sample that started before it, including - // ESRCH and helper failures. Only a complete read started in this new - // generation may prove the attempted target gone or authorize a next step. + const request = { target: "tool", processGroupId } as const; + const vetoedOutcome = this.#testOnlyBeforeTerminationRequest?.(request); + if (vetoedOutcome) { + return this.#recordTerminationRequest(request, vetoedOutcome); + } + if (this.#testOnlyRequestTermination) { + return this.#recordTerminationRequest( + request, + this.#testOnlyRequestTermination( + request.processGroupId, + "SIGKILL", + request.target, + ), + ); + } + const registration = (["parent", "child"] as const) + .map((role) => this.#toolProcessRegistrations.get(role)) + .find( + (candidate) => + candidate?.accepted && + !candidate.closed && + !candidate.socket.destroyed && + candidate.socket.writable, + ); + if (!registration) { + return this.#recordTerminationRequest(request, "failure"); + } + try { + // This retained socket is bound to the authenticated process instance, + // not its numeric PID. The receiver calls kill(0, SIGKILL), so the + // still-running member terminates its own current group without a host + // snapshot-to-signal PGID reuse window. + registration.socket.write('{"forceKill":true}\n'); + return this.#recordTerminationRequest(request, "sent"); + } catch { + return this.#recordTerminationRequest(request, "failure"); + } + } + + #requestRootGroupTermination( + root: OwnedRoot, + ): ManagedAgentTerminationRequestOutcome { + if (this.#sealed || this.#deadlineExpiredAndSeal()) return "failure"; + const request = { target: "root", processGroupId: root.pid } as const; + const vetoedOutcome = this.#testOnlyBeforeTerminationRequest?.(request); + if (vetoedOutcome) { + return this.#recordTerminationRequest(request, vetoedOutcome); + } + if (this.#testOnlyRequestTermination) { + return this.#recordTerminationRequest( + request, + this.#testOnlyRequestTermination( + request.processGroupId, + "SIGKILL", + request.target, + ), + ); + } + if (!childActive(root.child)) { + return this.#recordTerminationRequest(request, "gone"); + } + if (!root.child.connected) { + return this.#recordTerminationRequest(request, "failure"); + } + try { + // The IPC endpoint belongs to the retained supervisor process instance. + // Its disconnect handler terminates its own current group. PID reuse can + // therefore make this request fail, but can never redirect it. + root.child.disconnect(); + return this.#recordTerminationRequest(request, "sent"); + } catch { + return this.#recordTerminationRequest(request, "failure"); + } + } + + #invalidateSampleAfterTerminationRequest(): void { + if (this.#sealed || this.#deadlineExpiredAndSeal()) return; + // Every request invalidates every sample that started before it, including + // channel failures. Only a complete read started in this new generation + // may prove the target gone or authorize the next teardown step. this.#sampleGeneration += 1; this.#processTableNeedsRefresh = true; - return outcome; } - #killOwnedRootsSynchronously(): void { + #requestOwnedRootTerminationSynchronously(): void { if (this.#sealed) return; if (this.#platform !== "darwin" && this.#platform !== "linux") return; for (const root of this.#roots.values()) { @@ -1741,11 +1822,9 @@ export class LocalManagedAgentProcessObserver implements ManagedAgentProcessObse ) { continue; } - const killOutcome = this.#signalValidatedProcessGroup( - root.pid, - "SIGKILL", - ); - root.forceKillIssued = killOutcome === "sent"; + const requestOutcome = this.#requestRootGroupTermination(root); + this.#invalidateSampleAfterTerminationRequest(); + root.forceKillIssued = requestOutcome === "sent"; } } @@ -1754,8 +1833,8 @@ export class LocalManagedAgentProcessObserver implements ManagedAgentProcessObse if (!this.#fallbackCleanupRequested) { this.#fallbackCleanupRequested = true; // The cleanup request itself is a lifecycle boundary. Discard any - // earlier sample so the first numeric signal can only be authorized by - // a complete process-table read that began after teardown started. + // earlier sample so the first process-bound termination request can only + // follow a complete process-table read begun after teardown started. this.#sampleGeneration += 1; this.#processTableNeedsRefresh = true; } @@ -1773,7 +1852,7 @@ export class LocalManagedAgentProcessObserver implements ManagedAgentProcessObse return; } if (!this.#toolProcessContainmentArmed) { - this.#killOwnedRootsSynchronously(); + this.#requestOwnedRootTerminationSynchronously(); return; } // Once tool containment is armed, fail closed until the authenticated @@ -1791,7 +1870,7 @@ export class LocalManagedAgentProcessObserver implements ManagedAgentProcessObse if (liveToolGroupMembers && this.#toolProcessForceKillIssued) return; const groupLiveness = this.#processGroupLiveness(processGroupId); if (groupLiveness === "gone") { - this.#killOwnedRootsSynchronously(); + this.#requestOwnedRootTerminationSynchronously(); return; } if (groupLiveness !== "alive") return; @@ -1800,13 +1879,11 @@ export class LocalManagedAgentProcessObserver implements ManagedAgentProcessObse return; } if (!this.#toolProcessForceKillIssued) { - const killOutcome = this.#signalValidatedProcessGroup( - processGroupId, - "SIGKILL", - ); - this.#toolProcessForceKillIssued = killOutcome === "sent"; - if (killOutcome === "gone") { - this.#killOwnedRootsSynchronously(); + const requestOutcome = this.#requestToolGroupTermination(processGroupId); + this.#invalidateSampleAfterTerminationRequest(); + this.#toolProcessForceKillIssued = requestOutcome === "sent"; + if (requestOutcome === "gone") { + this.#requestOwnedRootTerminationSynchronously(); } } } diff --git a/packages/harness/src/experimental/managed-agent-spike/runtime-sdk-loopback.test.ts b/packages/harness/src/experimental/managed-agent-spike/runtime-sdk-loopback.test.ts index cee3f7d9f..5d42b4aba 100644 --- a/packages/harness/src/experimental/managed-agent-spike/runtime-sdk-loopback.test.ts +++ b/packages/harness/src/experimental/managed-agent-spike/runtime-sdk-loopback.test.ts @@ -1,4 +1,4 @@ -import { execFile, spawn } from "node:child_process"; +import { execFile, spawn, type ChildProcess } from "node:child_process"; import { once } from "node:events"; import { readFile } from "node:fs/promises"; import { createServer, type ServerResponse } from "node:http"; @@ -80,6 +80,36 @@ async function waitForProcessDeath( throw new Error(`Test process ${pid} survived cleanup`); } +function spawnCooperativeUnrelatedProcess(): ChildProcess { + return spawn( + process.execPath, + [ + "-e", + 'process.on("disconnect", () => process.exit(0)); setInterval(() => {}, 1000)', + ], + { + stdio: ["ignore", "ignore", "ignore", "ipc"], + windowsHide: true, + }, + ); +} + +async function stopCooperativeUnrelatedProcess( + child: ChildProcess, +): Promise { + if (typeof child.pid !== "number") return; + const pid = child.pid; + if (child.exitCode === null && child.signalCode === null) { + if (!child.connected) { + throw new Error( + `Refusing cleanup for unrelated process ${pid} without retained IPC`, + ); + } + child.disconnect(); + } + await waitForProcessDeath(pid); +} + interface LoopbackObservation { readonly headerNames: readonly string[]; readonly evalSourceMatches: boolean; @@ -283,7 +313,7 @@ it("enforces real-SDK built-in and in-process MCP calls with exact loopback corr const groupSignals: Array<{ readonly elapsedMs: number; readonly groupId: number; - readonly signal: "SIGSTOP" | "SIGKILL"; + readonly signal: "SIGKILL"; readonly outcome: "sent" | "gone" | "failure"; }> = []; const lifecycle: Array<{ @@ -299,24 +329,13 @@ it("enforces real-SDK built-in and in-process MCP calls with exact loopback corr }); return observation; }, - signalProcessGroup: (groupId, signal) => { - let outcome: "sent" | "gone" | "failure"; - try { - process.kill(-groupId, signal); - outcome = "sent"; - } catch (error) { - outcome = - (error as NodeJS.ErrnoException).code === "ESRCH" - ? "gone" - : "failure"; - } + onTerminationRequest: ({ processGroupId: groupId }, outcome) => { groupSignals.push({ elapsedMs: Date.now() - startedAt, groupId, - signal, + signal: "SIGKILL", outcome, }); - return outcome; }, }); let supervisorPid: number | undefined; @@ -650,7 +669,7 @@ it.skipIf( const groupSignals: Array<{ readonly elapsedMs: number; readonly groupId: number; - readonly signal: "SIGSTOP" | "SIGKILL"; + readonly signal: "SIGKILL"; readonly outcome: "sent" | "gone" | "failure"; }> = []; const observer = new LocalManagedAgentProcessObserver({ @@ -664,24 +683,13 @@ it.skipIf( }); return observation; }, - signalProcessGroup: (groupId, signal) => { - let outcome: "sent" | "gone" | "failure"; - try { - process.kill(-groupId, signal); - outcome = "sent"; - } catch (error) { - outcome = - (error as NodeJS.ErrnoException).code === "ESRCH" - ? "gone" - : "failure"; - } + onTerminationRequest: ({ processGroupId: groupId }, outcome) => { groupSignals.push({ elapsedMs: Date.now() - startedAt, groupId, - signal, + signal: "SIGKILL", outcome, }); - return outcome; }, }); const cleanupOrder: string[] = []; @@ -694,10 +702,10 @@ it.skipIf( { once: true }, ); const child = observer.spawn(options); - const logicalKill = child.kill.bind(child); + const nativeKill = child.kill.bind(child); Reflect.set(child, "kill", (signal: NodeJS.Signals = "SIGTERM") => { - cleanupOrder.push("sdk_logical_kill"); - return logicalKill(signal); + cleanupOrder.push("sdk_native_kill"); + return nativeKill(signal); }); const pid = Reflect.get(child, "pid"); supervisorPid = typeof pid === "number" ? pid : undefined; @@ -714,11 +722,7 @@ it.skipIf( }, dispose: () => observer.dispose(), }; - const unrelated = spawn( - process.execPath, - ["-e", "setInterval(() => {}, 1000)"], - { stdio: "ignore", windowsHide: true }, - ); + const unrelated = spawnCooperativeUnrelatedProcess(); await once(unrelated, "spawn"); let fixturePids: readonly number[] = []; let fixtureToolProcessGroupId: number | undefined; @@ -922,22 +926,22 @@ it.skipIf( cleanupOrder.indexOf("host_emergency_cleanup"), ); const forwardedSignalIndex = cleanupOrder.indexOf("sdk_forwarded_signal"); - const logicalKillIndexes = cleanupOrder.flatMap((step, index) => - step === "sdk_logical_kill" ? [index] : [], + const nativeKillIndexes = cleanupOrder.flatMap((step, index) => + step === "sdk_native_kill" ? [index] : [], ); expect(cleanupOrder).toEqual([ "sdk_close_called", "sdk_return_started", - "sdk_logical_kill", + "sdk_native_kill", "sdk_forwarded_signal", - "sdk_logical_kill", + "sdk_native_kill", "sdk_return_settled", "host_emergency_cleanup", ]); - expect(logicalKillIndexes).toEqual([2, 4]); + expect(nativeKillIndexes).toEqual([2, 4]); expect(forwardedSignalIndex).toBeGreaterThanOrEqual(0); - expect(logicalKillIndexes[0]).toBeLessThan(forwardedSignalIndex); - expect(logicalKillIndexes[1]).toBeGreaterThan(forwardedSignalIndex); + expect(nativeKillIndexes[0]).toBeLessThan(forwardedSignalIndex); + expect(nativeKillIndexes[1]).toBeGreaterThan(forwardedSignalIndex); expect(forwardedSignalIndex).toBeLessThan( cleanupOrder.indexOf("host_emergency_cleanup"), ); @@ -946,10 +950,7 @@ it.skipIf( server.closeAllConnections(); await new Promise((resolve) => server.close(() => resolve())); await Promise.all(fixturePids.map((pid) => waitForProcessDeath(pid))); - if (typeof unrelated.pid === "number" && processExists(unrelated.pid)) { - unrelated.kill("SIGKILL"); - await waitForProcessDeath(unrelated.pid); - } + await stopCooperativeUnrelatedProcess(unrelated); await fixture.cleanup(); } expect(fixturePids.every((pid) => !processExists(pid))).toBe(true); @@ -992,11 +993,7 @@ it.skipIf( }, dispose: () => observer.dispose(), }; - const unrelated = spawn( - process.execPath, - ["-e", "setInterval(() => {}, 1000)"], - { stdio: "ignore", windowsHide: true }, - ); + const unrelated = spawnCooperativeUnrelatedProcess(); await once(unrelated, "spawn"); let fixturePids: readonly number[] = []; let cancellationStartedAt: number | undefined; @@ -1138,10 +1135,7 @@ it.skipIf( server.closeAllConnections(); await new Promise((resolve) => server.close(() => resolve())); await Promise.all(fixturePids.map((pid) => waitForProcessDeath(pid))); - if (typeof unrelated.pid === "number" && processExists(unrelated.pid)) { - unrelated.kill("SIGKILL"); - await waitForProcessDeath(unrelated.pid); - } + await stopCooperativeUnrelatedProcess(unrelated); await fixture.cleanup(); } expect(fixturePids.every((pid) => !processExists(pid))).toBe(true); diff --git a/packages/harness/src/experimental/managed-agent-spike/types.ts b/packages/harness/src/experimental/managed-agent-spike/types.ts index 24ad90f6c..d5471505f 100644 --- a/packages/harness/src/experimental/managed-agent-spike/types.ts +++ b/packages/harness/src/experimental/managed-agent-spike/types.ts @@ -236,7 +236,7 @@ export interface ManagedAgentTeardownObservation { readonly containmentSupported: boolean; /** True after SDK-root authority and any required L2 observations are proven. */ readonly ownershipProven: boolean; - /** True only when SIGKILL was issued to every owned SDK supervisor root. */ + /** True only when forced termination was requested for every owned root. */ readonly forceKillIssued: boolean; /** True after both exact L2 fixture lifetime channels pass fresh observation. */ readonly toolProcessObservationComplete: boolean; From 6865e3a0d4d8879bff9962ba40b8bf944ab65a13 Mon Sep 17 00:00:00 2001 From: Yash Date: Mon, 17 Aug 2026 06:57:36 -0700 Subject: [PATCH 21/24] fix(harness): close agent termination channel races Broadcast tool self-termination over every authenticated live channel and make the supervisor handle IPC that disconnected before bootstrap. Refs SAP-2632 --- .../process-observer.test.ts | 78 +++++++++++++++++++ .../managed-agent-spike/process-observer.ts | 55 +++++++++---- 2 files changed, 119 insertions(+), 14 deletions(-) diff --git a/packages/harness/src/experimental/managed-agent-spike/process-observer.test.ts b/packages/harness/src/experimental/managed-agent-spike/process-observer.test.ts index b49f4c847..e1c49a684 100644 --- a/packages/harness/src/experimental/managed-agent-spike/process-observer.test.ts +++ b/packages/harness/src/experimental/managed-agent-spike/process-observer.test.ts @@ -960,6 +960,47 @@ describe("LocalManagedAgentProcessObserver", () => { 5_000, ); + it.skipIf(process.platform === "win32")( + "self-terminates when IPC disconnects before the supervisor installs its listener", + async () => { + const fixture = await createManagedAgentFixture( + () => "supervisor-bootstrap-disconnect", + ); + fixtures.push(fixture); + const disconnectPreload = join( + fixture.workspaceRoot, + "disconnect-supervisor-ipc.cjs", + ); + await writeFile( + disconnectPreload, + "if (process.connected) process.disconnect();\n", + ); + const observer = new LocalManagedAgentProcessObserver(); + const controller = new AbortController(); + const anchor = asChildProcess( + observer.spawn({ + command: "/usr/bin/true", + args: [], + cwd: fixture.workspaceRoot, + env: { + ...process.env, + NODE_OPTIONS: `--require=${disconnectPreload}`, + }, + signal: controller.signal, + }), + ); + try { + const [exitCode, signalCode] = await once(anchor, "exit"); + expect(exitCode).toBeNull(); + expect(signalCode).toBe("SIGKILL"); + } finally { + controller.abort(); + await observer.dispose(); + } + }, + 5_000, + ); + it.skipIf(process.platform === "win32")( "never signals a cached supervisor group through SDK kill after observed exit", async () => { @@ -1396,6 +1437,43 @@ describe("LocalManagedAgentProcessObserver", () => { 15_000, ); + it.skipIf(process.platform === "win32")( + "broadcasts tool termination when the parent channel write is unusable", + async () => { + const attemptedRoles: Array<"parent" | "child"> = []; + const observer = new LocalManagedAgentProcessObserver({ + testOnlyWriteToolTermination: (role) => { + attemptedRoles.push(role); + return role === "parent" ? "failure" : undefined; + }, + }); + let run: RegisteredDescendantToolRun | undefined; + try { + run = await startRegisteredDescendantToolRun( + observer, + "child-channel-termination-fallback", + ); + + const teardown = await observer.emergencyCleanup(deadlineAfter(3_000)); + + expect(attemptedRoles).toEqual(["parent", "child"]); + expect(teardown).toMatchObject({ + quiescent: true, + deadlineMet: true, + containmentSupported: true, + forceKillIssued: true, + toolProcessChannelsClosed: true, + alivePidsAtDeadline: [], + }); + expect(run.toolPids.every((pid) => !processExists(pid))).toBe(true); + } finally { + await cleanupRegisteredDescendantToolRun(run); + await observer.dispose(); + } + }, + 15_000, + ); + it.skipIf(process.platform === "win32")( "cleans exact fixture and anchor groups when setup fails after PID publication", async () => { diff --git a/packages/harness/src/experimental/managed-agent-spike/process-observer.ts b/packages/harness/src/experimental/managed-agent-spike/process-observer.ts index a990edd7f..73c26a9ba 100644 --- a/packages/harness/src/experimental/managed-agent-spike/process-observer.ts +++ b/packages/harness/src/experimental/managed-agent-spike/process-observer.ts @@ -96,6 +96,9 @@ function killOwnedGroup() { } process.on("disconnect", killOwnedGroup); +// The host can close IPC after spawn() succeeds but before this module starts. +// Register first, then close the already-disconnected bootstrap window. +if (!process.connected) killOwnedGroup(); function readOtherGroupMembers() { return new Promise((resolveMembers) => { @@ -290,6 +293,10 @@ export interface LocalManagedAgentProcessObserverOptions { readonly testOnlyBeforeTerminationRequest?: ( request: ManagedAgentTerminationRequest, ) => ManagedAgentTerminationRequestOutcome | undefined; + /** Simulate one role's channel write; undefined uses the real retained socket. */ + readonly testOnlyWriteToolTermination?: ( + role: ToolProcessRole, + ) => Exclude | undefined; /** Read-only test telemetry emitted after a channel request is attempted. */ readonly onTerminationRequest?: ( request: ManagedAgentTerminationRequest, @@ -556,6 +563,11 @@ export class LocalManagedAgentProcessObserver implements ManagedAgentProcessObse request: ManagedAgentTerminationRequest, ) => ManagedAgentTerminationRequestOutcome | undefined) | undefined; + readonly #testOnlyWriteToolTermination: + | (( + role: ToolProcessRole, + ) => Exclude | undefined) + | undefined; readonly #onTerminationRequest: | (( request: ManagedAgentTerminationRequest, @@ -617,6 +629,7 @@ export class LocalManagedAgentProcessObserver implements ManagedAgentProcessObse this.#testOnlyRequestTermination = options.testOnlyRequestTermination; this.#testOnlyBeforeTerminationRequest = options.testOnlyBeforeTerminationRequest; + this.#testOnlyWriteToolTermination = options.testOnlyWriteToolTermination; this.#onTerminationRequest = options.onTerminationRequest; this.#monotonicNow = options.monotonicNow ?? (() => performance.now()); this.#delay = options.delay ?? defaultDelay; @@ -1741,28 +1754,42 @@ export class LocalManagedAgentProcessObserver implements ManagedAgentProcessObse ), ); } - const registration = (["parent", "child"] as const) + const registrations = (["parent", "child"] as const) .map((role) => this.#toolProcessRegistrations.get(role)) - .find( - (candidate) => - candidate?.accepted && + .filter( + (candidate): candidate is ToolProcessRegistration => + candidate !== undefined && + candidate.accepted && !candidate.closed && !candidate.socket.destroyed && candidate.socket.writable, ); - if (!registration) { + if (registrations.length === 0) { return this.#recordTerminationRequest(request, "failure"); } - try { - // This retained socket is bound to the authenticated process instance, - // not its numeric PID. The receiver calls kill(0, SIGKILL), so the - // still-running member terminates its own current group without a host - // snapshot-to-signal PGID reuse window. - registration.socket.write('{"forceKill":true}\n'); - return this.#recordTerminationRequest(request, "sent"); - } catch { - return this.#recordTerminationRequest(request, "failure"); + let sent = false; + for (const registration of registrations) { + const simulatedOutcome = this.#testOnlyWriteToolTermination?.( + registration.role, + ); + if (simulatedOutcome) { + sent ||= simulatedOutcome === "sent"; + continue; + } + try { + // This retained socket is bound to the authenticated process instance, + // not its numeric PID. The receiver calls kill(0, SIGKILL), so the + // still-running member terminates its own current group without a host + // snapshot-to-signal PGID reuse window. Broadcast to every live role: + // a stale peer cannot prevent its surviving group-mate from receiving + // the same idempotent self-group termination request. + registration.socket.write('{"forceKill":true}\n'); + sent = true; + } catch { + // Try every independently authenticated channel before failing closed. + } } + return this.#recordTerminationRequest(request, sent ? "sent" : "failure"); } #requestRootGroupTermination( From 50e5462457c3cb5b555d11f6c8a85e707400dd93 Mon Sep 17 00:00:00 2001 From: Yash Date: Mon, 17 Aug 2026 07:48:40 -0700 Subject: [PATCH 22/24] fix(harness): fail closed on fixture channel loss Refs SAP-2632 --- .../managed-agent-spike/README.md | 16 ++++++++-- .../managed-agent-spike/fixture.ts | 26 ++++++++++++--- .../process-observer.test.ts | 32 +++++++++++++++++++ .../managed-agent-spike/process-observer.ts | 9 ++++++ 4 files changed, 76 insertions(+), 7 deletions(-) diff --git a/packages/harness/src/experimental/managed-agent-spike/README.md b/packages/harness/src/experimental/managed-agent-spike/README.md index 925ecb128..cac3cf0fc 100644 --- a/packages/harness/src/experimental/managed-agent-spike/README.md +++ b/packages/harness/src/experimental/managed-agent-spike/README.md @@ -195,6 +195,15 @@ deadline bounds the entire sequence. A PID or PGID can disappear and be reused between any sample and request without redirecting termination, because neither channel is addressed by that number. +The fixture also fails closed if its host disappears outside that orderly +sequence. After authentication, either lifetime channel closing makes the +receiving fixture process terminate its own current group. Before +authentication, connection failures retry only until a five-second monotonic +deadline and then terminate that same receiver-owned group. This receiver-side +behavior lets an outer process-bound supervisor close its own IPC channel and +cascade termination through nested detached fixture processes without sending +a host-side signal to a cached numeric PID or PGID. + Deadline expiry or a successful quiescence observation seals all evidence, closes the spawn gate, and permanently revokes numeric signal authority. Disposal never signals a cached PID or PGID, even if child exit delivery lags @@ -245,9 +254,10 @@ The disposable fixture uses a host-owned lifetime lease outside the writable workspace. The lease exists before launch; `shutdown` contents or a missing lease both make the fixture parent stop its child and exit. Cleanup can therefore remove the temporary root without turning a startup race into a permanently -The child also exits on IPC disconnect, and a failed readiness-file publication -shuts it down before the parent exits; a hermetic regression removes the real -fixture root during delayed readiness and verifies that neither process remains. +running process. The child also exits on IPC disconnect, and a failed +readiness-file publication shuts it down before the parent exits; a hermetic +regression removes the real fixture root during delayed readiness and verifies +that neither process remains. ## Pre-v2 live evidence diff --git a/packages/harness/src/experimental/managed-agent-spike/fixture.ts b/packages/harness/src/experimental/managed-agent-spike/fixture.ts index 7435fdd36..62beea119 100644 --- a/packages/harness/src/experimental/managed-agent-spike/fixture.ts +++ b/packages/harness/src/experimental/managed-agent-spike/fixture.ts @@ -88,6 +88,7 @@ import { spawn } from "node:child_process"; import { existsSync, readFileSync, unlinkSync, writeFileSync } from "node:fs"; import { createConnection } from "node:net"; import { resolve } from "node:path"; +import { performance } from "node:perf_hooks"; const pidFile = resolve(process.argv[2]); const requireControlRegistration = process.argv[3] === "--register-control"; @@ -105,15 +106,18 @@ const readinessDelayMs = Number.isSafeInteger(parsedReadinessDelay) && : 0; const controlSocket = process.env[${JSON.stringify(MANAGED_AGENT_TOOL_CONTROL_SOCKET_ENV)}]; const controlCapability = process.env[${JSON.stringify(MANAGED_AGENT_TOOL_CONTROL_CAPABILITY_ENV)}]; +const controlRegistrationDeadlineAt = performance.now() + 5_000; if (requireControlRegistration && (!controlSocket || !controlCapability)) { throw new Error("managed-agent tool control capability missing"); } process.on("SIGTERM", () => {}); const childProgram = [ 'const { createConnection } = require("node:net");', + 'const { performance } = require("node:perf_hooks");', 'const requireControlRegistration = ' + JSON.stringify(requireControlRegistration) + ';', 'const controlSocket = process.env["${MANAGED_AGENT_TOOL_CONTROL_SOCKET_ENV}"];', 'const controlCapability = process.env["${MANAGED_AGENT_TOOL_CONTROL_CAPABILITY_ENV}"];', + 'const controlRegistrationDeadlineAt = performance.now() + 5_000;', 'delete process.env["${MANAGED_AGENT_TOOL_CONTROL_SOCKET_ENV}"];', 'delete process.env["${MANAGED_AGENT_TOOL_CONTROL_CAPABILITY_ENV}"];', 'const readinessDelayMs = ' + JSON.stringify(readinessDelayMs) + ';', @@ -121,6 +125,9 @@ const childProgram = [ 'process.on("message", (message) => { if (message === "host-shutdown") process.exit(0); });', 'process.on("disconnect", () => process.exit(0));', 'let readyPublished = false;', + 'const terminateOwnedGroup = () => {', + ' try { process.kill(0, "SIGKILL"); } catch { process.exit(1); }', + '};', 'const publishReady = () => {', ' if (readyPublished) return;', ' readyPublished = true;', @@ -136,7 +143,11 @@ const childProgram = [ ' let registered = false;', ' let retryScheduled = false;', ' const retry = () => {', - ' if (registered || retryScheduled) return;', + ' if (registered || performance.now() >= controlRegistrationDeadlineAt) {', + ' terminateOwnedGroup();', + ' return;', + ' }', + ' if (retryScheduled) return;', ' retryScheduled = true;', ' setTimeout(connectControl, 10);', ' };', @@ -146,7 +157,7 @@ const childProgram = [ ' socket.on("data", (chunk) => {', ' response += chunk;', ' if (response.includes(' + JSON.stringify('"forceKill":true') + ')) {', - ' try { process.kill(0, "SIGKILL"); } catch { process.exit(1); }', + ' terminateOwnedGroup();', ' return;', ' }', ' if (response.includes(' + JSON.stringify('"shutdown":true') + ')) {', @@ -177,6 +188,9 @@ delete process.env[${JSON.stringify(MANAGED_AGENT_TOOL_CONTROL_SOCKET_ENV)}]; delete process.env[${JSON.stringify(MANAGED_AGENT_TOOL_CONTROL_CAPABILITY_ENV)}]; let childReady = false; let controlReady = !requireControlRegistration; +const terminateOwnedGroup = () => { + try { process.kill(0, "SIGKILL"); } catch { process.exit(1); } +}; const publishReadiness = () => { if (!childReady || !controlReady) return; try { @@ -200,7 +214,11 @@ const connectControl = () => { let registered = false; let retryScheduled = false; const retry = () => { - if (registered || retryScheduled) return; + if (registered || performance.now() >= controlRegistrationDeadlineAt) { + terminateOwnedGroup(); + return; + } + if (retryScheduled) return; retryScheduled = true; setTimeout(connectControl, 10); }; @@ -210,7 +228,7 @@ const connectControl = () => { socket.on("data", (chunk) => { response += chunk; if (response.includes('"forceKill":true')) { - try { process.kill(0, "SIGKILL"); } catch { process.exit(1); } + terminateOwnedGroup(); return; } if (response.includes('"shutdown":true')) { diff --git a/packages/harness/src/experimental/managed-agent-spike/process-observer.test.ts b/packages/harness/src/experimental/managed-agent-spike/process-observer.test.ts index e1c49a684..d3f14af12 100644 --- a/packages/harness/src/experimental/managed-agent-spike/process-observer.test.ts +++ b/packages/harness/src/experimental/managed-agent-spike/process-observer.test.ts @@ -1474,6 +1474,38 @@ describe("LocalManagedAgentProcessObserver", () => { 15_000, ); + it.skipIf(process.platform === "win32")( + "self-terminates the detached tool group when authenticated lifetime channels disappear", + async () => { + const hostKillSpy = vi.spyOn(process, "kill"); + const observer = new LocalManagedAgentProcessObserver(); + const unrelated = spawnCooperativeTestProcess(); + await once(unrelated, "spawn"); + let run: RegisteredDescendantToolRun | undefined; + try { + run = await startRegisteredDescendantToolRun( + observer, + "lost-tool-lifetime-channels", + ); + + observer.testOnlyDropToolLifetimeChannels(); + + await waitForExactTestProcessIdentitiesToExit(run.toolIdentities); + expect(run.toolPids.every((pid) => !processExists(pid))).toBe(true); + expect(processExists(unrelated.pid!)).toBe(true); + expect( + hostKillSpy.mock.calls.some(([, signal]) => signal === "SIGKILL"), + ).toBe(false); + } finally { + hostKillSpy.mockRestore(); + await cleanupRegisteredDescendantToolRun(run); + await observer.dispose(); + await stopExactTestProcess(unrelated); + } + }, + 15_000, + ); + it.skipIf(process.platform === "win32")( "cleans exact fixture and anchor groups when setup fails after PID publication", async () => { diff --git a/packages/harness/src/experimental/managed-agent-spike/process-observer.ts b/packages/harness/src/experimental/managed-agent-spike/process-observer.ts index 73c26a9ba..ef1414147 100644 --- a/packages/harness/src/experimental/managed-agent-spike/process-observer.ts +++ b/packages/harness/src/experimental/managed-agent-spike/process-observer.ts @@ -846,6 +846,15 @@ export class LocalManagedAgentProcessObserver implements ManagedAgentProcessObse } } + /** + * Simulates abrupt host loss without terminating the Vitest process. The + * exact fixture must treat authenticated lifetime-channel loss as a + * fail-closed instruction to terminate its own process group. + */ + public testOnlyDropToolLifetimeChannels(): void { + for (const socket of this.#toolControlSockets) socket.destroy(); + } + #invalidateRootContainment(root: OwnedRoot): void { root.containmentSupported = false; } From 6a0ae05fcbc289dc9ae51fae55421b678aeed21c Mon Sep 17 00:00:00 2001 From: Yash Date: Mon, 17 Aug 2026 08:04:56 -0700 Subject: [PATCH 23/24] fix(harness): bound fixture control handshake Refs SAP-2632 --- .../managed-agent-spike/README.md | 11 ++- .../managed-agent-spike/fixture.test.ts | 78 +++++++++++++++++++ .../managed-agent-spike/fixture.ts | 31 +++++++- 3 files changed, 114 insertions(+), 6 deletions(-) diff --git a/packages/harness/src/experimental/managed-agent-spike/README.md b/packages/harness/src/experimental/managed-agent-spike/README.md index cac3cf0fc..2345e4623 100644 --- a/packages/harness/src/experimental/managed-agent-spike/README.md +++ b/packages/harness/src/experimental/managed-agent-spike/README.md @@ -199,10 +199,13 @@ The fixture also fails closed if its host disappears outside that orderly sequence. After authentication, either lifetime channel closing makes the receiving fixture process terminate its own current group. Before authentication, connection failures retry only until a five-second monotonic -deadline and then terminate that same receiver-owned group. This receiver-side -behavior lets an outer process-bound supervisor close its own IPC channel and -cascade termination through nested detached fixture processes without sending -a host-side signal to a cached numeric PID or PGID. +deadline and then terminate that same receiver-owned group. An unconditional +timer enforces the same deadline when a controller accepts a connection but +never authenticates it, and every connect attempt and registration ACK rechecks +the deadline. This receiver-side behavior lets an outer process-bound +supervisor close its own IPC channel and cascade termination through nested +detached fixture processes without sending a host-side signal to a cached +numeric PID or PGID. Deadline expiry or a successful quiescence observation seals all evidence, closes the spawn gate, and permanently revokes numeric signal authority. diff --git a/packages/harness/src/experimental/managed-agent-spike/fixture.test.ts b/packages/harness/src/experimental/managed-agent-spike/fixture.test.ts index ee5ad6322..60e3bc237 100644 --- a/packages/harness/src/experimental/managed-agent-spike/fixture.test.ts +++ b/packages/harness/src/experimental/managed-agent-spike/fixture.test.ts @@ -1,8 +1,10 @@ import { execFileSync, spawn } from "node:child_process"; import { once } from "node:events"; import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import { createServer, type Socket as NetSocket } from "node:net"; import { tmpdir } from "node:os"; import { join } from "node:path"; +import { performance } from "node:perf_hooks"; import { afterEach, describe, expect, it } from "vitest"; @@ -16,6 +18,10 @@ import { verifyManagedAgentFixtureBytes, type ManagedAgentFixture, } from "./fixture.js"; +import { + MANAGED_AGENT_TOOL_CONTROL_CAPABILITY_ENV, + MANAGED_AGENT_TOOL_CONTROL_SOCKET_ENV, +} from "./process-observer.js"; const fixtures: ManagedAgentFixture[] = []; @@ -160,6 +166,78 @@ describe("managed-agent disposable git fixture", () => { }, ); + it.skipIf(process.platform === "win32")( + "self-terminates its detached group when a controller accepts but never authenticates", + { timeout: 12_000 }, + async () => { + const fixture = await createManagedAgentFixture( + () => "silent-control-registration", + ); + fixtures.push(fixture); + const controlRoot = await mkdtemp( + join(tmpdir(), "managed-agent-silent-control-"), + ); + const controlSocket = join(controlRoot, "control.sock"); + const acceptedSockets: NetSocket[] = []; + const server = createServer((socket) => { + acceptedSockets.push(socket); + socket.on("error", () => undefined); + socket.resume(); + }); + server.listen(controlSocket); + await once(server, "listening"); + const parent = spawn( + process.execPath, + [ + join(fixture.workspaceRoot, FIXTURE_PATHS.processScript), + join(fixture.workspaceRoot, FIXTURE_PATHS.processPidFile), + "--register-control", + ], + { + detached: true, + env: { + ...process.env, + [MANAGED_AGENT_TOOL_CONTROL_SOCKET_ENV]: controlSocket, + [MANAGED_AGENT_TOOL_CONTROL_CAPABILITY_ENV]: + "silent-control-capability", + }, + stdio: "ignore", + windowsHide: true, + }, + ); + await once(parent, "spawn"); + const spawnedAt = performance.now(); + let childPid: number | undefined; + + try { + childPid = await waitForDirectChildPid(parent.pid!); + const connectionDeadline = Date.now() + 2_000; + while (acceptedSockets.length < 2 && Date.now() < connectionDeadline) { + await new Promise((resolveDelay) => setTimeout(resolveDelay, 10)); + } + expect(acceptedSockets).toHaveLength(2); + expect(processExists(parent.pid!)).toBe(true); + expect(processExists(childPid)).toBe(true); + + const [exitCode, signal] = await once(parent, "exit"); + expect(exitCode).toBeNull(); + expect(signal).toBe("SIGKILL"); + expect(performance.now() - spawnedAt).toBeLessThan(6_000); + await waitForProcessExit(childPid); + } finally { + for (const socket of acceptedSockets) socket.destroy(); + await new Promise((resolveClose, rejectClose) => + server.close((error) => + error ? rejectClose(error) : resolveClose(), + ), + ); + await waitForProcessExit(parent.pid!, 7_000); + if (childPid !== undefined) await waitForProcessExit(childPid, 7_000); + await rm(controlRoot, { recursive: true, force: true }); + } + }, + ); + it("renders L1 as eleven exact ordered calls without resolving the escape link", async () => { const fixture = await createManagedAgentFixture(() => "prompt-contract"); fixtures.push(fixture); diff --git a/packages/harness/src/experimental/managed-agent-spike/fixture.ts b/packages/harness/src/experimental/managed-agent-spike/fixture.ts index 62beea119..dd63a4440 100644 --- a/packages/harness/src/experimental/managed-agent-spike/fixture.ts +++ b/packages/harness/src/experimental/managed-agent-spike/fixture.ts @@ -83,6 +83,8 @@ function shellQuote(value: string): string { return `'${value.split("'").join(`'"'"'`)}'`; } +const TOOL_CONTROL_REGISTRATION_TIMEOUT_MS = 5_000; + const LONG_RUNNING_SCRIPT = ` import { spawn } from "node:child_process"; import { existsSync, readFileSync, unlinkSync, writeFileSync } from "node:fs"; @@ -106,7 +108,8 @@ const readinessDelayMs = Number.isSafeInteger(parsedReadinessDelay) && : 0; const controlSocket = process.env[${JSON.stringify(MANAGED_AGENT_TOOL_CONTROL_SOCKET_ENV)}]; const controlCapability = process.env[${JSON.stringify(MANAGED_AGENT_TOOL_CONTROL_CAPABILITY_ENV)}]; -const controlRegistrationDeadlineAt = performance.now() + 5_000; +const controlRegistrationTimeoutMs = ${TOOL_CONTROL_REGISTRATION_TIMEOUT_MS}; +const controlRegistrationDeadlineAt = performance.now() + controlRegistrationTimeoutMs; if (requireControlRegistration && (!controlSocket || !controlCapability)) { throw new Error("managed-agent tool control capability missing"); } @@ -117,7 +120,8 @@ const childProgram = [ 'const requireControlRegistration = ' + JSON.stringify(requireControlRegistration) + ';', 'const controlSocket = process.env["${MANAGED_AGENT_TOOL_CONTROL_SOCKET_ENV}"];', 'const controlCapability = process.env["${MANAGED_AGENT_TOOL_CONTROL_CAPABILITY_ENV}"];', - 'const controlRegistrationDeadlineAt = performance.now() + 5_000;', + 'const controlRegistrationTimeoutMs = ' + JSON.stringify(controlRegistrationTimeoutMs) + ';', + 'const controlRegistrationDeadlineAt = performance.now() + controlRegistrationTimeoutMs;', 'delete process.env["${MANAGED_AGENT_TOOL_CONTROL_SOCKET_ENV}"];', 'delete process.env["${MANAGED_AGENT_TOOL_CONTROL_CAPABILITY_ENV}"];', 'const readinessDelayMs = ' + JSON.stringify(readinessDelayMs) + ';', @@ -128,6 +132,10 @@ const childProgram = [ 'const terminateOwnedGroup = () => {', ' try { process.kill(0, "SIGKILL"); } catch { process.exit(1); }', '};', + 'const controlRegistrationTimer = requireControlRegistration', + ' ? setTimeout(terminateOwnedGroup, Math.max(0, controlRegistrationDeadlineAt - performance.now()))', + ' : undefined;', + 'controlRegistrationTimer?.unref();', 'const publishReady = () => {', ' if (readyPublished) return;', ' readyPublished = true;', @@ -136,6 +144,7 @@ const childProgram = [ '};', 'const connectControl = () => {', ' if (!controlSocket || !controlCapability) { publishReady(); return; }', + ' if (performance.now() >= controlRegistrationDeadlineAt) { terminateOwnedGroup(); return; }', ' const socket = createConnection(controlSocket);', ' socket.unref();', ' socket.setEncoding("utf8");', @@ -166,7 +175,9 @@ const childProgram = [ ' }', ' if (!response.includes("\\\\n")) return;', ' if (!response.includes(' + JSON.stringify('"registered":true') + ')) { socket.destroy(); return; }', + ' if (performance.now() >= controlRegistrationDeadlineAt) { terminateOwnedGroup(); return; }', ' registered = true;', + ' if (controlRegistrationTimer) clearTimeout(controlRegistrationTimer);', ' publishReady();', ' });', ' socket.once("error", retry);', @@ -191,6 +202,13 @@ let controlReady = !requireControlRegistration; const terminateOwnedGroup = () => { try { process.kill(0, "SIGKILL"); } catch { process.exit(1); } }; +const controlRegistrationTimer = requireControlRegistration + ? setTimeout( + terminateOwnedGroup, + Math.max(0, controlRegistrationDeadlineAt - performance.now()), + ) + : undefined; +controlRegistrationTimer?.unref(); const publishReadiness = () => { if (!childReady || !controlReady) return; try { @@ -207,6 +225,10 @@ child.once("message", () => { }); const connectControl = () => { if (!controlSocket || !controlCapability) return; + if (performance.now() >= controlRegistrationDeadlineAt) { + terminateOwnedGroup(); + return; + } const socket = createConnection(controlSocket); socket.unref(); socket.setEncoding("utf8"); @@ -241,7 +263,12 @@ const connectControl = () => { if (!response.includes('"registered":true')) { throw new Error("managed-agent tool registration rejected"); } + if (performance.now() >= controlRegistrationDeadlineAt) { + terminateOwnedGroup(); + return; + } registered = true; + if (controlRegistrationTimer) clearTimeout(controlRegistrationTimer); controlReady = true; publishReadiness(); }); From 830811cc97386f83b0d95fd3ec6fa965e11df8b7 Mon Sep 17 00:00:00 2001 From: Yash Date: Mon, 17 Aug 2026 23:57:52 -0700 Subject: [PATCH 24/24] fix(harness): normalize SDK Bash permission input Accept only the pinned SDK Bash fields, reject unsafe controls, and canonicalize allowed execution to the exact allowlisted command. Refs: SAP-2632 --- .../managed-agent-spike/README.md | 7 +- .../managed-agent-spike/permissions.test.ts | 189 ++++++++++++++++-- .../managed-agent-spike/permissions.ts | 87 ++++++-- .../runtime-sdk-loopback.test.ts | 5 +- 4 files changed, 243 insertions(+), 45 deletions(-) diff --git a/packages/harness/src/experimental/managed-agent-spike/README.md b/packages/harness/src/experimental/managed-agent-spike/README.md index 2345e4623..409bdaf32 100644 --- a/packages/harness/src/experimental/managed-agent-spike/README.md +++ b/packages/harness/src/experimental/managed-agent-spike/README.md @@ -9,9 +9,10 @@ Codex flows. Every model-requested Read, Edit, Write, Bash, and in-process MCP call is gated by one programmatic `PreToolUse` hook registered without a matcher. The hook runs before the SDK's permission evaluation, applies canonical-path containment, -exact Bash equality, an exact Bash input shape of `{ command: string }`, and an -MCP allowlist, and returns a complete fresh input object only when allowing the -call. Extra SDK fields such as background execution or timeout controls fail +exact Bash equality, a pinned SDK-compatible Bash input shape, and an MCP +allowlist, and returns a complete fresh input object only when allowing the +call. Valid description and timeout metadata are stripped before execution; +background execution, sandbox bypass, malformed values, and unknown fields fail closed even when the command string itself matches. Unknown tools fail closed. The hook also requires a non-empty, bounded `tool_use_id` from the event. When diff --git a/packages/harness/src/experimental/managed-agent-spike/permissions.test.ts b/packages/harness/src/experimental/managed-agent-spike/permissions.test.ts index 59d7b4be6..2e451f459 100644 --- a/packages/harness/src/experimental/managed-agent-spike/permissions.test.ts +++ b/packages/harness/src/experimental/managed-agent-spike/permissions.test.ts @@ -238,11 +238,19 @@ describe("managed-agent universal policy boundary", () => { await expect( invoke( "Bash", - { command: "node .managed-agent-probe/long-running.mjs" }, + { + command: "node .managed-agent-probe/long-running.mjs", + description: "Run the cancellation fixture", + }, "l2-bash", ), ).resolves.toMatchObject({ - hookSpecificOutput: { permissionDecision: "allow" }, + hookSpecificOutput: { + permissionDecision: "allow", + updatedInput: { + command: "node .managed-agent-probe/long-running.mjs", + }, + }, }); expect( @@ -266,6 +274,109 @@ describe("managed-agent universal policy boundary", () => { ]); }); + it("accepts pinned SDK Bash metadata but strips it before execution", async () => { + const command = "git status --short"; + const descriptionMarker = "sdk-description-must-not-persist"; + const unknownMarker = "unknown-field-must-not-persist"; + const evidence: ManagedAgentPermissionEvidence[] = []; + const boundary = createManagedAgentPolicyBoundary({ + canonicalWorkspaceRoot: workspace, + allowedBashCommands: [command], + allowedMcpTools: [], + onDecision: (decision) => evidence.push(decision), + }); + const signal = new AbortController().signal; + let sequence = 0; + const invoke = (input: unknown) => { + const toolUseId = `bash-shape-${++sequence}`; + return boundary.preToolUseHook( + preToolUseInput("Bash", input, toolUseId), + toolUseId, + { signal }, + ); + }; + const acceptedInputs: Array> = [ + { command }, + { command, description: descriptionMarker }, + { command, timeout: 1 }, + { + command, + description: descriptionMarker, + timeout: 600_000, + run_in_background: false, + dangerouslyDisableSandbox: false, + }, + ]; + + for (const input of acceptedInputs) { + const originalInput = { ...input }; + await expect(invoke(input)).resolves.toEqual({ + hookSpecificOutput: { + hookEventName: "PreToolUse", + permissionDecision: "allow", + permissionDecisionReason: "Managed-agent policy: exact_bash_command", + updatedInput: { command }, + }, + }); + expect(input).toEqual(originalInput); + } + + const deniedInputs: Array> = [ + { command, unexpected: unknownMarker }, + { command, description: 123 }, + { command, timeout: "10" }, + { command, timeout: 0 }, + { command, timeout: 1.5 }, + { command, timeout: 600_001 }, + { command, run_in_background: "false" }, + { command, run_in_background: true }, + { command, dangerouslyDisableSandbox: "false" }, + { command, dangerouslyDisableSandbox: true }, + ]; + for (const input of deniedInputs) { + await expect(invoke(input)).resolves.toMatchObject({ + hookSpecificOutput: { + permissionDecision: "deny", + permissionDecisionReason: expect.stringContaining("invalid_input"), + }, + }); + } + + expect( + evidence + .slice(0, acceptedInputs.length) + .map(({ decision, reason, operationId }) => ({ + decision, + reason, + operationId, + })), + ).toEqual( + acceptedInputs.map(() => ({ + decision: "allow", + reason: "exact_bash_command", + operationId: "bash:exact_command", + })), + ); + expect( + evidence + .slice(acceptedInputs.length) + .map(({ decision, reason, operationId }) => ({ + decision, + reason, + operationId, + })), + ).toEqual( + deniedInputs.map(() => ({ + decision: "deny", + reason: "invalid_input", + operationId: "bash:unregistered", + })), + ); + const serializedEvidence = JSON.stringify(evidence); + expect(serializedEvidence).not.toContain(descriptionMarker); + expect(serializedEvidence).not.toContain(unknownMarker); + }); + it("uses exact Bash equality and emits content-free decisions", async () => { const evidence: ManagedAgentPermissionEvidence[] = []; const boundary = createManagedAgentPolicyBoundary({ @@ -316,7 +427,10 @@ describe("managed-agent universal policy boundary", () => { }, }); await expect( - invoke("Bash", { command: "git status --short", timeout: 10 }), + invoke("Bash", { + command: "git status --short", + dangerouslyDisableSandbox: true, + }), ).resolves.toMatchObject({ hookSpecificOutput: { permissionDecision: "deny", @@ -623,26 +737,63 @@ describe("managed-agent universal policy boundary", () => { }); expect(evidence).toHaveLength(1); + const fallbackInput = { + command: "git status --short", + description: "Show working tree status", + timeout: 10_000, + run_in_background: false, + dangerouslyDisableSandbox: false, + }; await expect( - boundary.canUseToolFallback( - "Bash", - { command: "git status --short" }, - { signal, toolUseID: "fallback-only", requestId: "request-2" }, - ), - ).resolves.toMatchObject({ behavior: "allow" }); + boundary.canUseToolFallback("Bash", fallbackInput, { + signal, + toolUseID: "fallback-only", + requestId: "request-2", + }), + ).resolves.toEqual({ + behavior: "allow", + toolUseID: "fallback-only", + updatedInput: { command: "git status --short" }, + }); + expect(fallbackInput).toEqual({ + command: "git status --short", + description: "Show working tree status", + timeout: 10_000, + run_in_background: false, + dangerouslyDisableSandbox: false, + }); expect(evidence).toHaveLength(2); expect(evidence[1]?.source).toBe("can_use_tool_fallback"); - await expect( - boundary.canUseToolFallback( - "Bash", - { command: "git status --short", run_in_background: true }, - { signal, toolUseID: "fallback-extra", requestId: "request-3" }, - ), - ).resolves.toMatchObject({ - behavior: "deny", - message: expect.stringContaining("invalid_input"), - }); + const deniedFallbackInputs = [ + { + toolUseID: "fallback-background", + input: { command: "git status --short", run_in_background: true }, + }, + { + toolUseID: "fallback-sandbox", + input: { + command: "git status --short", + dangerouslyDisableSandbox: true, + }, + }, + { + toolUseID: "fallback-unknown", + input: { command: "git status --short", unsupported: true }, + }, + ]; + for (const { toolUseID: deniedToolUseID, input } of deniedFallbackInputs) { + await expect( + boundary.canUseToolFallback("Bash", input, { + signal, + toolUseID: deniedToolUseID, + requestId: `request-${deniedToolUseID}`, + }), + ).resolves.toMatchObject({ + behavior: "deny", + message: expect.stringContaining("invalid_input"), + }); + } }); it("fails closed when aborted before or during asynchronous path validation", async () => { diff --git a/packages/harness/src/experimental/managed-agent-spike/permissions.ts b/packages/harness/src/experimental/managed-agent-spike/permissions.ts index 27535a0c8..996ad3f2f 100644 --- a/packages/harness/src/experimental/managed-agent-spike/permissions.ts +++ b/packages/harness/src/experimental/managed-agent-spike/permissions.ts @@ -232,17 +232,66 @@ function asRecord(value: unknown): Record | undefined { : undefined; } -function hasExactKeys( - value: Record, - expectedKeys: readonly string[], -): boolean { - const actualKeys = Object.keys(value); - return ( - actualKeys.length === expectedKeys.length && - expectedKeys.every((key) => - Object.prototype.hasOwnProperty.call(value, key), +const MANAGED_AGENT_BASH_INPUT_KEYS = new Set([ + "command", + "timeout", + "description", + "run_in_background", + "dangerouslyDisableSandbox", +]); +const MANAGED_AGENT_BASH_TIMEOUT_MAX_MS = 600_000; + +interface ManagedAgentNormalizedBashInput { + readonly command: string; +} + +/** + * Accept the pinned SDK's Bash input shape, but retain only the exact command + * that the host authorizes and executes. Model-authored execution controls are + * either harmless metadata that is stripped or unsafe values that fail closed. + */ +function normalizeManagedAgentBashInput( + rawInput: unknown, +): ManagedAgentNormalizedBashInput | undefined { + const input = asRecord(rawInput); + if ( + !input || + !Object.prototype.hasOwnProperty.call(input, "command") || + Reflect.ownKeys(input).some( + (key) => + typeof key !== "string" || !MANAGED_AGENT_BASH_INPUT_KEYS.has(key), ) - ); + ) { + return undefined; + } + + const command = input.command; + if (typeof command !== "string" || command.length === 0) return undefined; + if ( + input.description !== undefined && + typeof input.description !== "string" + ) { + return undefined; + } + if ( + input.timeout !== undefined && + (typeof input.timeout !== "number" || + !Number.isInteger(input.timeout) || + input.timeout <= 0 || + input.timeout > MANAGED_AGENT_BASH_TIMEOUT_MAX_MS) + ) { + return undefined; + } + if ( + (input.run_in_background !== undefined && + input.run_in_background !== false) || + (input.dangerouslyDisableSandbox !== undefined && + input.dangerouslyDisableSandbox !== false) + ) { + return undefined; + } + + return { command }; } function denied( @@ -275,10 +324,8 @@ function classifyManagedAgentOperation( ): ManagedAgentOperationId { const input = asRecord(rawInput); if (toolName === "Bash") { - return input && - hasExactKeys(input, ["command"]) && - typeof input.command === "string" && - allowedCommands.has(input.command) + const normalizedInput = normalizeManagedAgentBashInput(rawInput); + return normalizedInput && allowedCommands.has(normalizedInput.command) ? "bash:exact_command" : "bash:unregistered"; } @@ -332,13 +379,9 @@ async function evaluateManagedAgentPolicy( }; } if (toolName === "Bash") { - if (!hasExactKeys(input, ["command"])) { - return denied("invalid_input", operationId); - } - const command = - typeof input.command === "string" ? input.command : undefined; - if (!command) return denied("invalid_input", operationId); - if (!allowedCommands.has(command)) { + const normalizedInput = normalizeManagedAgentBashInput(input); + if (!normalizedInput) return denied("invalid_input", operationId); + if (!allowedCommands.has(normalizedInput.command)) { return denied("bash_command_not_allowed", operationId); } return signal.aborted @@ -347,7 +390,7 @@ async function evaluateManagedAgentPolicy( decision: "allow", reason: "exact_bash_command", operationId, - updatedInput: { ...input }, + updatedInput: { command: normalizedInput.command }, }; } if (toolName === "Read" || toolName === "Edit" || toolName === "Write") { diff --git a/packages/harness/src/experimental/managed-agent-spike/runtime-sdk-loopback.test.ts b/packages/harness/src/experimental/managed-agent-spike/runtime-sdk-loopback.test.ts index 5d42b4aba..9b3e96034 100644 --- a/packages/harness/src/experimental/managed-agent-spike/runtime-sdk-loopback.test.ts +++ b/packages/harness/src/experimental/managed-agent-spike/runtime-sdk-loopback.test.ts @@ -425,7 +425,10 @@ it("enforces real-SDK built-in and in-process MCP calls with exact loopback corr writeToolUseResponse(response, inferenceTurn, { id: "toolu_loopback_bash_allow", name: "Bash", - input: { command: ALLOWED_BASH_COMMAND }, + input: { + command: ALLOWED_BASH_COMMAND, + description: "Show working tree status", + }, }); } else if (inferenceTurn === 3) { writeToolUseResponse(response, inferenceTurn, {