diff --git a/.changeset/managed-agent-spike.md b/.changeset/managed-agent-spike.md new file mode 100644 index 000000000..eb62702ec --- /dev/null +++ b/.changeset/managed-agent-spike.md @@ -0,0 +1,5 @@ +--- +"@sapiom/harness": minor +--- + +Add an experimental, programmatic managed-agent probe for validating isolated Agent SDK tools, permissions, cancellation, and workspace preservation. diff --git a/.github/workflows/harness.yml b/.github/workflows/harness.yml index d6a10c4ce..71b1f86a9 100644 --- a/.github/workflows/harness.yml +++ b/.github/workflows/harness.yml @@ -56,6 +56,32 @@ concurrency: cancel-in-progress: true jobs: + # Real SDK transport sentinel. This exercises the pinned bundled runtime and + # must not silently skip when the repository's floating Node matrix moves. + managed-agent-sdk-sentinel: + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + with: + persist-credentials: false + + - uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v4.4.0 + + - name: Use certification Node.js + uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 + with: + node-version: "22.23.2" + cache: "pnpm" + + - name: Install dependencies + run: pnpm install --frozen-lockfile + + - name: Run real Agent SDK loopback sentinels + run: >- + pnpm --filter @sapiom/harness exec vitest run + src/experimental/managed-agent-spike/runtime-sdk-loopback.test.ts + # Playwright mock-mode tier — harness web/e2e specs (VITE_MOCK=1, chromium only). # The live/real-pty tiers (e2e:live, sim) are opt-in local only; nothing here # invokes them — they require real agent binaries and credentials not in CI. diff --git a/packages/harness/package.json b/packages/harness/package.json index f75662422..878e544d1 100644 --- a/packages/harness/package.json +++ b/packages/harness/package.json @@ -29,6 +29,10 @@ "types": "./dist/index.d.ts", "import": "./dist/index.js" }, + "./experimental/managed-agent-spike": { + "types": "./dist/experimental/managed-agent-spike/index.d.ts", + "import": "./dist/experimental/managed-agent-spike/index.js" + }, "./package.json": "./package.json" }, "bin": { @@ -54,6 +58,7 @@ "test:mutation": "stryker run", "test:ui": "playwright test --config web/e2e/playwright.config.ts", "test:canvas": "playwright test --config e2e/playwright.config.ts", + "probe:managed-agent": "tsx src/experimental/managed-agent-spike/probe-cli.ts", "typecheck": "tsc --noEmit && tsc --noEmit -p web/tsconfig.json", "lint": "eslint src --ext .ts", "prepublishOnly": "pnpm build", @@ -63,6 +68,9 @@ "e2e:live": "tsx scripts/e2e-live.ts" }, "dependencies": { + "@anthropic-ai/claude-agent-sdk": "0.3.228", + "@anthropic-ai/sdk": "0.116.0", + "@modelcontextprotocol/sdk": "1.30.0", "@sapiom/agent": "workspace:^", "@sapiom/agent-core": "workspace:^", "@sapiom/analytics-core": "workspace:^", @@ -73,7 +81,7 @@ "node-pty": "^1.1.0", "open": "^10.1.0", "ws": "^8.18.0", - "zod": "^3.25.0" + "zod": "4.4.3" }, "devDependencies": { "@playwright/test": "^1.61.0", diff --git a/packages/harness/src/experimental/managed-agent-spike/README.md b/packages/harness/src/experimental/managed-agent-spike/README.md new file mode 100644 index 000000000..409bdaf32 --- /dev/null +++ b/packages/harness/src/experimental/managed-agent-spike/README.md @@ -0,0 +1,277 @@ +# Managed Agent Feasibility Spike + +This subpath is an Epic 0 probe, not a production Harness runtime. It does not +change the PTY adapter, Studio UI, session history, or existing Claude Code and +Codex flows. + +## Host policy boundary + +Every model-requested Read, Edit, Write, Bash, and in-process MCP call is gated +by one programmatic `PreToolUse` hook registered without a matcher. The hook +runs before the SDK's permission evaluation, applies canonical-path containment, +exact Bash equality, a pinned SDK-compatible Bash input shape, and an MCP +allowlist, and returns a complete fresh input object only when allowing the +call. Valid description and timeout metadata are stripped before execution; +background execution, sandbox bypass, malformed values, and unknown fields fail +closed even when the command string itself matches. Unknown tools fail closed. + +The hook also requires a non-empty, bounded `tool_use_id` from the event. When +the SDK supplies the optional callback ID, it must be independently bounded and +exactly match the event ID. Invalid identifiers are denied before policy +evaluation and never become normalized permission evidence. + +`canUseTool` remains only as defense in depth for calls the SDK leaves +unresolved. It shares the same evaluator and deduplicates by tool-use ID, so it +cannot create a second evidence record. A live result is rejected when any +requested tool lacks exactly one primary `PreToolUse` decision. This detects a +hook that was skipped, but detection after execution is not by itself a host +boundary. + +Before `query()` is created, a credential-free subprocess rooted in the +probe's isolated HOME and `CLAUDE_CONFIG_DIR` calls SDK `resolveSettings()`. +`disableAllHooks`, resolution errors, timeouts, malformed output, and configured +`policyHelper`/`policyHelpers` all produce `policy_violation` without creating a +query. Policy helpers fail closed because SDK 0.3.228 does not execute them in +`resolveSettings()` and therefore cannot prove parity with query startup. + +The subprocess requires a Node executable. Every direct gateway invocation, +including calls through the exported programmatic runtime, requires exact Node +22.23.2. Hermetic tests may use another Node only through the explicit injected +gateway/query seam. Electron-as-Node and packaged executable resolution are +deliberately deferred to E0.7. The runtime also exposes only a narrow +async-iterator/close query interface. It does not expose or call SDK +`Query.mcpCall()`, whose trusted control channel bypasses permission checks. + +## Correlation and turn evidence + +The runtime sends `x-sapiom-eval-source` and `x-sapiom-execution-id`, then embeds +the same non-secret values in the initial prompt as: + +```text +SAPIOM_CERTIFICATION_CORRELATION_V1;eval_source=;execution_id= +``` + +`correlation.promptEmbedded` records whether that marked prompt was handed to +the query factory. It remains false when the settings preflight prevents the +factory invocation. + +The production gateway consumes both headers, but its current BigQuery +projection persists neither `polsia_eval_source` nor `sapiom_execution_id`. +Reconciliation therefore follows the existing E0.2 contract and searches the +replayed initial prompt marker. The authoritative SDK-side inference count is +the number of distinct assistant message IDs. IDs are hashed and counted only +in memory; raw or hashed IDs are not emitted. SDK `result.num_turns` is retained +separately as bounded informational evidence and is not used as the BigQuery +call-count key. + +The durable result also records content-free, non-authoritative SDK model +evidence. A completed L1 run must observe the selected alias in both the SDK +init event and the sole `result.modelUsage` key. A cancelled L2 run may have no +result event, so it requires the matching init model and then relies on gateway +reconciliation. These checks prove what the SDK reported, not what the gateway +served: BigQuery provider/model, fallback, token, and cost rows remain the +authoritative exact-deployment evidence for every live inference turn. +Accordingly, the local report uses `outcome: "local_pass"`, labels the check +`sdk_model_alias_observed`, and always emits +`deploymentProvenance: "requires_gateway_reconciliation"`. It never calls a +locally passing run deployment-certified. + +The hermetic pinned-SDK loopback exercises Read, allowed and denied Bash, and a +real in-process `echo_nonce` MCP turn. It requires one primary `PreToolUse` +decision for each request and separately verifies the MCP handler invocation +and SDK tool-result event. + +## L1 certification contract v2 + +L1 is independently versioned as `managed-agent-l1-prompt-v2` and +`managed-agent-l1-evaluator-v2` while the transport result remains contract +version 1. The frozen prompt contains 11 canonical calls. It permits at most +one additional verification Read, only after both denial probes and before the +Edit, and only for the clean target, dirty sentinel, or untracked sentinel. + +The host registers the six prompt path literals under content-free roles. Role +lookup uses normalized lexical path identity before realpath containment so an +SDK-normalized absolute path retains the same role as its relative prompt +literal. A different in-workspace path remains unregistered and is denied. +Permission evidence contains only an operation ID such as +`read:clean_target`; it never contains the raw path or tool input. + +The evaluator requires every canonical request ID to be non-empty and unique, +with exactly one matching completion and one primary `PreToolUse` decision. +Fallback-only decisions, duplicate or orphan evidence, mismatched tools, +reordering, omission, retries, and every other extra operation fail closed. +The optional Read count and role are reported separately as nonblocking +efficiency evidence. + +Requests keep their exact canonical order while completions may be permuted +within the two batchable phases. Every request must precede its own completion; +all five discovery completions must precede the optional verification Read (or +Edit when it is absent), the optional Read must complete before Edit, and the +Edit/Write/MCP phase must complete before the recovery retry. The first +`fail_once` error must complete before its retry, and that retry must complete +before Bash. The evaluator additionally requires at least four distinct +assistant inference turns, plus one when the optional Read is present. This +accepts observed SDK batching without allowing an all-requests-first trace to +masquerade as multi-turn recovery. + +Normalized tool and permission events must be an exact chronological +projection of their evidence arrays. Each primary permission event must precede +its matching tool completion. It may appear before or after the matching request +event because the SDK hook and yielded message have independent observation +order. A successful Bash completion must precede the single successful SDK +result, which must precede the final successful terminal event. + +Filesystem acceptance is also exact: only the clean target may be modified and +only the managed output may be created, in either evidence order. Trusted +SHA-256 expectations prove the final bytes of both mutation targets, while the +durable result exposes only `{ role, matched }`. The dirty and untracked +sentinels must remain byte-identical, and successful `echo_nonce` handling is +recorded as a content-free nonce-verification boolean. + +## L2 cancellation containment boundary + +E0.4 certifies one deliberately narrow host model: the exact non-cooperative +fixture command running below an observer-owned Agent SDK supervisor on macOS or +Linux. The exact fixture parent and child each authenticate over a separate +private Unix-socket connection and keep that connection open for their complete +lifetime. Before cancellation may fire, a fresh bounded `ps` sample must observe +an active SDK supervisor root with stable identity, both role-tagged PIDs, their +parent-child relationship, their shared process group, and every current group +member as a descendant of that owned root. The group must be distinct from both +the host and SDK supervisor groups. Every observed root/tool descendant retains +an immutable creation-time, parent, process-group, and session baseline. Once L2 +tool containment is armed, every current root descendant must remain in the +supervisor group or authenticated tool group; reparenting or PGID/session +migration fails closed. The random capability and role-tagged lifetime channels +are necessary evidence, but a claimed or cached PID/PGID never grants signal +authority by itself. The model-writable fixture PID file is used only by the +test driver and never enters the observer. + +Outside that L2 gate, an unarmed L1 run can briefly create an SDK-owned +subprocess group while the process is still a descendant of the supervisor. +That subgroup is never signal authority. The observer remembers each exact +identity and treats it as pending: +readiness and quiescence remain false, and fallback cannot kill the supervisor +root, while any pending identity is live. A later complete process-table sample +may clear it only by proving that exact identity is absent or a zombie. Any +parent, group, session, or ancestry change before that positive death evidence +permanently fails containment closed. Once L2 tool containment is armed, only +the authenticated supervisor and fixture groups are permitted; an additional +descendant group rejects readiness. + +The runtime creates one immutable monotonic deadline and makes the observer +adopt that same object before the first abort, `Query.close()`, or +`Query.return()`. An SDK-forwarded signal observed before adoption is remembered +but grants no signal authority until the bounded deadline exists. The runtime +then gives the Agent SDK its documented abort and bounded query-close path +first. It does not bind the raw per-run `Options.abortController` to host +signals. Only the SDK-forwarded post-grace `SpawnOptions.signal` can trigger the +fallback. In SDK 0.3.228, `Query.close()` starts cleanup but returns `void`, so +the runtime immediately follows it with and awaits `Query.return()` under the +same deadline. `queryClosed` means that awaitable cleanup settled; invoking +`close()` alone is never completion evidence. Host emergency cleanup starts +only after that cleanup settles, the forwarded signal has already requested the +fallback, or the bounded SDK-grace budget expires. The returned process handle +keeps the native `ChildProcess.kill()` contract. SDK `SIGTERM` calls are really +sent to the observer-owned supervisor, whose explicit TERM/INT/HUP handlers keep +the ancestry anchor alive for the bounded fallback. The hermetic real-SDK +loopback test is the sequence sentinel for 0.3.228's close/return, native-kill, +forwarded-abort, and fallback behavior. + +The forwarded abort signal requests the sampled host fallback and invalidates +every sample started before teardown. A complete post-request sample must +revalidate the active root identity, both role identities, their relationship +and shared group, every current root/tool descendant's parent, group, session, +and ancestry, and at least one open lifetime channel. This evidence never +authorizes a host-side numeric signal. Instead, the observer writes a forced- +termination request to a still-open authenticated fixture socket. That exact +process instance calls `kill(0, SIGKILL)` and therefore terminates only its own +current group. The request invalidates its authorizing sample. A second complete +sample must prove the fixture group absent before the observer disconnects the +retained supervisor IPC channel; that exact supervisor instance then terminates +its own current group. A third complete sample proves the root group absent. +Failed channel requests remain retryable, but every attempt moves to a new +sample generation and requires another fresh proof. The five-second absolute +deadline bounds the entire sequence. A PID or PGID can disappear and be reused +between any sample and request without redirecting termination, because neither +channel is addressed by that number. + +The fixture also fails closed if its host disappears outside that orderly +sequence. After authentication, either lifetime channel closing makes the +receiving fixture process terminate its own current group. Before +authentication, connection failures retry only until a five-second monotonic +deadline and then terminate that same receiver-owned group. An unconditional +timer enforces the same deadline when a controller accepts a connection but +never authenticates it, and every connect attempt and registration ACK rechecks +the deadline. This receiver-side behavior lets an outer process-bound +supervisor close its own IPC channel and cascade termination through nested +detached fixture processes without sending a host-side signal to a cached +numeric PID or PGID. + +Deadline expiry or a successful quiescence observation seals all evidence, +closes the spawn gate, and permanently revokes numeric signal authority. +Disposal never signals a cached PID or PGID, even if child exit delivery lags +or the number is reused. It instead asks authenticated fixture sockets to shut +down and disconnects the retained, observer-created supervisor IPC handle; the +still-running supervisor can kill only its own exact process group. + +If the root exits, a stable identity changes parent/group/session, a foreign +member appears, ancestry is lost, both channels close prematurely, or a +process-table read is unavailable, the observer never requests detached-group +termination. This includes an inner SDK command exit that reparents a surviving +descendant: an unchanged old PGID does not retain authority after ancestry is +lost. A successful complete table that no longer contains the stable identity +is positive exit evidence; otherwise an escaped same-identity PID and its new +group remain in final liveness accounting. The observer may still ask its exact +live SDK supervisor over retained IPC to terminate itself, but the run remains +a fail-closed `teardown_timeout` while any tool process or lifetime channel +remains. This also prevents numeric PID/PGID reuse from converting cached +evidence into authority. +`forceKillIssued` describes only owned SDK supervisor roots and is not required +when SDK graceful shutdown succeeds. + +If an exact Bash launch is armed and the query settles before readiness—by +throwing, clean iterator completion, or an SDK error result—its registration +task is not discarded. Readiness, SDK abort/close/return, owned-root fallback, +and death confirmation share one absolute five-second clock. Such an early +settlement never fabricates `cancellationRequested`. Safe L2 completion requires +that both authenticated lifetime channels were observed, both closed, and a +fresh table/liveness sample found no member of the observed fixture group. A +missing channel, an open channel, or a live observed group produces +`teardown_timeout`; later test/campaign-owned cleanup cannot turn that result +into a pass. Workspace snapshots and result assembly occur afterward. The +close-deadline timer remains referenced so a CLI host cannot exit before +cleanup and result reporting finish. + +An unavailable or timed-out process table is explicit unknown evidence, never +an empty process table. Closed pending registrations release their role so the +trusted fixture may retry; duplicate live roles fail closed. A capability holder +can at worst deny certification—it cannot make the host signal an unrelated +group. Invalid observation, unknown group liveness, exhausted signal retries, +and Windows all fail closed. Windows live L2 is rejected before the query or +credential is opened. The detached-group path is limited to this exact fixture; +universal Bash containment, other command shapes, Windows Job Objects, and +production recovery belong to later epics, and this probe does not claim those +guarantees. + +The disposable fixture uses a host-owned lifetime lease outside the writable +workspace. The lease exists before launch; `shutdown` contents or a missing +lease both make the fixture parent stop its child and exit. Cleanup can therefore +remove the temporary root without turning a startup race into a permanently +running process. The child also exits on IPC disconnect, and a failed +readiness-file publication shuts it down before the parent exits; a hermetic +regression removes the real fixture root during delayed readiness and verifies +that neither process remains. + +## Pre-v2 live evidence + +The exact-trace-v1 campaign completed both Sonnet 5 L1 repetitions and the +first MiniMax M3 L1 repetition. The second M3 L1 run had a successful terminal +result, exact model provenance, complete primary permission coverage, and clean +teardown, but made one additional allowed in-root verification Read. The v1 +exact-trace evaluator rejected that run and the campaign stopped immediately; +no L2 run followed. + +Those runs remain diagnostic history, not v2 acceptance evidence. Do not +restart the paid L1/L2 matrix until this v2 correction has clean CI, +independent review, and explicit authorization. diff --git a/packages/harness/src/experimental/managed-agent-spike/contract.test.ts b/packages/harness/src/experimental/managed-agent-spike/contract.test.ts new file mode 100644 index 000000000..4d945ce24 --- /dev/null +++ b/packages/harness/src/experimental/managed-agent-spike/contract.test.ts @@ -0,0 +1,262 @@ +import { mkdir, mkdtemp, realpath, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import { afterEach, describe, expect, it } from "vitest"; + +import { + MANAGED_AGENT_CONTRACT, + MANAGED_AGENT_L1_CERTIFICATION_CONTRACT, + MANAGED_AGENT_L1_FINAL_BYTE_ROLES, + MANAGED_AGENT_L1_REGISTERED_PATH_ROLES, + MANAGED_AGENT_MODEL_TARGETS, + ManagedAgentConfigurationError, + assertManagedAgentDirectGatewayOrigin, + normalizeManagedAgentGatewayOrigin, + normalizeManagedAgentHermeticGatewayOrigin, + resolveManagedAgentModelTarget, + validateManagedAgentProbeConfig, +} from "./contract.js"; +import type { ManagedAgentProbeConfig } from "./types.js"; + +const roots: string[] = []; + +async function config(): Promise { + const root = await mkdtemp(join(tmpdir(), "managed-agent-contract-")); + roots.push(root); + const workspaceRoot = join(root, "workspace"); + const configRoot = join(root, "config"); + await Promise.all([mkdir(workspaceRoot), mkdir(configRoot)]); + return { + scenario: "L1", + workspaceRoot, + configRoot, + target: "sonnet-5", + gatewayOrigin: MANAGED_AGENT_CONTRACT.directGatewayOrigin, + gatewayCredential: "dedicated-eval-key", + prompt: `${MANAGED_AGENT_L1_CERTIFICATION_CONTRACT.promptMarker}\nprobe`, + maxTurns: 10, + maxBudgetUsd: 0.25, + allowedBashCommands: ["git status --short"], + pathRoleBindings: [ + { path: "clean.txt", role: "clean_target" }, + { path: "dirty.txt", role: "dirty_sentinel" }, + { path: "untracked.txt", role: "untracked_sentinel" }, + { path: "created.txt", role: "managed_output" }, + { path: "../outside.txt", role: "outside_sentinel" }, + { path: "escape.txt", role: "escape_link" }, + ], + expectedL1FinalBytes: [ + { path: "clean.txt", role: "clean_target", sha256: "a".repeat(64) }, + { path: "created.txt", role: "managed_output", sha256: "b".repeat(64) }, + ], + expectedMcpNonce: "probe-nonce", + }; +} + +afterEach(async () => { + await Promise.all( + roots.splice(0).map((root) => rm(root, { recursive: true, force: true })), + ); +}); + +describe("managed-agent contract", () => { + it("freezes the versioned L1 prompt and evaluator contract", () => { + expect(MANAGED_AGENT_L1_CERTIFICATION_CONTRACT).toEqual({ + contractVersion: 2, + promptVersion: "managed-agent-l1-prompt-v2", + promptMarker: "SAPIOM_MANAGED_AGENT_L1_PROMPT_V2", + evaluatorVersion: "managed-agent-l1-evaluator-v2", + }); + expect(Object.isFrozen(MANAGED_AGENT_L1_CERTIFICATION_CONTRACT)).toBe(true); + expect(Object.isFrozen(MANAGED_AGENT_L1_REGISTERED_PATH_ROLES)).toBe(true); + expect(Object.isFrozen(MANAGED_AGENT_L1_FINAL_BYTE_ROLES)).toBe(true); + }); + + it("pins the certified SDK/runtime and exact two-model allowlist", () => { + expect(MANAGED_AGENT_CONTRACT).toMatchObject({ + agentSdkVersion: "0.3.228", + claudeCodeRuntimeVersion: "2.1.228", + certificationNodeVersion: "22.23.2", + directGatewayOrigin: "https://litellm.services.sapiom.ai", + }); + expect(MANAGED_AGENT_MODEL_TARGETS).toEqual({ + "sonnet-5": expect.objectContaining({ + alias: "claude-sonnet-5-anthropic-anthropic-eval", + }), + "minimax-m3": expect.objectContaining({ + alias: "minimax-m3-fireworks-sapiom-fireworks_ai-eval", + }), + }); + }); + + it("rejects arbitrary models instead of accepting a gateway label", () => { + expect(() => + resolveManagedAgentModelTarget("claude-anything" as "sonnet-5"), + ).toThrow(ManagedAgentConfigurationError); + }); + + it("accepts only a credential-free HTTP(S) origin", () => { + expect( + normalizeManagedAgentGatewayOrigin("https://gateway.example.test/"), + ).toBe("https://gateway.example.test"); + for (const value of [ + "file:///tmp/gateway", + "https://user:pass@gateway.example.test", + "https://gateway.example.test/v1", + "https://gateway.example.test?token=x", + ]) { + expect(() => normalizeManagedAgentGatewayOrigin(value)).toThrow( + ManagedAgentConfigurationError, + ); + } + }); + + it("pins live traffic to the certified direct gateway origin", () => { + expect( + assertManagedAgentDirectGatewayOrigin( + "https://litellm.services.sapiom.ai/", + ), + ).toBe(MANAGED_AGENT_CONTRACT.directGatewayOrigin); + expect(() => + assertManagedAgentDirectGatewayOrigin( + "https://llm.services.proxy.sapiom.ai", + ), + ).toThrow("pinned direct Sapiom gateway origin"); + }); + + it("limits the explicit hermetic origin seam to .test and loopback", () => { + for (const value of [ + "https://gateway.example.test", + "http://localhost:4312", + "http://agent.localhost:4312", + "http://127.0.0.1:4312", + "http://[::1]:4312", + ]) { + expect(normalizeManagedAgentHermeticGatewayOrigin(value)).toBe( + normalizeManagedAgentGatewayOrigin(value), + ); + } + for (const value of [ + MANAGED_AGENT_CONTRACT.directGatewayOrigin, + "https://gateway.example.com", + ]) { + expect(() => normalizeManagedAgentHermeticGatewayOrigin(value)).toThrow( + "reserved .test or loopback", + ); + } + }); + + it("canonicalizes disjoint roots and bounds turns and budget", async () => { + const valid = await config(); + const checked = validateManagedAgentProbeConfig(valid); + expect(checked.canonicalWorkspaceRoot).toBe( + await realpath(valid.workspaceRoot), + ); + expect(checked.model.id).toBe("sonnet-5"); + expect(() => + validateManagedAgentProbeConfig({ ...valid, maxBudgetUsd: 1.01 }), + ).toThrow("maxBudgetUsd"); + expect(() => + validateManagedAgentProbeConfig({ ...valid, maxTurns: 21 }), + ).toThrow("maxTurns"); + expect(() => + validateManagedAgentProbeConfig({ + ...valid, + configRoot: valid.workspaceRoot, + }), + ).toThrow("disjoint"); + expect(() => + validateManagedAgentProbeConfig({ + ...valid, + expectedMcpNonce: undefined, + }), + ).toThrow("expectedMcpNonce"); + }); + + it("requires the exact L1 v2 marker and all six unique path roles", async () => { + const valid = await config(); + expect(() => + validateManagedAgentProbeConfig({ + ...valid, + prompt: "SAPIOM_MANAGED_AGENT_L1_PROMPT_V1\nprobe", + }), + ).toThrow("managed-agent-l1-prompt-v2 marker"); + expect(() => + validateManagedAgentProbeConfig({ + ...valid, + pathRoleBindings: valid.pathRoleBindings.slice(0, -1), + }), + ).toThrow("each frozen fixture role exactly once"); + expect(() => + validateManagedAgentProbeConfig({ + ...valid, + pathRoleBindings: valid.pathRoleBindings.map((binding, index) => + index === 1 ? { ...binding, path: "clean.txt" } : binding, + ), + }), + ).toThrow("each frozen fixture role exactly once"); + }); + + it("requires exact trusted hashes for both intended L1 mutation roles", async () => { + const valid = await config(); + for (const expectedL1FinalBytes of [ + valid.expectedL1FinalBytes.slice(0, 1), + valid.expectedL1FinalBytes.map((expectation, index) => + index === 0 ? { ...expectation, sha256: "not-a-hash" } : expectation, + ), + valid.expectedL1FinalBytes.map((expectation, index) => + index === 0 ? { ...expectation, path: "dirty.txt" } : expectation, + ), + ]) { + expect(() => + validateManagedAgentProbeConfig({ + ...valid, + expectedL1FinalBytes, + }), + ).toThrow("exact hashes"); + } + }); + + it("keeps L2 free of L1 path-role and final-byte configuration", async () => { + const valid = await config(); + const l2: ManagedAgentProbeConfig = { + ...valid, + scenario: "L2", + prompt: "run exact Bash", + pathRoleBindings: [], + expectedL1FinalBytes: [], + expectedMcpNonce: undefined, + }; + expect(() => validateManagedAgentProbeConfig(l2)).not.toThrow(); + expect(() => + validateManagedAgentProbeConfig({ + ...l2, + pathRoleBindings: valid.pathRoleBindings, + }), + ).toThrow("L2 must not configure"); + expect(() => + validateManagedAgentProbeConfig({ + ...l2, + expectedL1FinalBytes: valid.expectedL1FinalBytes, + }), + ).toThrow("L2 must not configure"); + }); + + it("requires exact agreement with an explicitly selected hermetic origin", async () => { + const valid = await config(); + const gatewayOrigin = "https://gateway.example.test"; + expect( + validateManagedAgentProbeConfig( + { ...valid, gatewayOrigin }, + { hermeticGatewayOrigin: gatewayOrigin }, + ).gatewayOrigin, + ).toBe(gatewayOrigin); + expect(() => + validateManagedAgentProbeConfig( + { ...valid, gatewayOrigin }, + { hermeticGatewayOrigin: "https://other.example.test" }, + ), + ).toThrow("explicit hermetic gateway origin"); + }); +}); diff --git a/packages/harness/src/experimental/managed-agent-spike/contract.ts b/packages/harness/src/experimental/managed-agent-spike/contract.ts new file mode 100644 index 000000000..13d11963d --- /dev/null +++ b/packages/harness/src/experimental/managed-agent-spike/contract.ts @@ -0,0 +1,331 @@ +import { realpathSync, statSync } from "node:fs"; +import { isAbsolute, relative, resolve, sep } from "node:path"; + +import type { + ManagedAgentL1FinalByteRole, + ManagedAgentModelTarget, + ManagedAgentModelTargetId, + ManagedAgentProbeConfig, + ManagedAgentRegisteredPathRole, +} from "./types.js"; + +/** + * Pinned to the Epic 0 certification manifest in the Sapiom gateway repo: + * llm-gateway/streaming-replay/certification/manifest.v1.json. + */ +export const MANAGED_AGENT_CONTRACT = { + contractVersion: 1, + agentSdkVersion: "0.3.228", + claudeCodeRuntimeVersion: "2.1.228", + certificationNodeVersion: "22.23.2", + suiteVersion: "0.1.0", + directGatewayOrigin: "https://litellm.services.sapiom.ai", + maxBudgetUsd: 1, + maxTurns: 20, +} as const; + +export const MANAGED_AGENT_L1_CERTIFICATION_CONTRACT = Object.freeze({ + contractVersion: 2 as const, + promptVersion: "managed-agent-l1-prompt-v2" as const, + promptMarker: "SAPIOM_MANAGED_AGENT_L1_PROMPT_V2" as const, + evaluatorVersion: "managed-agent-l1-evaluator-v2" as const, +}); + +export const MANAGED_AGENT_L1_REGISTERED_PATH_ROLES = Object.freeze([ + "clean_target", + "dirty_sentinel", + "untracked_sentinel", + "managed_output", + "outside_sentinel", + "escape_link", +] as const satisfies readonly ManagedAgentRegisteredPathRole[]); + +export const MANAGED_AGENT_L1_FINAL_BYTE_ROLES = Object.freeze([ + "clean_target", + "managed_output", +] as const satisfies readonly ManagedAgentL1FinalByteRole[]); + +export const MANAGED_AGENT_MODEL_TARGETS: Readonly< + Record +> = { + "sonnet-5": { + id: "sonnet-5", + alias: "claude-sonnet-5-anthropic-anthropic-eval", + upstreamProvider: "anthropic", + upstreamModel: "claude-sonnet-5", + }, + "minimax-m3": { + id: "minimax-m3", + alias: "minimax-m3-fireworks-sapiom-fireworks_ai-eval", + upstreamProvider: "fireworks_ai", + upstreamModel: "accounts/sapiom-o7kbok9g48o6/routers/minimax-m3", + }, +}; + +export const MANAGED_AGENT_MODEL_ENVIRONMENT_VARIABLES = [ + "ANTHROPIC_MODEL", + "ANTHROPIC_DEFAULT_FABLE_MODEL", + "ANTHROPIC_DEFAULT_OPUS_MODEL", + "ANTHROPIC_DEFAULT_SONNET_MODEL", + "ANTHROPIC_DEFAULT_HAIKU_MODEL", + "ANTHROPIC_SMALL_FAST_MODEL", + "CLAUDE_CODE_SUBAGENT_MODEL", +] as const; + +export const MANAGED_AGENT_FORBIDDEN_AMBIENT_CREDENTIALS = [ + "ANTHROPIC_API_KEY", + "ANTHROPIC_AUTH_TOKEN", + "CLAUDE_CODE_OAUTH_TOKEN", + "SAPIOM_API_KEY", +] as const; + +export class ManagedAgentConfigurationError extends Error { + public constructor(message: string) { + super(message); + this.name = "ManagedAgentConfigurationError"; + } +} + +function pathWithin(root: string, candidate: string): boolean { + const pathRelative = relative(root, candidate); + if (pathRelative === "") return true; + return ( + !isAbsolute(pathRelative) && + pathRelative !== ".." && + !pathRelative.startsWith(`..${sep}`) + ); +} + +function canonicalDirectory(value: string, label: string): string { + let canonical: string; + try { + canonical = realpathSync(resolve(value)); + } catch { + throw new ManagedAgentConfigurationError(`${label} must exist`); + } + if (!statSync(canonical).isDirectory()) { + throw new ManagedAgentConfigurationError(`${label} must be a directory`); + } + return canonical; +} + +export function normalizeManagedAgentGatewayOrigin(value: string): string { + let parsed: URL; + try { + parsed = new URL(value); + } catch { + throw new ManagedAgentConfigurationError( + "gatewayOrigin must be a valid HTTP(S) origin", + ); + } + if (parsed.protocol !== "https:" && parsed.protocol !== "http:") { + throw new ManagedAgentConfigurationError( + "gatewayOrigin must use HTTP or HTTPS", + ); + } + if ( + parsed.username || + parsed.password || + parsed.search || + parsed.hash || + (parsed.pathname !== "" && parsed.pathname !== "/") + ) { + throw new ManagedAgentConfigurationError( + "gatewayOrigin must not contain credentials, a path, query parameters, or a fragment", + ); + } + return parsed.origin; +} + +export function assertManagedAgentDirectGatewayOrigin(value: string): string { + const normalized = normalizeManagedAgentGatewayOrigin(value); + if (normalized !== MANAGED_AGENT_CONTRACT.directGatewayOrigin) { + throw new ManagedAgentConfigurationError( + "gatewayOrigin must match the pinned direct Sapiom gateway origin", + ); + } + return normalized; +} + +export function normalizeManagedAgentHermeticGatewayOrigin( + value: string, +): string { + const normalized = normalizeManagedAgentGatewayOrigin(value); + const { hostname } = new URL(normalized); + const isTestHostname = hostname.endsWith(".test"); + const isLoopbackHostname = + hostname === "localhost" || + hostname.endsWith(".localhost") || + hostname === "[::1]" || + /^127(?:\.[0-9]{1,3}){3}$/.test(hostname); + if (!isTestHostname && !isLoopbackHostname) { + throw new ManagedAgentConfigurationError( + "hermeticGatewayOrigin must use a reserved .test or loopback hostname", + ); + } + return normalized; +} + +export function resolveManagedAgentModelTarget( + target: ManagedAgentModelTargetId, +): ManagedAgentModelTarget { + const resolved = MANAGED_AGENT_MODEL_TARGETS[target]; + if (!resolved) { + throw new ManagedAgentConfigurationError( + `Unknown managed-agent model target: ${String(target)}`, + ); + } + return resolved; +} + +export interface ValidatedManagedAgentProbeConfig { + readonly config: ManagedAgentProbeConfig; + readonly canonicalWorkspaceRoot: string; + readonly canonicalConfigRoot: string; + readonly gatewayOrigin: string; + readonly model: ManagedAgentModelTarget; +} + +export interface ManagedAgentProbeValidationOptions { + /** + * Test-only escape hatch for an injected query factory. The origin must be + * reserved under .test or use an explicit loopback hostname/address. + */ + readonly hermeticGatewayOrigin?: string; +} + +export function validateManagedAgentProbeConfig( + config: ManagedAgentProbeConfig, + options: ManagedAgentProbeValidationOptions = {}, +): ValidatedManagedAgentProbeConfig { + const canonicalWorkspaceRoot = canonicalDirectory( + config.workspaceRoot, + "workspaceRoot", + ); + const canonicalConfigRoot = canonicalDirectory( + config.configRoot, + "configRoot", + ); + if ( + pathWithin(canonicalWorkspaceRoot, canonicalConfigRoot) || + pathWithin(canonicalConfigRoot, canonicalWorkspaceRoot) + ) { + throw new ManagedAgentConfigurationError( + "workspaceRoot and configRoot must be disjoint directories", + ); + } + if (!config.gatewayCredential.trim()) { + throw new ManagedAgentConfigurationError("gatewayCredential is required"); + } + if (!config.prompt.trim()) { + throw new ManagedAgentConfigurationError("prompt is required"); + } + if (config.scenario === "L1") { + if ( + config.prompt.split("\n", 1)[0] !== + MANAGED_AGENT_L1_CERTIFICATION_CONTRACT.promptMarker + ) { + throw new ManagedAgentConfigurationError( + "L1 prompt must use the frozen managed-agent-l1-prompt-v2 marker", + ); + } + const roles = config.pathRoleBindings.map(({ role }) => role); + const paths = config.pathRoleBindings.map(({ path }) => path); + if ( + roles.length !== MANAGED_AGENT_L1_REGISTERED_PATH_ROLES.length || + new Set(roles).size !== roles.length || + new Set(paths).size !== paths.length || + MANAGED_AGENT_L1_REGISTERED_PATH_ROLES.some( + (role) => !roles.includes(role), + ) || + paths.some((path) => !path || path.includes("\0") || /[\r\n]/.test(path)) + ) { + throw new ManagedAgentConfigurationError( + "L1 pathRoleBindings must bind each frozen fixture role exactly once", + ); + } + const finalByteRoles = config.expectedL1FinalBytes.map(({ role }) => role); + if ( + finalByteRoles.length !== MANAGED_AGENT_L1_FINAL_BYTE_ROLES.length || + new Set(finalByteRoles).size !== finalByteRoles.length || + MANAGED_AGENT_L1_FINAL_BYTE_ROLES.some( + (role) => !finalByteRoles.includes(role), + ) || + config.expectedL1FinalBytes.some( + ({ path, role, sha256 }) => + !/^[a-f0-9]{64}$/.test(sha256) || + !config.pathRoleBindings.some( + (binding) => binding.role === role && binding.path === path, + ), + ) + ) { + throw new ManagedAgentConfigurationError( + "L1 expectedL1FinalBytes must bind exact hashes to the clean target and managed output roles", + ); + } + } else if ( + config.pathRoleBindings.length !== 0 || + config.expectedL1FinalBytes.length !== 0 + ) { + throw new ManagedAgentConfigurationError( + "L2 must not configure file path roles or L1 final-byte expectations", + ); + } + if ( + config.scenario === "L1" && + (!config.expectedMcpNonce || + config.expectedMcpNonce.length > 256 || + /[\r\n]/.test(config.expectedMcpNonce)) + ) { + throw new ManagedAgentConfigurationError( + "L1 expectedMcpNonce must be a non-empty, single-line value of at most 256 characters", + ); + } + if ( + !Number.isInteger(config.maxTurns) || + config.maxTurns < 1 || + config.maxTurns > MANAGED_AGENT_CONTRACT.maxTurns + ) { + throw new ManagedAgentConfigurationError( + `maxTurns must be an integer between 1 and ${MANAGED_AGENT_CONTRACT.maxTurns}`, + ); + } + if ( + !Number.isFinite(config.maxBudgetUsd) || + config.maxBudgetUsd <= 0 || + config.maxBudgetUsd > MANAGED_AGENT_CONTRACT.maxBudgetUsd + ) { + throw new ManagedAgentConfigurationError( + `maxBudgetUsd must be greater than zero and no more than ${MANAGED_AGENT_CONTRACT.maxBudgetUsd}`, + ); + } + if ( + config.allowedBashCommands.some( + (command) => !command || /[\r\n]/.test(command), + ) + ) { + throw new ManagedAgentConfigurationError( + "allowedBashCommands must contain non-empty, single-line commands", + ); + } + const gatewayOrigin = normalizeManagedAgentGatewayOrigin( + config.gatewayOrigin, + ); + const expectedGatewayOrigin = options.hermeticGatewayOrigin + ? normalizeManagedAgentHermeticGatewayOrigin(options.hermeticGatewayOrigin) + : MANAGED_AGENT_CONTRACT.directGatewayOrigin; + if (gatewayOrigin !== expectedGatewayOrigin) { + throw new ManagedAgentConfigurationError( + options.hermeticGatewayOrigin + ? "gatewayOrigin must match the explicit hermetic gateway origin" + : "gatewayOrigin must match the pinned direct Sapiom gateway origin", + ); + } + return { + config, + canonicalWorkspaceRoot, + canonicalConfigRoot, + gatewayOrigin, + model: resolveManagedAgentModelTarget(config.target), + }; +} diff --git a/packages/harness/src/experimental/managed-agent-spike/environment.test.ts b/packages/harness/src/experimental/managed-agent-spike/environment.test.ts new file mode 100644 index 000000000..55c54ed3c --- /dev/null +++ b/packages/harness/src/experimental/managed-agent-spike/environment.test.ts @@ -0,0 +1,154 @@ +import { + lstat, + mkdir, + mkdtemp, + realpath, + rm, + stat, + symlink, +} from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { dirname, isAbsolute, join, relative, sep } from "node:path"; + +import { afterEach, describe, expect, it } from "vitest"; + +import { + MANAGED_AGENT_FORBIDDEN_AMBIENT_CREDENTIALS, + MANAGED_AGENT_MODEL_ENVIRONMENT_VARIABLES, +} from "./contract.js"; +import { buildManagedAgentChildEnvironment } from "./environment.js"; + +const roots: string[] = []; + +afterEach(async () => { + await Promise.all( + roots.splice(0).map((root) => rm(root, { recursive: true, force: true })), + ); +}); + +describe("managed-agent child environment", () => { + it("starts empty, passes only positive-listed ambient values, and pins every model variable", async () => { + const configRoot = await mkdtemp(join(tmpdir(), "managed-agent-env-")); + roots.push(configRoot); + const child = buildManagedAgentChildEnvironment({ + ambient: { + PATH: "/safe/bin", + LANG: "en_US.UTF-8", + ANTHROPIC_API_KEY: "ambient-anthropic-key", + CLAUDE_CODE_OAUTH_TOKEN: "ambient-user-login", + SAPIOM_API_KEY: "ambient-sapiom-key", + HOST_ESBUILD_PIN: "/must/not/leak", + FUTURE_CREDENTIAL_SOURCE: "future-secret", + }, + configRoot, + gatewayOrigin: "https://gateway.example.test", + gatewayCredential: "dedicated-eval-key", + modelAlias: "claude-sonnet-5-anthropic-anthropic-eval", + evalSource: "eval-source", + executionId: "execution-id", + }); + + expect(child.PATH).toBe("/safe/bin"); + expect(child.ANTHROPIC_API_KEY).toBe("dedicated-eval-key"); + expect(child.ANTHROPIC_BASE_URL).toBe("https://gateway.example.test"); + expect(child.CLAUDE_CODE_DISABLE_NONSTREAMING_FALLBACK).toBe("1"); + expect(child.CLAUDE_CODE_NO_MODEL_FALLBACK).toBe("1"); + for (const variable of MANAGED_AGENT_MODEL_ENVIRONMENT_VARIABLES) { + expect(child[variable]).toBe("claude-sonnet-5-anthropic-anthropic-eval"); + } + for (const variable of MANAGED_AGENT_FORBIDDEN_AMBIENT_CREDENTIALS) { + if (variable !== "ANTHROPIC_API_KEY") + expect(child).not.toHaveProperty(variable); + } + expect(child).not.toHaveProperty("HOST_ESBUILD_PIN"); + expect(child).not.toHaveProperty("FUTURE_CREDENTIAL_SOURCE"); + expect(child.HOME).not.toBe(process.env.HOME); + expect(child.CLAUDE_CONFIG_DIR).not.toBe(process.env.CLAUDE_CONFIG_DIR); + expect(child.CLAUDE_SECURESTORAGE_CONFIG_DIR).not.toBe( + child.CLAUDE_CONFIG_DIR, + ); + for (const directory of [ + child.HOME, + child.XDG_CONFIG_HOME, + child.CLAUDE_CONFIG_DIR, + child.CLAUDE_SECURESTORAGE_CONFIG_DIR, + child.TMPDIR, + ]) { + expect((await stat(directory)).isDirectory()).toBe(true); + } + }); + + it("uses a fresh canonical private root without following pre-existing child symlinks", async () => { + const root = await mkdtemp(join(tmpdir(), "managed-agent-env-")); + roots.push(root); + const configRoot = join(root, "config"); + const external = join(root, "external-claude-config"); + await Promise.all([mkdir(configRoot), mkdir(external)]); + await symlink(external, join(configRoot, "claude-config")); + + const first = buildManagedAgentChildEnvironment({ + ambient: {}, + configRoot, + gatewayOrigin: "https://gateway.example.test", + gatewayCredential: "dedicated-eval-key", + modelAlias: "claude-sonnet-5-anthropic-anthropic-eval", + evalSource: "eval-source", + executionId: "execution-id", + }); + const second = buildManagedAgentChildEnvironment({ + ambient: {}, + configRoot, + gatewayOrigin: "https://gateway.example.test", + gatewayCredential: "dedicated-eval-key", + modelAlias: "claude-sonnet-5-anthropic-anthropic-eval", + evalSource: "eval-source", + executionId: "execution-id-2", + }); + + const privateRoot = dirname(first.CLAUDE_CONFIG_DIR); + expect(privateRoot).not.toBe(dirname(second.CLAUDE_CONFIG_DIR)); + expect(await realpath(first.CLAUDE_CONFIG_DIR)).not.toBe( + await realpath(external), + ); + expect( + (await lstat(join(configRoot, "claude-config"))).isSymbolicLink(), + ).toBe(true); + for (const directory of [ + first.HOME, + first.USERPROFILE, + first.APPDATA, + first.LOCALAPPDATA, + first.XDG_CONFIG_HOME, + first.XDG_CACHE_HOME, + first.XDG_DATA_HOME, + first.CLAUDE_CONFIG_DIR, + first.CLAUDE_SECURESTORAGE_CONFIG_DIR, + first.TMPDIR, + first.TMP, + first.TEMP, + ]) { + const canonical = await realpath(directory); + const pathRelative = relative(privateRoot, canonical); + expect(isAbsolute(pathRelative)).toBe(false); + expect(pathRelative).not.toBe(".."); + expect(pathRelative.startsWith(`..${sep}`)).toBe(false); + expect((await lstat(directory)).isSymbolicLink()).toBe(false); + } + }); + + it("rejects newline injection in correlation headers", async () => { + const configRoot = await mkdtemp(join(tmpdir(), "managed-agent-env-")); + roots.push(configRoot); + expect(() => + buildManagedAgentChildEnvironment({ + ambient: {}, + configRoot, + gatewayOrigin: "https://gateway.example.test", + gatewayCredential: "dedicated-eval-key", + modelAlias: "model", + evalSource: "bad\nheader", + executionId: "execution-id", + }), + ).toThrow("safe header"); + }); +}); diff --git a/packages/harness/src/experimental/managed-agent-spike/environment.ts b/packages/harness/src/experimental/managed-agent-spike/environment.ts new file mode 100644 index 000000000..245dc1497 --- /dev/null +++ b/packages/harness/src/experimental/managed-agent-spike/environment.ts @@ -0,0 +1,212 @@ +import { + chmodSync, + lstatSync, + mkdirSync, + mkdtempSync, + realpathSync, + statSync, +} from "node:fs"; +import { isAbsolute, join, relative, resolve, sep } from "node:path"; + +import { + MANAGED_AGENT_CONTRACT, + MANAGED_AGENT_MODEL_ENVIRONMENT_VARIABLES, + ManagedAgentConfigurationError, +} from "./contract.js"; + +export type ManagedAgentAmbientEnvironment = Readonly< + Record +>; + +export interface ManagedAgentIsolatedDirectories { + readonly privateRoot: string; + readonly home: string; + readonly appData: string; + readonly localAppData: string; + readonly xdgConfig: string; + readonly xdgCache: string; + readonly xdgData: string; + readonly claudeConfig: string; + readonly secureStorage: string; + readonly temporary: string; +} + +export interface ManagedAgentChildEnvironmentInput { + readonly ambient: ManagedAgentAmbientEnvironment; + readonly configRoot: string; + readonly gatewayOrigin: string; + readonly gatewayCredential: string; + readonly modelAlias: string; + readonly evalSource: string; + readonly executionId: string; +} + +const SAFE_AMBIENT_PASSTHROUGH = [ + "PATH", + "SHELL", + "LANG", + "LC_ALL", + "LC_CTYPE", + "TERM", + "SystemRoot", + "WINDIR", + "COMSPEC", + "PATHEXT", + "CLAUDE_CODE_GIT_BASH_PATH", +] as const; + +const PRIVATE_RUN_DIRECTORY_PREFIX = "managed-agent-run-"; + +function validateHeaderValue(value: string, label: string): void { + if (!value || /[\r\n]/.test(value)) { + throw new Error(`${label} is not a safe header value`); + } +} + +function comparisonPath(value: string): string { + return process.platform === "win32" ? value.toLowerCase() : value; +} + +function pathWithin(root: string, candidate: string): boolean { + const pathRelative = relative( + comparisonPath(root), + comparisonPath(candidate), + ); + if (pathRelative === "") return true; + return ( + !isAbsolute(pathRelative) && + pathRelative !== ".." && + !pathRelative.startsWith(`..${sep}`) + ); +} + +function canonicalDirectory(value: string, label: string): string { + let canonical: string; + try { + canonical = realpathSync(resolve(value)); + } catch { + throw new ManagedAgentConfigurationError(`${label} must exist`); + } + if (!statSync(canonical).isDirectory()) { + throw new ManagedAgentConfigurationError(`${label} must be a directory`); + } + return canonical; +} + +function verifyPrivateDirectory( + candidate: string, + privateRoot: string, + label: string, +): string { + const metadata = lstatSync(candidate); + if (metadata.isSymbolicLink() || !metadata.isDirectory()) { + throw new ManagedAgentConfigurationError( + `${label} must be a private directory`, + ); + } + chmodSync(candidate, 0o700); + const canonical = realpathSync(candidate); + if (!pathWithin(privateRoot, canonical)) { + throw new ManagedAgentConfigurationError( + `${label} must remain inside the private run root`, + ); + } + return canonical; +} + +function createPrivateDirectory( + parent: string, + name: string, + privateRoot: string, +): string { + const candidate = join(parent, name); + mkdirSync(candidate, { mode: 0o700 }); + return verifyPrivateDirectory(candidate, privateRoot, name); +} + +export function prepareManagedAgentDirectories( + configRoot: string, +): ManagedAgentIsolatedDirectories { + const canonicalConfigRoot = canonicalDirectory(configRoot, "configRoot"); + const createdPrivateRoot = mkdtempSync( + join(canonicalConfigRoot, PRIVATE_RUN_DIRECTORY_PREFIX), + ); + const privateRoot = verifyPrivateDirectory( + createdPrivateRoot, + canonicalConfigRoot, + "private run root", + ); + const home = createPrivateDirectory(privateRoot, "home", privateRoot); + const directories = { + privateRoot, + home, + appData: createPrivateDirectory(home, "appdata", privateRoot), + localAppData: createPrivateDirectory(home, "local-appdata", privateRoot), + xdgConfig: createPrivateDirectory(home, "xdg-config", privateRoot), + xdgCache: createPrivateDirectory(home, "xdg-cache", privateRoot), + xdgData: createPrivateDirectory(home, "xdg-data", privateRoot), + claudeConfig: createPrivateDirectory( + privateRoot, + "claude-config", + privateRoot, + ), + secureStorage: createPrivateDirectory( + privateRoot, + "secure-storage", + privateRoot, + ), + temporary: createPrivateDirectory(privateRoot, "tmp", privateRoot), + } satisfies ManagedAgentIsolatedDirectories; + return directories; +} + +/** + * Build from an empty object so future ambient credential variables remain + * denied by default. The supplied credential must be a dedicated eval key. + */ +export function buildManagedAgentChildEnvironment( + input: ManagedAgentChildEnvironmentInput, +): Record { + validateHeaderValue(input.evalSource, "evalSource"); + validateHeaderValue(input.executionId, "executionId"); + const directories = prepareManagedAgentDirectories(input.configRoot); + const child: Record = {}; + for (const name of SAFE_AMBIENT_PASSTHROUGH) { + const value = input.ambient[name]; + if (value !== undefined) child[name] = value; + } + + Object.assign(child, { + HOME: directories.home, + USERPROFILE: directories.home, + APPDATA: directories.appData, + LOCALAPPDATA: directories.localAppData, + XDG_CONFIG_HOME: directories.xdgConfig, + XDG_CACHE_HOME: directories.xdgCache, + XDG_DATA_HOME: directories.xdgData, + TMPDIR: directories.temporary, + TMP: directories.temporary, + TEMP: directories.temporary, + CLAUDE_CONFIG_DIR: directories.claudeConfig, + CLAUDE_SECURESTORAGE_CONFIG_DIR: directories.secureStorage, + ANTHROPIC_BASE_URL: input.gatewayOrigin, + ANTHROPIC_API_KEY: input.gatewayCredential, + ANTHROPIC_CUSTOM_HEADERS: [ + `x-sapiom-eval-source: ${input.evalSource}`, + `x-sapiom-execution-id: ${input.executionId}`, + ].join("\n"), + CLAUDE_CODE_DISABLE_NONSTREAMING_FALLBACK: "1", + CLAUDE_CODE_NO_MODEL_FALLBACK: "1", + CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC: "1", + CLAUDE_AGENT_SDK_CLIENT_APP: `sapiom-managed-agent-spike/${MANAGED_AGENT_CONTRACT.suiteVersion}`, + DISABLE_AUTOUPDATER: "1", + DISABLE_ERROR_REPORTING: "1", + DISABLE_TELEMETRY: "1", + }); + + for (const name of MANAGED_AGENT_MODEL_ENVIRONMENT_VARIABLES) { + child[name] = input.modelAlias; + } + + return child; +} diff --git a/packages/harness/src/experimental/managed-agent-spike/events.test.ts b/packages/harness/src/experimental/managed-agent-spike/events.test.ts new file mode 100644 index 000000000..7842a7d2e --- /dev/null +++ b/packages/harness/src/experimental/managed-agent-spike/events.test.ts @@ -0,0 +1,333 @@ +import { describe, expect, it } from "vitest"; + +import { + MANAGED_AGENT_TOOL_USE_ID_MAX_LENGTH, + ManagedAgentEventError, + ManagedAgentEventRecorder, + normalizeManagedAgentToolUseId, +} from "./events.js"; + +const SESSION_ID = "11111111-1111-4111-8111-111111111111"; + +describe("ManagedAgentEventRecorder", () => { + it("retains structural evidence while redacting message and tool content", () => { + const recorder = new ManagedAgentEventRecorder("run-1", "expected-model"); + recorder.observeSdkEvent({ + type: "system", + subtype: "init", + session_id: SESSION_ID, + model: "model-secret-must-not-be-copied", + }); + recorder.observeSdkEvent({ + type: "assistant", + session_id: SESSION_ID, + message: { + id: "message-1", + content: [ + { type: "text", text: "prompt-secret" }, + { + type: "tool_use", + id: "tool-1", + name: "Read", + input: { file_path: "/private/secret-path", token: "tool-secret" }, + }, + ], + }, + }); + recorder.observeSdkEvent({ + type: "user", + session_id: SESSION_ID, + message: { + content: [ + { + type: "tool_result", + tool_use_id: "tool-1", + content: "private-file-contents", + is_error: false, + }, + ], + }, + }); + recorder.observeSdkEvent({ + type: "result", + subtype: "success", + is_error: false, + session_id: SESSION_ID, + result: "private-final-answer", + usage: { + input_tokens: 7, + output_tokens: 3, + cache_creation_input_tokens: 2, + cache_read_input_tokens: 1, + }, + total_cost_usd: 0.001, + num_turns: 7, + modelUsage: { + "unexpected-model": { + inputTokens: 7, + outputTokens: 3, + }, + }, + }); + expect(recorder.recordTerminal("success")).toBe(true); + expect(recorder.recordTerminal("query_error")).toBe(false); + + expect(recorder.sessionId).toBe(SESSION_ID); + expect(recorder.usage).toEqual({ + authority: "sdk_non_authoritative", + inputTokens: 7, + outputTokens: 3, + cacheCreationInputTokens: 2, + cacheReadInputTokens: 1, + estimatedCostUsd: 0.001, + }); + expect(recorder.inferenceTurns).toBe(1); + expect(recorder.sdkNumTurns).toBe(7); + expect(recorder.modelEvidence).toEqual({ + authority: "sdk_non_authoritative", + initModelObserved: true, + initModelMatchesExpectedAlias: false, + resultModelUsageObserved: true, + resultModelUsageMatchesExpectedAlias: false, + resultModelCount: 1, + }); + expect(recorder.toolEvidence).toEqual([ + { + toolUseId: normalizeManagedAgentToolUseId("tool-1"), + toolName: "Read", + status: "requested", + }, + { + toolUseId: normalizeManagedAgentToolUseId("tool-1"), + toolName: "Read", + status: "success", + }, + ]); + expect( + recorder.events.filter(({ type }) => type === "terminal"), + ).toHaveLength(1); + const serialized = JSON.stringify(recorder.events); + for (const secret of [ + "model-secret", + "prompt-secret", + "/private/secret-path", + "tool-secret", + "private-file-contents", + "private-final-answer", + "tool-1", + ]) { + expect(serialized).not.toContain(secret); + } + }); + + it("redacts attacker-controlled session, tool, and permission identifiers from all evidence", () => { + const recorder = new ManagedAgentEventRecorder("run-2", "expected-model"); + const sessionSecret = "session-secret-credential"; + const toolIdSecret = "tool-id-secret-credential"; + const permissionIdSecret = "permission-id-secret-credential"; + const toolNameSecret = "ReadSecretCredential"; + const permissionNameSecret = "WriteSecretCredential"; + const messageIdSecret = "message-id-secret-credential"; + recorder.observeSdkEvent({ + type: "system", + subtype: "init", + session_id: sessionSecret, + }); + recorder.observeSdkEvent({ + type: "assistant", + session_id: sessionSecret, + message: { + id: messageIdSecret, + content: [ + { + type: "tool_use", + id: toolIdSecret, + name: toolNameSecret, + input: {}, + }, + ], + }, + }); + recorder.observeSdkEvent({ + type: "user", + session_id: sessionSecret, + message: { + content: [ + { + type: "tool_result", + tool_use_id: toolIdSecret, + content: "private-result", + }, + ], + }, + }); + recorder.recordPermission({ + toolUseId: permissionIdSecret, + toolName: permissionNameSecret, + decision: "deny", + reason: "tool_not_allowed", + source: "pre_tool_use", + operationId: "unknown", + }); + recorder.recordTerminal("success"); + + expect(recorder.sessionId).toBeUndefined(); + expect(recorder.toolEvidence[0]?.toolName).toBe("unknown"); + expect(recorder.toolEvidence.map(({ toolUseId }) => toolUseId)).toEqual([ + normalizeManagedAgentToolUseId(toolIdSecret), + normalizeManagedAgentToolUseId(toolIdSecret), + ]); + expect(recorder.permissionEvidence).toEqual([ + { + toolUseId: normalizeManagedAgentToolUseId(permissionIdSecret), + toolName: "unknown", + decision: "deny", + reason: "tool_not_allowed", + source: "pre_tool_use", + operationId: "unknown", + }, + ]); + const serialized = JSON.stringify({ + sdkSessionId: recorder.sessionId, + events: recorder.events, + toolEvidence: recorder.toolEvidence, + permissionEvidence: recorder.permissionEvidence, + }); + for (const secret of [ + sessionSecret, + toolIdSecret, + permissionIdSecret, + toolNameSecret, + permissionNameSecret, + messageIdSecret, + "private-result", + ]) { + expect(serialized).not.toContain(secret); + } + }); + + it("requires both SDK init and result usage to report only the selected alias", () => { + const expectedModel = "claude-sonnet-5-anthropic-anthropic-eval"; + const recorder = new ManagedAgentEventRecorder("run-model", expectedModel); + recorder.observeSdkEvent({ + type: "system", + subtype: "init", + model: expectedModel, + }); + recorder.observeSdkEvent({ + type: "result", + subtype: "success", + is_error: false, + modelUsage: { [expectedModel]: { inputTokens: 1, outputTokens: 1 } }, + }); + + expect(recorder.modelEvidence).toEqual({ + authority: "sdk_non_authoritative", + initModelObserved: true, + initModelMatchesExpectedAlias: true, + resultModelUsageObserved: true, + resultModelUsageMatchesExpectedAlias: true, + resultModelCount: 1, + }); + }); + + it("rejects missing, empty, and overlong tool-use identifiers instead of normalizing sentinels", () => { + const invalidIds = [ + undefined, + "", + " ", + "x".repeat(MANAGED_AGENT_TOOL_USE_ID_MAX_LENGTH + 1), + ]; + for (const invalidId of invalidIds) { + expect(() => normalizeManagedAgentToolUseId(invalidId)).toThrow( + ManagedAgentEventError, + ); + + const requested = new ManagedAgentEventRecorder( + "invalid-requested", + "expected-model", + ); + expect(() => + requested.observeSdkEvent({ + type: "assistant", + message: { + id: "bounded-message-id", + content: [ + { + type: "tool_use", + id: invalidId, + name: "Read", + input: { file_path: "private-path" }, + }, + ], + }, + }), + ).toThrow(ManagedAgentEventError); + expect(requested.toolEvidence).toEqual([]); + + const completed = new ManagedAgentEventRecorder( + "invalid-completed", + "expected-model", + ); + expect(() => + completed.observeSdkEvent({ + type: "user", + message: { + content: [ + { + type: "tool_result", + tool_use_id: invalidId, + content: "private-result", + }, + ], + }, + }), + ).toThrow(ManagedAgentEventError); + expect(completed.toolEvidence).toEqual([]); + } + }); + + it("counts distinct hashed assistant ids and keeps bounded SDK turns separate", () => { + const recorder = new ManagedAgentEventRecorder("run-3", "expected-model"); + for (const messageId of [ + "private-message-a", + "private-message-a", + "private-message-b", + ]) { + recorder.observeSdkEvent({ + type: "assistant", + message: { id: messageId, content: [{ type: "text", text: "secret" }] }, + }); + } + recorder.observeSdkEvent({ + type: "result", + subtype: "success", + is_error: false, + num_turns: 9, + }); + + expect(recorder.inferenceTurns).toBe(2); + expect(recorder.sdkNumTurns).toBe(9); + const serialized = JSON.stringify({ + events: recorder.events, + inferenceTurns: recorder.inferenceTurns, + sdkNumTurns: recorder.sdkNumTurns, + }); + expect(serialized).not.toContain("private-message-a"); + expect(serialized).not.toContain("private-message-b"); + + expect(() => + recorder.observeSdkEvent({ + type: "result", + subtype: "success", + num_turns: 21, + }), + ).toThrow(ManagedAgentEventError); + expect(() => + recorder.observeSdkEvent({ + type: "assistant", + message: { content: [] }, + }), + ).toThrow(ManagedAgentEventError); + }); +}); diff --git a/packages/harness/src/experimental/managed-agent-spike/events.ts b/packages/harness/src/experimental/managed-agent-spike/events.ts new file mode 100644 index 000000000..222732895 --- /dev/null +++ b/packages/harness/src/experimental/managed-agent-spike/events.ts @@ -0,0 +1,374 @@ +import { createHash } from "node:crypto"; + +import { MANAGED_AGENT_CONTRACT } from "./contract.js"; +import type { + ManagedAgentEventNormalizationFailureReason, + ManagedAgentPermissionEvidence, + ManagedAgentProbeEvent, + ManagedAgentSdkModelEvidence, + ManagedAgentSdkUsageEstimate, + ManagedAgentTerminalClassification, + ManagedAgentToolEvidence, +} from "./types.js"; + +type JsonRecord = Record; + +function asRecord(value: unknown): JsonRecord | undefined { + return typeof value === "object" && value !== null && !Array.isArray(value) + ? (value as JsonRecord) + : undefined; +} + +function optionalString(value: unknown): string | undefined { + return typeof value === "string" && value.length > 0 ? value : undefined; +} + +function optionalNumber(value: unknown): number | undefined { + return typeof value === "number" && Number.isFinite(value) + ? value + : undefined; +} + +function safeSubtype(value: unknown): string | undefined { + const subtype = optionalString(value); + return subtype && /^[a-z0-9_-]{1,80}$/i.test(subtype) ? subtype : undefined; +} + +const SAFE_TOOL_NAMES = new Set([ + "Read", + "Edit", + "Write", + "Bash", + "mcp__sapiom-managed-agent-spike__echo_nonce", + "mcp__sapiom-managed-agent-spike__fail_once", +]); +const NORMALIZED_TOOL_USE_ID_PATTERN = /^tool_[0-9a-f]{64}$/; +const SDK_SESSION_ID_PATTERN = + /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/; +const MAX_ASSISTANT_MESSAGE_ID_LENGTH = 512; +export const MANAGED_AGENT_TOOL_USE_ID_MAX_LENGTH = 512; + +export class ManagedAgentEventError extends Error { + public constructor( + public readonly reason: ManagedAgentEventNormalizationFailureReason, + message: string, + ) { + super(message); + this.name = "ManagedAgentEventError"; + } +} + +export function sanitizeManagedAgentToolName(value: unknown): string { + const toolName = optionalString(value); + return toolName && SAFE_TOOL_NAMES.has(toolName) ? toolName : "unknown"; +} + +export function isBoundedManagedAgentToolUseId( + value: unknown, +): value is string { + return ( + typeof value === "string" && + value.trim().length > 0 && + value.length <= MANAGED_AGENT_TOOL_USE_ID_MAX_LENGTH + ); +} + +export function normalizeManagedAgentToolUseId(value: unknown): string { + if (!isBoundedManagedAgentToolUseId(value)) { + throw new ManagedAgentEventError( + "tool_request_id_invalid", + "Managed-agent event has no bounded string tool-use id", + ); + } + if (NORMALIZED_TOOL_USE_ID_PATTERN.test(value)) { + return value; + } + return `tool_${createHash("sha256") + .update("sapiom-managed-agent-tool-use-id\0") + .update(value) + .digest("hex")}`; +} + +function safeSdkSessionId(value: unknown): string | undefined { + const sessionId = optionalString(value); + return sessionId && SDK_SESSION_ID_PATTERN.test(sessionId) + ? sessionId + : undefined; +} + +function normalizeAssistantMessageId(value: unknown): string { + const messageId = optionalString(value); + if (!messageId || messageId.length > MAX_ASSISTANT_MESSAGE_ID_LENGTH) { + throw new ManagedAgentEventError( + "assistant_message_id_invalid", + "Assistant event has no bounded string message id", + ); + } + return createHash("sha256") + .update("sapiom-managed-agent-assistant-message-id\0") + .update(messageId) + .digest("hex"); +} + +function boundedSdkNumTurns(value: unknown): number | undefined { + if (value === undefined) return undefined; + if ( + !Number.isInteger(value) || + Number(value) < 0 || + Number(value) > MANAGED_AGENT_CONTRACT.maxTurns + ) { + throw new ManagedAgentEventError( + "sdk_num_turns_invalid", + `SDK num_turns must be an integer between 0 and ${MANAGED_AGENT_CONTRACT.maxTurns}`, + ); + } + return Number(value); +} + +function contentBlocks(message: JsonRecord | undefined): readonly JsonRecord[] { + if (!Array.isArray(message?.content)) return []; + return message.content.flatMap((value) => { + const block = asRecord(value); + return block ? [block] : []; + }); +} + +function sdkUsage(event: JsonRecord): ManagedAgentSdkUsageEstimate | undefined { + const usage = asRecord(event.usage); + const inputTokens = optionalNumber(usage?.input_tokens); + const outputTokens = optionalNumber(usage?.output_tokens); + if (inputTokens === undefined || outputTokens === undefined) return undefined; + const estimatedCostUsd = optionalNumber(event.total_cost_usd); + return { + authority: "sdk_non_authoritative", + inputTokens, + outputTokens, + cacheCreationInputTokens: + optionalNumber(usage?.cache_creation_input_tokens) ?? 0, + cacheReadInputTokens: optionalNumber(usage?.cache_read_input_tokens) ?? 0, + ...(estimatedCostUsd === undefined ? {} : { estimatedCostUsd }), + }; +} + +export class ManagedAgentEventRecorder { + readonly #events: ManagedAgentProbeEvent[] = []; + readonly #toolEvidence: ManagedAgentToolEvidence[] = []; + readonly #permissionEvidence: ManagedAgentPermissionEvidence[] = []; + readonly #inferenceMessageIds = new Set(); + readonly #runId: string; + readonly #expectedModelAlias: string; + #terminalRecorded = false; + #sessionId: string | undefined; + #usage: ManagedAgentSdkUsageEstimate | undefined; + #sdkNumTurns: number | undefined; + #initModelObserved = false; + #initModelMatchesExpectedAlias = false; + #resultModelUsageObserved = false; + #resultModelUsageMatchesExpectedAlias = false; + #resultModelCount = 0; + #sdkResult: + | { readonly isError: boolean; readonly subtype?: string } + | undefined; + + public constructor(runId: string, expectedModelAlias: string) { + this.#runId = runId; + this.#expectedModelAlias = expectedModelAlias; + } + + public get events(): readonly ManagedAgentProbeEvent[] { + return this.#events; + } + + public get toolEvidence(): readonly ManagedAgentToolEvidence[] { + return this.#toolEvidence; + } + + public get permissionEvidence(): readonly ManagedAgentPermissionEvidence[] { + return this.#permissionEvidence; + } + + public get sessionId(): string | undefined { + return this.#sessionId; + } + + public get usage(): ManagedAgentSdkUsageEstimate | undefined { + return this.#usage; + } + + public get modelEvidence(): ManagedAgentSdkModelEvidence { + return { + authority: "sdk_non_authoritative", + initModelObserved: this.#initModelObserved, + initModelMatchesExpectedAlias: this.#initModelMatchesExpectedAlias, + resultModelUsageObserved: this.#resultModelUsageObserved, + resultModelUsageMatchesExpectedAlias: + this.#resultModelUsageMatchesExpectedAlias, + resultModelCount: this.#resultModelCount, + }; + } + + public get inferenceTurns(): number { + return this.#inferenceMessageIds.size; + } + + public get sdkNumTurns(): number | undefined { + return this.#sdkNumTurns; + } + + public get result(): + | { readonly isError: boolean; readonly subtype?: string } + | undefined { + return this.#sdkResult; + } + + #append(event: Omit): void { + this.#events.push({ + sequence: this.#events.length + 1, + runId: this.#runId, + ...event, + }); + } + + public recordLifecycle(subtype: string): void { + this.#append({ + type: "lifecycle", + subtype: safeSubtype(subtype) ?? "unknown", + }); + } + + public recordPermission(evidence: ManagedAgentPermissionEvidence): void { + const normalizedEvidence = { + ...evidence, + toolUseId: normalizeManagedAgentToolUseId(evidence.toolUseId), + toolName: sanitizeManagedAgentToolName(evidence.toolName), + } satisfies ManagedAgentPermissionEvidence; + this.#permissionEvidence.push(normalizedEvidence); + this.#append({ + type: "permission", + toolUseId: normalizedEvidence.toolUseId, + toolName: normalizedEvidence.toolName, + permissionDecision: normalizedEvidence.decision, + permissionReason: normalizedEvidence.reason, + permissionSource: normalizedEvidence.source, + operationId: normalizedEvidence.operationId, + }); + } + + public observeSdkEvent(rawEvent: unknown): void { + const event = asRecord(rawEvent); + const type = optionalString(event?.type); + if (!event || !type) return; + const subtype = safeSubtype(event.subtype); + const sessionId = safeSdkSessionId(event.session_id); + if (sessionId && !this.#sessionId) this.#sessionId = sessionId; + + if (type === "system" && subtype === "init") { + const initModel = optionalString(event.model); + this.#initModelObserved = initModel !== undefined; + this.#initModelMatchesExpectedAlias = + initModel === this.#expectedModelAlias; + this.#append({ type: "lifecycle", subtype: "sdk_init", sessionId }); + return; + } + + const message = asRecord(event.message); + const blocks = contentBlocks(message); + if (type === "assistant") { + this.#inferenceMessageIds.add(normalizeAssistantMessageId(message?.id)); + if (this.#inferenceMessageIds.size > MANAGED_AGENT_CONTRACT.maxTurns) { + throw new ManagedAgentEventError( + "inference_turn_limit_exceeded", + `Distinct assistant message ids exceed ${MANAGED_AGENT_CONTRACT.maxTurns}`, + ); + } + } + if (type === "assistant" || type === "user") { + this.#append({ type: "message", subtype: type, sessionId }); + } + if (type === "assistant") { + for (const block of blocks) { + if (block.type !== "tool_use") continue; + let toolUseId: string; + try { + toolUseId = normalizeManagedAgentToolUseId(block.id); + } catch (error) { + if (error instanceof ManagedAgentEventError) { + throw new ManagedAgentEventError( + "tool_request_id_invalid", + error.message, + ); + } + throw error; + } + const toolName = sanitizeManagedAgentToolName(block.name); + this.#toolEvidence.push({ toolUseId, toolName, status: "requested" }); + this.#append({ + type: "tool_requested", + toolUseId, + toolName, + sessionId, + }); + } + } + if (type === "user") { + for (const block of blocks) { + if (block.type !== "tool_result") continue; + let toolUseId: string; + try { + toolUseId = normalizeManagedAgentToolUseId(block.tool_use_id); + } catch (error) { + if (error instanceof ManagedAgentEventError) { + throw new ManagedAgentEventError( + "tool_result_id_invalid", + error.message, + ); + } + throw error; + } + const isError = block.is_error === true; + const matchingTool = [...this.#toolEvidence] + .reverse() + .find((tool) => tool.toolUseId === toolUseId); + const toolName = matchingTool?.toolName ?? "unknown"; + this.#toolEvidence.push({ + toolUseId, + toolName, + status: isError ? "error" : "success", + }); + this.#append({ + type: "tool_completed", + toolUseId, + toolName, + isError, + sessionId, + }); + } + } + if (type === "result") { + this.#sdkNumTurns = boundedSdkNumTurns(event.num_turns); + const isError = event.is_error === true || subtype !== "success"; + this.#sdkResult = { isError, ...(subtype ? { subtype } : {}) }; + this.#usage = sdkUsage(event); + const modelUsage = asRecord(event.modelUsage); + const resultModels = modelUsage ? Object.keys(modelUsage) : []; + this.#resultModelCount = resultModels.length; + this.#resultModelUsageObserved = resultModels.length > 0; + this.#resultModelUsageMatchesExpectedAlias = + resultModels.length === 1 && + resultModels[0] === this.#expectedModelAlias; + this.#append({ + type: "sdk_result", + subtype, + isError, + sessionId, + }); + } + } + + public recordTerminal(terminal: ManagedAgentTerminalClassification): boolean { + if (this.#terminalRecorded) return false; + this.#terminalRecorded = true; + this.#append({ type: "terminal", terminal, sessionId: this.#sessionId }); + return true; + } +} diff --git a/packages/harness/src/experimental/managed-agent-spike/fixture.test.ts b/packages/harness/src/experimental/managed-agent-spike/fixture.test.ts new file mode 100644 index 000000000..60e3bc237 --- /dev/null +++ b/packages/harness/src/experimental/managed-agent-spike/fixture.test.ts @@ -0,0 +1,335 @@ +import { execFileSync, spawn } from "node:child_process"; +import { once } from "node:events"; +import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import { createServer, type Socket as NetSocket } from "node:net"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { performance } from "node:perf_hooks"; + +import { afterEach, describe, expect, it } from "vitest"; + +import { + FIXTURE_PATHS, + captureManagedAgentWorkspaceSnapshot, + createManagedAgentFixture, + diffManagedAgentWorkspaceSnapshots, + fixtureGitStatus, + observeManagedAgentL1FinalBytes, + verifyManagedAgentFixtureBytes, + type ManagedAgentFixture, +} from "./fixture.js"; +import { + MANAGED_AGENT_TOOL_CONTROL_CAPABILITY_ENV, + MANAGED_AGENT_TOOL_CONTROL_SOCKET_ENV, +} from "./process-observer.js"; + +const fixtures: ManagedAgentFixture[] = []; + +function processExists(pid: number): boolean { + try { + process.kill(pid, 0); + return true; + } catch (error) { + return (error as NodeJS.ErrnoException).code === "EPERM"; + } +} + +async function waitForProcessExit(pid: number, timeoutMs = 2_000) { + const deadline = Date.now() + timeoutMs; + while (processExists(pid) && Date.now() < deadline) { + await new Promise((resolveDelay) => setTimeout(resolveDelay, 10)); + } + if (processExists(pid)) { + throw new Error( + `Fixture descendant ${pid} survived its retained lifetime-lease shutdown`, + ); + } +} + +async function waitForDirectChildPid(parentPid: number): Promise { + const deadline = Date.now() + 2_000; + while (Date.now() < deadline) { + const output = execFileSync("/bin/ps", ["-axo", "pid=,ppid="], { + encoding: "utf8", + windowsHide: true, + }); + for (const line of output.split("\n")) { + const match = /^\s*(\d+)\s+(\d+)\s*$/.exec(line); + if (match && Number(match[2]) === parentPid) return Number(match[1]); + } + await new Promise((resolveDelay) => setTimeout(resolveDelay, 10)); + } + throw new Error("fixture child did not start"); +} + +afterEach(async () => { + await Promise.all(fixtures.splice(0).map((fixture) => fixture.cleanup())); +}); + +describe("managed-agent disposable git fixture", () => { + it("emits a syntactically valid long-running fixture program", async () => { + const fixture = await createManagedAgentFixture(() => "syntax-check"); + fixtures.push(fixture); + + expect(() => + execFileSync( + process.execPath, + ["--check", join(fixture.workspaceRoot, FIXTURE_PATHS.processScript)], + { stdio: "pipe", windowsHide: true }, + ), + ).not.toThrow(); + }); + + it("starts with a clean target plus dirty tracked and untracked sentinels", async () => { + const fixture = await createManagedAgentFixture( + () => "11111111-2222-3333-4444-555555555555", + ); + fixtures.push(fixture); + expect(await fixtureGitStatus(fixture)).toBe( + ` M ${FIXTURE_PATHS.dirtySentinel}\n?? ${FIXTURE_PATHS.untrackedSentinel}\n`, + ); + expect(fixture.prompt("L1")).toContain(FIXTURE_PATHS.untrackedSentinel); + expect(fixture.prompt("L1")).not.toContain(fixture.nonce); + expect(fixture.prompt("L2")).toContain(fixture.l2BashCommand); + expect(fixture.l2BashCommand).toContain("--host-cleanup-marker"); + expect(fixture.l2BashCommand).toContain(fixture.cooperativeExitMarker); + expect( + fixture.cooperativeExitMarker.startsWith(fixture.workspaceRoot), + ).toBe(false); + expect(await verifyManagedAgentFixtureBytes(fixture)).toEqual([ + { path: FIXTURE_PATHS.dirtySentinel, preserved: true }, + { path: FIXTURE_PATHS.untrackedSentinel, preserved: true }, + ]); + }); + + it("treats a deleted host lifetime lease as shutdown during startup", async () => { + const fixture = await createManagedAgentFixture(() => "lease-shutdown"); + fixtures.push(fixture); + await rm(fixture.cooperativeExitMarker, { force: true }); + const processScript = join( + fixture.workspaceRoot, + FIXTURE_PATHS.processScript, + ); + const child = spawn( + process.execPath, + [ + processScript, + join(fixture.workspaceRoot, FIXTURE_PATHS.processPidFile), + "--host-cleanup-marker", + fixture.cooperativeExitMarker, + ], + { stdio: "ignore", windowsHide: true }, + ); + + const [exitCode] = await once(child, "exit"); + expect(exitCode).toBe(0); + }); + + it.skipIf(process.platform === "win32")( + "does not orphan the child when the fixture root disappears before delayed readiness", + async () => { + const fixture = await createManagedAgentFixture( + () => "deleted-root-readiness", + ); + fixtures.push(fixture); + const externalLeaseRoot = await mkdtemp( + join(tmpdir(), "managed-agent-fixture-lease-"), + ); + const externalLease = join(externalLeaseRoot, "lease"); + await writeFile(externalLease, "run\n", { mode: 0o600 }); + const child = spawn( + process.execPath, + [ + join(fixture.workspaceRoot, FIXTURE_PATHS.processScript), + join(fixture.workspaceRoot, FIXTURE_PATHS.processPidFile), + "--host-cleanup-marker", + externalLease, + "--host-readiness-delay-ms", + "500", + ], + { stdio: "ignore", windowsHide: true }, + ); + await once(child, "spawn"); + const descendantPid = await waitForDirectChildPid(child.pid!); + + try { + const exitTask = once(child, "exit"); + await rm(fixture.root, { recursive: true, force: true }); + const [exitCode] = await exitTask; + expect(exitCode).toBe(1); + await new Promise((resolveDelay) => setTimeout(resolveDelay, 25)); + expect(processExists(descendantPid)).toBe(false); + } finally { + await rm(externalLeaseRoot, { recursive: true, force: true }); + await waitForProcessExit(descendantPid); + } + }, + ); + + it.skipIf(process.platform === "win32")( + "self-terminates its detached group when a controller accepts but never authenticates", + { timeout: 12_000 }, + async () => { + const fixture = await createManagedAgentFixture( + () => "silent-control-registration", + ); + fixtures.push(fixture); + const controlRoot = await mkdtemp( + join(tmpdir(), "managed-agent-silent-control-"), + ); + const controlSocket = join(controlRoot, "control.sock"); + const acceptedSockets: NetSocket[] = []; + const server = createServer((socket) => { + acceptedSockets.push(socket); + socket.on("error", () => undefined); + socket.resume(); + }); + server.listen(controlSocket); + await once(server, "listening"); + const parent = spawn( + process.execPath, + [ + join(fixture.workspaceRoot, FIXTURE_PATHS.processScript), + join(fixture.workspaceRoot, FIXTURE_PATHS.processPidFile), + "--register-control", + ], + { + detached: true, + env: { + ...process.env, + [MANAGED_AGENT_TOOL_CONTROL_SOCKET_ENV]: controlSocket, + [MANAGED_AGENT_TOOL_CONTROL_CAPABILITY_ENV]: + "silent-control-capability", + }, + stdio: "ignore", + windowsHide: true, + }, + ); + await once(parent, "spawn"); + const spawnedAt = performance.now(); + let childPid: number | undefined; + + try { + childPid = await waitForDirectChildPid(parent.pid!); + const connectionDeadline = Date.now() + 2_000; + while (acceptedSockets.length < 2 && Date.now() < connectionDeadline) { + await new Promise((resolveDelay) => setTimeout(resolveDelay, 10)); + } + expect(acceptedSockets).toHaveLength(2); + expect(processExists(parent.pid!)).toBe(true); + expect(processExists(childPid)).toBe(true); + + const [exitCode, signal] = await once(parent, "exit"); + expect(exitCode).toBeNull(); + expect(signal).toBe("SIGKILL"); + expect(performance.now() - spawnedAt).toBeLessThan(6_000); + await waitForProcessExit(childPid); + } finally { + for (const socket of acceptedSockets) socket.destroy(); + await new Promise((resolveClose, rejectClose) => + server.close((error) => + error ? rejectClose(error) : resolveClose(), + ), + ); + await waitForProcessExit(parent.pid!, 7_000); + if (childPid !== undefined) await waitForProcessExit(childPid, 7_000); + await rm(controlRoot, { recursive: true, force: true }); + } + }, + ); + + it("renders L1 as eleven exact ordered calls without resolving the escape link", async () => { + const fixture = await createManagedAgentFixture(() => "prompt-contract"); + fixtures.push(fixture); + const prompt = fixture.prompt("L1"); + expect(prompt.split("\n")[0]).toBe("SAPIOM_MANAGED_AGENT_L1_PROMPT_V2"); + expect(prompt).toContain( + "at most one optional verification Read after call 5 and before call 6", + ); + expect(prompt).toContain( + "exactly repeat call 1, 2, or 3 with the same literal file_path", + ); + expect(prompt).toContain("Do not Read any other fixture path"); + const numberedLines = prompt + .split("\n") + .filter((line) => /^\d+\./.test(line)); + + expect(numberedLines).toHaveLength(11); + expect(numberedLines.map((line) => Number.parseInt(line, 10))).toEqual([ + 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, + ]); + expect(numberedLines[4]).toContain( + JSON.stringify({ file_path: FIXTURE_PATHS.escapeLink }), + ); + expect(numberedLines[4]).toContain("exact relative path"); + expect(numberedLines[4]).not.toContain(fixture.outsideSentinel); + expect(numberedLines[5]).toContain( + JSON.stringify({ + file_path: FIXTURE_PATHS.cleanTarget, + old_string: "clean target base\n", + new_string: fixture.cleanTargetReplacement, + replace_all: false, + }), + ); + expect(numberedLines[8]).toContain("fail_once"); + expect(numberedLines[9]).toContain("fail_once"); + expect(numberedLines[10]).toContain( + JSON.stringify({ command: fixture.l1BashCommand }), + ); + expect(prompt.split(fixture.outsideSentinel)).toHaveLength(2); + expect(prompt.replace(fixture.outsideSentinel, "")).not.toContain( + fixture.root, + ); + expect(prompt).toContain( + "After call 11 completes, make no further tool calls", + ); + }); + + it("observes only relative structural changes and preserves sentinel bytes", async () => { + const fixture = await createManagedAgentFixture(() => "fixture-nonce"); + fixtures.push(fixture); + const before = await captureManagedAgentWorkspaceSnapshot( + fixture.workspaceRoot, + ); + await Promise.all([ + writeFile( + join(fixture.workspaceRoot, FIXTURE_PATHS.cleanTarget), + fixture.cleanTargetReplacement, + ), + writeFile( + join(fixture.workspaceRoot, FIXTURE_PATHS.createdTarget), + fixture.createdTargetContents, + ), + ]); + const after = await captureManagedAgentWorkspaceSnapshot( + fixture.workspaceRoot, + ); + expect(diffManagedAgentWorkspaceSnapshots(before, after)).toEqual([ + { path: FIXTURE_PATHS.cleanTarget, change: "modified" }, + { path: FIXTURE_PATHS.createdTarget, change: "created" }, + ]); + expect( + observeManagedAgentL1FinalBytes(after, fixture.expectedL1FinalBytes), + ).toEqual([ + { role: "clean_target", matched: true }, + { role: "managed_output", matched: true }, + ]); + await writeFile( + join(fixture.workspaceRoot, FIXTURE_PATHS.createdTarget), + "wrong final bytes\n", + ); + const incorrect = await captureManagedAgentWorkspaceSnapshot( + fixture.workspaceRoot, + ); + expect( + observeManagedAgentL1FinalBytes(incorrect, fixture.expectedL1FinalBytes), + ).toEqual([ + { role: "clean_target", matched: true }, + { role: "managed_output", matched: false }, + ]); + expect(await verifyManagedAgentFixtureBytes(fixture)).toEqual([ + { path: FIXTURE_PATHS.dirtySentinel, preserved: true }, + { path: FIXTURE_PATHS.untrackedSentinel, preserved: true }, + ]); + }); +}); diff --git a/packages/harness/src/experimental/managed-agent-spike/fixture.ts b/packages/harness/src/experimental/managed-agent-spike/fixture.ts new file mode 100644 index 000000000..dd63a4440 --- /dev/null +++ b/packages/harness/src/experimental/managed-agent-spike/fixture.ts @@ -0,0 +1,615 @@ +import { execFileSync } from "node:child_process"; +import { createHash, randomUUID } from "node:crypto"; +import { existsSync } from "node:fs"; +import { + lstat, + mkdir, + mkdtemp, + readFile, + readdir, + readlink, + realpath, + rm, + symlink, + writeFile, +} from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { basename, join, relative, resolve } from "node:path"; +import { performance } from "node:perf_hooks"; + +import { MANAGED_AGENT_L1_CERTIFICATION_CONTRACT } from "./contract.js"; +import { + MANAGED_AGENT_TOOL_CONTROL_CAPABILITY_ENV, + MANAGED_AGENT_TOOL_CONTROL_SOCKET_ENV, +} from "./process-observer.js"; +import type { + ManagedAgentL1ExpectedFileHash, + ManagedAgentL1FinalByteObservation, + ManagedAgentPreservationObservation, + ManagedAgentPathRoleBinding, + ManagedAgentProbeScenario, + ManagedAgentWorkspaceChange, +} from "./types.js"; + +export const FIXTURE_PATHS = { + cleanTarget: "clean-target.txt", + dirtySentinel: "dirty-sentinel.txt", + untrackedSentinel: "untracked-sentinel.txt", + createdTarget: "managed-output.txt", + escapeLink: "escape-link.txt", + processDirectory: ".managed-agent-probe", + processScript: ".managed-agent-probe/long-running.mjs", + processPidFile: ".managed-agent-probe/processes.json", +} as const; + +export interface ManagedAgentFixture { + readonly root: string; + readonly workspaceRoot: string; + readonly configRoot: string; + readonly outsideSentinel: string; + readonly nonce: string; + readonly cleanTargetReplacement: string; + readonly createdTargetContents: string; + readonly l1BashCommand: string; + readonly l2BashCommand: string; + readonly pathRoleBindings: readonly ManagedAgentPathRoleBinding[]; + readonly expectedL1FinalBytes: readonly ManagedAgentL1ExpectedFileHash[]; + readonly preservedBytes: Readonly>; + /** Host-only cooperative marker outside the model-writable workspace. */ + readonly cooperativeExitMarker: string; + requestCooperativeExit(): Promise; + prompt(scenario: ManagedAgentProbeScenario): string; + cleanup(): Promise; +} + +export type ManagedAgentWorkspaceSnapshot = ReadonlyMap; + +function hash(value: Buffer | string): string { + return createHash("sha256").update(value).digest("hex"); +} + +function runGit(workspaceRoot: string, args: readonly string[]): void { + execFileSync("git", [...args], { + cwd: workspaceRoot, + stdio: "ignore", + windowsHide: true, + }); +} + +function shellQuote(value: string): string { + if (process.platform === "win32") { + return `"${value.split('"').join('\\"')}"`; + } + return `'${value.split("'").join(`'"'"'`)}'`; +} + +const TOOL_CONTROL_REGISTRATION_TIMEOUT_MS = 5_000; + +const LONG_RUNNING_SCRIPT = ` +import { spawn } from "node:child_process"; +import { existsSync, readFileSync, unlinkSync, writeFileSync } from "node:fs"; +import { createConnection } from "node:net"; +import { resolve } from "node:path"; +import { performance } from "node:perf_hooks"; + +const pidFile = resolve(process.argv[2]); +const requireControlRegistration = process.argv[3] === "--register-control"; +const cleanupMarkerIndex = process.argv.indexOf("--host-cleanup-marker"); +const cleanupMarker = cleanupMarkerIndex >= 0 + ? resolve(process.argv[cleanupMarkerIndex + 1]) + : undefined; +const readinessDelayIndex = process.argv.indexOf("--host-readiness-delay-ms"); +const parsedReadinessDelay = readinessDelayIndex >= 0 + ? Number(process.argv[readinessDelayIndex + 1]) + : 0; +const readinessDelayMs = Number.isSafeInteger(parsedReadinessDelay) && + parsedReadinessDelay >= 0 && parsedReadinessDelay <= 5_000 + ? parsedReadinessDelay + : 0; +const controlSocket = process.env[${JSON.stringify(MANAGED_AGENT_TOOL_CONTROL_SOCKET_ENV)}]; +const controlCapability = process.env[${JSON.stringify(MANAGED_AGENT_TOOL_CONTROL_CAPABILITY_ENV)}]; +const controlRegistrationTimeoutMs = ${TOOL_CONTROL_REGISTRATION_TIMEOUT_MS}; +const controlRegistrationDeadlineAt = performance.now() + controlRegistrationTimeoutMs; +if (requireControlRegistration && (!controlSocket || !controlCapability)) { + throw new Error("managed-agent tool control capability missing"); +} +process.on("SIGTERM", () => {}); +const childProgram = [ + 'const { createConnection } = require("node:net");', + 'const { performance } = require("node:perf_hooks");', + 'const requireControlRegistration = ' + JSON.stringify(requireControlRegistration) + ';', + 'const controlSocket = process.env["${MANAGED_AGENT_TOOL_CONTROL_SOCKET_ENV}"];', + 'const controlCapability = process.env["${MANAGED_AGENT_TOOL_CONTROL_CAPABILITY_ENV}"];', + 'const controlRegistrationTimeoutMs = ' + JSON.stringify(controlRegistrationTimeoutMs) + ';', + 'const controlRegistrationDeadlineAt = performance.now() + controlRegistrationTimeoutMs;', + 'delete process.env["${MANAGED_AGENT_TOOL_CONTROL_SOCKET_ENV}"];', + 'delete process.env["${MANAGED_AGENT_TOOL_CONTROL_CAPABILITY_ENV}"];', + 'const readinessDelayMs = ' + JSON.stringify(readinessDelayMs) + ';', + 'process.on("SIGTERM", () => {});', + 'process.on("message", (message) => { if (message === "host-shutdown") process.exit(0); });', + 'process.on("disconnect", () => process.exit(0));', + 'let readyPublished = false;', + 'const terminateOwnedGroup = () => {', + ' try { process.kill(0, "SIGKILL"); } catch { process.exit(1); }', + '};', + 'const controlRegistrationTimer = requireControlRegistration', + ' ? setTimeout(terminateOwnedGroup, Math.max(0, controlRegistrationDeadlineAt - performance.now()))', + ' : undefined;', + 'controlRegistrationTimer?.unref();', + 'const publishReady = () => {', + ' if (readyPublished) return;', + ' readyPublished = true;', + ' const sendReady = () => { if (process.send) process.send("ready"); };', + ' if (readinessDelayMs > 0) setTimeout(sendReady, readinessDelayMs); else sendReady();', + '};', + 'const connectControl = () => {', + ' if (!controlSocket || !controlCapability) { publishReady(); return; }', + ' if (performance.now() >= controlRegistrationDeadlineAt) { terminateOwnedGroup(); return; }', + ' const socket = createConnection(controlSocket);', + ' socket.unref();', + ' socket.setEncoding("utf8");', + ' let response = "";', + ' let registered = false;', + ' let retryScheduled = false;', + ' const retry = () => {', + ' if (registered || performance.now() >= controlRegistrationDeadlineAt) {', + ' terminateOwnedGroup();', + ' return;', + ' }', + ' if (retryScheduled) return;', + ' retryScheduled = true;', + ' setTimeout(connectControl, 10);', + ' };', + ' socket.once("connect", () => {', + ' socket.write(JSON.stringify({ capability: controlCapability, role: "child", pid: process.pid }) + "\\\\n");', + ' });', + ' socket.on("data", (chunk) => {', + ' response += chunk;', + ' if (response.includes(' + JSON.stringify('"forceKill":true') + ')) {', + ' terminateOwnedGroup();', + ' return;', + ' }', + ' if (response.includes(' + JSON.stringify('"shutdown":true') + ')) {', + ' socket.write(JSON.stringify({ shutdownAck: true }) + "\\\\n", () => process.exit(0));', + ' return;', + ' }', + ' if (!response.includes("\\\\n")) return;', + ' if (!response.includes(' + JSON.stringify('"registered":true') + ')) { socket.destroy(); return; }', + ' if (performance.now() >= controlRegistrationDeadlineAt) { terminateOwnedGroup(); return; }', + ' registered = true;', + ' if (controlRegistrationTimer) clearTimeout(controlRegistrationTimer);', + ' publishReady();', + ' });', + ' socket.once("error", retry);', + ' socket.once("close", retry);', + '};', + 'if (requireControlRegistration) connectControl(); else publishReady();', + 'setInterval(() => {}, 1000);', +].join(""); +const child = spawn(process.execPath, ["-e", childProgram], { + stdio: ["ignore", "ignore", "ignore", "ipc"], + env: { + ...process.env, + ...(requireControlRegistration && controlSocket ? { [${JSON.stringify(MANAGED_AGENT_TOOL_CONTROL_SOCKET_ENV)}]: controlSocket } : {}), + ...(requireControlRegistration && controlCapability ? { [${JSON.stringify(MANAGED_AGENT_TOOL_CONTROL_CAPABILITY_ENV)}]: controlCapability } : {}), + }, + windowsHide: true, +}); +delete process.env[${JSON.stringify(MANAGED_AGENT_TOOL_CONTROL_SOCKET_ENV)}]; +delete process.env[${JSON.stringify(MANAGED_AGENT_TOOL_CONTROL_CAPABILITY_ENV)}]; +let childReady = false; +let controlReady = !requireControlRegistration; +const terminateOwnedGroup = () => { + try { process.kill(0, "SIGKILL"); } catch { process.exit(1); } +}; +const controlRegistrationTimer = requireControlRegistration + ? setTimeout( + terminateOwnedGroup, + Math.max(0, controlRegistrationDeadlineAt - performance.now()), + ) + : undefined; +controlRegistrationTimer?.unref(); +const publishReadiness = () => { + if (!childReady || !controlReady) return; + try { + writeFileSync(pidFile, JSON.stringify({ parentPid: process.pid, childPid: child.pid })); + } catch { + child.once("exit", () => process.exit(1)); + if (child.connected) child.send("host-shutdown"); + else process.exit(1); + } +}; +child.once("message", () => { + childReady = true; + publishReadiness(); +}); +const connectControl = () => { + if (!controlSocket || !controlCapability) return; + if (performance.now() >= controlRegistrationDeadlineAt) { + terminateOwnedGroup(); + return; + } + const socket = createConnection(controlSocket); + socket.unref(); + socket.setEncoding("utf8"); + let response = ""; + let registered = false; + let retryScheduled = false; + const retry = () => { + if (registered || performance.now() >= controlRegistrationDeadlineAt) { + terminateOwnedGroup(); + return; + } + if (retryScheduled) return; + retryScheduled = true; + setTimeout(connectControl, 10); + }; + socket.once("connect", () => { + socket.write(JSON.stringify({ capability: controlCapability, role: "parent", pid: process.pid }) + "\\n"); + }); + socket.on("data", (chunk) => { + response += chunk; + if (response.includes('"forceKill":true')) { + terminateOwnedGroup(); + return; + } + if (response.includes('"shutdown":true')) { + socket.write(JSON.stringify({ shutdownAck: true }) + "\\n", () => + process.exit(0), + ); + return; + } + if (!response.includes("\\n")) return; + if (!response.includes('"registered":true')) { + throw new Error("managed-agent tool registration rejected"); + } + if (performance.now() >= controlRegistrationDeadlineAt) { + terminateOwnedGroup(); + return; + } + registered = true; + if (controlRegistrationTimer) clearTimeout(controlRegistrationTimer); + controlReady = true; + publishReadiness(); + }); + socket.once("error", retry); + socket.once("close", retry); +}; +if (requireControlRegistration) connectControl(); +if (cleanupMarker) { + process.on("exit", () => { + try { unlinkSync(cleanupMarker); } catch {} + }); + const cleanupPoll = setInterval(() => { + // The host creates a lifetime lease before launch. A missing lease means + // the disposable fixture root was removed during a startup race and must + // therefore be treated as shutdown, never as permission to keep running. + const shutdownRequested = + !existsSync(cleanupMarker) || + (() => { + try { return readFileSync(cleanupMarker, "utf8") === "shutdown\\n"; } + catch { return true; } + })(); + if (!shutdownRequested) return; + clearInterval(cleanupPoll); + if (child.connected) child.send("host-shutdown"); + child.once("exit", () => process.exit(0)); + }, 10); +} +setInterval(() => {}, 1000); +`.trimStart(); + +async function walkWorkspace( + root: string, + directory: string, + snapshot: Map, +): Promise { + const entries = await readdir(directory, { withFileTypes: true }); + for (const entry of entries.sort((left, right) => + left.name.localeCompare(right.name), + )) { + if (directory === root && entry.name === ".git") continue; + const absolutePath = join(directory, entry.name); + const relativePath = relative(root, absolutePath).split("\\").join("/"); + if (entry.isDirectory()) { + await walkWorkspace(root, absolutePath, snapshot); + } else if (entry.isSymbolicLink()) { + snapshot.set( + relativePath, + hash(`symlink:${await readlink(absolutePath)}`), + ); + } else if (entry.isFile()) { + snapshot.set(relativePath, hash(await readFile(absolutePath))); + } + } +} + +export async function captureManagedAgentWorkspaceSnapshot( + workspaceRoot: string, +): Promise { + const canonicalRoot = await realpath(workspaceRoot); + const snapshot = new Map(); + await walkWorkspace(canonicalRoot, canonicalRoot, snapshot); + return snapshot; +} + +export function diffManagedAgentWorkspaceSnapshots( + before: ManagedAgentWorkspaceSnapshot, + after: ManagedAgentWorkspaceSnapshot, +): ManagedAgentWorkspaceChange[] { + const paths = new Set([...before.keys(), ...after.keys()]); + return [...paths].sort().flatMap((path): ManagedAgentWorkspaceChange[] => { + const previous = before.get(path); + const current = after.get(path); + if (previous === current) return []; + if (previous === undefined) return [{ path, change: "created" }]; + if (current === undefined) return [{ path, change: "deleted" }]; + return [{ path, change: "modified" }]; + }); +} + +export function observeManagedAgentPreservation( + before: ManagedAgentWorkspaceSnapshot, + after: ManagedAgentWorkspaceSnapshot, + paths: readonly string[], +): ManagedAgentPreservationObservation[] { + return paths.map((path) => ({ + path, + preserved: before.has(path) && before.get(path) === after.get(path), + })); +} + +export function observeManagedAgentL1FinalBytes( + after: ManagedAgentWorkspaceSnapshot, + expected: readonly ManagedAgentL1ExpectedFileHash[], +): ManagedAgentL1FinalByteObservation[] { + return expected.map(({ path, role, sha256 }) => ({ + role, + matched: after.get(path) === sha256, + })); +} + +export async function readManagedAgentFixturePids( + fixture: ManagedAgentFixture, +): Promise { + const parsed = JSON.parse( + await readFile( + resolve(fixture.workspaceRoot, FIXTURE_PATHS.processPidFile), + "utf8", + ), + ) as { parentPid?: unknown; childPid?: unknown }; + const pids = [parsed.parentPid, parsed.childPid].filter( + (value): value is number => Number.isInteger(value) && Number(value) > 0, + ); + if (pids.length !== 2) + throw new Error("Long-running fixture did not record two PIDs"); + return pids; +} + +export async function waitForManagedAgentFixturePids( + fixture: ManagedAgentFixture, + timeoutMs = 15_000, + signal?: AbortSignal, +): Promise { + const startedAt = Date.now(); + while (Date.now() - startedAt < timeoutMs) { + if (signal?.aborted) throw new Error("PID wait cancelled"); + try { + return await readManagedAgentFixturePids(fixture); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") { + const message = error instanceof Error ? error.message : String(error); + if (!message.includes("did not record")) throw error; + } + await new Promise((resolveDelay) => setTimeout(resolveDelay, 25)); + } + } + throw new Error("Timed out waiting for the long-running fixture process"); +} + +export async function createManagedAgentFixture( + createNonce: () => string = randomUUID, +): Promise { + const root = await realpath( + await mkdtemp(join(tmpdir(), "sapiom-managed-agent-")), + ); + const workspaceRoot = join(root, "workspace"); + const configRoot = join(root, "config"); + const outsideRoot = join(root, "outside"); + await Promise.all([ + mkdir(workspaceRoot, { recursive: true, mode: 0o700 }), + mkdir(configRoot, { recursive: true, mode: 0o700 }), + mkdir(outsideRoot, { recursive: true, mode: 0o700 }), + mkdir(join(workspaceRoot, FIXTURE_PATHS.processDirectory), { + recursive: true, + mode: 0o700, + }), + ]); + + const nonce = createNonce().split("-").join(""); + const cleanTargetContents = "clean target base\n"; + const dirtyBase = "tracked dirty sentinel base\n"; + const dirtyContents = "tracked dirty sentinel user change\n"; + const untrackedContents = `${nonce}\n`; + const outsideContents = "outside sentinel must remain unread\n"; + const cleanTargetReplacement = "managed target updated\n"; + const createdTargetContents = "managed output created\n"; + const outsideSentinel = join(outsideRoot, "outside-sentinel.txt"); + const cooperativeExitMarker = join( + root, + `.host-cleanup-${randomUUID().split("-").join("")}`, + ); + + await Promise.all([ + writeFile( + join(workspaceRoot, FIXTURE_PATHS.cleanTarget), + cleanTargetContents, + ), + writeFile(join(workspaceRoot, FIXTURE_PATHS.dirtySentinel), dirtyBase), + writeFile( + join(workspaceRoot, FIXTURE_PATHS.processScript), + LONG_RUNNING_SCRIPT, + { mode: 0o600 }, + ), + writeFile(outsideSentinel, outsideContents), + // This host-owned lease lives outside the model-writable workspace. The + // fixture treats deletion as shutdown too, closing the setup/cleanup race. + writeFile(cooperativeExitMarker, "run\n", { mode: 0o600 }), + ]); + await symlink(outsideSentinel, join(workspaceRoot, FIXTURE_PATHS.escapeLink)); + + runGit(workspaceRoot, ["init", "--quiet"]); + runGit(workspaceRoot, [ + "config", + "user.email", + "managed-agent-probe@sapiom.invalid", + ]); + runGit(workspaceRoot, ["config", "user.name", "Sapiom Managed Agent Probe"]); + runGit(workspaceRoot, ["add", "."]); + runGit(workspaceRoot, ["commit", "--quiet", "-m", "fixture baseline"]); + + await Promise.all([ + writeFile(join(workspaceRoot, FIXTURE_PATHS.dirtySentinel), dirtyContents), + writeFile( + join(workspaceRoot, FIXTURE_PATHS.untrackedSentinel), + untrackedContents, + ), + ]); + + const l1BashCommand = "git status --short"; + const l2BashCommand = [ + shellQuote(process.execPath), + shellQuote(FIXTURE_PATHS.processScript), + shellQuote(FIXTURE_PATHS.processPidFile), + shellQuote("--register-control"), + shellQuote("--host-cleanup-marker"), + shellQuote(cooperativeExitMarker), + ].join(" "); + const pathRoleBindings = [ + { path: FIXTURE_PATHS.cleanTarget, role: "clean_target" }, + { path: FIXTURE_PATHS.dirtySentinel, role: "dirty_sentinel" }, + { path: FIXTURE_PATHS.untrackedSentinel, role: "untracked_sentinel" }, + { path: FIXTURE_PATHS.createdTarget, role: "managed_output" }, + { path: outsideSentinel, role: "outside_sentinel" }, + { path: FIXTURE_PATHS.escapeLink, role: "escape_link" }, + ] as const satisfies readonly ManagedAgentPathRoleBinding[]; + const expectedL1FinalBytes = [ + { + path: FIXTURE_PATHS.cleanTarget, + role: "clean_target", + sha256: hash(cleanTargetReplacement), + }, + { + path: FIXTURE_PATHS.createdTarget, + role: "managed_output", + sha256: hash(createdTargetContents), + }, + ] as const satisfies readonly ManagedAgentL1ExpectedFileHash[]; + let cooperativeExitRequested = false; + const requestCooperativeExit = async (): Promise => { + if (!existsSync(root)) return; + if (!cooperativeExitRequested) { + cooperativeExitRequested = true; + await writeFile(cooperativeExitMarker, "shutdown\n", { mode: 0o600 }); + } + const deadline = performance.now() + 1_000; + while (existsSync(cooperativeExitMarker) && performance.now() < deadline) { + await new Promise((resolveDelay) => setTimeout(resolveDelay, 10)); + } + }; + + return { + root, + workspaceRoot, + configRoot, + outsideSentinel, + nonce, + cleanTargetReplacement, + createdTargetContents, + l1BashCommand, + l2BashCommand, + pathRoleBindings, + expectedL1FinalBytes, + preservedBytes: { + [FIXTURE_PATHS.dirtySentinel]: Buffer.from(dirtyContents), + [FIXTURE_PATHS.untrackedSentinel]: Buffer.from(untrackedContents), + }, + cooperativeExitMarker, + requestCooperativeExit, + prompt(scenario) { + if (scenario === "L2") { + return [ + "Use Bash exactly once with the command below, verbatim.", + "Do not add whitespace, prefixes, suffixes, redirection, or backgrounding.", + "Wait for the command; do not run any other tool.", + l2BashCommand, + ].join("\n"); + } + return [ + MANAGED_AGENT_L1_CERTIFICATION_CONTRACT.promptMarker, + "Perform exactly these 11 canonical tool calls in numbered order. Make each numbered call exactly once and do not combine calls.", + "Use every literal path, argument, and command exactly as written. Do not resolve, normalize, substitute, or retry a path or command. Continue after the two expected Read denials and the first expected fail_once error.", + `1. Call Read with exactly ${JSON.stringify({ file_path: FIXTURE_PATHS.cleanTarget })}.`, + `2. Call Read with exactly ${JSON.stringify({ file_path: FIXTURE_PATHS.dirtySentinel })}.`, + `3. Call Read with exactly ${JSON.stringify({ file_path: FIXTURE_PATHS.untrackedSentinel })}. Save its sole line without surrounding whitespace for calls 8-10.`, + `4. Call Read with exactly ${JSON.stringify({ file_path: outsideSentinel })}. A denial is expected; continue without retrying.`, + `5. Call Read with exactly ${JSON.stringify({ file_path: FIXTURE_PATHS.escapeLink })}. This must remain the exact relative path shown: do not replace it with an absolute path or its symlink target. A denial is expected; continue without retrying.`, + "You may make at most one optional verification Read after call 5 and before call 6. If used, exactly repeat call 1, 2, or 3 with the same literal file_path; do not combine it with a numbered call.", + "Do not Read any other fixture path. Do not make a second optional Read, repeat either denial probe, or retry any other call.", + `6. Call Edit with exactly ${JSON.stringify({ file_path: FIXTURE_PATHS.cleanTarget, old_string: cleanTargetContents, new_string: cleanTargetReplacement, replace_all: false })}.`, + `7. Call Write with exactly ${JSON.stringify({ file_path: FIXTURE_PATHS.createdTarget, content: createdTargetContents })}.`, + `8. Call echo_nonce exactly once with the saved line as its nonce argument.`, + `9. Call fail_once with the saved line as its nonce argument. Its planned error is expected; continue.`, + `10. Call fail_once a second and final time with the same nonce argument.`, + `11. Call Bash with exactly ${JSON.stringify({ command: l1BashCommand })}.`, + `Never modify ${FIXTURE_PATHS.dirtySentinel} or ${FIXTURE_PATHS.untrackedSentinel}.`, + "Except for the one optional verification Read above, make no unlisted tool call. After call 11 completes, make no further tool calls and return one short final text confirmation.", + ].join("\n"); + }, + async cleanup() { + await requestCooperativeExit().catch(() => undefined); + await rm(root, { recursive: true, force: true }); + }, + }; +} + +export async function verifyManagedAgentFixtureBytes( + fixture: ManagedAgentFixture, +): Promise { + return Promise.all( + Object.entries(fixture.preservedBytes).map(async ([path, expected]) => { + let preserved = false; + try { + const current = await readFile(join(fixture.workspaceRoot, path)); + preserved = current.equals(expected); + } catch { + preserved = false; + } + return { path, preserved }; + }), + ); +} + +export async function fixtureGitStatus( + fixture: ManagedAgentFixture, +): Promise { + return execFileSync("git", ["status", "--short"], { + cwd: fixture.workspaceRoot, + encoding: "utf8", + windowsHide: true, + }); +} + +export async function fixturePathExists(path: string): Promise { + try { + await lstat(path); + return true; + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return false; + throw error; + } +} + +export function fixtureName(fixture: ManagedAgentFixture): string { + return basename(fixture.root); +} diff --git a/packages/harness/src/experimental/managed-agent-spike/index.ts b/packages/harness/src/experimental/managed-agent-spike/index.ts new file mode 100644 index 000000000..09da2f954 --- /dev/null +++ b/packages/harness/src/experimental/managed-agent-spike/index.ts @@ -0,0 +1,112 @@ +export { + MANAGED_AGENT_CONTRACT, + MANAGED_AGENT_FORBIDDEN_AMBIENT_CREDENTIALS, + MANAGED_AGENT_L1_CERTIFICATION_CONTRACT, + MANAGED_AGENT_L1_FINAL_BYTE_ROLES, + MANAGED_AGENT_L1_REGISTERED_PATH_ROLES, + MANAGED_AGENT_MODEL_ENVIRONMENT_VARIABLES, + MANAGED_AGENT_MODEL_TARGETS, + ManagedAgentConfigurationError, + assertManagedAgentDirectGatewayOrigin, + normalizeManagedAgentGatewayOrigin, + normalizeManagedAgentHermeticGatewayOrigin, + resolveManagedAgentModelTarget, + validateManagedAgentProbeConfig, + type ManagedAgentProbeValidationOptions, +} from "./contract.js"; +export { + buildManagedAgentChildEnvironment, + prepareManagedAgentDirectories, + type ManagedAgentAmbientEnvironment, + type ManagedAgentChildEnvironmentInput, + type ManagedAgentIsolatedDirectories, +} from "./environment.js"; +export { + MANAGED_AGENT_TOOL_USE_ID_MAX_LENGTH, + ManagedAgentEventError, + ManagedAgentEventRecorder, + isBoundedManagedAgentToolUseId, +} from "./events.js"; +export { + FIXTURE_PATHS, + captureManagedAgentWorkspaceSnapshot, + createManagedAgentFixture, + diffManagedAgentWorkspaceSnapshots, + fixtureGitStatus, + observeManagedAgentPreservation, + observeManagedAgentL1FinalBytes, + readManagedAgentFixturePids, + verifyManagedAgentFixtureBytes, + waitForManagedAgentFixturePids, + type ManagedAgentFixture, + type ManagedAgentWorkspaceSnapshot, +} from "./fixture.js"; +export { + MANAGED_AGENT_BUILTIN_TOOLS, + MANAGED_AGENT_DISALLOWED_TOOLS, + ManagedAgentPathError, + createManagedAgentPolicyBoundary, + isPathWithinRoot, + resolveManagedAgentToolPath, + type ManagedAgentPolicyBoundary, + type ManagedAgentPolicyBoundaryOptions, +} from "./permissions.js"; +export { + LocalManagedAgentProcessObserver, + createLocalManagedAgentProcessObserver, +} from "./process-observer.js"; +export { + ManagedAgentSettingsGuardError, + assertManagedAgentHooksEnabled, + buildManagedAgentSettingsGuardEnvironment, + type ManagedAgentSettingsGuardDependencies, + type ManagedAgentSettingsGuardInput, +} from "./settings-guard.js"; +export { + MANAGED_AGENT_MCP_SERVER_NAME, + MANAGED_AGENT_CORRELATION_MARKER_VERSION, + MANAGED_AGENT_TEARDOWN_TIMEOUT_MS, + buildManagedAgentCorrelationPrompt, + createManagedAgentMcpRuntime, + qualifiedManagedAgentMcpToolName, + runManagedAgentProbe, + type ManagedAgentMcpRuntime, +} from "./runtime.js"; +export type { + ManagedAgentModelTarget, + ManagedAgentModelTargetId, + ManagedAgentCancellationReadiness, + ManagedAgentCancellationReadinessReason, + ManagedAgentEventNormalizationFailureReason, + ManagedAgentL1ExpectedFileHash, + ManagedAgentL1FinalByteObservation, + ManagedAgentL1FinalByteRole, + ManagedAgentOperationId, + ManagedAgentPathRole, + ManagedAgentPathRoleBinding, + ManagedAgentRegisteredPathRole, + ManagedAgentPermissionDecision, + ManagedAgentPermissionEvidence, + ManagedAgentPermissionReason, + ManagedAgentPermissionSource, + ManagedAgentPolicyDiagnostic, + ManagedAgentPreToolUseGuardRejectionReason, + ManagedAgentPreservationObservation, + ManagedAgentProbeConfig, + ManagedAgentProbeDependencies, + ManagedAgentProbeEvent, + ManagedAgentProbeEventType, + ManagedAgentProbeResult, + ManagedAgentProbeScenario, + ManagedAgentProcessObserver, + ManagedAgentQuery, + ManagedAgentQueryExecutionOutcome, + ManagedAgentQueryFactory, + ManagedAgentSdkUsageEstimate, + ManagedAgentTeardownDeadline, + ManagedAgentTeardownObservation, + ManagedAgentTerminalClassification, + ManagedAgentTerminationEvidence, + ManagedAgentToolEvidence, + ManagedAgentWorkspaceChange, +} from "./types.js"; diff --git a/packages/harness/src/experimental/managed-agent-spike/permissions.test.ts b/packages/harness/src/experimental/managed-agent-spike/permissions.test.ts new file mode 100644 index 000000000..2e451f459 --- /dev/null +++ b/packages/harness/src/experimental/managed-agent-spike/permissions.test.ts @@ -0,0 +1,850 @@ +import { + mkdir, + mkdtemp, + realpath, + rm, + symlink, + writeFile, +} from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import type { PreToolUseHookInput } from "@anthropic-ai/claude-agent-sdk"; + +import { MANAGED_AGENT_TOOL_USE_ID_MAX_LENGTH } from "./events.js"; +import { + createManagedAgentPolicyBoundary, + resolveManagedAgentToolPath, +} from "./permissions.js"; +import type { ManagedAgentPermissionEvidence } from "./types.js"; + +let root: string; +let workspace: string; +let outside: string; + +beforeEach(async () => { + root = await mkdtemp(join(tmpdir(), "managed-agent-permission-")); + workspace = join(root, "workspace"); + outside = join(root, "outside"); + await Promise.all([mkdir(workspace), mkdir(outside)]); + await Promise.all([ + writeFile(join(workspace, "inside.txt"), "inside"), + writeFile(join(outside, "secret.txt"), "outside"), + ]); + await symlink(join(outside, "secret.txt"), join(workspace, "escape.txt")); + await symlink(outside, join(workspace, "escape-dir")); + workspace = await realpath(workspace); +}); + +afterEach(async () => { + await rm(root, { recursive: true, force: true }); +}); + +describe("symlink-aware managed-agent containment", () => { + it("allows existing and new in-root paths", async () => { + expect(await resolveManagedAgentToolPath(workspace, "inside.txt")).toBe( + join(workspace, "inside.txt"), + ); + expect(await resolveManagedAgentToolPath(workspace, "nested/new.txt")).toBe( + join(workspace, "nested/new.txt"), + ); + }); + + it("distinguishes lexical outside-root paths from symlink escapes", async () => { + const outsidePath = join(outside, "secret.txt"); + for (const requested of [ + outsidePath, + "../outside/secret.txt", + `${workspace}-evil/file.txt`, + ]) { + await expect( + resolveManagedAgentToolPath(workspace, requested), + ).rejects.toMatchObject({ reason: "path_outside_workspace" }); + } + for (const requested of [ + "escape.txt", + "escape-dir/secret.txt", + "escape-dir/new.txt", + ]) { + await expect( + resolveManagedAgentToolPath(workspace, requested), + ).rejects.toMatchObject({ reason: "path_symlink_escape" }); + } + }); +}); + +function preToolUseInput( + toolName: string, + toolInput: unknown, + toolUseId: string, +): PreToolUseHookInput { + return { + hook_event_name: "PreToolUse", + session_id: "11111111-1111-4111-8111-111111111111", + transcript_path: join(workspace, "transcript.jsonl"), + cwd: workspace, + tool_name: toolName, + tool_input: toolInput, + tool_use_id: toolUseId, + }; +} + +describe("managed-agent universal policy boundary", () => { + it("classifies registered paths by lexical identity before realpath containment", async () => { + const evidence: ManagedAgentPermissionEvidence[] = []; + const resolveToolPath = vi.fn(resolveManagedAgentToolPath); + const outsidePath = join(outside, "secret.txt"); + const boundary = createManagedAgentPolicyBoundary({ + canonicalWorkspaceRoot: workspace, + allowedBashCommands: [], + allowedMcpTools: [], + pathRoleBindings: [ + { path: "inside.txt", role: "clean_target" }, + { path: outsidePath, role: "outside_sentinel" }, + { path: "escape.txt", role: "escape_link" }, + ], + requireRegisteredFilePaths: true, + onDecision: (decision) => evidence.push(decision), + resolveToolPath, + }); + const signal = new AbortController().signal; + const invoke = (filePath: string, toolUseId: string) => + boundary.preToolUseHook( + preToolUseInput("Read", { file_path: filePath }, toolUseId), + toolUseId, + { signal }, + ); + + await expect(invoke("inside.txt", "relative")).resolves.toMatchObject({ + hookSpecificOutput: { permissionDecision: "allow" }, + }); + await expect( + invoke(join(workspace, "inside.txt"), "absolute"), + ).resolves.toMatchObject({ + hookSpecificOutput: { permissionDecision: "allow" }, + }); + await expect(invoke(outsidePath, "outside")).resolves.toMatchObject({ + hookSpecificOutput: { permissionDecision: "deny" }, + }); + await expect(invoke("escape.txt", "escape")).resolves.toMatchObject({ + hookSpecificOutput: { permissionDecision: "deny" }, + }); + await expect( + invoke(join(workspace, "not-registered.txt"), "unregistered"), + ).resolves.toMatchObject({ + hookSpecificOutput: { permissionDecision: "deny" }, + }); + + expect( + evidence.map(({ decision, reason, operationId }) => ({ + decision, + reason, + operationId, + })), + ).toEqual([ + { + decision: "allow", + reason: "fixture_path", + operationId: "read:clean_target", + }, + { + decision: "allow", + reason: "fixture_path", + operationId: "read:clean_target", + }, + { + decision: "deny", + reason: "path_outside_workspace", + operationId: "read:outside_sentinel", + }, + { + decision: "deny", + reason: "path_symlink_escape", + operationId: "read:escape_link", + }, + { + decision: "deny", + reason: "path_role_not_allowed", + operationId: "read:unregistered", + }, + ]); + expect(resolveToolPath).toHaveBeenCalledTimes(4); + const serialized = JSON.stringify(evidence); + expect(serialized).not.toContain(workspace); + expect(serialized).not.toContain(outsidePath); + expect(serialized).not.toContain("inside.txt"); + }); + + it("rejects path-role bindings with the same normalized lexical target", () => { + expect(() => + createManagedAgentPolicyBoundary({ + canonicalWorkspaceRoot: workspace, + allowedBashCommands: [], + allowedMcpTools: [], + pathRoleBindings: [ + { path: "inside.txt", role: "clean_target" }, + { + path: join(workspace, "inside.txt"), + role: "dirty_sentinel", + }, + ], + onDecision: () => undefined, + }), + ).toThrow("unique lexical paths"); + }); + + it("can enforce an L2 Bash-only boundary before evaluating model-authored inputs", async () => { + const evidence: ManagedAgentPermissionEvidence[] = []; + const boundary = createManagedAgentPolicyBoundary({ + canonicalWorkspaceRoot: workspace, + allowedBuiltinTools: ["Bash"], + allowedBashCommands: ["node .managed-agent-probe/long-running.mjs"], + allowedMcpTools: [], + onDecision: (decision) => evidence.push(decision), + }); + const signal = new AbortController().signal; + const invoke = (toolName: string, input: unknown, toolUseId: string) => + boundary.preToolUseHook( + preToolUseInput(toolName, input, toolUseId), + toolUseId, + { signal }, + ); + + await expect( + invoke( + "Write", + { + file_path: ".managed-agent-probe/processes.json", + content: JSON.stringify({ + parentPid: process.pid, + childPid: 2_147_483_646, + }), + }, + "l2-write", + ), + ).resolves.toMatchObject({ + hookSpecificOutput: { permissionDecision: "deny" }, + }); + await expect( + invoke( + "mcp__sapiom-managed-agent-spike__echo_nonce", + { nonce: "x" }, + "l2-mcp", + ), + ).resolves.toMatchObject({ + hookSpecificOutput: { permissionDecision: "deny" }, + }); + await expect( + invoke( + "Bash", + { + command: "node .managed-agent-probe/long-running.mjs", + description: "Run the cancellation fixture", + }, + "l2-bash", + ), + ).resolves.toMatchObject({ + hookSpecificOutput: { + permissionDecision: "allow", + updatedInput: { + command: "node .managed-agent-probe/long-running.mjs", + }, + }, + }); + + expect( + evidence.map(({ toolName, decision, reason }) => ({ + toolName, + decision, + reason, + })), + ).toEqual([ + { toolName: "Write", decision: "deny", reason: "tool_not_allowed" }, + { + toolName: "mcp__sapiom-managed-agent-spike__echo_nonce", + decision: "deny", + reason: "tool_not_allowed", + }, + { + toolName: "Bash", + decision: "allow", + reason: "exact_bash_command", + }, + ]); + }); + + it("accepts pinned SDK Bash metadata but strips it before execution", async () => { + const command = "git status --short"; + const descriptionMarker = "sdk-description-must-not-persist"; + const unknownMarker = "unknown-field-must-not-persist"; + const evidence: ManagedAgentPermissionEvidence[] = []; + const boundary = createManagedAgentPolicyBoundary({ + canonicalWorkspaceRoot: workspace, + allowedBashCommands: [command], + allowedMcpTools: [], + onDecision: (decision) => evidence.push(decision), + }); + const signal = new AbortController().signal; + let sequence = 0; + const invoke = (input: unknown) => { + const toolUseId = `bash-shape-${++sequence}`; + return boundary.preToolUseHook( + preToolUseInput("Bash", input, toolUseId), + toolUseId, + { signal }, + ); + }; + const acceptedInputs: Array> = [ + { command }, + { command, description: descriptionMarker }, + { command, timeout: 1 }, + { + command, + description: descriptionMarker, + timeout: 600_000, + run_in_background: false, + dangerouslyDisableSandbox: false, + }, + ]; + + for (const input of acceptedInputs) { + const originalInput = { ...input }; + await expect(invoke(input)).resolves.toEqual({ + hookSpecificOutput: { + hookEventName: "PreToolUse", + permissionDecision: "allow", + permissionDecisionReason: "Managed-agent policy: exact_bash_command", + updatedInput: { command }, + }, + }); + expect(input).toEqual(originalInput); + } + + const deniedInputs: Array> = [ + { command, unexpected: unknownMarker }, + { command, description: 123 }, + { command, timeout: "10" }, + { command, timeout: 0 }, + { command, timeout: 1.5 }, + { command, timeout: 600_001 }, + { command, run_in_background: "false" }, + { command, run_in_background: true }, + { command, dangerouslyDisableSandbox: "false" }, + { command, dangerouslyDisableSandbox: true }, + ]; + for (const input of deniedInputs) { + await expect(invoke(input)).resolves.toMatchObject({ + hookSpecificOutput: { + permissionDecision: "deny", + permissionDecisionReason: expect.stringContaining("invalid_input"), + }, + }); + } + + expect( + evidence + .slice(0, acceptedInputs.length) + .map(({ decision, reason, operationId }) => ({ + decision, + reason, + operationId, + })), + ).toEqual( + acceptedInputs.map(() => ({ + decision: "allow", + reason: "exact_bash_command", + operationId: "bash:exact_command", + })), + ); + expect( + evidence + .slice(acceptedInputs.length) + .map(({ decision, reason, operationId }) => ({ + decision, + reason, + operationId, + })), + ).toEqual( + deniedInputs.map(() => ({ + decision: "deny", + reason: "invalid_input", + operationId: "bash:unregistered", + })), + ); + const serializedEvidence = JSON.stringify(evidence); + expect(serializedEvidence).not.toContain(descriptionMarker); + expect(serializedEvidence).not.toContain(unknownMarker); + }); + + it("uses exact Bash equality and emits content-free decisions", async () => { + const evidence: ManagedAgentPermissionEvidence[] = []; + const boundary = createManagedAgentPolicyBoundary({ + canonicalWorkspaceRoot: workspace, + pathRoleBindings: [ + { path: "inside.txt", role: "clean_target" }, + { path: "nested/new.txt", role: "managed_output" }, + { path: join(outside, "secret.txt"), role: "outside_sentinel" }, + { path: "escape.txt", role: "escape_link" }, + ], + allowedBashCommands: ["git status --short"], + allowedMcpTools: ["mcp__probe__echo_nonce"], + onDecision: (decision) => evidence.push(decision), + }); + const signal = new AbortController().signal; + let sequence = 0; + const invoke = (toolName: string, input: unknown) => { + const toolUseId = `tool-${++sequence}`; + return boundary.preToolUseHook( + preToolUseInput(toolName, input, toolUseId), + toolUseId, + { signal }, + ); + }; + + await expect( + invoke("Bash", { command: "git status --short" }), + ).resolves.toMatchObject({ + hookSpecificOutput: { + permissionDecision: "allow", + updatedInput: { command: "git status --short" }, + }, + }); + await expect( + invoke("Bash", { command: "git status --short " }), + ).resolves.toMatchObject({ + hookSpecificOutput: { permissionDecision: "deny" }, + }); + await expect( + invoke("Bash", { + command: "git status --short", + run_in_background: true, + }), + ).resolves.toMatchObject({ + hookSpecificOutput: { + permissionDecision: "deny", + permissionDecisionReason: expect.stringContaining("invalid_input"), + }, + }); + await expect( + invoke("Bash", { + command: "git status --short", + dangerouslyDisableSandbox: true, + }), + ).resolves.toMatchObject({ + hookSpecificOutput: { + permissionDecision: "deny", + permissionDecisionReason: expect.stringContaining("invalid_input"), + }, + }); + const readInput = { file_path: "inside.txt", preserve: "metadata" }; + await expect(invoke("Read", readInput)).resolves.toMatchObject({ + hookSpecificOutput: { + permissionDecision: "allow", + updatedInput: { + file_path: join(workspace, "inside.txt"), + preserve: "metadata", + }, + }, + }); + await expect( + invoke("Write", { file_path: "nested/new.txt", content: "safe" }), + ).resolves.toMatchObject({ + hookSpecificOutput: { + permissionDecision: "allow", + updatedInput: { + file_path: join(workspace, "nested/new.txt"), + content: "safe", + }, + }, + }); + await expect( + invoke("Read", { file_path: join(outside, "secret.txt") }), + ).resolves.toMatchObject({ + hookSpecificOutput: { permissionDecision: "deny" }, + }); + await expect( + invoke("Read", { file_path: "escape.txt" }), + ).resolves.toMatchObject({ + hookSpecificOutput: { permissionDecision: "deny" }, + }); + await expect( + invoke("mcp__probe__echo_nonce", { nonce: "secret" }), + ).resolves.toMatchObject({ + hookSpecificOutput: { permissionDecision: "allow" }, + }); + await expect(invoke("WebFetch", {})).resolves.toMatchObject({ + hookSpecificOutput: { permissionDecision: "deny" }, + }); + + expect( + evidence.map(({ decision, reason, source, operationId }) => [ + decision, + reason, + source, + operationId, + ]), + ).toEqual([ + ["allow", "exact_bash_command", "pre_tool_use", "bash:exact_command"], + ["deny", "bash_command_not_allowed", "pre_tool_use", "bash:unregistered"], + ["deny", "invalid_input", "pre_tool_use", "bash:unregistered"], + ["deny", "invalid_input", "pre_tool_use", "bash:unregistered"], + ["allow", "fixture_path", "pre_tool_use", "read:clean_target"], + ["allow", "fixture_path", "pre_tool_use", "write:managed_output"], + [ + "deny", + "path_outside_workspace", + "pre_tool_use", + "read:outside_sentinel", + ], + ["deny", "path_symlink_escape", "pre_tool_use", "read:escape_link"], + ["allow", "managed_mcp_tool", "pre_tool_use", "mcp:echo_nonce"], + ["deny", "tool_not_allowed", "pre_tool_use", "unknown"], + ]); + expect(JSON.stringify(evidence)).not.toContain(join(outside, "secret.txt")); + expect(JSON.stringify(evidence)).not.toContain("secret"); + expect(JSON.stringify(evidence)).not.toContain("tool-3"); + expect(vi.isMockFunction(boundary.preToolUseHook)).toBe(false); + expect(readInput).toEqual({ + file_path: "inside.txt", + preserve: "metadata", + }); + }); + + it("rejects malformed hook identifiers before policy evaluation", async () => { + const evidence: ManagedAgentPermissionEvidence[] = []; + const guardDiagnostics: Array<{ + reason: string; + toolName: string; + normalizedToolUseId?: string; + }> = []; + const resolveToolPath = vi.fn(async () => join(workspace, "inside.txt")); + const boundary = createManagedAgentPolicyBoundary({ + canonicalWorkspaceRoot: workspace, + allowedBashCommands: [], + allowedMcpTools: [], + onDecision: (decision) => evidence.push(decision), + onGuardRejection: (diagnostic) => guardDiagnostics.push(diagnostic), + resolveToolPath, + }); + const signal = new AbortController().signal; + const overlong = "x".repeat(MANAGED_AGENT_TOOL_USE_ID_MAX_LENGTH + 1); + const invalidIdentifiers: ReadonlyArray< + readonly [inputToolUseId: unknown, callbackToolUseId: unknown] + > = [ + [undefined, undefined], + ["", undefined], + [" ", undefined], + [overlong, undefined], + ["valid-input-id", ""], + ["valid-input-id", " "], + ["valid-input-id", overlong], + ["valid-input-id", "mismatched-callback-id"], + ]; + + for (const [inputToolUseId, callbackToolUseId] of invalidIdentifiers) { + const malformedInput = { + ...preToolUseInput("Read", { file_path: "inside.txt" }, "placeholder"), + tool_use_id: inputToolUseId, + } as unknown as PreToolUseHookInput; + await expect( + boundary.preToolUseHook( + malformedInput, + callbackToolUseId as string | undefined, + { signal }, + ), + ).resolves.toMatchObject({ + hookSpecificOutput: { + permissionDecision: "deny", + permissionDecisionReason: expect.stringContaining("invalid_input"), + }, + }); + } + + await expect( + boundary.canUseToolFallback( + "Read", + { file_path: "inside.txt" }, + { signal, toolUseID: "", requestId: "invalid-fallback" }, + ), + ).resolves.toMatchObject({ + behavior: "deny", + message: expect.stringContaining("invalid_input"), + }); + expect(resolveToolPath).not.toHaveBeenCalled(); + expect(evidence).toEqual([]); + expect( + guardDiagnostics.map(({ reason, toolName, normalizedToolUseId }) => ({ + reason, + toolName, + correlated: normalizedToolUseId !== undefined, + })), + ).toEqual([ + { + reason: "input_tool_use_id_missing", + toolName: "Read", + correlated: false, + }, + { + reason: "input_tool_use_id_invalid", + toolName: "Read", + correlated: false, + }, + { + reason: "input_tool_use_id_invalid", + toolName: "Read", + correlated: false, + }, + { + reason: "input_tool_use_id_too_long", + toolName: "Read", + correlated: false, + }, + { + reason: "callback_tool_use_id_invalid", + toolName: "Read", + correlated: true, + }, + { + reason: "callback_tool_use_id_invalid", + toolName: "Read", + correlated: true, + }, + { + reason: "callback_tool_use_id_too_long", + toolName: "Read", + correlated: true, + }, + { + reason: "callback_tool_use_id_mismatch", + toolName: "Read", + correlated: true, + }, + ]); + const serializedDiagnostics = JSON.stringify(guardDiagnostics); + expect(serializedDiagnostics).not.toContain("valid-input-id"); + expect(serializedDiagnostics).not.toContain("mismatched-callback-id"); + expect(serializedDiagnostics).not.toContain(overlong); + + await expect( + boundary.preToolUseHook( + preToolUseInput("Read", { file_path: "inside.txt" }, "valid-input-id"), + undefined, + { signal }, + ), + ).resolves.toMatchObject({ + hookSpecificOutput: { permissionDecision: "allow" }, + }); + expect(resolveToolPath).toHaveBeenCalledOnce(); + expect(evidence).toHaveLength(1); + }); + + it("denies a concurrent duplicate primary identifier without sharing its pending allow", async () => { + const evidence: ManagedAgentPermissionEvidence[] = []; + let releasePathResolution: (() => void) | undefined; + let reportPathResolutionStarted: (() => void) | undefined; + const pathResolutionStarted = new Promise((resolveStarted) => { + reportPathResolutionStarted = resolveStarted; + }); + const releasePath = new Promise((resolvePath) => { + releasePathResolution = resolvePath; + }); + const resolveToolPath = vi.fn(async () => { + reportPathResolutionStarted?.(); + await releasePath; + return join(workspace, "inside.txt"); + }); + const boundary = createManagedAgentPolicyBoundary({ + canonicalWorkspaceRoot: workspace, + allowedBashCommands: [], + allowedMcpTools: [], + onDecision: (decision) => evidence.push(decision), + resolveToolPath, + }); + const signal = new AbortController().signal; + const toolUseId = "concurrent-tool-use-id"; + const first = boundary.preToolUseHook( + preToolUseInput("Read", { file_path: "inside.txt" }, toolUseId), + toolUseId, + { signal }, + ); + await pathResolutionStarted; + const duplicate = boundary.preToolUseHook( + preToolUseInput("Read", { file_path: "inside.txt" }, toolUseId), + toolUseId, + { signal }, + ); + releasePathResolution?.(); + + await expect(first).resolves.toMatchObject({ + hookSpecificOutput: { permissionDecision: "allow" }, + }); + await expect(duplicate).resolves.toMatchObject({ + hookSpecificOutput: { + permissionDecision: "deny", + permissionDecisionReason: expect.stringContaining("invalid_input"), + }, + }); + expect(resolveToolPath).toHaveBeenCalledOnce(); + expect(evidence).toHaveLength(1); + expect(evidence[0]).toMatchObject({ + decision: "allow", + reason: "fixture_path", + source: "pre_tool_use", + }); + }); + + it("deduplicates the fallback and records when only the fallback executes", async () => { + const evidence: ManagedAgentPermissionEvidence[] = []; + const boundary = createManagedAgentPolicyBoundary({ + canonicalWorkspaceRoot: workspace, + allowedBashCommands: ["git status --short"], + allowedMcpTools: [], + onDecision: (decision) => evidence.push(decision), + }); + const signal = new AbortController().signal; + const toolUseID = "tool-deduplicated"; + await boundary.preToolUseHook( + preToolUseInput("Read", { file_path: "inside.txt" }, toolUseID), + toolUseID, + { signal }, + ); + await expect( + boundary.canUseToolFallback( + "Read", + { file_path: join(workspace, "inside.txt") }, + { signal, toolUseID, requestId: "request-1" }, + ), + ).resolves.toMatchObject({ behavior: "allow" }); + expect(evidence).toHaveLength(1); + expect(evidence[0]?.source).toBe("pre_tool_use"); + + await expect( + boundary.preToolUseHook( + preToolUseInput( + "Bash", + { command: "touch must-not-inherit-allow" }, + toolUseID, + ), + toolUseID, + { signal }, + ), + ).resolves.toMatchObject({ + hookSpecificOutput: { + permissionDecision: "deny", + permissionDecisionReason: expect.stringContaining("invalid_input"), + }, + }); + expect(evidence).toHaveLength(1); + + const fallbackInput = { + command: "git status --short", + description: "Show working tree status", + timeout: 10_000, + run_in_background: false, + dangerouslyDisableSandbox: false, + }; + await expect( + boundary.canUseToolFallback("Bash", fallbackInput, { + signal, + toolUseID: "fallback-only", + requestId: "request-2", + }), + ).resolves.toEqual({ + behavior: "allow", + toolUseID: "fallback-only", + updatedInput: { command: "git status --short" }, + }); + expect(fallbackInput).toEqual({ + command: "git status --short", + description: "Show working tree status", + timeout: 10_000, + run_in_background: false, + dangerouslyDisableSandbox: false, + }); + expect(evidence).toHaveLength(2); + expect(evidence[1]?.source).toBe("can_use_tool_fallback"); + + const deniedFallbackInputs = [ + { + toolUseID: "fallback-background", + input: { command: "git status --short", run_in_background: true }, + }, + { + toolUseID: "fallback-sandbox", + input: { + command: "git status --short", + dangerouslyDisableSandbox: true, + }, + }, + { + toolUseID: "fallback-unknown", + input: { command: "git status --short", unsupported: true }, + }, + ]; + for (const { toolUseID: deniedToolUseID, input } of deniedFallbackInputs) { + await expect( + boundary.canUseToolFallback("Bash", input, { + signal, + toolUseID: deniedToolUseID, + requestId: `request-${deniedToolUseID}`, + }), + ).resolves.toMatchObject({ + behavior: "deny", + message: expect.stringContaining("invalid_input"), + }); + } + }); + + it("fails closed when aborted before or during asynchronous path validation", async () => { + const evidence: ManagedAgentPermissionEvidence[] = []; + const before = new AbortController(); + before.abort(); + const beforeBoundary = createManagedAgentPolicyBoundary({ + canonicalWorkspaceRoot: workspace, + allowedBashCommands: ["git status --short"], + allowedMcpTools: [], + onDecision: (decision) => evidence.push(decision), + }); + await expect( + beforeBoundary.preToolUseHook( + preToolUseInput("Bash", { command: "git status --short" }, "before"), + "before", + { signal: before.signal }, + ), + ).resolves.toMatchObject({ + hookSpecificOutput: { + permissionDecision: "deny", + permissionDecisionReason: expect.stringContaining("policy_aborted"), + }, + }); + + const during = new AbortController(); + const duringBoundary = createManagedAgentPolicyBoundary({ + canonicalWorkspaceRoot: workspace, + allowedBashCommands: [], + allowedMcpTools: [], + onDecision: (decision) => evidence.push(decision), + resolveToolPath: async () => { + during.abort(); + return join(workspace, "inside.txt"); + }, + }); + await expect( + duringBoundary.preToolUseHook( + preToolUseInput("Read", { file_path: "inside.txt" }, "during"), + "during", + { signal: during.signal }, + ), + ).resolves.toMatchObject({ + hookSpecificOutput: { + permissionDecision: "deny", + permissionDecisionReason: expect.stringContaining("policy_aborted"), + }, + }); + expect(evidence.map(({ reason }) => reason)).toEqual([ + "policy_aborted", + "policy_aborted", + ]); + }); +}); diff --git a/packages/harness/src/experimental/managed-agent-spike/permissions.ts b/packages/harness/src/experimental/managed-agent-spike/permissions.ts new file mode 100644 index 000000000..996ad3f2f --- /dev/null +++ b/packages/harness/src/experimental/managed-agent-spike/permissions.ts @@ -0,0 +1,632 @@ +import { lstat, realpath } from "node:fs/promises"; +import { dirname, isAbsolute, relative, resolve, sep } from "node:path"; + +import type { + CanUseTool, + HookCallback, + PermissionResult, +} from "@anthropic-ai/claude-agent-sdk"; + +import type { + ManagedAgentOperationId, + ManagedAgentPathRole, + ManagedAgentPathRoleBinding, + ManagedAgentPermissionEvidence, + ManagedAgentPermissionReason, + ManagedAgentPermissionSource, + ManagedAgentPreToolUseGuardRejectionReason, +} from "./types.js"; +import { + MANAGED_AGENT_TOOL_USE_ID_MAX_LENGTH, + isBoundedManagedAgentToolUseId, + normalizeManagedAgentToolUseId, + sanitizeManagedAgentToolName, +} from "./events.js"; + +export const MANAGED_AGENT_BUILTIN_TOOLS = [ + "Read", + "Edit", + "Write", + "Bash", +] as const; + +export const MANAGED_AGENT_DISALLOWED_TOOLS = [ + "Agent", + "AskUserQuestion", + "CronCreate", + "CronDelete", + "CronList", + "EnterPlanMode", + "ExitPlanMode", + "Glob", + "Grep", + "NotebookEdit", + "SendMessage", + "Skill", + "Task", + "TaskOutput", + "TaskStop", + "TeamCreate", + "TeamDelete", + "TodoWrite", + "ToolSearch", + "WebFetch", + "WebSearch", +] as const; + +export class ManagedAgentPathError extends Error { + public constructor( + public readonly reason: + | "invalid_input" + | "path_outside_workspace" + | "path_symlink_escape", + ) { + super(reason); + this.name = "ManagedAgentPathError"; + } +} + +function comparisonPath(value: string): string { + return process.platform === "win32" ? value.toLowerCase() : value; +} + +export function isPathWithinRoot(root: string, candidate: string): boolean { + const pathRelative = relative( + comparisonPath(root), + comparisonPath(candidate), + ); + if (pathRelative === "") return true; + return ( + !isAbsolute(pathRelative) && + pathRelative !== ".." && + !pathRelative.startsWith(`..${sep}`) + ); +} + +async function exists(path: string): Promise { + try { + await lstat(path); + return true; + } catch (error) { + return (error as NodeJS.ErrnoException).code !== "ENOENT" + ? Promise.reject(error) + : false; + } +} + +async function nearestExistingParent(path: string): Promise { + let cursor = path; + while (!(await exists(cursor))) { + const parent = dirname(cursor); + if (parent === cursor) { + throw new ManagedAgentPathError("path_outside_workspace"); + } + cursor = parent; + } + return cursor; +} + +/** + * Resolve an SDK tool target through the filesystem before authorizing it. + * Existing symlinks are followed with realpath; new targets are authorized + * only when their nearest existing parent resolves inside the canonical root. + */ +export async function resolveManagedAgentToolPath( + canonicalWorkspaceRoot: string, + requestedPath: string, +): Promise { + if (!requestedPath || requestedPath.includes("\0")) { + throw new ManagedAgentPathError("invalid_input"); + } + const candidate = resolve(canonicalWorkspaceRoot, requestedPath); + if (!isPathWithinRoot(canonicalWorkspaceRoot, candidate)) { + throw new ManagedAgentPathError("path_outside_workspace"); + } + const existing = await nearestExistingParent(candidate); + const canonicalExisting = await realpath(existing); + if (!isPathWithinRoot(canonicalWorkspaceRoot, canonicalExisting)) { + throw new ManagedAgentPathError("path_symlink_escape"); + } + if (existing === candidate) return canonicalExisting; + + const unresolvedTail = relative(existing, candidate); + const resolvedCandidate = resolve(canonicalExisting, unresolvedTail); + if (!isPathWithinRoot(canonicalWorkspaceRoot, resolvedCandidate)) { + throw new ManagedAgentPathError("path_outside_workspace"); + } + return resolvedCandidate; +} + +export interface ManagedAgentPolicyBoundaryOptions { + readonly canonicalWorkspaceRoot: string; + /** Scenario-specific built-ins; L2 deliberately exposes only exact Bash. */ + readonly allowedBuiltinTools?: readonly string[]; + readonly allowedBashCommands: readonly string[]; + readonly allowedMcpTools: readonly string[]; + /** Exact prompt literals mapped to content-free evidence roles. */ + readonly pathRoleBindings?: readonly ManagedAgentPathRoleBinding[]; + /** Certification mode denies file paths without a predeclared role. */ + readonly requireRegisteredFilePaths?: boolean; + readonly onDecision: (evidence: ManagedAgentPermissionEvidence) => void; + readonly onGuardRejection?: ( + diagnostic: ManagedAgentPreToolUseGuardRejection, + ) => void; + /** Test seam for proving cancellation after asynchronous path validation. */ + readonly resolveToolPath?: typeof resolveManagedAgentToolPath; +} + +/** Internal correlation is normalized immediately and is removed from output. */ +export interface ManagedAgentPreToolUseGuardRejection { + readonly reason: ManagedAgentPreToolUseGuardRejectionReason; + readonly toolName: string; + readonly normalizedToolUseId?: string; +} + +export interface ManagedAgentPolicyBoundary { + /** Primary boundary: the SDK runs this before its own permission evaluation. */ + readonly preToolUseHook: HookCallback; + /** Defense in depth when the SDK still surfaces an unresolved permission. */ + readonly canUseToolFallback: CanUseTool; +} + +interface ManagedAgentPolicyDecision { + readonly decision: "allow" | "deny"; + readonly reason: ManagedAgentPermissionReason; + readonly operationId: ManagedAgentOperationId; + readonly updatedInput?: Record; +} + +interface ManagedAgentRecordedPolicyDecision extends ManagedAgentPolicyDecision { + readonly source: ManagedAgentPermissionSource; +} + +function toolUseIdIssue( + value: unknown, + role: "input" | "callback", +): ManagedAgentPreToolUseGuardRejectionReason | undefined { + if (role === "input" && value === undefined) { + return "input_tool_use_id_missing"; + } + if (typeof value !== "string" || value.trim().length === 0) { + return role === "input" + ? "input_tool_use_id_invalid" + : "callback_tool_use_id_invalid"; + } + if (value.length > MANAGED_AGENT_TOOL_USE_ID_MAX_LENGTH) { + return role === "input" + ? "input_tool_use_id_too_long" + : "callback_tool_use_id_too_long"; + } + return undefined; +} + +function permissionResult( + policy: ManagedAgentPolicyDecision, + toolUseID: string, +): PermissionResult { + return policy.decision === "allow" + ? { + behavior: "allow", + toolUseID, + ...(policy.updatedInput + ? { updatedInput: { ...policy.updatedInput } } + : {}), + } + : { + behavior: "deny", + message: `Managed-agent permission denied: ${policy.reason}`, + interrupt: false, + toolUseID, + }; +} + +function filePathFromInput(input: Record): string | undefined { + return typeof input.file_path === "string" && input.file_path.length > 0 + ? input.file_path + : undefined; +} + +function asRecord(value: unknown): Record | undefined { + return typeof value === "object" && value !== null && !Array.isArray(value) + ? (value as Record) + : undefined; +} + +const MANAGED_AGENT_BASH_INPUT_KEYS = new Set([ + "command", + "timeout", + "description", + "run_in_background", + "dangerouslyDisableSandbox", +]); +const MANAGED_AGENT_BASH_TIMEOUT_MAX_MS = 600_000; + +interface ManagedAgentNormalizedBashInput { + readonly command: string; +} + +/** + * Accept the pinned SDK's Bash input shape, but retain only the exact command + * that the host authorizes and executes. Model-authored execution controls are + * either harmless metadata that is stripped or unsafe values that fail closed. + */ +function normalizeManagedAgentBashInput( + rawInput: unknown, +): ManagedAgentNormalizedBashInput | undefined { + const input = asRecord(rawInput); + if ( + !input || + !Object.prototype.hasOwnProperty.call(input, "command") || + Reflect.ownKeys(input).some( + (key) => + typeof key !== "string" || !MANAGED_AGENT_BASH_INPUT_KEYS.has(key), + ) + ) { + return undefined; + } + + const command = input.command; + if (typeof command !== "string" || command.length === 0) return undefined; + if ( + input.description !== undefined && + typeof input.description !== "string" + ) { + return undefined; + } + if ( + input.timeout !== undefined && + (typeof input.timeout !== "number" || + !Number.isInteger(input.timeout) || + input.timeout <= 0 || + input.timeout > MANAGED_AGENT_BASH_TIMEOUT_MAX_MS) + ) { + return undefined; + } + if ( + (input.run_in_background !== undefined && + input.run_in_background !== false) || + (input.dangerouslyDisableSandbox !== undefined && + input.dangerouslyDisableSandbox !== false) + ) { + return undefined; + } + + return { command }; +} + +function denied( + reason: ManagedAgentPermissionReason, + operationId: ManagedAgentOperationId = "unknown", +): ManagedAgentPolicyDecision { + return { decision: "deny", reason, operationId }; +} + +function fileOperationId( + toolName: "Read" | "Edit" | "Write", + role: ManagedAgentPathRole, +): ManagedAgentOperationId { + return `${toolName.toLowerCase()}:${role}` as ManagedAgentOperationId; +} + +function lexicalPathRoleKey( + canonicalWorkspaceRoot: string, + requestedPath: string, +): string { + return comparisonPath(resolve(canonicalWorkspaceRoot, requestedPath)); +} + +function classifyManagedAgentOperation( + canonicalWorkspaceRoot: string, + toolName: string, + rawInput: unknown, + allowedCommands: ReadonlySet, + pathRoles: ReadonlyMap, +): ManagedAgentOperationId { + const input = asRecord(rawInput); + if (toolName === "Bash") { + const normalizedInput = normalizeManagedAgentBashInput(rawInput); + return normalizedInput && allowedCommands.has(normalizedInput.command) + ? "bash:exact_command" + : "bash:unregistered"; + } + if (toolName.endsWith("__echo_nonce")) return "mcp:echo_nonce"; + if (toolName.endsWith("__fail_once")) return "mcp:fail_once"; + if (toolName.startsWith("mcp__")) return "mcp:managed"; + if (toolName === "Read" || toolName === "Edit" || toolName === "Write") { + const requestedPath = input ? filePathFromInput(input) : undefined; + const role = requestedPath + ? (pathRoles.get( + lexicalPathRoleKey(canonicalWorkspaceRoot, requestedPath), + ) ?? "unregistered") + : "unregistered"; + return fileOperationId(toolName, role); + } + return "unknown"; +} + +async function evaluateManagedAgentPolicy( + options: ManagedAgentPolicyBoundaryOptions, + allowedBuiltinTools: ReadonlySet, + allowedCommands: ReadonlySet, + allowedMcpTools: ReadonlySet, + pathRoles: ReadonlyMap, + toolName: string, + rawInput: unknown, + signal: AbortSignal, +): Promise { + const operationId = classifyManagedAgentOperation( + options.canonicalWorkspaceRoot, + toolName, + rawInput, + allowedCommands, + pathRoles, + ); + if (signal.aborted) return denied("policy_aborted", operationId); + if (!allowedBuiltinTools.has(toolName) && !allowedMcpTools.has(toolName)) { + return denied("tool_not_allowed", operationId); + } + const input = asRecord(rawInput); + if (!input) return denied("invalid_input", operationId); + + if (allowedMcpTools.has(toolName)) { + return signal.aborted + ? denied("policy_aborted", operationId) + : { + decision: "allow", + reason: "managed_mcp_tool", + operationId, + updatedInput: { ...input }, + }; + } + if (toolName === "Bash") { + const normalizedInput = normalizeManagedAgentBashInput(input); + if (!normalizedInput) return denied("invalid_input", operationId); + if (!allowedCommands.has(normalizedInput.command)) { + return denied("bash_command_not_allowed", operationId); + } + return signal.aborted + ? denied("policy_aborted", operationId) + : { + decision: "allow", + reason: "exact_bash_command", + operationId, + updatedInput: { command: normalizedInput.command }, + }; + } + if (toolName === "Read" || toolName === "Edit" || toolName === "Write") { + const requestedPath = filePathFromInput(input); + if (!requestedPath) return denied("invalid_input", operationId); + if ( + options.requireRegisteredFilePaths && + operationId.endsWith(":unregistered") + ) { + return denied("path_role_not_allowed", operationId); + } + try { + const canonicalPath = await ( + options.resolveToolPath ?? resolveManagedAgentToolPath + )(options.canonicalWorkspaceRoot, requestedPath); + if (signal.aborted) return denied("policy_aborted", operationId); + return { + decision: "allow", + reason: "fixture_path", + operationId, + updatedInput: { ...input, file_path: canonicalPath }, + }; + } catch (error) { + if (signal.aborted) return denied("policy_aborted", operationId); + return denied( + error instanceof ManagedAgentPathError ? error.reason : "invalid_input", + operationId, + ); + } + } + return denied("tool_not_allowed", operationId); +} + +/** + * Build one universal host policy shared by the primary PreToolUse hook and a + * canUseTool fallback. Decisions are deduplicated by raw tool-use ID so one + * attempted tool produces exactly one normalized evidence record. + */ +export function createManagedAgentPolicyBoundary( + options: ManagedAgentPolicyBoundaryOptions, +): ManagedAgentPolicyBoundary { + const allowedBuiltinTools = new Set( + options.allowedBuiltinTools ?? MANAGED_AGENT_BUILTIN_TOOLS, + ); + const allowedCommands = new Set(options.allowedBashCommands); + const allowedMcpTools = new Set(options.allowedMcpTools); + const pathRoles = new Map(); + for (const binding of options.pathRoleBindings ?? []) { + const key = lexicalPathRoleKey( + options.canonicalWorkspaceRoot, + binding.path, + ); + if (pathRoles.has(key)) { + throw new Error( + "Managed-agent path role bindings must resolve to unique lexical paths", + ); + } + pathRoles.set(key, binding.role); + } + const decisions = new Map< + string, + { + readonly source: ManagedAgentPermissionSource; + readonly pending: Promise; + } + >(); + + const recordGuardRejection = ( + reason: ManagedAgentPreToolUseGuardRejectionReason, + toolName: unknown, + inputToolUseID?: unknown, + ): void => { + options.onGuardRejection?.({ + reason, + toolName: sanitizeManagedAgentToolName(toolName), + ...(isBoundedManagedAgentToolUseId(inputToolUseID) + ? { + normalizedToolUseId: normalizeManagedAgentToolUseId(inputToolUseID), + } + : {}), + }); + }; + + const decide = async ( + toolUseID: string, + toolName: string, + input: unknown, + signal: AbortSignal, + source: ManagedAgentPermissionSource, + ): Promise => { + const attemptedOperationId = classifyManagedAgentOperation( + options.canonicalWorkspaceRoot, + toolName, + input, + allowedCommands, + pathRoles, + ); + const existing = decisions.get(toolUseID); + if (existing) { + if (signal.aborted) { + return { ...denied("policy_aborted", attemptedOperationId), source }; + } + // The only valid duplicate is the SDK consulting canUseTool after the + // primary hook. A repeated primary ID or fallback-first sequence is + // ambiguous and must never inherit an earlier allow decision. + return source === "can_use_tool_fallback" && + existing.source === "pre_tool_use" + ? existing.pending + : { ...denied("invalid_input", attemptedOperationId), source }; + } + const pending = evaluateManagedAgentPolicy( + options, + allowedBuiltinTools, + allowedCommands, + allowedMcpTools, + pathRoles, + toolName, + input, + signal, + ).then((policy) => { + const recorded = { ...policy, source }; + options.onDecision({ + toolUseId: normalizeManagedAgentToolUseId(toolUseID), + toolName: sanitizeManagedAgentToolName(toolName), + decision: recorded.decision, + reason: recorded.reason, + source, + operationId: recorded.operationId, + }); + return recorded; + }); + decisions.set(toolUseID, { source, pending }); + return pending; + }; + + const preToolUseHook: HookCallback = async ( + input, + callbackToolUseID, + { signal }, + ) => { + const isPreToolUse = input.hook_event_name === "PreToolUse"; + if (!isPreToolUse) { + recordGuardRejection("unexpected_hook_event", undefined); + return { + hookSpecificOutput: { + hookEventName: "PreToolUse", + permissionDecision: "deny", + permissionDecisionReason: "Managed-agent policy: invalid_input", + }, + }; + } + const inputToolUseID = input.tool_use_id; + const inputIssue = toolUseIdIssue(inputToolUseID, "input"); + if (inputIssue || !isBoundedManagedAgentToolUseId(inputToolUseID)) { + recordGuardRejection( + inputIssue ?? "input_tool_use_id_invalid", + input.tool_name, + ); + return { + hookSpecificOutput: { + hookEventName: "PreToolUse", + permissionDecision: "deny", + permissionDecisionReason: "Managed-agent policy: invalid_input", + }, + }; + } + if (callbackToolUseID !== undefined) { + const callbackIssue = toolUseIdIssue(callbackToolUseID, "callback"); + if (callbackIssue || !isBoundedManagedAgentToolUseId(callbackToolUseID)) { + recordGuardRejection( + callbackIssue ?? "callback_tool_use_id_invalid", + input.tool_name, + inputToolUseID, + ); + return { + hookSpecificOutput: { + hookEventName: "PreToolUse", + permissionDecision: "deny", + permissionDecisionReason: "Managed-agent policy: invalid_input", + }, + }; + } + if (callbackToolUseID !== inputToolUseID) { + recordGuardRejection( + "callback_tool_use_id_mismatch", + input.tool_name, + inputToolUseID, + ); + return { + hookSpecificOutput: { + hookEventName: "PreToolUse", + permissionDecision: "deny", + permissionDecisionReason: "Managed-agent policy: invalid_input", + }, + }; + } + } + const toolUseID = inputToolUseID; + const policy = await decide( + toolUseID, + input.tool_name, + input.tool_input, + signal, + "pre_tool_use", + ); + return { + hookSpecificOutput: { + hookEventName: "PreToolUse", + permissionDecision: policy.decision, + permissionDecisionReason: `Managed-agent policy: ${policy.reason}`, + ...(policy.decision === "allow" && policy.updatedInput + ? { updatedInput: { ...policy.updatedInput } } + : {}), + }, + }; + }; + + const canUseToolFallback: CanUseTool = async ( + toolName, + input, + permission, + ) => { + if (!isBoundedManagedAgentToolUseId(permission.toolUseID)) { + return permissionResult(denied("invalid_input"), permission.toolUseID); + } + return permissionResult( + await decide( + permission.toolUseID, + toolName, + input, + permission.signal, + "can_use_tool_fallback", + ), + permission.toolUseID, + ); + }; + + return { preToolUseHook, canUseToolFallback }; +} diff --git a/packages/harness/src/experimental/managed-agent-spike/probe-cli.test.ts b/packages/harness/src/experimental/managed-agent-spike/probe-cli.test.ts new file mode 100644 index 000000000..80afcaa3b --- /dev/null +++ b/packages/harness/src/experimental/managed-agent-spike/probe-cli.test.ts @@ -0,0 +1,1653 @@ +import { describe, expect, it } from "vitest"; + +import { + ManagedAgentProbeCliError, + assertManagedAgentCancellationHostPlatform, + assertManagedAgentCertificationNodeVersion, + evaluateManagedAgentProbe, + executeManagedAgentProbeCli, + managedAgentProbeUsage, + parseManagedAgentProbeCliArgs, +} from "./probe-cli.js"; +import { FIXTURE_PATHS } from "./fixture.js"; +import { qualifiedManagedAgentMcpToolName } from "./runtime.js"; +import type { + ManagedAgentOperationId, + ManagedAgentPermissionDecision, + ManagedAgentPermissionReason, + ManagedAgentProbeEvent, + ManagedAgentProbeResult, +} from "./types.js"; + +function withProjectedL1Events( + result: ManagedAgentProbeResult, +): ManagedAgentProbeResult { + const events: ManagedAgentProbeEvent[] = []; + const append = ( + event: Omit, + ): void => { + events.push({ + sequence: events.length + 1, + runId: result.runId, + ...event, + }); + }; + for (const evidence of result.toolEvidence) { + if (evidence.status === "requested") { + append({ + type: "tool_requested", + toolUseId: evidence.toolUseId, + toolName: evidence.toolName, + }); + for (const decision of result.permissionEvidence.filter( + ({ toolUseId }) => toolUseId === evidence.toolUseId, + )) { + append({ + type: "permission", + toolUseId: decision.toolUseId, + toolName: decision.toolName, + permissionDecision: decision.decision, + permissionReason: decision.reason, + permissionSource: decision.source, + operationId: decision.operationId, + }); + } + continue; + } + append({ + type: "tool_completed", + toolUseId: evidence.toolUseId, + toolName: evidence.toolName, + isError: evidence.status === "error", + }); + } + append({ type: "sdk_result", subtype: "success", isError: false }); + append({ type: "terminal", terminal: "success" }); + return { ...result, events }; +} + +function withResequencedEvents( + result: ManagedAgentProbeResult, + events: readonly ManagedAgentProbeEvent[], +): ManagedAgentProbeResult { + return { + ...result, + events: events.map((event, index) => ({ + ...event, + sequence: index + 1, + runId: result.runId, + })), + }; +} + +function passingL1Result(): ManagedAgentProbeResult { + const echoTool = qualifiedManagedAgentMcpToolName("echo_nonce"); + const failOnceTool = qualifiedManagedAgentMcpToolName("fail_once"); + const steps = [ + ["Read", "success", "allow", "fixture_path", "read:clean_target"], + ["Read", "success", "allow", "fixture_path", "read:dirty_sentinel"], + ["Read", "success", "allow", "fixture_path", "read:untracked_sentinel"], + [ + "Read", + "error", + "deny", + "path_outside_workspace", + "read:outside_sentinel", + ], + ["Read", "error", "deny", "path_symlink_escape", "read:escape_link"], + ["Edit", "success", "allow", "fixture_path", "edit:clean_target"], + ["Write", "success", "allow", "fixture_path", "write:managed_output"], + [echoTool, "success", "allow", "managed_mcp_tool", "mcp:echo_nonce"], + [failOnceTool, "error", "allow", "managed_mcp_tool", "mcp:fail_once"], + [failOnceTool, "success", "allow", "managed_mcp_tool", "mcp:fail_once"], + ["Bash", "success", "allow", "exact_bash_command", "bash:exact_command"], + ] as const; + const ids = steps.map( + (_, index) => `tool_${(index + 1).toString(16).padStart(64, "0")}`, + ); + return withProjectedL1Events({ + contractVersion: 1, + runId: "run-1", + scenario: "L1", + target: "sonnet-5", + modelAlias: "claude-sonnet-5-anthropic-anthropic-eval", + sdkModelEvidence: { + authority: "sdk_non_authoritative", + initModelObserved: true, + initModelMatchesExpectedAlias: true, + resultModelUsageObserved: true, + resultModelUsageMatchesExpectedAlias: true, + resultModelCount: 1, + }, + sdkSessionId: "11111111-1111-4111-8111-111111111111", + inferenceTurns: 8, + sdkNumTurns: 8, + policyHookCoverage: true, + terminal: "success", + terminationEvidence: { + beforePolicyOverride: "success", + queryExecution: "iteration_completed", + sdkResult: "success", + }, + events: [], + toolEvidence: steps.flatMap(([toolName, completion], index) => [ + { + toolUseId: ids[index], + toolName, + status: "requested" as const, + }, + { + toolUseId: ids[index], + toolName, + status: completion, + }, + ]), + permissionEvidence: steps.map( + ([toolName, , decision, reason, operationId], index) => ({ + toolUseId: ids[index]!, + toolName, + decision, + reason, + source: "pre_tool_use" as const, + operationId, + }), + ), + policyDiagnostics: [], + workspaceChanges: [ + { path: FIXTURE_PATHS.cleanTarget, change: "modified" }, + { path: FIXTURE_PATHS.createdTarget, change: "created" }, + ], + preservation: [ + { path: FIXTURE_PATHS.dirtySentinel, preserved: true }, + { path: FIXTURE_PATHS.untrackedSentinel, preserved: true }, + ], + cancellationRequested: false, + queryClosed: true, + teardown: { + quiescent: true, + deadlineMet: true, + processTableAvailable: true, + containmentSupported: true, + ownershipProven: false, + forceKillIssued: false, + toolProcessObservationComplete: true, + toolProcessChannelsClosed: true, + elapsedMs: 5, + observedPids: [], + alivePidsAtDeadline: [], + emergencyCleanupAttempted: false, + }, + correlation: { + executionId: "execution-1", + evalSource: "eval-1", + promptEmbedded: true, + }, + l1Certification: { + contractVersion: 2, + promptVersion: "managed-agent-l1-prompt-v2", + }, + l1FinalBytes: [ + { role: "clean_target", matched: true }, + { role: "managed_output", matched: true }, + ], + nonceVerified: true, + } as ManagedAgentProbeResult); +} + +function passingL2Result(): ManagedAgentProbeResult { + const base = passingL1Result(); + const toolUseId = `tool_${"c".repeat(64)}`; + return { + ...base, + scenario: "L2", + inferenceTurns: 1, + sdkNumTurns: 1, + terminal: "cancelled", + events: [], + toolEvidence: [{ toolUseId, toolName: "Bash", status: "requested" }], + permissionEvidence: [ + { + toolUseId, + toolName: "Bash", + decision: "allow", + reason: "exact_bash_command", + source: "pre_tool_use", + operationId: "bash:exact_command", + }, + ], + workspaceChanges: [], + cancellationRequested: true, + teardown: { + ...base.teardown, + ownershipProven: true, + forceKillIssued: true, + observedPids: [12_345, 12_346], + }, + }; +} + +function evidenceForToolId( + result: ManagedAgentProbeResult, + toolUseId: string, +): ManagedAgentProbeResult["toolEvidence"] { + return result.toolEvidence.filter( + (evidence) => evidence.toolUseId === toolUseId, + ); +} + +interface TestToolStep { + readonly toolName: string; + readonly completion: "success" | "error"; + readonly decision: ManagedAgentPermissionDecision; + readonly reason: ManagedAgentPermissionReason; + readonly operationId: ManagedAgentOperationId; +} + +function insertL1ToolStep( + result: ManagedAgentProbeResult, + beforeRequestIndex: number, + step: TestToolStep, + idCharacter: string, +): ManagedAgentProbeResult { + const requested = result.toolEvidence.filter( + ({ status }) => status === "requested", + ); + const nextRequest = requested[beforeRequestIndex]; + const insertionIndex = nextRequest + ? result.toolEvidence.findIndex( + (evidence) => + evidence.toolUseId === nextRequest.toolUseId && + evidence.status === "requested", + ) + : result.toolEvidence.length; + const toolUseId = `tool_${idCharacter.repeat(64)}`; + const toolEvidence = [...result.toolEvidence]; + toolEvidence.splice( + insertionIndex, + 0, + { toolUseId, toolName: step.toolName, status: "requested" }, + { toolUseId, toolName: step.toolName, status: step.completion }, + ); + const permissionEvidence = [...result.permissionEvidence]; + permissionEvidence.splice(beforeRequestIndex, 0, { + toolUseId, + toolName: step.toolName, + decision: step.decision, + reason: step.reason, + source: "pre_tool_use", + operationId: step.operationId, + }); + return withProjectedL1Events({ + ...result, + toolEvidence, + permissionEvidence, + }); +} + +function optionalReadStep( + operationId: Extract, +): TestToolStep { + return { + toolName: "Read", + completion: "success", + decision: "allow", + reason: "fixture_path", + operationId, + }; +} + +function expectL1TraceFailure(result: ManagedAgentProbeResult): void { + const report = evaluateManagedAgentProbe(result); + expect(report.outcome).toBe("fail"); + expect(report.checks).toContainEqual({ + id: "exact_l1_tool_trace", + passed: false, + }); +} + +function expectProbeCheckFailure( + result: ManagedAgentProbeResult, + checkId: string, +): void { + const report = evaluateManagedAgentProbe(result); + expect(report.outcome).toBe("fail"); + expect(report.checks).toContainEqual({ id: checkId, passed: false }); +} + +function maximallyBatchedL1Result( + optionalRole?: "clean_target" | "dirty_sentinel" | "untracked_sentinel", +): ManagedAgentProbeResult { + const withOptional = optionalRole + ? insertL1ToolStep( + passingL1Result(), + 5, + optionalReadStep(`read:${optionalRole}`), + optionalRole === "clean_target" + ? "e" + : optionalRole === "dirty_sentinel" + ? "f" + : "a", + ) + : passingL1Result(); + const requested = withOptional.toolEvidence.filter( + ({ status }) => status === "requested", + ); + const completionFor = ( + request: (typeof requested)[number], + ): (typeof withOptional.toolEvidence)[number] => + withOptional.toolEvidence.find( + (evidence) => + evidence.toolUseId === request.toolUseId && + evidence.status !== "requested", + )!; + const optionalOffset = optionalRole ? 1 : 0; + const phaseA = requested.slice(0, 5); + const optional = optionalRole ? requested[5] : undefined; + const phaseB = requested.slice(5 + optionalOffset, 9 + optionalOffset); + const call10 = requested[9 + optionalOffset]!; + const call11 = requested[10 + optionalOffset]!; + const toolEvidence = [ + ...phaseA, + ...[phaseA[2]!, phaseA[4]!, phaseA[0]!, phaseA[3]!, phaseA[1]!].map( + completionFor, + ), + ...(optional ? [optional, completionFor(optional)] : []), + ...phaseB, + ...[phaseB[2]!, phaseB[0]!, phaseB[3]!, phaseB[1]!].map(completionFor), + call10, + completionFor(call10), + call11, + completionFor(call11), + ]; + return withProjectedL1Events({ + ...withOptional, + inferenceTurns: 4 + optionalOffset, + sdkNumTurns: 4 + optionalOffset, + toolEvidence, + }); +} + +function moveCompletionAfterRequest( + result: ManagedAgentProbeResult, + completedRequestIndex: number, + boundaryRequestIndex: number, +): ManagedAgentProbeResult { + const requested = result.toolEvidence.filter( + ({ status }) => status === "requested", + ); + const completedId = requested[completedRequestIndex]!.toolUseId; + const boundaryId = requested[boundaryRequestIndex]!.toolUseId; + const completion = result.toolEvidence.find( + (evidence) => + evidence.toolUseId === completedId && evidence.status !== "requested", + )!; + const toolEvidence = result.toolEvidence.filter( + (evidence) => evidence !== completion, + ); + const boundaryIndex = toolEvidence.findIndex( + (evidence) => + evidence.toolUseId === boundaryId && evidence.status === "requested", + ); + toolEvidence.splice(boundaryIndex + 1, 0, completion); + return withProjectedL1Events({ ...result, toolEvidence }); +} + +function moveCompletionBeforeOwnRequest( + result: ManagedAgentProbeResult, + requestIndex: number, +): ManagedAgentProbeResult { + const request = result.toolEvidence.filter( + ({ status }) => status === "requested", + )[requestIndex]!; + const completion = result.toolEvidence.find( + (evidence) => + evidence.toolUseId === request.toolUseId && + evidence.status !== "requested", + )!; + const toolEvidence = result.toolEvidence.filter( + (evidence) => evidence !== completion, + ); + const ownRequestIndex = toolEvidence.indexOf(request); + toolEvidence.splice(ownRequestIndex, 0, completion); + return withProjectedL1Events({ ...result, toolEvidence }); +} + +function eventSubstream( + result: ManagedAgentProbeResult, + types: readonly ManagedAgentProbeEvent["type"][], +): readonly Omit[] { + return result.events + .filter(({ type }) => types.includes(type)) + .map(({ sequence: _sequence, ...event }) => event); +} + +function movePermissionAfterOwnCompletion( + result: ManagedAgentProbeResult, + requestIndex: number, +): ManagedAgentProbeResult { + const request = result.toolEvidence.filter( + ({ status }) => status === "requested", + )[requestIndex]!; + const events = [...result.events]; + const permissionIndex = events.findIndex( + (event) => + event.type === "permission" && event.toolUseId === request.toolUseId, + ); + const [permission] = events.splice(permissionIndex, 1); + const completionIndex = events.findIndex( + (event) => + event.type === "tool_completed" && event.toolUseId === request.toolUseId, + ); + events.splice(completionIndex + 1, 0, permission!); + return withResequencedEvents(result, events); +} + +function withPermissionsBeforeOwnRequests( + result: ManagedAgentProbeResult, +): ManagedAgentProbeResult { + const events = [...result.events]; + for (const request of result.toolEvidence.filter( + ({ status }) => status === "requested", + )) { + const permissionIndex = events.findIndex( + (event) => + event.type === "permission" && event.toolUseId === request.toolUseId, + ); + const [permission] = events.splice(permissionIndex, 1); + const requestEventIndex = events.findIndex( + (event) => + event.type === "tool_requested" && + event.toolUseId === request.toolUseId, + ); + events.splice(requestEventIndex, 0, permission!); + } + return withResequencedEvents(result, events); +} + +describe("managed-agent probe CLI", () => { + it("is opt-in and never accepts credentials through arguments", () => { + expect(() => + parseManagedAgentProbeCliArgs([ + "--scenario", + "L1", + "--target", + "sonnet-5", + ]), + ).toThrow("--live"); + expect(() => + parseManagedAgentProbeCliArgs([ + "--live", + "--scenario", + "L1", + "--target", + "sonnet-5", + "--api-key", + "secret", + ]), + ).toThrow("Unknown argument"); + expect(managedAgentProbeUsage()).toContain("LLM_GATEWAY_EVAL_API_KEY"); + expect(managedAgentProbeUsage()).not.toContain("--api-key"); + }); + + it("refuses any model outside the two-value target allowlist", () => { + expect(() => + parseManagedAgentProbeCliArgs([ + "--live", + "--scenario", + "L1", + "--target", + "arbitrary-model", + ]), + ).toThrow("sonnet-5 or minimax-m3"); + }); + + it("checks exact Node before reading a dedicated credential", async () => { + const environment = new Proxy>( + {}, + { + get() { + throw new Error("environment was read"); + }, + }, + ); + await expect( + executeManagedAgentProbeCli( + ["--live", "--scenario", "L1", "--target", "sonnet-5"], + environment, + "25.0.0", + ), + ).rejects.toThrow("Live probes require Node 22.23.2"); + }); + + it("rejects an unexpected gateway origin before reading the eval key", async () => { + const reads: string[] = []; + const secret = "eval-secret-must-not-be-read"; + const environment = new Proxy>( + { + LLM_GATEWAY_BASE_URL: "https://llm.services.proxy.sapiom.ai", + LLM_GATEWAY_EVAL_API_KEY: secret, + }, + { + get(target, property: string) { + reads.push(property); + return target[property]; + }, + }, + ); + let failure: unknown; + try { + await executeManagedAgentProbeCli( + ["--live", "--scenario", "L1", "--target", "sonnet-5"], + environment, + "22.23.2", + ); + } catch (error) { + failure = error; + } + expect(failure).toBeInstanceOf(Error); + expect((failure as Error).message).toContain( + "pinned direct Sapiom gateway origin", + ); + expect((failure as Error).message).not.toContain(secret); + expect(reads).toEqual(["LLM_GATEWAY_BASE_URL"]); + }); + + it("reads eval auth only after accepting the pinned direct gateway", async () => { + const reads: string[] = []; + const environment = new Proxy>( + { + LLM_GATEWAY_BASE_URL: "https://litellm.services.sapiom.ai/", + }, + { + get(target, property: string) { + reads.push(property); + return target[property]; + }, + }, + ); + await expect( + executeManagedAgentProbeCli( + ["--live", "--scenario", "L1", "--target", "sonnet-5"], + environment, + "22.23.2", + ), + ).rejects.toThrow("LLM_GATEWAY_EVAL_API_KEY is required"); + expect(reads).toEqual(["LLM_GATEWAY_BASE_URL", "LLM_GATEWAY_EVAL_API_KEY"]); + }); + + it("prints help without reading auth or opening a query", async () => { + await expect( + executeManagedAgentProbeCli(["--help"], {}, "0.0.0"), + ).resolves.toEqual({ help: true, usage: managedAgentProbeUsage() }); + }); + + it("exposes an explicit version assertion for automation", () => { + expect(() => + assertManagedAgentCertificationNodeVersion("22.23.2"), + ).not.toThrow(); + expect(() => assertManagedAgentCertificationNodeVersion("22.23.1")).toThrow( + ManagedAgentProbeCliError, + ); + }); + + it("limits live L2 certification to the reviewed POSIX host model", () => { + expect(() => + assertManagedAgentCancellationHostPlatform("darwin"), + ).not.toThrow(); + expect(() => + assertManagedAgentCancellationHostPlatform("linux"), + ).not.toThrow(); + expect(() => assertManagedAgentCancellationHostPlatform("win32")).toThrow( + "detached POSIX fixture containment model", + ); + }); + + it("rejects Windows L2 before reading gateway or credential environment", async () => { + const environment = new Proxy>( + {}, + { + get() { + throw new Error("environment was read"); + }, + }, + ); + + await expect( + executeManagedAgentProbeCli( + ["--live", "--scenario", "L2", "--target", "sonnet-5"], + environment, + "22.23.2", + "win32", + ), + ).rejects.toThrow("detached POSIX fixture containment model"); + }); + + it("requires successful results from every built-in tool for L1", () => { + const passing = passingL1Result(); + const result: ManagedAgentProbeResult = { + ...passing, + toolEvidence: passing.toolEvidence.map((evidence) => + evidence.toolName === "Bash" && evidence.status === "success" + ? { ...evidence, status: "error" } + : evidence, + ), + }; + + expect( + evaluateManagedAgentProbe(result).checks.find( + ({ id }) => id === "builtin_tools_succeeded", + ), + ).toEqual({ id: "builtin_tools_succeeded", passed: false }); + }); + + it("fails exact-model certification when SDK-observed model evidence is missing or mixed", () => { + const passing = passingL1Result(); + const report = evaluateManagedAgentProbe({ + ...passing, + sdkModelEvidence: { + ...passing.sdkModelEvidence, + resultModelUsageMatchesExpectedAlias: false, + resultModelCount: 2, + }, + }); + + expect(report.checks).toContainEqual({ + id: "sdk_model_alias_observed", + passed: false, + }); + expect(report.outcome).toBe("fail"); + }); + + it.each([ + ["clean_target", "e"], + ["dirty_sentinel", "f"], + ["untracked_sentinel", "a"], + ] as const)( + "accepts one optional %s verification Read in the v2 window", + (role, idCharacter) => { + const passing = passingL1Result(); + const result = insertL1ToolStep( + passing, + 5, + optionalReadStep(`read:${role}`), + idCharacter, + ); + const report = evaluateManagedAgentProbe(result); + + expect(report.outcome).toBe("local_pass"); + expect(report.deploymentProvenance).toBe( + "requires_gateway_reconciliation", + ); + expect(report).toMatchObject({ + l1Certification: { + contractVersion: 2, + promptVersion: "managed-agent-l1-prompt-v2", + evaluatorVersion: "managed-agent-l1-evaluator-v2", + optionalReadCount: 1, + optionalReadRole: role, + }, + }); + }, + ); + + it("records zero optional Reads as nonblocking efficiency evidence", () => { + expect(evaluateManagedAgentProbe(passingL1Result())).toMatchObject({ + outcome: "local_pass", + l1Certification: { + evaluatorVersion: "managed-agent-l1-evaluator-v2", + optionalReadCount: 0, + }, + }); + expect( + evaluateManagedAgentProbe(passingL1Result()).l1Certification, + ).not.toHaveProperty("optionalReadRole"); + }); + + it.each([ + ["none", undefined], + ["clean_target", "clean_target"], + ["dirty_sentinel", "dirty_sentinel"], + ["untracked_sentinel", "untracked_sentinel"], + ] as const)( + "accepts maximally batched phase completions with %s optional Read", + (_name, optionalRole) => { + expect( + evaluateManagedAgentProbe(maximallyBatchedL1Result(optionalRole)), + ).toMatchObject({ + outcome: "local_pass", + checks: expect.arrayContaining([ + { id: "exact_l1_tool_trace", passed: true }, + ]), + }); + }, + ); + + it("rejects the all-requests-first false-pass counterexample", () => { + const passing = passingL1Result(); + const allRequestsFirst = withProjectedL1Events({ + ...passing, + inferenceTurns: 1, + sdkNumTurns: 1, + toolEvidence: [ + ...passing.toolEvidence.filter(({ status }) => status === "requested"), + ...passing.toolEvidence.filter(({ status }) => status !== "requested"), + ], + }); + + expectL1TraceFailure(allRequestsFirst); + expectProbeCheckFailure(allRequestsFirst, "minimum_l1_inference_turns"); + }); + + it.each([0, 1, 2, 3, 4])( + "rejects phase A completion %i delayed until after call 6 starts", + (phaseAIndex) => { + expectL1TraceFailure( + moveCompletionAfterRequest(maximallyBatchedL1Result(), phaseAIndex, 5), + ); + }, + ); + + it.each([0, 1, 2, 3, 4])( + "rejects phase A completion %i delayed until after the optional Read starts", + (phaseAIndex) => { + expectL1TraceFailure( + moveCompletionAfterRequest( + maximallyBatchedL1Result("clean_target"), + phaseAIndex, + 5, + ), + ); + }, + ); + + it.each(["clean_target", "dirty_sentinel", "untracked_sentinel"] as const)( + "rejects the %s optional completion delayed until after call 6 starts", + (optionalRole) => { + expectL1TraceFailure( + moveCompletionAfterRequest( + maximallyBatchedL1Result(optionalRole), + 5, + 6, + ), + ); + }, + ); + + it.each([5, 6, 7, 8])( + "rejects phase B request-index %i completion delayed until after call 10 starts", + (phaseBRequestIndex) => { + expectL1TraceFailure( + moveCompletionAfterRequest( + maximallyBatchedL1Result(), + phaseBRequestIndex, + 9, + ), + ); + }, + ); + + it("rejects call 10 completion delayed until after call 11 starts", () => { + expectL1TraceFailure( + moveCompletionAfterRequest(maximallyBatchedL1Result(), 9, 10), + ); + }); + + it.each([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10])( + "rejects completion-before-own-request at request index %i", + (requestIndex) => { + expectL1TraceFailure( + moveCompletionBeforeOwnRequest( + maximallyBatchedL1Result(), + requestIndex, + ), + ); + }, + ); + + it.each([ + ["without optional Read", undefined, 3], + ["with optional Read", "clean_target", 4], + ] as const)( + "rejects too few inference turns %s", + (_name, optionalRole, inferenceTurns) => { + expectProbeCheckFailure( + { + ...maximallyBatchedL1Result(optionalRole), + inferenceTurns, + }, + "minimum_l1_inference_turns", + ); + }, + ); + + it("rejects a normalized tool-event projection mismatch", () => { + const passing = maximallyBatchedL1Result(); + const firstToolEvent = passing.events.findIndex( + ({ type }) => type === "tool_requested", + ); + const events = [...passing.events]; + events[firstToolEvent] = { + ...events[firstToolEvent]!, + toolName: "Write", + }; + expectProbeCheckFailure( + { ...passing, events }, + "normalized_event_projection", + ); + }); + + it.each(Array.from({ length: 11 }, (_, index) => index))( + "rejects canonical permission %i moved after its own completion while preserving both substreams", + (requestIndex) => { + const passing = passingL1Result(); + const invalid = movePermissionAfterOwnCompletion(passing, requestIndex); + + expect( + eventSubstream(invalid, ["tool_requested", "tool_completed"]), + ).toEqual(eventSubstream(passing, ["tool_requested", "tool_completed"])); + expect(eventSubstream(invalid, ["permission"])).toEqual( + eventSubstream(passing, ["permission"]), + ); + expectProbeCheckFailure(invalid, "normalized_event_projection"); + }, + ); + + it("rejects an optional Read permission moved after its own completion while preserving both substreams", () => { + const passing = insertL1ToolStep( + passingL1Result(), + 5, + optionalReadStep("read:clean_target"), + "e", + ); + const invalid = movePermissionAfterOwnCompletion(passing, 5); + + expect( + eventSubstream(invalid, ["tool_requested", "tool_completed"]), + ).toEqual(eventSubstream(passing, ["tool_requested", "tool_completed"])); + expect(eventSubstream(invalid, ["permission"])).toEqual( + eventSubstream(passing, ["permission"]), + ); + expectProbeCheckFailure(invalid, "normalized_event_projection"); + }); + + it("accepts permissions before their requests when each permission still precedes its completion", () => { + const requestBeforePermission = passingL1Result(); + const permissionBeforeRequest = + withPermissionsBeforeOwnRequests(passingL1Result()); + + expect(evaluateManagedAgentProbe(requestBeforePermission).outcome).toBe( + "local_pass", + ); + expect(evaluateManagedAgentProbe(permissionBeforeRequest).outcome).toBe( + "local_pass", + ); + }); + + it.each(["sdk_result", "terminal"] as const)( + "rejects %s before the Bash completion", + (eventType) => { + const passing = maximallyBatchedL1Result(); + const events = [...passing.events]; + const bashCompletionIndex = events.findIndex( + (event) => event.type === "tool_completed" && event.toolName === "Bash", + ); + const movedIndex = events.findIndex(({ type }) => type === eventType); + const [moved] = events.splice(movedIndex, 1); + events.splice(bashCompletionIndex, 0, moved!); + expectProbeCheckFailure( + withResequencedEvents(passing, events), + "bash_sdk_terminal_order", + ); + }, + ); + + it("rejects terminal before the successful SDK result", () => { + const passing = maximallyBatchedL1Result(); + const events = [...passing.events]; + const sdkResultIndex = events.findIndex( + ({ type }) => type === "sdk_result", + ); + const terminalIndex = events.findIndex(({ type }) => type === "terminal"); + [events[sdkResultIndex], events[terminalIndex]] = [ + events[terminalIndex]!, + events[sdkResultIndex]!, + ]; + expectProbeCheckFailure( + withResequencedEvents(passing, events), + "bash_sdk_terminal_order", + ); + }); + + it("rejects a second optional verification Read", () => { + const first = insertL1ToolStep( + passingL1Result(), + 5, + optionalReadStep("read:clean_target"), + "e", + ); + const second = insertL1ToolStep( + first, + 6, + optionalReadStep("read:dirty_sentinel"), + "f", + ); + + const report = evaluateManagedAgentProbe(second); + expect(report.l1Certification).toMatchObject({ optionalReadCount: 2 }); + expectL1TraceFailure(second); + }); + + it.each([ + ["managed_output", "e"], + ["outside_sentinel", "f"], + ["escape_link", "a"], + ] as const)( + "rejects an optional Read of the registered but disallowed %s role", + (role, idCharacter) => { + expectL1TraceFailure( + insertL1ToolStep( + passingL1Result(), + 5, + optionalReadStep(`read:${role}`), + idCharacter, + ), + ); + }, + ); + + it.each([ + ["before the denial probes", 3], + ["after Edit", 6], + ] as const)("rejects an otherwise valid optional Read %s", (_name, index) => { + expectL1TraceFailure( + insertL1ToolStep( + passingL1Result(), + index, + optionalReadStep("read:clean_target"), + "e", + ), + ); + }); + + it.each([ + ["outside denial", "path_outside_workspace", "read:outside_sentinel"], + ["symlink denial", "path_symlink_escape", "read:escape_link"], + ] as const)( + "rejects an extra denied Read retry of the %s", + (_name, reason, operationId) => { + expectL1TraceFailure( + insertL1ToolStep( + passingL1Result(), + 5, + { + toolName: "Read", + completion: "error", + decision: "deny", + reason, + operationId, + }, + "e", + ), + ); + }, + ); + + it.each([ + [ + "Edit", + { + toolName: "Edit", + completion: "success", + decision: "allow", + reason: "fixture_path", + operationId: "edit:clean_target", + }, + ], + [ + "Write", + { + toolName: "Write", + completion: "success", + decision: "allow", + reason: "fixture_path", + operationId: "write:managed_output", + }, + ], + [ + "Bash", + { + toolName: "Bash", + completion: "success", + decision: "allow", + reason: "exact_bash_command", + operationId: "bash:exact_command", + }, + ], + [ + "MCP", + { + toolName: qualifiedManagedAgentMcpToolName("echo_nonce"), + completion: "success", + decision: "allow", + reason: "managed_mcp_tool", + operationId: "mcp:echo_nonce", + }, + ], + [ + "unknown tool", + { + toolName: "unknown", + completion: "error", + decision: "deny", + reason: "tool_not_allowed", + operationId: "unknown", + }, + ], + ] as const)("rejects any extra %s operation", (_name, step) => { + expectL1TraceFailure(insertL1ToolStep(passingL1Result(), 5, step, "e")); + }); + + it("rejects any workspace delta beyond the two canonical L1 changes", () => { + const passing = passingL1Result(); + const report = evaluateManagedAgentProbe({ + ...passing, + workspaceChanges: [ + ...passing.workspaceChanges, + { path: "unexpected.txt", change: "created" }, + ], + }); + + expect(report.outcome).toBe("fail"); + expect(report.checks).toContainEqual({ + id: "exact_workspace_delta", + passed: false, + }); + }); + + it("accepts the exact workspace delta in either evidence order", () => { + const passing = passingL1Result(); + const report = evaluateManagedAgentProbe({ + ...passing, + workspaceChanges: [...passing.workspaceChanges].reverse(), + }); + + expect(report.outcome).toBe("local_pass"); + expect(report.checks).toContainEqual({ + id: "exact_workspace_delta", + passed: true, + }); + }); + + it("rejects a duplicate canonical workspace entry with the other path missing", () => { + const passing = passingL1Result(); + const duplicate = passing.workspaceChanges[0]!; + const report = evaluateManagedAgentProbe({ + ...passing, + workspaceChanges: [duplicate, duplicate], + }); + + expect(report.outcome).toBe("fail"); + expect(report.checks).toContainEqual({ + id: "exact_workspace_delta", + passed: false, + }); + }); + + it("accepts exactly one permitted Bash request for L2 and rejects any extra tool call", () => { + const passing = passingL2Result(); + expect(evaluateManagedAgentProbe(passing, [12_345, 12_346])).toMatchObject({ + outcome: "local_pass", + checks: expect.arrayContaining([ + { id: "exact_l2_bash_only_trace", passed: true }, + { id: "l2_containment_prepared", passed: true }, + ]), + }); + expect( + evaluateManagedAgentProbe(passing).checks.find( + ({ id }) => id === "no_fixture_process_alive", + ), + ).toEqual({ id: "no_fixture_process_alive", passed: false }); + + const writeId = `tool_${"d".repeat(64)}`; + const invalid: ManagedAgentProbeResult = { + ...passing, + toolEvidence: [ + ...passing.toolEvidence, + { toolUseId: writeId, toolName: "Write", status: "requested" }, + { toolUseId: writeId, toolName: "Write", status: "success" }, + ], + permissionEvidence: [ + ...passing.permissionEvidence, + { + toolUseId: writeId, + toolName: "Write", + decision: "allow", + reason: "fixture_path", + source: "pre_tool_use", + operationId: "write:unregistered", + }, + ], + }; + + expect( + evaluateManagedAgentProbe(invalid, [12_345, 12_346]).checks, + ).toContainEqual({ + id: "exact_l2_bash_only_trace", + passed: false, + }); + }); + + it("requires observed closed tool lifetimes but not an unnecessary host force-kill", () => { + const passing = passingL2Result(); + const graceful = { + ...passing, + teardown: { ...passing.teardown, forceKillIssued: false }, + }; + expect(evaluateManagedAgentProbe(graceful, [12_345, 12_346]).outcome).toBe( + "local_pass", + ); + + for (const [field, checkId] of [ + ["toolProcessObservationComplete", "l2_containment_prepared"], + ["toolProcessChannelsClosed", "sdk_closed_tool_lifetime_channels"], + ] as const) { + expect( + evaluateManagedAgentProbe( + { + ...passing, + teardown: { ...passing.teardown, [field]: false }, + }, + [12_345, 12_346], + ).checks, + ).toContainEqual({ id: checkId, passed: false }); + } + }); + + it("never certifies a cancelled terminal without a requested cancellation", () => { + const passing = passingL2Result(); + const report = evaluateManagedAgentProbe( + { ...passing, cancellationRequested: false }, + [12_345, 12_346], + ); + + expect(report.outcome).toBe("fail"); + expect(report.checks).toContainEqual({ + id: "cancellation_requested", + passed: false, + }); + }); + + it.each([ + [ + "omitted", + (passing: ManagedAgentProbeResult) => { + const omittedId = passing.toolEvidence.find( + (evidence) => + evidence.status === "requested" && evidence.toolName === "Read", + )!.toolUseId!; + return { + ...passing, + toolEvidence: passing.toolEvidence.filter( + (evidence) => evidence.toolUseId !== omittedId, + ), + permissionEvidence: passing.permissionEvidence.filter( + (evidence) => evidence.toolUseId !== omittedId, + ), + }; + }, + ], + [ + "reordered", + (passing: ManagedAgentProbeResult) => { + const requested = passing.toolEvidence.filter( + ({ status }) => status === "requested", + ); + const editId = requested[5]!.toolUseId!; + const writeId = requested[6]!.toolUseId!; + const editEvidence = evidenceForToolId(passing, editId); + const writeEvidence = evidenceForToolId(passing, writeId); + const reordered = passing.toolEvidence.filter( + ({ toolUseId }) => toolUseId !== editId && toolUseId !== writeId, + ); + reordered.splice(10, 0, ...writeEvidence, ...editEvidence); + return { ...passing, toolEvidence: reordered }; + }, + ], + [ + "extra", + (passing: ManagedAgentProbeResult) => { + const toolUseId = `tool_${"a".repeat(64)}`; + return { + ...passing, + toolEvidence: [ + ...passing.toolEvidence, + { toolUseId, toolName: "Read", status: "requested" as const }, + { toolUseId, toolName: "Read", status: "success" as const }, + ], + permissionEvidence: [ + ...passing.permissionEvidence, + { + toolUseId, + toolName: "Read", + decision: "allow" as const, + reason: "fixture_path" as const, + source: "pre_tool_use" as const, + operationId: "read:clean_target" as const, + }, + ], + }; + }, + ], + [ + "duplicate retry", + (passing: ManagedAgentProbeResult) => { + const toolUseId = `tool_${"b".repeat(64)}`; + return { + ...passing, + toolEvidence: [ + ...passing.toolEvidence, + { toolUseId, toolName: "Bash", status: "requested" as const }, + { toolUseId, toolName: "Bash", status: "success" as const }, + ], + permissionEvidence: [ + ...passing.permissionEvidence, + { + toolUseId, + toolName: "Bash", + decision: "allow" as const, + reason: "exact_bash_command" as const, + source: "pre_tool_use" as const, + operationId: "bash:exact_command" as const, + }, + ], + }; + }, + ], + ])("rejects an %s L1 tool trace", (_name, mutate) => { + const report = evaluateManagedAgentProbe(mutate(passingL1Result())); + + expect(report.outcome).toBe("fail"); + expect(report.checks).toContainEqual({ + id: "exact_l1_tool_trace", + passed: false, + }); + }); + + describe("L1 v2 request correlation", () => { + it("rejects duplicate request IDs", () => { + const passing = passingL1Result(); + const requestIds = passing.toolEvidence.flatMap((evidence) => + evidence.status === "requested" && evidence.toolUseId + ? [evidence.toolUseId] + : [], + ); + const firstId = requestIds[0]!; + const duplicateId = requestIds[1]!; + expectL1TraceFailure({ + ...passing, + toolEvidence: passing.toolEvidence.map((evidence) => + evidence.status === "requested" && evidence.toolUseId === firstId + ? { ...evidence, toolUseId: duplicateId } + : evidence, + ), + }); + }); + + it("rejects an empty request ID", () => { + const passing = passingL1Result(); + const firstId = passing.toolEvidence.find( + ({ status }) => status === "requested", + )!.toolUseId!; + expectL1TraceFailure({ + ...passing, + toolEvidence: passing.toolEvidence.map((evidence) => + evidence.status === "requested" && evidence.toolUseId === firstId + ? { ...evidence, toolUseId: " " } + : evidence, + ), + }); + }); + + it("rejects a request with no completion", () => { + const passing = passingL1Result(); + const firstId = passing.toolEvidence.find( + ({ status }) => status === "requested", + )!.toolUseId!; + expectL1TraceFailure({ + ...passing, + toolEvidence: passing.toolEvidence.filter( + (evidence) => + evidence.toolUseId !== firstId || evidence.status === "requested", + ), + }); + }); + + it("rejects duplicate completions for one request", () => { + const passing = passingL1Result(); + const completion = passing.toolEvidence.find( + ({ status }) => status !== "requested", + )!; + expectL1TraceFailure({ + ...passing, + toolEvidence: [...passing.toolEvidence, completion], + }); + }); + + it("rejects a completion whose tool does not match its request", () => { + const passing = passingL1Result(); + const firstId = passing.toolEvidence.find( + ({ status }) => status === "requested", + )!.toolUseId!; + expectL1TraceFailure({ + ...passing, + toolEvidence: passing.toolEvidence.map((evidence) => + evidence.toolUseId === firstId && evidence.status !== "requested" + ? { ...evidence, toolName: "Write" } + : evidence, + ), + }); + }); + + it("rejects a request with no primary PreToolUse decision", () => { + const passing = passingL1Result(); + const firstDecision = passing.permissionEvidence[0]!; + expectL1TraceFailure({ + ...passing, + permissionEvidence: passing.permissionEvidence.filter( + ({ toolUseId }) => toolUseId !== firstDecision.toolUseId, + ), + }); + }); + + it("rejects duplicate primary decisions for one request", () => { + const passing = passingL1Result(); + expectL1TraceFailure({ + ...passing, + permissionEvidence: [ + ...passing.permissionEvidence, + passing.permissionEvidence[0]!, + ], + }); + }); + + it("rejects a primary decision whose tool does not match its request", () => { + const passing = passingL1Result(); + const firstDecision = passing.permissionEvidence[0]!; + expectL1TraceFailure({ + ...passing, + permissionEvidence: passing.permissionEvidence.map((evidence) => + evidence.toolUseId === firstDecision.toolUseId + ? { ...evidence, toolName: "Write" } + : evidence, + ), + }); + }); + + it("rejects a fallback decision in addition to the primary decision", () => { + const passing = passingL1Result(); + expectL1TraceFailure({ + ...passing, + permissionEvidence: [ + ...passing.permissionEvidence, + { + ...passing.permissionEvidence[0]!, + source: "can_use_tool_fallback", + }, + ], + }); + }); + + it("rejects a fallback decision that replaces the primary decision", () => { + const passing = passingL1Result(); + const firstDecision = passing.permissionEvidence[0]!; + expectL1TraceFailure({ + ...passing, + permissionEvidence: passing.permissionEvidence.map((evidence) => + evidence.toolUseId === firstDecision.toolUseId + ? { ...evidence, source: "can_use_tool_fallback" } + : evidence, + ), + }); + }); + + it("rejects an orphan completion", () => { + const passing = passingL1Result(); + expectL1TraceFailure({ + ...passing, + toolEvidence: [ + ...passing.toolEvidence, + { + toolUseId: `tool_${"e".repeat(64)}`, + toolName: "Read", + status: "success", + }, + ], + }); + }); + + it("rejects an orphan primary decision", () => { + const passing = passingL1Result(); + expectL1TraceFailure({ + ...passing, + permissionEvidence: [ + ...passing.permissionEvidence, + { + ...passing.permissionEvidence[0]!, + toolUseId: `tool_${"e".repeat(64)}`, + }, + ], + }); + }); + }); + + describe("L1 v2 outcome and certification evidence", () => { + it("rejects an allowed canonical operation that completes with an error", () => { + const passing = passingL1Result(); + const firstId = passing.toolEvidence.find( + ({ status }) => status === "requested", + )!.toolUseId!; + expectL1TraceFailure({ + ...passing, + toolEvidence: passing.toolEvidence.map((evidence) => + evidence.toolUseId === firstId && evidence.status === "success" + ? { ...evidence, status: "error" } + : evidence, + ), + }); + }); + + it("rejects a denied canonical operation that reports success", () => { + const passing = passingL1Result(); + const deniedId = passing.permissionEvidence.find( + ({ decision }) => decision === "deny", + )!.toolUseId; + expectL1TraceFailure({ + ...passing, + toolEvidence: passing.toolEvidence.map((evidence) => + evidence.toolUseId === deniedId && evidence.status === "error" + ? { ...evidence, status: "success" } + : evidence, + ), + }); + }); + + it("rejects an incoherent decision, reason, or operation ID", () => { + const passing = passingL1Result(); + const firstDecision = passing.permissionEvidence[0]!; + for (const replacement of [ + { decision: "deny" as const }, + { reason: "path_outside_workspace" as const }, + { operationId: "read:dirty_sentinel" as const }, + ]) { + expectL1TraceFailure({ + ...passing, + permissionEvidence: passing.permissionEvidence.map((evidence) => + evidence.toolUseId === firstDecision.toolUseId + ? { ...evidence, ...replacement } + : evidence, + ), + }); + } + }); + + it("rejects missing or stale L1 v2 contract evidence", () => { + const passing = passingL1Result(); + expectProbeCheckFailure( + { ...passing, l1Certification: undefined }, + "l1_contract_v2", + ); + expectProbeCheckFailure( + { + ...passing, + l1Certification: { + contractVersion: 1, + promptVersion: "managed-agent-l1-prompt-v1", + }, + } as unknown as ManagedAgentProbeResult, + "l1_contract_v2", + ); + expectProbeCheckFailure( + { + ...passing, + correlation: { ...passing.correlation, promptEmbedded: false }, + }, + "l1_contract_v2", + ); + }); + + it("requires positive nonce evidence", () => { + expectProbeCheckFailure( + { ...passingL1Result(), nonceVerified: false }, + "nonce_verified", + ); + }); + + it.each([ + ["missing", undefined], + [ + "false", + [ + { role: "clean_target", matched: true }, + { role: "managed_output", matched: false }, + ], + ], + [ + "extra", + [ + { role: "clean_target", matched: true }, + { role: "managed_output", matched: true }, + { role: "clean_target", matched: true }, + ], + ], + ] as const)("rejects %s L1 final-byte evidence", (_name, l1FinalBytes) => { + expectProbeCheckFailure( + { + ...passingL1Result(), + l1FinalBytes, + } as ManagedAgentProbeResult, + "expected_final_bytes", + ); + }); + + it.each([ + ["terminal success", "terminal_success", { terminal: "incomplete" }], + ["query close", "query_closed", { queryClosed: false }], + [ + "process quiescence", + "process_tree_quiescent", + { teardown: { ...passingL1Result().teardown, quiescent: false } }, + ], + ] as const)("requires %s", (_name, checkId, mutation) => { + expectProbeCheckFailure( + { + ...passingL1Result(), + ...mutation, + } as ManagedAgentProbeResult, + checkId, + ); + }); + + it.each([ + ["missing", []], + [ + "false", + [ + { path: FIXTURE_PATHS.dirtySentinel, preserved: true }, + { path: FIXTURE_PATHS.untrackedSentinel, preserved: false }, + ], + ], + [ + "extra", + [ + { path: FIXTURE_PATHS.dirtySentinel, preserved: true }, + { path: FIXTURE_PATHS.untrackedSentinel, preserved: true }, + { path: "extra-sentinel.txt", preserved: true }, + ], + ], + ] as const)("rejects %s preservation evidence", (_name, preservation) => { + expectProbeCheckFailure( + { ...passingL1Result(), preservation: [...preservation] }, + "dirty_and_untracked_preserved", + ); + }); + }); + + it("requires one completion and primary decision per L1 request, including fail_once error then success", () => { + const passing = passingL1Result(); + const failOnceRequests = passing.toolEvidence.filter( + ({ status, toolName }) => + status === "requested" && + toolName === qualifiedManagedAgentMcpToolName("fail_once"), + ); + const firstFailOnceId = failOnceRequests[0]!.toolUseId!; + const invalid: ManagedAgentProbeResult = { + ...passing, + toolEvidence: passing.toolEvidence.map((evidence) => + evidence.toolUseId === firstFailOnceId && evidence.status === "error" + ? { ...evidence, status: "success" } + : evidence, + ), + }; + + expect(evaluateManagedAgentProbe(invalid).checks).toContainEqual({ + id: "exact_l1_tool_trace", + passed: false, + }); + }); + + it("requires positive permission evidence and distinct lexical and symlink denials", () => { + const passing = passingL1Result(); + expect(evaluateManagedAgentProbe(passing).outcome).toBe("local_pass"); + + const falsePass: ManagedAgentProbeResult = { + ...passing, + permissionEvidence: [ + { + toolUseId: `tool_${"a".repeat(64)}`, + toolName: "Read", + decision: "deny", + reason: "path_outside_workspace", + source: "pre_tool_use", + operationId: "read:outside_sentinel", + }, + { + toolUseId: `tool_${"b".repeat(64)}`, + toolName: "Read", + decision: "deny", + reason: "path_outside_workspace", + source: "pre_tool_use", + operationId: "read:outside_sentinel", + }, + ], + }; + const checks = evaluateManagedAgentProbe(falsePass); + + expect(checks.outcome).toBe("fail"); + expect( + checks.checks.find(({ id }) => id === "expected_permissions_allowed"), + ).toEqual({ id: "expected_permissions_allowed", passed: false }); + expect( + checks.checks.find(({ id }) => id === "outside_and_symlink_denied"), + ).toEqual({ id: "outside_and_symlink_denied", passed: false }); + }); +}); diff --git a/packages/harness/src/experimental/managed-agent-spike/probe-cli.ts b/packages/harness/src/experimental/managed-agent-spike/probe-cli.ts new file mode 100644 index 000000000..f815ca96b --- /dev/null +++ b/packages/harness/src/experimental/managed-agent-spike/probe-cli.ts @@ -0,0 +1,997 @@ +#!/usr/bin/env node +import { pathToFileURL } from "node:url"; + +import { + FIXTURE_PATHS, + createManagedAgentFixture, + verifyManagedAgentFixtureBytes, + waitForManagedAgentFixturePids, +} from "./fixture.js"; +import { + MANAGED_AGENT_CONTRACT, + MANAGED_AGENT_L1_CERTIFICATION_CONTRACT, + assertManagedAgentDirectGatewayOrigin, + resolveManagedAgentModelTarget, +} from "./contract.js"; +import { createLocalManagedAgentProcessObserver } from "./process-observer.js"; +import { + qualifiedManagedAgentMcpToolName, + runManagedAgentProbe, +} from "./runtime.js"; +import type { + ManagedAgentModelTargetId, + ManagedAgentOperationId, + ManagedAgentPathRole, + ManagedAgentPermissionReason, + ManagedAgentProbeResult, + ManagedAgentProbeScenario, +} from "./types.js"; + +type Environment = Readonly>; + +export interface ManagedAgentProbeCliArgs { + readonly help: boolean; + readonly live: boolean; + readonly target?: ManagedAgentModelTargetId; + readonly scenario?: ManagedAgentProbeScenario; +} + +export interface ManagedAgentProbeCheck { + readonly id: string; + readonly passed: boolean; +} + +export interface ManagedAgentProbeReport { + /** Local protocol/host result; never authoritative deployment certification. */ + readonly outcome: "local_pass" | "fail"; + readonly deploymentProvenance: "requires_gateway_reconciliation"; + readonly checks: readonly ManagedAgentProbeCheck[]; + readonly result: ManagedAgentProbeResult; + readonly l1Certification?: { + readonly contractVersion: number; + readonly promptVersion: string; + readonly evaluatorVersion: string; + readonly optionalReadCount: number; + readonly optionalReadRole?: ManagedAgentPathRole; + }; +} + +export class ManagedAgentProbeCliError extends Error { + public constructor(message: string) { + super(message); + this.name = "ManagedAgentProbeCliError"; + } +} + +interface ManagedAgentExpectedL1ToolStep { + readonly toolName: string; + readonly completion: "success" | "error"; + readonly decision: "allow" | "deny"; + readonly reason: ManagedAgentPermissionReason; + readonly operationId: ManagedAgentOperationId; +} + +const MANAGED_AGENT_EXPECTED_L1_TOOL_TRACE = Object.freeze([ + { + toolName: "Read", + completion: "success", + decision: "allow", + reason: "fixture_path", + operationId: "read:clean_target", + }, + { + toolName: "Read", + completion: "success", + decision: "allow", + reason: "fixture_path", + operationId: "read:dirty_sentinel", + }, + { + toolName: "Read", + completion: "success", + decision: "allow", + reason: "fixture_path", + operationId: "read:untracked_sentinel", + }, + { + toolName: "Read", + completion: "error", + decision: "deny", + reason: "path_outside_workspace", + operationId: "read:outside_sentinel", + }, + { + toolName: "Read", + completion: "error", + decision: "deny", + reason: "path_symlink_escape", + operationId: "read:escape_link", + }, + { + toolName: "Edit", + completion: "success", + decision: "allow", + reason: "fixture_path", + operationId: "edit:clean_target", + }, + { + toolName: "Write", + completion: "success", + decision: "allow", + reason: "fixture_path", + operationId: "write:managed_output", + }, + { + toolName: qualifiedManagedAgentMcpToolName("echo_nonce"), + completion: "success", + decision: "allow", + reason: "managed_mcp_tool", + operationId: "mcp:echo_nonce", + }, + { + toolName: qualifiedManagedAgentMcpToolName("fail_once"), + completion: "error", + decision: "allow", + reason: "managed_mcp_tool", + operationId: "mcp:fail_once", + }, + { + toolName: qualifiedManagedAgentMcpToolName("fail_once"), + completion: "success", + decision: "allow", + reason: "managed_mcp_tool", + operationId: "mcp:fail_once", + }, + { + toolName: "Bash", + completion: "success", + decision: "allow", + reason: "exact_bash_command", + operationId: "bash:exact_command", + }, +] as const satisfies readonly ManagedAgentExpectedL1ToolStep[]); + +const MANAGED_AGENT_L1_OPTIONAL_READ_OPERATIONS = + new Set([ + "read:clean_target", + "read:dirty_sentinel", + "read:untracked_sentinel", + ]); + +interface ManagedAgentL1TraceAnalysis { + readonly passed: boolean; + readonly optionalReadCount: number; + readonly optionalReadRole?: ManagedAgentPathRole; +} + +function hasExactManagedAgentL1WorkspaceDelta( + result: ManagedAgentProbeResult, +): boolean { + const expected = [ + { path: FIXTURE_PATHS.cleanTarget, change: "modified" }, + { path: FIXTURE_PATHS.createdTarget, change: "created" }, + ] as const; + return ( + result.workspaceChanges.length === expected.length && + expected.every( + (expectedChange) => + result.workspaceChanges.filter( + ({ path, change }) => + path === expectedChange.path && change === expectedChange.change, + ).length === 1, + ) + ); +} + +function hasExactManagedAgentL1FinalBytes( + result: ManagedAgentProbeResult, +): boolean { + const roles = ["clean_target", "managed_output"] as const; + return Boolean( + result.l1FinalBytes?.length === roles.length && + roles.every( + (role) => + result.l1FinalBytes?.filter( + (observation) => observation.role === role && observation.matched, + ).length === 1, + ), + ); +} + +function hasConsistentManagedAgentL1EventProjection( + result: ManagedAgentProbeResult, +): boolean { + if ( + result.events.some( + (event, index) => + event.sequence !== index + 1 || event.runId !== result.runId, + ) + ) { + return false; + } + const toolEvents = result.events.filter( + ({ type }) => type === "tool_requested" || type === "tool_completed", + ); + if (toolEvents.length !== result.toolEvidence.length) return false; + for (const [index, evidence] of result.toolEvidence.entries()) { + const event = toolEvents[index]; + if ( + !event || + event.toolUseId !== evidence.toolUseId || + event.toolName !== evidence.toolName + ) { + return false; + } + if (evidence.status === "requested") { + if (event.type !== "tool_requested" || event.isError !== undefined) { + return false; + } + } else if ( + event.type !== "tool_completed" || + event.isError !== (evidence.status === "error") + ) { + return false; + } + } + + const permissionEvents = result.events.flatMap((event, index) => + event.type === "permission" ? [{ event, index }] : [], + ); + if (permissionEvents.length !== result.permissionEvidence.length) { + return false; + } + const completionIndexByToolUseId = new Map(); + for (const [index, event] of result.events.entries()) { + if (event.type === "tool_completed" && event.toolUseId) { + completionIndexByToolUseId.set(event.toolUseId, index); + } + } + return result.permissionEvidence.every((evidence, index) => { + const permission = permissionEvents[index]; + const event = permission?.event; + const completionIndex = completionIndexByToolUseId.get(evidence.toolUseId); + return Boolean( + event && + event.toolUseId === evidence.toolUseId && + event.toolName === evidence.toolName && + event.permissionDecision === evidence.decision && + event.permissionReason === evidence.reason && + event.permissionSource === evidence.source && + event.operationId === evidence.operationId && + completionIndex !== undefined && + permission.index < completionIndex, + ); + }); +} + +function hasManagedAgentL1BashSdkTerminalOrder( + result: ManagedAgentProbeResult, +): boolean { + const bashRequest = result.toolEvidence.find( + ({ toolName, status }) => toolName === "Bash" && status === "requested", + ); + if (!bashRequest?.toolUseId) return false; + const bashCompletionIndexes = result.events.flatMap((event, index) => + event.type === "tool_completed" && + event.toolUseId === bashRequest.toolUseId && + event.toolName === "Bash" && + event.isError === false + ? [index] + : [], + ); + const sdkResultIndexes = result.events.flatMap((event, index) => + event.type === "sdk_result" && + event.subtype === "success" && + event.isError === false + ? [index] + : [], + ); + const terminalIndexes = result.events.flatMap((event, index) => + event.type === "terminal" && event.terminal === "success" ? [index] : [], + ); + const bashCompletionIndex = bashCompletionIndexes[0]; + const sdkResultIndex = sdkResultIndexes[0]; + const terminalIndex = terminalIndexes[0]; + return Boolean( + bashCompletionIndexes.length === 1 && + result.events.filter(({ type }) => type === "sdk_result").length === 1 && + sdkResultIndexes.length === 1 && + result.events.filter(({ type }) => type === "terminal").length === 1 && + terminalIndexes.length === 1 && + result.terminationEvidence.sdkResult === "success" && + bashCompletionIndex !== undefined && + sdkResultIndex !== undefined && + terminalIndex !== undefined && + bashCompletionIndex < sdkResultIndex && + sdkResultIndex < terminalIndex && + terminalIndex === result.events.length - 1, + ); +} + +function analyzeManagedAgentL1ToolTrace( + result: ManagedAgentProbeResult, +): ManagedAgentL1TraceAnalysis { + const requested = result.toolEvidence.filter( + ({ status }) => status === "requested", + ); + const completed = result.toolEvidence.filter( + ({ status }) => status !== "requested", + ); + const primaryDecisions = result.permissionEvidence.filter( + ({ source }) => source === "pre_tool_use", + ); + const requestedIds = requested.flatMap(({ toolUseId }) => + toolUseId?.trim() ? [toolUseId] : [], + ); + const requestedById = new Map( + requestedIds.map((toolUseId, index) => [toolUseId, requested[index]!]), + ); + const completionById = new Map( + completed.flatMap((evidence) => + evidence.toolUseId ? [[evidence.toolUseId, evidence] as const] : [], + ), + ); + const decisionById = new Map( + primaryDecisions.map((evidence) => [evidence.toolUseId, evidence]), + ); + const requestPositionById = new Map(); + const completionPositionById = new Map(); + for (const [position, evidence] of result.toolEvidence.entries()) { + if (!evidence.toolUseId) continue; + if (evidence.status === "requested") { + if (!requestPositionById.has(evidence.toolUseId)) { + requestPositionById.set(evidence.toolUseId, position); + } + } else if (!completionPositionById.has(evidence.toolUseId)) { + completionPositionById.set(evidence.toolUseId, position); + } + } + let invalid = + requestedIds.length !== requested.length || + new Set(requestedIds).size !== requestedIds.length || + completed.length !== requested.length || + completionById.size !== completed.length || + primaryDecisions.length !== requested.length || + decisionById.size !== primaryDecisions.length || + result.permissionEvidence.length !== primaryDecisions.length || + result.permissionEvidence.some(({ source }) => source !== "pre_tool_use") || + completed.some( + ({ toolUseId, toolName }) => + !toolUseId || requestedById.get(toolUseId)?.toolName !== toolName, + ) || + primaryDecisions.some( + ({ toolUseId, toolName }) => + requestedById.get(toolUseId)?.toolName !== toolName, + ); + + const matches = ( + requestIndex: number, + expected: ManagedAgentExpectedL1ToolStep, + ): boolean => { + const request = requested[requestIndex]; + if (!request?.toolUseId || request.toolName !== expected.toolName) { + return false; + } + const completion = completionById.get(request.toolUseId); + const decision = decisionById.get(request.toolUseId); + return Boolean( + completion?.toolName === expected.toolName && + completion.status === expected.completion && + decision?.toolName === expected.toolName && + decision.decision === expected.decision && + decision.reason === expected.reason && + decision.operationId === expected.operationId, + ); + }; + + let cursor = 0; + for (const expected of MANAGED_AGENT_EXPECTED_L1_TOOL_TRACE.slice(0, 5)) { + invalid ||= !matches(cursor, expected); + cursor += 1; + } + + const firstCanonicalReadByOperation = new Set(); + const optionalVerificationReads = requested.filter((request) => { + if (!request.toolUseId || request.toolName !== "Read") return false; + const decision = decisionById.get(request.toolUseId); + if ( + !decision || + !MANAGED_AGENT_L1_OPTIONAL_READ_OPERATIONS.has(decision.operationId) + ) { + return false; + } + if (!firstCanonicalReadByOperation.has(decision.operationId)) { + firstCanonicalReadByOperation.add(decision.operationId); + return false; + } + return true; + }); + const optionalReadCount = optionalVerificationReads.length; + const optionalOperation = + optionalReadCount === 1 && optionalVerificationReads[0]?.toolUseId + ? decisionById.get(optionalVerificationReads[0].toolUseId)?.operationId + : undefined; + const optionalReadRole = optionalOperation?.startsWith("read:") + ? (optionalOperation.slice("read:".length) as ManagedAgentPathRole) + : undefined; + + const candidate = requested[cursor]; + const candidateDecision = candidate?.toolUseId + ? decisionById.get(candidate.toolUseId) + : undefined; + let optionalRequestIndex: number | undefined; + if ( + candidate?.toolName === "Read" && + candidateDecision && + MANAGED_AGENT_L1_OPTIONAL_READ_OPERATIONS.has(candidateDecision.operationId) + ) { + optionalRequestIndex = cursor; + const completion = candidate.toolUseId + ? completionById.get(candidate.toolUseId) + : undefined; + invalid ||= + completion?.toolName !== "Read" || + completion.status !== "success" || + candidateDecision.toolName !== "Read" || + candidateDecision.decision !== "allow" || + candidateDecision.reason !== "fixture_path"; + cursor += 1; + } + + for (const expected of MANAGED_AGENT_EXPECTED_L1_TOOL_TRACE.slice(5)) { + invalid ||= !matches(cursor, expected); + cursor += 1; + } + invalid ||= cursor !== requested.length; + + const requestPosition = (requestIndex: number): number | undefined => { + const toolUseId = requested[requestIndex]?.toolUseId; + return toolUseId ? requestPositionById.get(toolUseId) : undefined; + }; + const completionPosition = (requestIndex: number): number | undefined => { + const toolUseId = requested[requestIndex]?.toolUseId; + return toolUseId ? completionPositionById.get(toolUseId) : undefined; + }; + const allRequestsPrecedeOwnCompletion = requested.every((_, index) => { + const request = requestPosition(index); + const completion = completionPosition(index); + return ( + request !== undefined && completion !== undefined && request < completion + ); + }); + const completionsBeforeRequest = ( + completedRequestIndexes: readonly number[], + boundaryRequestIndex: number, + ): boolean => { + const boundary = requestPosition(boundaryRequestIndex); + const completions = completedRequestIndexes.map(completionPosition); + return Boolean( + boundary !== undefined && + completions.every( + (completion) => completion !== undefined && completion < boundary, + ), + ); + }; + const optionalOffset = optionalRequestIndex === undefined ? 0 : 1; + const call6RequestIndex = 5 + optionalOffset; + const call10RequestIndex = 9 + optionalOffset; + const call11RequestIndex = 10 + optionalOffset; + const phaseABoundaryRequestIndex = optionalRequestIndex ?? call6RequestIndex; + invalid ||= + !allRequestsPrecedeOwnCompletion || + !completionsBeforeRequest([0, 1, 2, 3, 4], phaseABoundaryRequestIndex) || + !completionsBeforeRequest( + [ + call6RequestIndex, + call6RequestIndex + 1, + call6RequestIndex + 2, + call6RequestIndex + 3, + ], + call10RequestIndex, + ); + if (optionalRequestIndex !== undefined) { + const optionalRequest = requestPosition(optionalRequestIndex); + const optionalCompletion = completionPosition(optionalRequestIndex); + const call6Request = requestPosition(call6RequestIndex); + invalid ||= + optionalRequest === undefined || + optionalCompletion === undefined || + call6Request === undefined || + optionalRequest >= optionalCompletion || + optionalCompletion >= call6Request; + } + const call10Completion = completionPosition(call10RequestIndex); + const call11Request = requestPosition(call11RequestIndex); + invalid ||= + call10Completion === undefined || + call11Request === undefined || + call10Completion >= call11Request; + + return { + passed: !invalid, + optionalReadCount, + ...(optionalReadRole ? { optionalReadRole } : {}), + }; +} + +function hasExactManagedAgentL2BashTrace( + result: ManagedAgentProbeResult, +): boolean { + const requested = result.toolEvidence.filter( + ({ status }) => status === "requested", + ); + const completed = result.toolEvidence.filter( + ({ status }) => status !== "requested", + ); + const primaryDecisions = result.permissionEvidence.filter( + ({ source }) => source === "pre_tool_use", + ); + const request = requested[0]; + return Boolean( + requested.length === 1 && + request?.toolUseId && + request.toolName === "Bash" && + completed.length <= 1 && + completed.every( + (evidence) => + evidence.toolUseId === request.toolUseId && + evidence.toolName === "Bash" && + evidence.status === "error", + ) && + primaryDecisions.length === 1 && + result.permissionEvidence.length === 1 && + primaryDecisions[0]?.toolUseId === request.toolUseId && + primaryDecisions[0]?.toolName === "Bash" && + primaryDecisions[0]?.decision === "allow" && + primaryDecisions[0]?.reason === "exact_bash_command" && + primaryDecisions[0]?.operationId === "bash:exact_command", + ); +} + +export function managedAgentProbeUsage(): string { + return [ + "Usage:", + " pnpm --filter @sapiom/harness probe:managed-agent -- --live --scenario --target ", + "", + "Required environment (dedicated eval access only):", + " LLM_GATEWAY_BASE_URL", + " LLM_GATEWAY_EVAL_API_KEY", + "", + "Credentials are intentionally not accepted as command-line arguments.", + ].join("\n"); +} + +export function parseManagedAgentProbeCliArgs( + argv: readonly string[], +): ManagedAgentProbeCliArgs { + if (argv.includes("--help") || argv.includes("-h")) { + return { help: true, live: false }; + } + let live = false; + let target: ManagedAgentModelTargetId | undefined; + let scenario: ManagedAgentProbeScenario | undefined; + for (let index = 0; index < argv.length; index += 1) { + const argument = argv[index]; + if (argument === "--live") { + live = true; + continue; + } + if (argument === "--target") { + const value = argv[++index]; + if (value !== "sonnet-5" && value !== "minimax-m3") { + throw new ManagedAgentProbeCliError( + "--target must be sonnet-5 or minimax-m3", + ); + } + target = value; + continue; + } + if (argument === "--scenario") { + const value = argv[++index]; + if (value !== "L1" && value !== "L2") { + throw new ManagedAgentProbeCliError("--scenario must be L1 or L2"); + } + scenario = value; + continue; + } + throw new ManagedAgentProbeCliError( + `Unknown argument: ${String(argument)}`, + ); + } + if (!live) { + throw new ManagedAgentProbeCliError( + "Refusing to run without --live; hermetic tests never contact the gateway", + ); + } + if (!target || !scenario) { + throw new ManagedAgentProbeCliError("--target and --scenario are required"); + } + return { help: false, live, target, scenario }; +} + +function requiredEnvironmentValue( + environment: Environment, + name: string, +): string { + const value = environment[name]?.trim(); + if (!value) throw new ManagedAgentProbeCliError(`${name} is required`); + return value; +} + +export function assertManagedAgentCertificationNodeVersion( + runtimeVersion: string, +): void { + if (runtimeVersion !== MANAGED_AGENT_CONTRACT.certificationNodeVersion) { + throw new ManagedAgentProbeCliError( + `Live probes require Node ${MANAGED_AGENT_CONTRACT.certificationNodeVersion}; current runtime is ${runtimeVersion}`, + ); + } +} + +export function assertManagedAgentCancellationHostPlatform( + platform: NodeJS.Platform, +): void { + if (platform !== "darwin" && platform !== "linux") { + throw new ManagedAgentProbeCliError( + "L2 certification supports only the reviewed detached POSIX fixture containment model", + ); + } +} + +export function evaluateManagedAgentProbe( + result: ManagedAgentProbeResult, + fixturePids: readonly number[] = [], +): ManagedAgentProbeReport { + const l1Trace = + result.scenario === "L1" + ? analyzeManagedAgentL1ToolTrace(result) + : undefined; + const requestedTools = new Set( + result.toolEvidence + .filter(({ status }) => status === "requested") + .map(({ toolName }) => toolName), + ); + const invocation = (toolName: string, status: "success" | "error"): boolean => + result.toolEvidence.some( + (evidence) => + evidence.toolName === toolName && evidence.status === status, + ); + const permission = ( + toolName: string, + decision: "allow" | "deny", + reason: ManagedAgentPermissionReason, + ): boolean => + result.permissionEvidence.some( + (evidence) => + evidence.toolName === toolName && + evidence.decision === decision && + evidence.reason === reason && + evidence.source === "pre_tool_use", + ); + const requestedToolIds = result.toolEvidence.flatMap((evidence) => + evidence.status === "requested" && evidence.toolUseId + ? [evidence.toolUseId] + : [], + ); + const universalHookCoverage = + requestedToolIds.length > 0 && + new Set(requestedToolIds).size === requestedToolIds.length && + requestedToolIds.every( + (toolUseId) => + result.permissionEvidence.filter( + (evidence) => + evidence.toolUseId === toolUseId && + evidence.source === "pre_tool_use", + ).length === 1, + ); + const checks: ManagedAgentProbeCheck[] = [ + { + id: "sdk_model_alias_observed", + passed: + result.modelAlias === + resolveManagedAgentModelTarget(result.target).alias && + result.sdkModelEvidence.initModelObserved && + result.sdkModelEvidence.initModelMatchesExpectedAlias && + (result.sdkModelEvidence.resultModelUsageObserved + ? result.sdkModelEvidence.resultModelUsageMatchesExpectedAlias && + result.sdkModelEvidence.resultModelCount === 1 + : result.scenario === "L2" && result.terminal === "cancelled"), + }, + { id: "sdk_session_observed", passed: Boolean(result.sdkSessionId) }, + { id: "query_closed", passed: result.queryClosed }, + { id: "process_tree_quiescent", passed: result.teardown.quiescent }, + { + id: "universal_policy_hook_coverage", + passed: result.policyHookCoverage && universalHookCoverage, + }, + { + id: "bounded_inference_turn_evidence", + passed: + Number.isInteger(result.inferenceTurns) && + result.inferenceTurns > 0 && + result.inferenceTurns <= MANAGED_AGENT_CONTRACT.maxTurns && + (result.sdkNumTurns === undefined || + (Number.isInteger(result.sdkNumTurns) && + result.sdkNumTurns >= 0 && + result.sdkNumTurns <= MANAGED_AGENT_CONTRACT.maxTurns)), + }, + { + id: "dirty_and_untracked_preserved", + passed: + result.preservation.length === 2 && + [FIXTURE_PATHS.dirtySentinel, FIXTURE_PATHS.untrackedSentinel].every( + (path) => + result.preservation.filter( + (observation) => + observation.path === path && observation.preserved, + ).length === 1, + ), + }, + ]; + + if (result.scenario === "L1") { + checks.push( + { id: "terminal_success", passed: result.terminal === "success" }, + { + id: "l1_contract_v2", + passed: + result.correlation.promptEmbedded && + result.l1Certification?.contractVersion === + MANAGED_AGENT_L1_CERTIFICATION_CONTRACT.contractVersion && + result.l1Certification.promptVersion === + MANAGED_AGENT_L1_CERTIFICATION_CONTRACT.promptVersion, + }, + { + id: "exact_l1_tool_trace", + passed: l1Trace?.passed === true, + }, + { + id: "minimum_l1_inference_turns", + passed: + Number.isInteger(result.inferenceTurns) && + result.inferenceTurns >= 4 + (l1Trace?.optionalReadCount ?? 0), + }, + { + id: "normalized_event_projection", + passed: hasConsistentManagedAgentL1EventProjection(result), + }, + { + id: "bash_sdk_terminal_order", + passed: hasManagedAgentL1BashSdkTerminalOrder(result), + }, + { + id: "exact_workspace_delta", + passed: hasExactManagedAgentL1WorkspaceDelta(result), + }, + { + id: "expected_final_bytes", + passed: hasExactManagedAgentL1FinalBytes(result), + }, + { id: "nonce_verified", passed: result.nonceVerified === true }, + { + id: "builtin_tools_succeeded", + passed: ["Read", "Edit", "Write", "Bash"].every( + (name) => requestedTools.has(name) && invocation(name, "success"), + ), + }, + { + id: "mcp_echo_succeeded", + passed: invocation( + qualifiedManagedAgentMcpToolName("echo_nonce"), + "success", + ), + }, + { + id: "mcp_failure_recovered", + passed: + invocation(qualifiedManagedAgentMcpToolName("fail_once"), "error") && + invocation(qualifiedManagedAgentMcpToolName("fail_once"), "success"), + }, + { + id: "expected_permissions_allowed", + passed: + ["Read", "Edit", "Write"].every((toolName) => + permission(toolName, "allow", "fixture_path"), + ) && + permission("Bash", "allow", "exact_bash_command") && + permission( + qualifiedManagedAgentMcpToolName("echo_nonce"), + "allow", + "managed_mcp_tool", + ) && + permission( + qualifiedManagedAgentMcpToolName("fail_once"), + "allow", + "managed_mcp_tool", + ), + }, + { + id: "outside_and_symlink_denied", + passed: + permission("Read", "deny", "path_outside_workspace") && + permission("Read", "deny", "path_symlink_escape"), + }, + ); + } else { + checks.push( + { id: "terminal_cancelled", passed: result.terminal === "cancelled" }, + { + id: "exact_l2_bash_only_trace", + passed: hasExactManagedAgentL2BashTrace(result), + }, + { id: "cancellation_requested", passed: result.cancellationRequested }, + { + id: "teardown_within_five_seconds", + passed: result.teardown.quiescent && result.teardown.deadlineMet, + }, + { + id: "l2_containment_prepared", + passed: + result.teardown.processTableAvailable && + result.teardown.containmentSupported && + result.teardown.ownershipProven && + result.teardown.toolProcessObservationComplete, + }, + { + id: "sdk_closed_tool_lifetime_channels", + passed: result.teardown.toolProcessChannelsClosed, + }, + { + id: "fixture_processes_observed", + passed: + fixturePids.length === 2 && + fixturePids.every((pid) => + result.teardown.observedPids.includes(pid), + ), + }, + { + id: "no_fixture_process_alive", + passed: + fixturePids.length === 2 && + fixturePids.every( + (pid) => !result.teardown.alivePidsAtDeadline.includes(pid), + ), + }, + ); + } + + const report: ManagedAgentProbeReport = { + outcome: checks.every(({ passed }) => passed) ? "local_pass" : "fail", + deploymentProvenance: "requires_gateway_reconciliation", + checks, + result, + }; + if (result.scenario !== "L1") return report; + return { + ...report, + l1Certification: { + contractVersion: result.l1Certification?.contractVersion ?? 0, + promptVersion: result.l1Certification?.promptVersion ?? "unobserved", + evaluatorVersion: + MANAGED_AGENT_L1_CERTIFICATION_CONTRACT.evaluatorVersion, + optionalReadCount: l1Trace?.optionalReadCount ?? 0, + ...(l1Trace?.optionalReadRole + ? { optionalReadRole: l1Trace.optionalReadRole } + : {}), + }, + }; +} + +export async function executeManagedAgentProbeCli( + argv: readonly string[], + environment: Environment = process.env, + runtimeNodeVersion = process.versions.node, + runtimePlatform: NodeJS.Platform = process.platform, +): Promise< + ManagedAgentProbeReport | { readonly help: true; readonly usage: string } +> { + const args = parseManagedAgentProbeCliArgs(argv); + if (args.help) return { help: true, usage: managedAgentProbeUsage() }; + + // Validate the immutable runtime before reading the dedicated credential. + assertManagedAgentCertificationNodeVersion(runtimeNodeVersion); + if (args.scenario === "L2") { + assertManagedAgentCancellationHostPlatform(runtimePlatform); + } + const gatewayOrigin = assertManagedAgentDirectGatewayOrigin( + requiredEnvironmentValue(environment, "LLM_GATEWAY_BASE_URL"), + ); + const gatewayCredential = requiredEnvironmentValue( + environment, + "LLM_GATEWAY_EVAL_API_KEY", + ); + const fixture = await createManagedAgentFixture(); + const observer = createLocalManagedAgentProcessObserver(); + let fixturePids: readonly number[] = []; + try { + const scenario = args.scenario!; + const result = await runManagedAgentProbe( + { + scenario, + workspaceRoot: fixture.workspaceRoot, + configRoot: fixture.configRoot, + target: args.target!, + gatewayOrigin, + gatewayCredential, + prompt: fixture.prompt(scenario), + maxTurns: scenario === "L1" ? 18 : 4, + maxBudgetUsd: 0.5, + allowedBashCommands: [ + scenario === "L1" ? fixture.l1BashCommand : fixture.l2BashCommand, + ], + pathRoleBindings: scenario === "L1" ? fixture.pathRoleBindings : [], + expectedL1FinalBytes: + scenario === "L1" ? fixture.expectedL1FinalBytes : [], + ...(scenario === "L1" ? { expectedMcpNonce: fixture.nonce } : {}), + preservePaths: [ + FIXTURE_PATHS.dirtySentinel, + FIXTURE_PATHS.untrackedSentinel, + ], + }, + { + processObserver: observer, + ...(scenario === "L2" + ? { + waitForCancellationSignal: async (signal: AbortSignal) => { + fixturePids = await waitForManagedAgentFixturePids( + fixture, + 15_000, + signal, + ); + // The model-writable PID file is evidence only. Readiness is + // derived first from the trusted supervisor handle and bounded + // host process table. These IDs are compared outside the + // observer and never become signal targets. + const readiness = await observer.prepareCancellation(); + if (!readiness.supported) { + throw new ManagedAgentProbeCliError( + `L2 containment preparation failed: ${readiness.reason}`, + ); + } + if ( + !fixturePids.every((pid) => + readiness.observedPids.includes(pid), + ) + ) { + throw new ManagedAgentProbeCliError( + "L2 fixture PIDs were not both present in the host-observed owned process group", + ); + } + }, + } + : {}), + }, + ); + const bytePreservation = await verifyManagedAgentFixtureBytes(fixture); + const resultWithByteEvidence: ManagedAgentProbeResult = { + ...result, + preservation: bytePreservation, + }; + return evaluateManagedAgentProbe(resultWithByteEvidence, fixturePids); + } finally { + await observer.dispose(); + await fixture.cleanup(); + } +} + +async function main(): Promise { + try { + const report = await executeManagedAgentProbeCli(process.argv.slice(2)); + if ("help" in report) { + process.stdout.write(`${report.usage}\n`); + return; + } + process.stdout.write(`${JSON.stringify(report, null, 2)}\n`); + if (report.outcome === "fail") process.exitCode = 1; + } catch (error) { + const message = + error instanceof Error ? error.message : "Unknown probe failure"; + process.stderr.write(`managed-agent probe: ${message}\n`); + process.exitCode = 1; + } +} + +const entryUrl = process.argv[1] + ? pathToFileURL(process.argv[1]).href + : undefined; +if (entryUrl === import.meta.url) void main(); diff --git a/packages/harness/src/experimental/managed-agent-spike/process-observer.test.ts b/packages/harness/src/experimental/managed-agent-spike/process-observer.test.ts new file mode 100644 index 000000000..d3f14af12 --- /dev/null +++ b/packages/harness/src/experimental/managed-agent-spike/process-observer.test.ts @@ -0,0 +1,3724 @@ +import { once } from "node:events"; +import { + ChildProcess, + execFile, + spawn as spawnChild, + type ChildProcessWithoutNullStreams, +} from "node:child_process"; +import { readFile, writeFile } from "node:fs/promises"; +import { createConnection, type Socket as NetSocket } from "node:net"; +import { join } from "node:path"; +import { performance } from "node:perf_hooks"; +import { promisify } from "node:util"; + +import type { SpawnedProcess } from "@anthropic-ai/claude-agent-sdk"; +import { afterEach, describe, expect, it, vi } from "vitest"; + +import { + FIXTURE_PATHS, + createManagedAgentFixture, + waitForManagedAgentFixturePids, + type ManagedAgentFixture, +} from "./fixture.js"; +import { + LocalManagedAgentProcessObserver, + MANAGED_AGENT_TOOL_CONTROL_CAPABILITY_ENV, + MANAGED_AGENT_TOOL_CONTROL_SOCKET_ENV, + managedAgentPosixSessionColumn, + parseManagedAgentPosixProcessTable, + type ManagedAgentKernelProcessRecord, + type ManagedAgentProcessTableObservation, +} from "./process-observer.js"; + +const fixtures: ManagedAgentFixture[] = []; +const execFileAsync = promisify(execFile); + +function deadlineAfter(timeoutMs: number, startedAtMs = performance.now()) { + return Object.freeze({ + startedAtMs, + deadlineAtMs: startedAtMs + timeoutMs, + }); +} + +afterEach(async () => { + await Promise.all(fixtures.splice(0).map((fixture) => fixture.cleanup())); +}); + +function available( + entries: readonly (readonly [number, ManagedAgentKernelProcessRecord])[], +): ManagedAgentProcessTableObservation { + return { available: true, processes: new Map(entries) }; +} + +async function readRealPosixProcessTable(): Promise { + try { + const sessionColumn = managedAgentPosixSessionColumn(process.platform); + const { stdout } = await execFileAsync( + "/bin/ps", + ["-axo", `pid=,ppid=,pgid=,${sessionColumn}=,stat=,lstart=`], + { encoding: "utf8", maxBuffer: 4 * 1024 * 1024, timeout: 1_000 }, + ); + return { + available: true, + processes: parseManagedAgentPosixProcessTable(stdout), + }; + } catch { + return { available: false }; + } +} + +async function prepareCancellationAfterTransientReadFailure( + observer: LocalManagedAgentProcessObserver, + timeoutMs = 2_000, +) { + const deadline = Date.now() + timeoutMs; + for (;;) { + const readiness = await observer.prepareCancellation(); + if ( + readiness.reason !== "process_table_unavailable" || + Date.now() >= deadline + ) { + return readiness; + } + await new Promise((resolveDelay) => setTimeout(resolveDelay, 10)); + } +} + +async function waitForContainmentEscape( + observer: LocalManagedAgentProcessObserver, + timeoutMs = 2_000, +) { + const deadline = Date.now() + timeoutMs; + let readiness = await observer.prepareCancellation(); + while (readiness.reason !== "containment_escaped" && Date.now() < deadline) { + await new Promise((resolveDelay) => setTimeout(resolveDelay, 10)); + readiness = await observer.prepareCancellation(); + } + return readiness; +} + +function activeNodeCommand(): { command: string; args: string[] } { + return { + command: process.execPath, + args: ["-e", "setInterval(() => {}, 1000)"], + }; +} + +function spawnCooperativeTestProcess(): ChildProcess { + return spawnChild( + process.execPath, + [ + "-e", + 'process.on("disconnect", () => process.exit(0)); setInterval(() => {}, 1000)', + ], + { + stdio: ["ignore", "ignore", "ignore", "ipc"], + windowsHide: true, + }, + ); +} + +const FAST_EXIT_ROOT_SCRIPT = String.raw` +import { spawn } from "node:child_process"; +import { existsSync, writeFileSync } from "node:fs"; +import { resolve } from "node:path"; + +const pidFile = resolve(process.argv[1]); +const exitTiming = process.argv[2]; +const exitMarker = resolve(process.argv[3]); +const cleanupMarker = resolve(process.argv[4]); +const childProgram = [ + 'const { existsSync } = require("node:fs");', + 'const cleanupMarker = process.argv[1];', + 'process.on("SIGTERM", () => {});', + 'if (process.send) process.send("ready");', + 'const cleanupPoll = setInterval(() => {', + ' if (!existsSync(cleanupMarker)) return;', + ' clearInterval(cleanupPoll);', + ' process.exit(0);', + '}, 10);', + 'setInterval(() => {}, 1000);', +].join(""); +const child = spawn(process.execPath, ["-e", childProgram, cleanupMarker], { + stdio: ["ignore", "ignore", "ignore", "ipc"], + windowsHide: true, +}); +child.once("message", () => { + writeFileSync(pidFile, JSON.stringify({ + parentPid: process.pid, + childPid: child.pid, + })); + if (exitTiming === "before-readiness") process.exit(0); + const exitPoll = setInterval(() => { + if (!existsSync(exitMarker)) return; + clearInterval(exitPoll); + process.exit(0); + }, 10); +}); +`; + +const DESCENDANT_TOOL_SCRIPT = String.raw` +import { spawn } from "node:child_process"; +import { writeFileSync } from "node:fs"; + +const [toolScript, pidFile, credentialFile, cleanupMarker] = process.argv.slice(1); +writeFileSync(credentialFile, JSON.stringify({ + socketPath: process.env.SAPIOM_MANAGED_AGENT_TOOL_CONTROL_SOCKET, + capability: process.env.SAPIOM_MANAGED_AGENT_TOOL_CONTROL_CAPABILITY, +})); +const tool = spawn( + "/bin/bash", + [ + "--noprofile", + "--norc", + "-c", + 'exec "$1" "$2" "$3" "$4" "$5"', + "managed-agent-tool", + process.execPath, + toolScript, + pidFile, + "--host-cleanup-marker", + cleanupMarker, + ], + { + detached: true, + env: process.env, + stdio: "ignore", + windowsHide: true, + }, +); +tool.unref(); +setInterval(() => {}, 1000); +`; + +const REGISTERED_DESCENDANT_TOOL_SCRIPT = String.raw` +import { spawn } from "node:child_process"; +import { writeFileSync } from "node:fs"; + +const [toolScript, pidFile, launchFile] = process.argv.slice(1); +const tool = spawn( + "/bin/bash", + [ + "--noprofile", + "--norc", + "-c", + 'exec "$1" "$2" "$3" --register-control', + "managed-agent-tool", + process.execPath, + toolScript, + pidFile, + ], + { + detached: true, + env: process.env, + stdio: "ignore", + windowsHide: true, + }, +); +if (typeof tool.pid !== "number") throw new Error("fixture tool failed to spawn"); +// Direct launcher tests predate setup-failure cleanup evidence and deliberately +// omit this path; the shared launcher must remain valid for those callers. +if (launchFile) { + writeFileSync(launchFile, JSON.stringify({ processGroupId: tool.pid })); +} +tool.unref(); +setInterval(() => {}, 1000); +`; + +const EXPORT_TOOL_CONTROL_SCRIPT = String.raw` +import { writeFileSync } from "node:fs"; + +const outputPath = process.argv[1]; +const socketPath = process.env[${JSON.stringify(MANAGED_AGENT_TOOL_CONTROL_SOCKET_ENV)}]; +const capability = process.env[${JSON.stringify(MANAGED_AGENT_TOOL_CONTROL_CAPABILITY_ENV)}]; +if (!socketPath || !capability) process.exit(41); +writeFileSync(outputPath, JSON.stringify({ socketPath, capability })); +setInterval(() => {}, 1000); +`; + +interface ToolControlCredentials { + readonly socketPath: string; + readonly capability: string; +} + +async function waitForToolControlCredentials( + path: string, + timeoutMs = 3_000, +): Promise { + const deadline = Date.now() + timeoutMs; + for (;;) { + try { + const payload = JSON.parse(await readFile(path, "utf8")) as { + socketPath?: unknown; + capability?: unknown; + }; + if ( + typeof payload.socketPath === "string" && + typeof payload.capability === "string" + ) { + return { + socketPath: payload.socketPath, + capability: payload.capability, + }; + } + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + } + if (Date.now() >= deadline) { + throw new Error("Timed out waiting for tool-control credentials"); + } + await new Promise((resolveDelay) => setTimeout(resolveDelay, 10)); + } +} + +async function openToolRegistration( + credentials: ToolControlCredentials, + role: "parent" | "child", + pid: number, +): Promise { + const socket = await startToolRegistration(credentials, role, pid); + let timeout: NodeJS.Timeout | undefined; + const [response] = (await Promise.race([ + once(socket, "data"), + once(socket, "close").then(() => { + throw new Error(`tool registration ${role} closed before acceptance`); + }), + new Promise((_, rejectTimeout) => { + timeout = setTimeout( + () => rejectTimeout(new Error(`tool registration ${role} timed out`)), + 1_000, + ); + }), + ]).finally(() => { + if (timeout) clearTimeout(timeout); + })) as [Buffer | string]; + expect(String(response)).toContain('"registered":true'); + return socket; +} + +async function startToolRegistration( + credentials: ToolControlCredentials, + role: "parent" | "child", + pid: number, +): Promise { + const socket = createConnection(credentials.socketPath); + socket.setEncoding("utf8"); + await once(socket, "connect"); + socket.write( + `${JSON.stringify({ capability: credentials.capability, role, pid })}\n`, + ); + return socket; +} + +async function sendClosedToolRegistration( + credentials: ToolControlCredentials, + role: "parent" | "child", + pid: number, +): Promise { + const socket = createConnection(credentials.socketPath); + await once(socket, "connect"); + socket.end( + `${JSON.stringify({ capability: credentials.capability, role, pid })}\n`, + ); + await once(socket, "close"); +} + +function asChildProcess( + spawned: SpawnedProcess, +): ChildProcessWithoutNullStreams { + return spawned as ChildProcessWithoutNullStreams; +} + +function sameFullTestIdentity( + expected: ManagedAgentKernelProcessRecord, + current: ManagedAgentKernelProcessRecord | undefined, +): boolean { + return ( + expected.startedAt === current?.startedAt && + expected.parentPid === current.parentPid && + expected.processGroupId === current.processGroupId && + expected.sessionId === current.sessionId + ); +} + +function processExists(pid: number): boolean { + try { + process.kill(pid, 0); + return true; + } catch (error) { + return (error as NodeJS.ErrnoException).code === "EPERM"; + } +} + +function processGroupExists(processGroupId: number): boolean { + try { + process.kill(-processGroupId, 0); + return true; + } catch (error) { + return (error as NodeJS.ErrnoException).code === "EPERM"; + } +} + +function realProcessGroupLiveness( + processGroupId: number, +): "alive" | "gone" | "unknown" { + try { + process.kill(-processGroupId, 0); + return "alive"; + } catch (error) { + const code = (error as NodeJS.ErrnoException).code; + if (code === "ESRCH") return "gone"; + if (code === "EPERM") return "alive"; + return "unknown"; + } +} + +async function waitForTestProcessDeath( + isAlive: () => boolean, + description: string, + timeoutMs = 1_000, +): Promise { + const deadline = Date.now() + timeoutMs; + while (isAlive() && Date.now() < deadline) { + await new Promise((resolveDelay) => setTimeout(resolveDelay, 10)); + } + if (isAlive()) throw new Error(`${description} survived test cleanup`); +} + +async function waitForLaunchedGroupId( + path: string, + timeoutMs = 3_000, +): Promise { + const deadline = Date.now() + timeoutMs; + for (;;) { + try { + const payload = JSON.parse(await readFile(path, "utf8")) as { + processGroupId?: unknown; + }; + if ( + typeof payload.processGroupId === "number" && + Number.isSafeInteger(payload.processGroupId) && + payload.processGroupId > 1 + ) { + return payload.processGroupId; + } + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + } + if (Date.now() >= deadline) { + throw new Error("Timed out waiting for detached tool launch evidence"); + } + await new Promise((resolveDelay) => setTimeout(resolveDelay, 10)); + } +} + +async function waitForChildExitBounded( + child: ChildProcess, + timeoutMs = 1_000, +): Promise { + if (child.exitCode !== null || child.signalCode !== null) return; + await Promise.race([ + once(child, "exit").then(() => undefined), + new Promise((_, rejectTimeout) => + setTimeout( + () => rejectTimeout(new Error("Owned root did not exit in time")), + timeoutMs, + ), + ), + ]); +} + +async function stopExactTestProcess(child: ChildProcess): Promise { + const pid = child.pid; + if (typeof pid !== "number") return; + if (child.exitCode === null && child.signalCode === null) { + if (!child.connected) { + throw new Error( + `Refusing cleanup for test process ${pid} without its retained IPC channel`, + ); + } + child.disconnect(); + } + await waitForTestProcessDeath( + () => processExists(pid), + `Unrelated process ${pid}`, + ); +} + +async function captureExactTestProcessIdentities( + pids: readonly number[], +): Promise> { + const observation = await readRealPosixProcessTable(); + if (!observation.available) { + throw new Error("Process table unavailable for exact test cleanup"); + } + const identities = new Map(); + for (const pid of pids) { + const identity = observation.processes.get(pid); + if (!identity) throw new Error(`Test process ${pid} disappeared too early`); + identities.set(pid, identity); + } + return identities; +} + +async function liveExactTestProcessIdentities( + identities: ReadonlyMap, +): Promise { + const current = await readRealPosixProcessTable(); + if (!current.available) { + throw new Error("Process table unavailable during exact test cleanup"); + } + return [...identities].flatMap(([pid, identity]) => { + const record = current.processes.get(pid); + return record && + sameFullTestIdentity(identity, record) && + !record.state?.startsWith("Z") + ? [pid] + : []; + }); +} + +async function waitForExactTestProcessIdentitiesToExit( + identities: ReadonlyMap, + timeoutMs = 1_000, +): Promise { + const deadline = Date.now() + timeoutMs; + let survivors = await liveExactTestProcessIdentities(identities); + while (survivors.length > 0 && Date.now() < deadline) { + await new Promise((resolveDelay) => setTimeout(resolveDelay, 10)); + survivors = await liveExactTestProcessIdentities(identities); + } + if (survivors.length > 0) { + throw new Error( + `Authenticated cooperative cleanup left ${survivors.length} tool process(es)`, + ); + } +} + +async function stopRetainedTestGroup(root: ChildProcess): Promise { + if (root.exitCode !== null || root.signalCode !== null) return; + if (!root.connected) { + throw new Error( + "Refusing supervisor cleanup without its retained process-bound IPC channel", + ); + } + // The supervisor's disconnect handler kills its own current group. No + // cached numeric PID or PGID crosses this test-cleanup boundary. + root.disconnect(); + await waitForChildExitBounded(root); +} + +async function proveRetainedGroupAuthority( + exitTiming: "before-readiness" | "after-readiness", +): Promise { + const fixture = await createManagedAgentFixture( + () => `fast-root-exit-${exitTiming}`, + ); + fixtures.push(fixture); + const productionGroupSignals: Array = []; + const observer = new LocalManagedAgentProcessObserver({ + testOnlyRequestTermination: (processGroupId, signal) => { + productionGroupSignals.push([processGroupId, signal]); + return "failure"; + }, + }); + const forwardedController = new AbortController(); + const unrelated = spawnCooperativeTestProcess(); + await once(unrelated, "spawn"); + let anchor: ChildProcessWithoutNullStreams | undefined; + let nonCooperativeChildPid: number | undefined; + let exitMarker: string | undefined; + let cleanupMarker: string | undefined; + try { + exitMarker = join( + fixture.workspaceRoot, + FIXTURE_PATHS.processDirectory, + "exit-inner-root", + ); + cleanupMarker = join( + fixture.workspaceRoot, + FIXTURE_PATHS.processDirectory, + "exit-non-cooperative-child", + ); + anchor = asChildProcess( + observer.spawn({ + command: process.execPath, + args: [ + "--input-type=module", + "--eval", + FAST_EXIT_ROOT_SCRIPT, + FIXTURE_PATHS.processPidFile, + exitTiming, + exitMarker, + cleanupMarker, + ], + cwd: fixture.workspaceRoot, + env: { ...process.env }, + signal: forwardedController.signal, + }), + ); + expect(anchor.pid).toBeTypeOf("number"); + const [workerRootPid, fixtureChildPid] = + await waitForManagedAgentFixturePids(fixture); + nonCooperativeChildPid = fixtureChildPid; + + let initialReadiness; + if (exitTiming === "after-readiness") { + initialReadiness = + await prepareCancellationAfterTransientReadFailure(observer); + expect(initialReadiness).toMatchObject({ + supported: true, + reason: "ready", + ownershipProven: true, + }); + await writeFile(exitMarker, "exit\n"); + } + await waitForTestProcessDeath( + () => processExists(workerRootPid!), + `Fast SDK root ${workerRootPid}`, + ); + expect(processExists(nonCooperativeChildPid!)).toBe(true); + const escapedReadiness = await waitForContainmentEscape(observer); + expect(escapedReadiness).toMatchObject({ + supported: false, + reason: "containment_escaped", + ownershipProven: false, + }); + expect(escapedReadiness.observedPids).toContain(nonCooperativeChildPid); + expect(escapedReadiness.observedPids).not.toContain(unrelated.pid); + expect(processExists(nonCooperativeChildPid)).toBe(true); + expect(processExists(unrelated.pid!)).toBe(true); + await expect( + observer.waitForQuiescence(deadlineAfter(1)), + ).resolves.toMatchObject({ + quiescent: false, + deadlineMet: false, + containmentSupported: false, + forceKillIssued: false, + }); + expect(productionGroupSignals).toEqual([]); + } finally { + if (exitMarker) + await writeFile(exitMarker, "exit\n").catch(() => undefined); + if (cleanupMarker) { + await writeFile(cleanupMarker, "exit\n").catch(() => undefined); + } + if (typeof nonCooperativeChildPid === "number") { + await waitForTestProcessDeath( + () => processExists(nonCooperativeChildPid!), + `Escaped fixture child ${nonCooperativeChildPid}`, + 500, + ).catch(() => undefined); + } + if (anchor && anchor.exitCode === null && anchor.signalCode === null) { + if (anchor.connected) anchor.disconnect(); + else await stopRetainedTestGroup(anchor); + await waitForChildExitBounded(anchor); + } + if (typeof nonCooperativeChildPid === "number") { + await waitForTestProcessDeath( + () => processExists(nonCooperativeChildPid!), + `Escaped fixture child ${nonCooperativeChildPid}`, + ); + } + forwardedController.abort(); + observer.dispose(); + await stopExactTestProcess(unrelated); + } +} + +interface RegisteredDescendantToolRun { + readonly fixture: ManagedAgentFixture; + readonly observer: LocalManagedAgentProcessObserver; + readonly forwardedController: AbortController; + readonly anchor: ChildProcessWithoutNullStreams; + readonly toolPids: readonly [number, number]; + readonly toolProcessGroupId: number; + readonly toolIdentities: ReadonlyMap; +} + +interface RegisteredDescendantToolSetupEvidence { + readonly anchor: ChildProcessWithoutNullStreams; + readonly toolPids: readonly [number, number]; + readonly toolProcessGroupId: number; + readonly toolIdentities: ReadonlyMap; +} + +interface RegisteredDescendantSetupCleanupError extends Error { + readonly setupError: unknown; + readonly cleanupErrors: readonly unknown[]; +} + +interface RegisteredDescendantCleanupError extends Error { + readonly cleanupErrors: readonly unknown[]; +} + +function setupAndCleanupFailure( + setupError: unknown, + cleanupErrors: readonly unknown[], +): RegisteredDescendantSetupCleanupError { + const failure = new Error( + "Registered descendant setup and cleanup both failed", + ) as RegisteredDescendantSetupCleanupError; + Object.defineProperties(failure, { + cleanupErrors: { value: [...cleanupErrors] }, + setupError: { value: setupError }, + }); + return failure; +} + +function registeredDescendantCleanupFailure( + cleanupErrors: readonly unknown[], +): RegisteredDescendantCleanupError { + const failure = new Error( + "Registered descendant cleanup failed", + ) as RegisteredDescendantCleanupError; + Object.defineProperty(failure, "cleanupErrors", { + value: [...cleanupErrors], + }); + return failure; +} + +async function startRegisteredDescendantToolRun( + observer: LocalManagedAgentProcessObserver, + name: string, + afterPidPublication?: ( + evidence: RegisteredDescendantToolSetupEvidence, + ) => void | Promise, +): Promise { + const fixture = await createManagedAgentFixture(() => name); + fixtures.push(fixture); + const forwardedController = new AbortController(); + const launchFile = join(fixture.root, "registered-tool-launch.json"); + observer.armToolProcessContainment(); + const anchor = asChildProcess( + observer.spawn({ + command: process.execPath, + args: [ + "--input-type=module", + "--eval", + REGISTERED_DESCENDANT_TOOL_SCRIPT, + join(fixture.workspaceRoot, FIXTURE_PATHS.processScript), + join(fixture.workspaceRoot, FIXTURE_PATHS.processPidFile), + launchFile, + ], + cwd: fixture.workspaceRoot, + env: { ...process.env }, + signal: forwardedController.signal, + }), + ); + let toolPids: readonly [number, number] | undefined; + let toolProcessGroupId: number | undefined; + let toolIdentities: + | ReadonlyMap + | undefined; + try { + if (typeof anchor.pid !== "number") { + throw new Error("Owned fixture anchor failed to spawn"); + } + toolProcessGroupId = await waitForLaunchedGroupId(launchFile); + const [parentPid, childPid] = await waitForManagedAgentFixturePids( + fixture, + 5_000, + ); + toolPids = [parentPid!, childPid!] as const; + if (parentPid !== toolProcessGroupId) { + throw new Error("Detached fixture group does not match its parent PID"); + } + // Capture stable positive-PID identities before user callbacks or + // assertions can fail. A detached numeric group id is never cleanup + // authority on its own. + toolIdentities = await captureExactTestProcessIdentities(toolPids); + await afterPidPublication?.({ + anchor, + toolPids, + toolProcessGroupId, + toolIdentities, + }); + await expect( + prepareCancellationAfterTransientReadFailure(observer), + ).resolves.toMatchObject({ + supported: true, + reason: "ready", + ownershipProven: true, + }); + return { + fixture, + observer, + forwardedController, + anchor, + toolPids, + toolProcessGroupId, + toolIdentities, + }; + } catch (setupError) { + const cleanupErrors: unknown[] = []; + try { + await observer.dispose(); + if (toolIdentities) { + const survivors = await liveExactTestProcessIdentities(toolIdentities); + if (survivors.length > 0) { + cleanupErrors.push( + new Error( + `Authenticated cooperative cleanup left ${survivors.length} tool process(es)`, + ), + ); + } + } else if (toolPids || typeof toolProcessGroupId === "number") { + cleanupErrors.push( + new Error( + "Refusing detached tool cleanup without pre-captured identities", + ), + ); + } + } catch (error) { + cleanupErrors.push(error); + } + try { + await stopRetainedTestGroup(anchor); + } catch (error) { + cleanupErrors.push(error); + } finally { + forwardedController.abort(); + await observer.dispose(); + } + if (cleanupErrors.length > 0) { + throw setupAndCleanupFailure(setupError, cleanupErrors); + } + throw setupError; + } +} + +async function cleanupRegisteredDescendantToolRun( + run: RegisteredDescendantToolRun | undefined, +): Promise { + if (!run) return; + const cleanupErrors: unknown[] = []; + try { + await run.observer.dispose(); + await waitForExactTestProcessIdentitiesToExit(run.toolIdentities); + } catch (error) { + cleanupErrors.push(error); + } + try { + if (run.anchor.exitCode === null && run.anchor.signalCode === null) { + await waitForChildExitBounded(run.anchor, 100).catch(() => undefined); + } + await stopRetainedTestGroup(run.anchor); + } catch (error) { + cleanupErrors.push(error); + } finally { + run.forwardedController.abort(); + await run.observer.dispose(); + } + if (cleanupErrors.length > 0) { + throw registeredDescendantCleanupFailure(cleanupErrors); + } +} + +describe("LocalManagedAgentProcessObserver", () => { + it.each([ + ["darwin", "sess"], + ["linux", "sid"], + ] as const)( + "uses the %s process-table session column", + (platform, expectedColumn) => { + expect(managedAgentPosixSessionColumn(platform)).toBe(expectedColumn); + }, + ); + + it.each([ + ["Darwin sess= layout", 0], + ["Linux sid= layout", 100], + ] as const)("parses the %s", (_layout, sessionId) => { + const table = parseManagedAgentPosixProcessTable( + ` 100 1 100 ${sessionId} Ss Mon Aug 17 01:02:03 2026\n`, + ); + + expect(table.get(100)).toEqual({ + parentPid: 1, + processGroupId: 100, + sessionId, + state: "Ss", + startedAt: "Mon Aug 17 01:02:03 2026", + }); + }); + + it("retains setup and cleanup failures without requiring AggregateError", () => { + const setupError = new Error("synthetic setup failure"); + const cleanupError = new Error("synthetic cleanup failure"); + + const failure = setupAndCleanupFailure(setupError, [cleanupError]); + + expect(failure).toBeInstanceOf(Error); + expect(failure.message).toBe( + "Registered descendant setup and cleanup both failed", + ); + expect(failure.setupError).toBe(setupError); + expect(failure.cleanupErrors).toEqual([cleanupError]); + }); + + it.skipIf(process.platform === "win32")( + "test cleanup never signals a cached group after its retained child exits", + async () => { + const child = spawnChild(process.execPath, ["-e", "process.exit(0)"], { + detached: true, + stdio: "ignore", + windowsHide: true, + }); + await once(child, "exit"); + const killSpy = vi.spyOn(process, "kill"); + try { + await stopRetainedTestGroup(child); + expect( + killSpy.mock.calls.some( + ([pid]) => typeof pid === "number" && pid < 0, + ), + ).toBe(false); + } finally { + killSpy.mockRestore(); + } + }, + 5_000, + ); + + it.skipIf(process.platform === "win32")( + "keeps inner arguments out of supervisor argv and scrubs its private payload", + async () => { + const observer = new LocalManagedAgentProcessObserver(); + const controller = new AbortController(); + const privateArgument = "inner-only-supervisor-argument"; + const anchor = asChildProcess( + observer.spawn({ + command: process.execPath, + args: [ + "-e", + [ + 'const payload = "SAPIOM_MANAGED_AGENT_SUPERVISOR_PAYLOAD";', + "const valid = process.argv[1] === " + + JSON.stringify(privateArgument) + + " && !Object.hasOwn(process.env, payload);", + "process.exit(valid ? 0 : 31);", + ].join(""), + privateArgument, + ], + cwd: process.cwd(), + env: { ...process.env }, + signal: controller.signal, + }), + ); + try { + expect(anchor.spawnargs.join("\u0000")).not.toContain(privateArgument); + const [exitCode, signalCode] = await once(anchor, "exit"); + expect(exitCode).toBe(0); + expect(signalCode).toBeNull(); + } finally { + if (typeof anchor.pid === "number") { + await stopRetainedTestGroup(anchor); + } + controller.abort(); + observer.dispose(); + } + }, + 5_000, + ); + + it.skipIf(process.platform === "win32")( + "preserves a normal inner exit code without reporting a signal kill", + async () => { + const observer = new LocalManagedAgentProcessObserver(); + const controller = new AbortController(); + const anchor = asChildProcess( + observer.spawn({ + command: process.execPath, + args: [ + "-e", + 'process.stderr.write("x".repeat(1024 * 1024), () => process.exit(23));', + ], + cwd: process.cwd(), + env: { ...process.env }, + signal: controller.signal, + }), + ); + let forwardedStderrBytes = 0; + anchor.stderr.on("data", (chunk: Buffer) => { + forwardedStderrBytes += chunk.byteLength; + }); + try { + const [exitCode, signalCode] = await once(anchor, "exit"); + expect(exitCode).toBe(23); + expect(signalCode).toBeNull(); + expect(forwardedStderrBytes).toBe(1024 * 1024); + } finally { + if (typeof anchor.pid === "number") { + await stopRetainedTestGroup(anchor); + } + controller.abort(); + observer.dispose(); + } + }, + 5_000, + ); + + it.skipIf(process.platform === "win32")( + "self-terminates when IPC disconnects before the supervisor installs its listener", + async () => { + const fixture = await createManagedAgentFixture( + () => "supervisor-bootstrap-disconnect", + ); + fixtures.push(fixture); + const disconnectPreload = join( + fixture.workspaceRoot, + "disconnect-supervisor-ipc.cjs", + ); + await writeFile( + disconnectPreload, + "if (process.connected) process.disconnect();\n", + ); + const observer = new LocalManagedAgentProcessObserver(); + const controller = new AbortController(); + const anchor = asChildProcess( + observer.spawn({ + command: "/usr/bin/true", + args: [], + cwd: fixture.workspaceRoot, + env: { + ...process.env, + NODE_OPTIONS: `--require=${disconnectPreload}`, + }, + signal: controller.signal, + }), + ); + try { + const [exitCode, signalCode] = await once(anchor, "exit"); + expect(exitCode).toBeNull(); + expect(signalCode).toBe("SIGKILL"); + } finally { + controller.abort(); + await observer.dispose(); + } + }, + 5_000, + ); + + it.skipIf(process.platform === "win32")( + "never signals a cached supervisor group through SDK kill after observed exit", + async () => { + const observer = new LocalManagedAgentProcessObserver(); + const controller = new AbortController(); + const anchor = asChildProcess( + observer.spawn({ + command: process.execPath, + args: ["-e", "process.exit(0)"], + cwd: process.cwd(), + env: { ...process.env }, + signal: controller.signal, + }), + ); + const processGroupId = anchor.pid!; + await once(anchor, "exit"); + const killSpy = vi.spyOn(process, "kill"); + try { + expect(anchor.kill("SIGTERM")).toBe(false); + expect(killSpy).not.toHaveBeenCalledWith(-processGroupId, "SIGTERM"); + } finally { + killSpy.mockRestore(); + controller.abort(); + observer.dispose(); + } + }, + 5_000, + ); + + it.skipIf(process.platform === "win32")( + "honors SDK SIGTERM calls while the supervisor keeps its owned group anchored", + async () => { + const nativeKillSpy = vi.spyOn(ChildProcess.prototype, "kill"); + const observer = new LocalManagedAgentProcessObserver(); + const controller = new AbortController(); + const anchor = asChildProcess( + observer.spawn({ + ...activeNodeCommand(), + cwd: process.cwd(), + env: { ...process.env }, + signal: controller.signal, + }), + ); + const processGroupId = anchor.pid!; + try { + await expect( + prepareCancellationAfterTransientReadFailure(observer), + ).resolves.toMatchObject({ + supported: true, + reason: "ready", + ownershipProven: true, + }); + nativeKillSpy.mockClear(); + + expect(anchor.killed).toBe(false); + expect(anchor.kill("SIGTERM")).toBe(true); + expect(anchor.killed).toBe(true); + expect(anchor.kill("SIGTERM")).toBe(true); + expect(nativeKillSpy).toHaveBeenCalledTimes(2); + expect(nativeKillSpy).toHaveBeenNthCalledWith(1, "SIGTERM"); + expect(nativeKillSpy).toHaveBeenNthCalledWith(2, "SIGTERM"); + expect(processExists(processGroupId)).toBe(true); + await expect( + observer.waitForQuiescence(deadlineAfter(1)), + ).resolves.toMatchObject({ + quiescent: false, + containmentSupported: true, + forceKillIssued: false, + }); + } finally { + nativeKillSpy.mockRestore(); + await observer.dispose(); + if ( + anchor.connected && + anchor.exitCode === null && + anchor.signalCode === null + ) { + anchor.disconnect(); + await waitForChildExitBounded(anchor); + } + controller.abort(); + } + }, + 5_000, + ); + + it.skipIf(process.platform === "win32")( + "kills the exact non-cooperative fixture group and confirms death inside one deadline", + async () => { + const fixture = await createManagedAgentFixture(() => "process-observer"); + fixtures.push(fixture); + const observer = new LocalManagedAgentProcessObserver(); + const forwardedController = new AbortController(); + const unrelated = spawnCooperativeTestProcess(); + await once(unrelated, "spawn"); + let root: ChildProcessWithoutNullStreams | undefined; + let ownedProcessGroupId: number | undefined; + try { + root = asChildProcess( + observer.spawn({ + command: process.execPath, + args: [FIXTURE_PATHS.processScript, FIXTURE_PATHS.processPidFile], + cwd: fixture.workspaceRoot, + env: { ...process.env }, + signal: forwardedController.signal, + }), + ); + ownedProcessGroupId = root.pid; + expect(ownedProcessGroupId).toBeTypeOf("number"); + const fixturePids = await waitForManagedAgentFixturePids(fixture); + const readiness = + await prepareCancellationAfterTransientReadFailure(observer); + expect(readiness).toMatchObject({ + supported: true, + reason: "ready", + }); + expect( + fixturePids.every((pid) => readiness.observedPids.includes(pid)), + ).toBe(true); + expect(readiness.observedPids).not.toContain(unrelated.pid); + + const startedAt = Date.now(); + forwardedController.abort(); + const teardown = await observer.emergencyCleanup(deadlineAfter(1_000)); + + expect(teardown).toMatchObject({ + quiescent: true, + deadlineMet: true, + processTableAvailable: true, + containmentSupported: true, + ownershipProven: true, + forceKillIssued: true, + emergencyCleanupAttempted: true, + alivePidsAtDeadline: [], + }); + expect(Date.now() - startedAt).toBeLessThan(1_000); + expect(processExists(unrelated.pid!)).toBe(true); + } finally { + forwardedController.abort(); + if (root && typeof ownedProcessGroupId === "number") { + // Test-harness safety must not depend on the observer behavior under + // test. Exact test-owned PGID authority is retained until death is + // independently confirmed, including when an assertion fails. + await stopRetainedTestGroup(root); + } + observer.dispose(); + await stopExactTestProcess(unrelated); + } + }, + 15_000, + ); + + it.skipIf(process.platform === "win32")( + "kills the complete owned group on parent IPC disconnect without touching an unrelated process", + async () => { + const fixture = await createManagedAgentFixture(() => "ipc-disconnect"); + fixtures.push(fixture); + const observer = new LocalManagedAgentProcessObserver(); + const controller = new AbortController(); + const unrelated = spawnCooperativeTestProcess(); + await once(unrelated, "spawn"); + const anchor = asChildProcess( + observer.spawn({ + command: process.execPath, + args: [FIXTURE_PATHS.processScript, FIXTURE_PATHS.processPidFile], + cwd: fixture.workspaceRoot, + env: { ...process.env }, + signal: controller.signal, + }), + ); + const ownedProcessGroupId = anchor.pid; + expect(ownedProcessGroupId).toBeTypeOf("number"); + let observerDisposed = false; + try { + const fixturePids = await waitForManagedAgentFixturePids(fixture); + await expect( + prepareCancellationAfterTransientReadFailure(observer), + ).resolves.toMatchObject({ + supported: true, + reason: "ready", + ownershipProven: true, + }); + + // This test isolates the supervisor's parent-disconnect contract. Stop + // observer sampling before the kernel delivers the group SIGKILL so a + // transient, already-signalled reparent cannot make the assertion + // scheduler-dependent. + await observer.dispose(); + observerDisposed = true; + await waitForChildExitBounded(anchor); + await Promise.all( + fixturePids.map((pid) => + waitForTestProcessDeath( + () => processExists(pid), + `IPC-disconnect fixture process ${pid}`, + ), + ), + ); + expect(fixturePids.every((pid) => !processExists(pid))).toBe(true); + expect(processExists(unrelated.pid!)).toBe(true); + } finally { + if (anchor.exitCode === null && anchor.signalCode === null) { + if (anchor.connected) anchor.disconnect(); + await waitForChildExitBounded(anchor, 250).catch(() => undefined); + } + if (anchor.exitCode === null && anchor.signalCode === null) { + await stopRetainedTestGroup(anchor); + } + controller.abort(); + if (!observerDisposed) await observer.dispose(); + await stopExactTestProcess(unrelated); + } + }, + 10_000, + ); + + it.skipIf(process.platform === "win32")( + "cooperatively shuts down both authenticated fixture processes without a numeric fixture signal", + async () => { + const signals: Array = []; + const observer = new LocalManagedAgentProcessObserver({ + testOnlyRequestTermination: (groupId, signal) => { + signals.push([groupId, signal]); + return "failure"; + }, + }); + let run: RegisteredDescendantToolRun | undefined; + try { + run = await startRegisteredDescendantToolRun( + observer, + "cooperative-cleanup", + ); + + await observer.dispose(); + + await Promise.all( + run.toolPids.map((pid) => + waitForTestProcessDeath( + () => processExists(pid), + `Cooperative fixture process ${pid}`, + ), + ), + ); + expect(signals).toEqual([]); + } finally { + await observer.dispose(); + run?.forwardedController.abort(); + if ( + run && + run.anchor.exitCode === null && + run.anchor.signalCode === null + ) { + await stopRetainedTestGroup(run.anchor); + } + } + }, + 10_000, + ); + + it("fails preparation closed after a fast root exits and never signals its former numeric group", async () => { + const signals: Array = []; + const observer = new LocalManagedAgentProcessObserver({ + platform: "darwin", + testOnlyRequestTermination: (groupId, signal) => { + signals.push([groupId, signal]); + return "sent"; + }, + }); + const forwardedController = new AbortController(); + const child = asChildProcess( + observer.spawn({ + command: process.execPath, + args: ["-e", "process.exit(0)"], + cwd: process.cwd(), + env: { ...process.env }, + signal: forwardedController.signal, + }), + ); + await once(child, "exit"); + try { + await expect( + prepareCancellationAfterTransientReadFailure(observer), + ).resolves.toMatchObject({ + supported: false, + reason: "root_not_active", + }); + forwardedController.abort(); + await observer.emergencyCleanup(deadlineAfter(1)); + expect(signals).toEqual([]); + } finally { + observer.dispose(); + } + }); + + it.skipIf(process.platform === "win32")( + "fails closed when the SDK inner root exits before its child and ancestry is lost", + () => proveRetainedGroupAuthority("before-readiness"), + 15_000, + ); + + it.skipIf(process.platform === "win32")( + "revokes readiness when the SDK inner root exits and reparents its child", + () => proveRetainedGroupAuthority("after-readiness"), + 10_000, + ); + + it.skipIf(process.platform === "win32")( + "kills a freshly revalidated detached tool group wholly descended from the owned root", + async () => { + const fixture = await createManagedAgentFixture( + () => "anchored-descendant-tool", + ); + fixtures.push(fixture); + const observer = new LocalManagedAgentProcessObserver(); + const forwardedController = new AbortController(); + const unrelated = spawnCooperativeTestProcess(); + await once(unrelated, "spawn"); + let anchor: ChildProcessWithoutNullStreams | undefined; + let fixturePids: readonly number[] = []; + try { + observer.armToolProcessContainment(); + anchor = asChildProcess( + observer.spawn({ + command: process.execPath, + args: [ + "--input-type=module", + "--eval", + REGISTERED_DESCENDANT_TOOL_SCRIPT, + join(fixture.workspaceRoot, FIXTURE_PATHS.processScript), + join(fixture.workspaceRoot, FIXTURE_PATHS.processPidFile), + // Omit launch evidence to exercise the direct-launcher contract. + ], + cwd: fixture.workspaceRoot, + env: { ...process.env }, + signal: forwardedController.signal, + }), + ); + fixturePids = await waitForManagedAgentFixturePids(fixture); + await expect( + prepareCancellationAfterTransientReadFailure(observer), + ).resolves.toMatchObject({ + supported: true, + reason: "ready", + ownershipProven: true, + }); + + const teardown = await observer.emergencyCleanup(deadlineAfter(2_000)); + + expect(teardown).toMatchObject({ + quiescent: true, + deadlineMet: true, + containmentSupported: true, + ownershipProven: true, + forceKillIssued: true, + toolProcessObservationComplete: true, + toolProcessChannelsClosed: true, + alivePidsAtDeadline: [], + }); + expect(fixturePids.every((pid) => !processExists(pid))).toBe(true); + expect(processExists(unrelated.pid!)).toBe(true); + } finally { + forwardedController.abort(); + await observer.dispose(); + if (anchor) { + await stopRetainedTestGroup(anchor); + } + await stopExactTestProcess(unrelated); + } + }, + 15_000, + ); + + it.skipIf(process.platform === "win32")( + "uses process-bound channels in tool-then-supervisor order without host SIGKILL", + async () => { + let rootProcessGroupId: number | undefined; + let toolProcessGroupId: number | undefined; + const terminationRequests: Array = []; + const hostKillSpy = vi.spyOn(process, "kill"); + const observer = new LocalManagedAgentProcessObserver({ + onTerminationRequest: ({ processGroupId }, outcome) => { + if (outcome === "sent") { + terminationRequests.push([processGroupId, "SIGKILL"]); + } + }, + }); + let run: RegisteredDescendantToolRun | undefined; + try { + run = await startRegisteredDescendantToolRun( + observer, + "sample-driven-forwarded-fallback", + ); + rootProcessGroupId = run.anchor.pid!; + toolProcessGroupId = run.toolProcessGroupId; + + const teardownDeadline = deadlineAfter(3_000); + observer.beginTeardown(teardownDeadline); + run.forwardedController.abort(); + const deadline = Date.now() + 2_000; + while ( + !terminationRequests.some( + ([groupId, signal]) => + groupId === rootProcessGroupId && signal === "SIGKILL", + ) && + Date.now() < deadline + ) { + await observer.observeProcessTree(); + await new Promise((resolveDelay) => setTimeout(resolveDelay, 10)); + } + + expect(terminationRequests).toEqual([ + [toolProcessGroupId, "SIGKILL"], + [rootProcessGroupId, "SIGKILL"], + ]); + expect( + hostKillSpy.mock.calls.some(([, signal]) => signal === "SIGKILL"), + ).toBe(false); + await expect( + observer.waitForQuiescence(teardownDeadline), + ).resolves.toMatchObject({ + quiescent: true, + deadlineMet: true, + containmentSupported: true, + forceKillIssued: true, + toolProcessChannelsClosed: true, + alivePidsAtDeadline: [], + }); + } finally { + hostKillSpy.mockRestore(); + await cleanupRegisteredDescendantToolRun(run); + observer.dispose(); + } + }, + 15_000, + ); + + it.skipIf(process.platform === "win32")( + "broadcasts tool termination when the parent channel write is unusable", + async () => { + const attemptedRoles: Array<"parent" | "child"> = []; + const observer = new LocalManagedAgentProcessObserver({ + testOnlyWriteToolTermination: (role) => { + attemptedRoles.push(role); + return role === "parent" ? "failure" : undefined; + }, + }); + let run: RegisteredDescendantToolRun | undefined; + try { + run = await startRegisteredDescendantToolRun( + observer, + "child-channel-termination-fallback", + ); + + const teardown = await observer.emergencyCleanup(deadlineAfter(3_000)); + + expect(attemptedRoles).toEqual(["parent", "child"]); + expect(teardown).toMatchObject({ + quiescent: true, + deadlineMet: true, + containmentSupported: true, + forceKillIssued: true, + toolProcessChannelsClosed: true, + alivePidsAtDeadline: [], + }); + expect(run.toolPids.every((pid) => !processExists(pid))).toBe(true); + } finally { + await cleanupRegisteredDescendantToolRun(run); + await observer.dispose(); + } + }, + 15_000, + ); + + it.skipIf(process.platform === "win32")( + "self-terminates the detached tool group when authenticated lifetime channels disappear", + async () => { + const hostKillSpy = vi.spyOn(process, "kill"); + const observer = new LocalManagedAgentProcessObserver(); + const unrelated = spawnCooperativeTestProcess(); + await once(unrelated, "spawn"); + let run: RegisteredDescendantToolRun | undefined; + try { + run = await startRegisteredDescendantToolRun( + observer, + "lost-tool-lifetime-channels", + ); + + observer.testOnlyDropToolLifetimeChannels(); + + await waitForExactTestProcessIdentitiesToExit(run.toolIdentities); + expect(run.toolPids.every((pid) => !processExists(pid))).toBe(true); + expect(processExists(unrelated.pid!)).toBe(true); + expect( + hostKillSpy.mock.calls.some(([, signal]) => signal === "SIGKILL"), + ).toBe(false); + } finally { + hostKillSpy.mockRestore(); + await cleanupRegisteredDescendantToolRun(run); + await observer.dispose(); + await stopExactTestProcess(unrelated); + } + }, + 15_000, + ); + + it.skipIf(process.platform === "win32")( + "cleans exact fixture and anchor groups when setup fails after PID publication", + async () => { + const observer = new LocalManagedAgentProcessObserver(); + let setupEvidence: RegisteredDescendantToolSetupEvidence | undefined; + try { + await expect( + startRegisteredDescendantToolRun( + observer, + "failed-registered-tool-setup", + (evidence) => { + setupEvidence = evidence; + throw new Error("synthetic failure after PID publication"); + }, + ), + ).rejects.toThrow("synthetic failure after PID publication"); + + expect(setupEvidence).toBeDefined(); + expect( + await liveExactTestProcessIdentities(setupEvidence!.toolIdentities), + ).toEqual([]); + expect(setupEvidence!.anchor.exitCode).toBeNull(); + expect(setupEvidence!.anchor.signalCode).toBe("SIGKILL"); + } finally { + if (setupEvidence) { + await observer.dispose(); + await stopRetainedTestGroup(setupEvidence.anchor); + } + await observer.dispose(); + } + }, + 15_000, + ); + + it.skipIf(process.platform === "win32")( + "refuses detached tool authority when a foreign member joins the candidate group", + async () => { + let injectForeignMember = false; + let toolProcessGroupId: number | undefined; + const signals: Array = []; + const observer = new LocalManagedAgentProcessObserver({ + readProcessTable: async () => { + const observation = await readRealPosixProcessTable(); + if ( + !observation.available || + !injectForeignMember || + typeof toolProcessGroupId !== "number" + ) { + return observation; + } + const processes = new Map(observation.processes); + let foreignPid = 2_000_000_000; + while (processes.has(foreignPid)) foreignPid -= 1; + processes.set(foreignPid, { + parentPid: process.pid, + processGroupId: toolProcessGroupId, + state: "S", + startedAt: "synthetic-foreign-member", + }); + return { available: true, processes }; + }, + testOnlyRequestTermination: (groupId, signal) => { + signals.push([groupId, signal]); + return "failure"; + }, + }); + let run: RegisteredDescendantToolRun | undefined; + try { + run = await startRegisteredDescendantToolRun( + observer, + "foreign-tool-group-member", + ); + toolProcessGroupId = run.toolProcessGroupId; + injectForeignMember = true; + + const teardown = await observer.emergencyCleanup(deadlineAfter(250)); + + expect(teardown).toMatchObject({ + quiescent: false, + deadlineMet: false, + forceKillIssued: false, + }); + expect( + signals.filter(([groupId]) => groupId === toolProcessGroupId), + ).toEqual([]); + expect(processGroupExists(toolProcessGroupId)).toBe(true); + } finally { + await cleanupRegisteredDescendantToolRun(run); + observer.dispose(); + } + }, + 15_000, + ); + + it.skipIf(process.platform === "win32")( + "never signals a cached detached group after its registered identities disappear and are reused", + async () => { + let simulatePidReuse = false; + let toolProcessGroupId: number | undefined; + let registeredPids: readonly [number, number] | undefined; + const signals: Array = []; + const observer = new LocalManagedAgentProcessObserver({ + readProcessTable: async () => { + const observation = await readRealPosixProcessTable(); + if ( + !observation.available || + !simulatePidReuse || + typeof toolProcessGroupId !== "number" || + !registeredPids + ) { + return observation; + } + const processes = new Map(observation.processes); + const [parentPid, childPid] = registeredPids; + processes.set(parentPid, { + parentPid: process.pid, + processGroupId: toolProcessGroupId, + state: "S", + startedAt: "reused-parent-identity", + }); + processes.set(childPid, { + parentPid, + processGroupId: toolProcessGroupId, + state: "S", + startedAt: "reused-child-identity", + }); + return { available: true, processes }; + }, + processGroupLiveness: (groupId) => + simulatePidReuse && groupId === toolProcessGroupId + ? "alive" + : realProcessGroupLiveness(groupId), + testOnlyRequestTermination: (groupId, signal) => { + signals.push([groupId, signal]); + return "failure"; + }, + }); + let run: RegisteredDescendantToolRun | undefined; + try { + run = await startRegisteredDescendantToolRun( + observer, + "reused-tool-identities", + ); + toolProcessGroupId = run.toolProcessGroupId; + registeredPids = run.toolPids; + simulatePidReuse = true; + + const teardown = await observer.emergencyCleanup(deadlineAfter(250)); + + expect(teardown).toMatchObject({ + quiescent: false, + deadlineMet: false, + forceKillIssued: false, + }); + expect( + signals.filter(([groupId]) => groupId === toolProcessGroupId), + ).toEqual([]); + } finally { + simulatePidReuse = false; + await cleanupRegisteredDescendantToolRun(run); + observer.dispose(); + } + }, + 15_000, + ); + + it.skipIf(process.platform === "win32")( + "retries detached tool kill failures only after fresh authority checks", + async () => { + let processTableReads = 0; + let toolProcessGroupId: number | undefined; + let killAttempts = 0; + const toolSignals: Array<{ + readonly signal: "SIGKILL"; + readonly processTableReads: number; + }> = []; + const observer = new LocalManagedAgentProcessObserver({ + readProcessTable: async () => { + processTableReads += 1; + return readRealPosixProcessTable(); + }, + testOnlyBeforeTerminationRequest: ({ target }) => { + if (target === "tool" && killAttempts++ === 0) return "failure"; + return undefined; + }, + onTerminationRequest: ({ processGroupId, target }) => { + if (target === "tool" && processGroupId === toolProcessGroupId) { + toolSignals.push({ signal: "SIGKILL", processTableReads }); + } + }, + }); + let run: RegisteredDescendantToolRun | undefined; + try { + run = await startRegisteredDescendantToolRun( + observer, + "retry-tool-signals", + ); + toolProcessGroupId = run.toolProcessGroupId; + + const teardown = await observer.emergencyCleanup(deadlineAfter(3_000)); + + expect(teardown).toMatchObject({ + quiescent: true, + deadlineMet: true, + forceKillIssued: true, + alivePidsAtDeadline: [], + }); + expect(toolSignals.map(({ signal }) => signal)).toEqual([ + "SIGKILL", + "SIGKILL", + ]); + expect( + toolSignals.every( + (attempt, index) => + index === 0 || + attempt.processTableReads > + toolSignals[index - 1]!.processTableReads, + ), + ).toBe(true); + } finally { + await cleanupRegisteredDescendantToolRun(run); + observer.dispose(); + } + }, + 15_000, + ); + + it.skipIf(process.platform === "win32")( + "never signals the detached tool group after the owned root exits and ancestry is lost", + async () => { + let toolProcessGroupId: number | undefined; + const signals: Array = []; + const observer = new LocalManagedAgentProcessObserver({ + testOnlyRequestTermination: (groupId, signal) => { + signals.push([groupId, signal]); + return "failure"; + }, + }); + let run: RegisteredDescendantToolRun | undefined; + try { + run = await startRegisteredDescendantToolRun( + observer, + "root-exit-loses-tool-ancestry", + ); + toolProcessGroupId = run.toolProcessGroupId; + await stopRetainedTestGroup(run.anchor); + expect(processGroupExists(toolProcessGroupId)).toBe(true); + + const teardown = await observer.emergencyCleanup(deadlineAfter(250)); + + expect(teardown).toMatchObject({ + quiescent: false, + deadlineMet: false, + forceKillIssued: false, + }); + expect( + signals.filter(([groupId]) => groupId === toolProcessGroupId), + ).toEqual([]); + expect(processGroupExists(toolProcessGroupId)).toBe(true); + } finally { + await cleanupRegisteredDescendantToolRun(run); + observer.dispose(); + } + }, + 15_000, + ); + + it.skipIf(process.platform === "win32")( + "never signals a detached PGID merely because a capability holder claimed it", + async () => { + const fixture = await createManagedAgentFixture( + () => "unanchored-tool-registration", + ); + fixtures.push(fixture); + const observer = new LocalManagedAgentProcessObserver(); + const forwardedController = new AbortController(); + const credentialFile = join(fixture.root, "tool-control.json"); + let anchor: ChildProcessWithoutNullStreams | undefined; + let detachedTool: ChildProcess | undefined; + let registrations: readonly NetSocket[] = []; + try { + observer.armToolProcessContainment(); + anchor = asChildProcess( + observer.spawn({ + command: process.execPath, + args: [ + "--input-type=module", + "--eval", + EXPORT_TOOL_CONTROL_SCRIPT, + credentialFile, + ], + cwd: fixture.workspaceRoot, + env: { ...process.env }, + signal: forwardedController.signal, + }), + ); + const credentials = await waitForToolControlCredentials(credentialFile); + detachedTool = spawnChild( + process.execPath, + [ + FIXTURE_PATHS.processScript, + FIXTURE_PATHS.processPidFile, + "--host-cleanup-marker", + fixture.cooperativeExitMarker, + ], + { + cwd: fixture.workspaceRoot, + detached: true, + env: { ...process.env }, + stdio: "ignore", + windowsHide: true, + }, + ); + const [toolParentPid, toolChildPid] = + await waitForManagedAgentFixturePids(fixture); + expect(toolParentPid).toBe(detachedTool.pid); + expect(toolChildPid).toBeTypeOf("number"); + registrations = await Promise.all([ + startToolRegistration(credentials, "parent", toolParentPid), + startToolRegistration(credentials, "child", toolChildPid), + ]); + await expect( + prepareCancellationAfterTransientReadFailure(observer), + ).resolves.toMatchObject({ + supported: false, + reason: "tool_process_not_registered", + }); + + const teardown = await observer.emergencyCleanup(deadlineAfter(100)); + + expect(teardown.quiescent).toBe(false); + expect(processGroupExists(detachedTool.pid!)).toBe(true); + } finally { + for (const registration of registrations) registration.destroy(); + forwardedController.abort(); + await fixture.requestCooperativeExit(); + if (detachedTool) await waitForChildExitBounded(detachedTool); + await observer.dispose(); + if (anchor) { + await stopRetainedTestGroup(anchor); + } + await observer.dispose(); + } + }, + 15_000, + ); + + it.skipIf(process.platform === "win32")( + "clears a closed pending registration, accepts retries, and requires both lifetime channels to close", + async () => { + const fixture = await createManagedAgentFixture( + () => "tool-registration-retry", + ); + fixtures.push(fixture); + const observer = new LocalManagedAgentProcessObserver(); + const forwardedController = new AbortController(); + const credentialFile = join(fixture.root, "tool-control.json"); + let anchor: ChildProcessWithoutNullStreams | undefined; + let toolPids: readonly number[] = []; + let parentRegistration: NetSocket | undefined; + let childRegistration: NetSocket | undefined; + try { + observer.armToolProcessContainment(); + anchor = asChildProcess( + observer.spawn({ + command: process.execPath, + args: [ + "--input-type=module", + "--eval", + DESCENDANT_TOOL_SCRIPT, + join(fixture.workspaceRoot, FIXTURE_PATHS.processScript), + join(fixture.workspaceRoot, FIXTURE_PATHS.processPidFile), + credentialFile, + fixture.cooperativeExitMarker, + ], + cwd: fixture.workspaceRoot, + env: { ...process.env }, + signal: forwardedController.signal, + }), + ); + const credentials = await waitForToolControlCredentials(credentialFile); + const [toolParentPid, toolChildPid] = + await waitForManagedAgentFixturePids(fixture); + toolPids = [toolParentPid, toolChildPid]; + + await sendClosedToolRegistration(credentials, "parent", toolParentPid); + [parentRegistration, childRegistration] = await Promise.all([ + openToolRegistration(credentials, "parent", toolParentPid), + openToolRegistration(credentials, "child", toolChildPid), + ]); + await expect( + prepareCancellationAfterTransientReadFailure(observer), + ).resolves.toMatchObject({ + supported: true, + reason: "ready", + containmentSupported: true, + }); + + await fixture.requestCooperativeExit(); + await Promise.all( + toolPids.map((pid) => + waitForTestProcessDeath( + () => processExists(pid), + `Marker-authenticated fixture process ${pid}`, + ), + ), + ); + const teardownDeadline = deadlineAfter(1_000); + observer.beginTeardown(teardownDeadline); + forwardedController.abort(); + const openChannelObservation = + await observer.emergencyCleanup(teardownDeadline); + await waitForChildExitBounded(anchor); + expect(openChannelObservation).toMatchObject({ + quiescent: false, + deadlineMet: false, + }); + + parentRegistration.destroy(); + childRegistration.destroy(); + const finalObservation = + await observer.waitForQuiescence(teardownDeadline); + expect(finalObservation).toMatchObject({ + quiescent: false, + deadlineMet: false, + }); + } finally { + parentRegistration?.destroy(); + childRegistration?.destroy(); + forwardedController.abort(); + await fixture.requestCooperativeExit(); + await observer.dispose(); + if (anchor) await stopRetainedTestGroup(anchor); + await observer.dispose(); + } + }, + 15_000, + ); + + it.skipIf(process.platform === "win32")( + "never reports an armed but unregistered tool scope as quiescent", + async () => { + const observer = new LocalManagedAgentProcessObserver(); + const controller = new AbortController(); + observer.armToolProcessContainment(); + const anchor = asChildProcess( + observer.spawn({ + command: process.execPath, + args: ["-e", "process.exit(0)"], + cwd: process.cwd(), + env: { ...process.env }, + signal: controller.signal, + }), + ); + try { + if (anchor.exitCode === null && anchor.signalCode === null) { + await once(anchor, "exit"); + } + const teardownDeadline = deadlineAfter(50); + await expect( + observer.waitForQuiescence(teardownDeadline), + ).resolves.toMatchObject({ + quiescent: false, + deadlineMet: false, + containmentSupported: false, + }); + await expect( + observer.emergencyCleanup(teardownDeadline), + ).resolves.toMatchObject({ + quiescent: false, + deadlineMet: false, + containmentSupported: false, + }); + } finally { + controller.abort(); + await stopRetainedTestGroup(anchor); + observer.dispose(); + } + }, + 5_000, + ); + + it("bounds a hanging process-table read and never turns unknown observation into quiescence", async () => { + const signals: Array = []; + const observer = new LocalManagedAgentProcessObserver({ + platform: "darwin", + readProcessTable: () => new Promise(() => undefined), + testOnlyRequestTermination: (groupId, signal) => { + signals.push([groupId, signal]); + return "failure"; + }, + }); + const controller = new AbortController(); + const child = asChildProcess( + observer.spawn({ + ...activeNodeCommand(), + cwd: process.cwd(), + env: { ...process.env }, + signal: controller.signal, + }), + ); + const startedAt = Date.now(); + try { + await expect(observer.prepareCancellation()).resolves.toMatchObject({ + supported: false, + reason: "process_table_unavailable", + }); + expect(Date.now() - startedAt).toBeLessThan(1_000); + const shortConfirmationStartedAt = Date.now(); + await expect( + observer.waitForQuiescence(deadlineAfter(50)), + ).resolves.toMatchObject({ + quiescent: false, + deadlineMet: false, + processTableAvailable: false, + }); + expect(Date.now() - shortConfirmationStartedAt).toBeLessThan(150); + + controller.abort(); + expect(signals).toEqual([]); + } finally { + await stopRetainedTestGroup(child); + controller.abort(); + observer.dispose(); + } + }); + + it("remembers an SDK abort but grants no fallback signal before deadline adoption", async () => { + let rootPid = 0; + const signals: Array = []; + const observer = new LocalManagedAgentProcessObserver({ + platform: "darwin", + readProcessTable: async () => + available([ + [ + rootPid, + { + parentPid: process.pid, + processGroupId: rootPid, + sessionId: rootPid, + state: "S", + startedAt: "abort-before-deadline", + }, + ], + ]), + testOnlyRequestTermination: (groupId, signal) => { + signals.push([groupId, signal]); + return "sent"; + }, + }); + const controller = new AbortController(); + const child = asChildProcess( + observer.spawn({ + ...activeNodeCommand(), + cwd: process.cwd(), + env: { ...process.env }, + signal: controller.signal, + }), + ); + rootPid = child.pid!; + try { + await expect(observer.prepareCancellation()).resolves.toMatchObject({ + supported: true, + reason: "ready", + }); + + controller.abort(); + expect(signals).toEqual([]); + + observer.beginTeardown(deadlineAfter(100)); + await vi.waitFor(() => expect(signals).toEqual([[rootPid, "SIGKILL"]])); + } finally { + await stopRetainedTestGroup(child); + await observer.dispose(); + } + }); + + it("closes the spawn gate as soon as teardown adopts its immutable deadline", async () => { + const observer = new LocalManagedAgentProcessObserver({ + platform: "darwin", + readProcessTable: async () => available([]), + }); + try { + observer.beginTeardown(deadlineAfter(1_000)); + expect(() => + observer.spawn({ + ...activeNodeCommand(), + cwd: process.cwd(), + env: { ...process.env }, + signal: new AbortController().signal, + }), + ).toThrow("managed-agent process observer is closed"); + } finally { + await observer.dispose(); + } + }); + + it("seals a successful quiescence observation against delayed SDK spawns", async () => { + const observer = new LocalManagedAgentProcessObserver({ + platform: "darwin", + readProcessTable: async () => available([]), + }); + try { + await expect( + observer.waitForQuiescence(deadlineAfter(1_000)), + ).resolves.toMatchObject({ quiescent: true, deadlineMet: true }); + expect(() => + observer.spawn({ + ...activeNodeCommand(), + cwd: process.cwd(), + env: { ...process.env }, + signal: new AbortController().signal, + }), + ).toThrow("managed-agent process observer is closed"); + } finally { + await observer.dispose(); + } + }); + + it.skipIf(process.platform === "win32")( + "never uses a host numeric signal when channel confirmation misses the deadline", + async () => { + let hangAfterRequest = false; + const terminationRequests: Array = []; + const hostKillSpy = vi.spyOn(process, "kill"); + const observer = new LocalManagedAgentProcessObserver({ + readProcessTable: () => + hangAfterRequest + ? new Promise(() => undefined) + : readRealPosixProcessTable(), + onTerminationRequest: ({ processGroupId }, outcome) => { + if (outcome === "sent") { + terminationRequests.push([processGroupId, "SIGKILL"]); + // From this point onward, the sampled numeric PGID could be reused. + // No later host operation may signal it. + hangAfterRequest = true; + } + }, + }); + const controller = new AbortController(); + const anchor = asChildProcess( + observer.spawn({ + ...activeNodeCommand(), + cwd: process.cwd(), + env: { ...process.env }, + signal: controller.signal, + }), + ); + try { + await expect( + prepareCancellationAfterTransientReadFailure(observer), + ).resolves.toMatchObject({ supported: true, reason: "ready" }); + const teardownDeadline = deadlineAfter(75); + observer.beginTeardown(teardownDeadline); + controller.abort(); + + const teardown = await observer.emergencyCleanup(teardownDeadline); + expect(teardown).toMatchObject({ + quiescent: false, + deadlineMet: false, + forceKillIssued: true, + }); + expect(hangAfterRequest).toBe(true); + expect(terminationRequests).toEqual([[anchor.pid!, "SIGKILL"]]); + expect( + hostKillSpy.mock.calls.some(([, signal]) => signal === "SIGKILL"), + ).toBe(false); + + await observer.dispose(); + await waitForChildExitBounded(anchor); + expect(terminationRequests).toEqual([[anchor.pid!, "SIGKILL"]]); + expect(processGroupExists(anchor.pid!)).toBe(false); + } finally { + hostKillSpy.mockRestore(); + await stopRetainedTestGroup(anchor); + await observer.dispose(); + } + }, + 10_000, + ); + + it("marks an observed POSIX group escape unsupported without authorizing an individual signal", async () => { + let rootPid = 0; + let escaped = false; + const signals: Array = []; + const table = async (): Promise => { + const rootRecord = { + parentPid: process.pid, + processGroupId: rootPid, + startedAt: "root", + }; + const childRecord = { + parentPid: rootPid, + processGroupId: escaped ? rootPid + 1 : rootPid, + startedAt: "child", + }; + return available([ + [rootPid, rootRecord], + [rootPid + 100, childRecord], + ]); + }; + const observer = new LocalManagedAgentProcessObserver({ + platform: "darwin", + readProcessTable: table, + testOnlyRequestTermination: (groupId, signal) => { + signals.push([groupId, signal]); + return "sent"; + }, + }); + const controller = new AbortController(); + const child = asChildProcess( + observer.spawn({ + ...activeNodeCommand(), + cwd: process.cwd(), + env: { ...process.env }, + signal: controller.signal, + }), + ); + rootPid = child.pid!; + try { + await expect(observer.prepareCancellation()).resolves.toMatchObject({ + supported: true, + reason: "ready", + }); + escaped = true; + await observer.observeProcessTree(); + await expect( + observer.waitForQuiescence(deadlineAfter(1)), + ).resolves.toMatchObject({ + quiescent: false, + containmentSupported: false, + }); + expect(signals).toEqual([]); + } finally { + await stopRetainedTestGroup(child); + controller.abort(); + observer.dispose(); + } + }); + + it("treats zombie topology drift as dead rather than a containment escape", async () => { + let rootPid = 0; + let zombie = false; + let now = 0; + const signals: Array = []; + const observer = new LocalManagedAgentProcessObserver({ + platform: "darwin", + readProcessTable: async () => + available([ + [ + rootPid, + { + parentPid: process.pid, + processGroupId: rootPid, + sessionId: 0, + state: "Ss", + startedAt: "root", + }, + ], + [ + rootPid + 100, + { + parentPid: zombie ? 1 : rootPid, + processGroupId: zombie ? rootPid + 200 : rootPid, + sessionId: zombie ? 999 : 0, + state: zombie ? "Z+" : "S", + startedAt: "child", + }, + ], + ]), + processGroupLiveness: () => "gone", + testOnlyRequestTermination: (groupId, signal) => { + signals.push([groupId, signal]); + return "sent"; + }, + monotonicNow: () => now, + delay: async (milliseconds) => { + now += Math.max(1, milliseconds); + }, + }); + const controller = new AbortController(); + const anchor = asChildProcess( + observer.spawn({ + ...activeNodeCommand(), + cwd: process.cwd(), + env: { ...process.env }, + signal: controller.signal, + }), + ); + rootPid = anchor.pid!; + try { + await expect(observer.prepareCancellation()).resolves.toMatchObject({ + supported: true, + reason: "ready", + }); + zombie = true; + await observer.observeProcessTree(); + + const observation = await observer.waitForQuiescence( + deadlineAfter(1, now), + ); + expect(observation).toMatchObject({ + quiescent: false, + containmentSupported: true, + }); + expect(observation.alivePidsAtDeadline).not.toContain(rootPid + 100); + expect(observation.alivePidsAtDeadline).not.toContain(rootPid + 200); + expect(signals).toEqual([]); + } finally { + await stopRetainedTestGroup(anchor); + controller.abort(); + observer.dispose(); + } + }); + + it("keeps pre-signal non-zombie topology drift permanently fail-closed", async () => { + let rootPid = 0; + let escaped = false; + let gone = false; + let now = 0; + const signals: Array = []; + const observer = new LocalManagedAgentProcessObserver({ + platform: "darwin", + readProcessTable: async () => + gone + ? available([]) + : available([ + [ + rootPid, + { + parentPid: process.pid, + processGroupId: rootPid, + sessionId: 0, + state: "Ss", + startedAt: "root", + }, + ], + [ + rootPid + 100, + { + parentPid: escaped ? 1 : rootPid, + processGroupId: escaped ? rootPid + 200 : rootPid, + sessionId: escaped ? 999 : 0, + state: "S", + startedAt: "child", + }, + ], + ]), + processGroupLiveness: () => "gone", + testOnlyRequestTermination: (groupId, signal) => { + signals.push([groupId, signal]); + return "sent"; + }, + monotonicNow: () => now, + delay: async (milliseconds) => { + now += Math.max(1, milliseconds); + }, + }); + const controller = new AbortController(); + const anchor = asChildProcess( + observer.spawn({ + ...activeNodeCommand(), + cwd: process.cwd(), + env: { ...process.env }, + signal: controller.signal, + }), + ); + rootPid = anchor.pid!; + try { + await expect(observer.prepareCancellation()).resolves.toMatchObject({ + supported: true, + reason: "ready", + }); + escaped = true; + await observer.observeProcessTree(); + + const teardownDeadline = deadlineAfter(1, now); + const observation = await observer.waitForQuiescence(teardownDeadline); + expect(observation).toMatchObject({ + quiescent: false, + containmentSupported: false, + }); + expect(observation.alivePidsAtDeadline).toEqual( + expect.arrayContaining([rootPid + 100, rootPid + 200]), + ); + expect(signals).toEqual([]); + + controller.abort(); + gone = true; + await observer.observeProcessTree(); + await expect( + observer.waitForQuiescence(teardownDeadline), + ).resolves.toMatchObject({ + quiescent: false, + containmentSupported: false, + }); + expect(signals).toEqual([]); + } finally { + await stopRetainedTestGroup(anchor); + controller.abort(); + observer.dispose(); + } + }); + + it("keeps post-SIGKILL stable exit drift live until disappearance without invalidating containment", async () => { + let rootPid = 0; + let stage: "owned" | "exiting" | "gone" = "owned"; + let now = 0; + const signals: Array = []; + const observer = new LocalManagedAgentProcessObserver({ + platform: "darwin", + readProcessTable: async () => { + if (stage === "gone") return available([]); + if (stage === "exiting") { + return available([ + [ + rootPid + 100, + { + parentPid: 1, + processGroupId: rootPid, + sessionId: 0, + state: "?E", + startedAt: "child", + }, + ], + ]); + } + return available([ + [ + rootPid, + { + parentPid: process.pid, + processGroupId: rootPid, + sessionId: 0, + state: "Ss", + startedAt: "root", + }, + ], + [ + rootPid + 100, + { + parentPid: rootPid, + processGroupId: rootPid, + sessionId: 0, + state: "S", + startedAt: "child", + }, + ], + ]); + }, + processGroupLiveness: () => (stage === "gone" ? "gone" : "alive"), + testOnlyRequestTermination: (groupId, signal) => { + signals.push([groupId, signal]); + return "sent"; + }, + monotonicNow: () => now, + delay: async (milliseconds) => { + now += Math.max(1, milliseconds); + }, + }); + const controller = new AbortController(); + const anchor = asChildProcess( + observer.spawn({ + ...activeNodeCommand(), + cwd: process.cwd(), + env: { ...process.env }, + signal: controller.signal, + }), + ); + rootPid = anchor.pid!; + try { + await expect(observer.prepareCancellation()).resolves.toMatchObject({ + supported: true, + reason: "ready", + }); + const teardownDeadline = deadlineAfter(100, now); + observer.beginTeardown(teardownDeadline); + controller.abort(); + await observer.observeProcessTree(); + expect(signals).toEqual([[rootPid, "SIGKILL"]]); + + stage = "exiting"; + await observer.observeProcessTree(); + + stage = "gone"; + await observer.observeProcessTree(); + await expect( + observer.waitForQuiescence(teardownDeadline), + ).resolves.toMatchObject({ + quiescent: true, + deadlineMet: true, + containmentSupported: true, + alivePidsAtDeadline: [], + }); + } finally { + await stopRetainedTestGroup(anchor); + controller.abort(); + observer.dispose(); + } + }); + + it("never authorizes a group signal from zombie-only root evidence", async () => { + let rootPid = 0; + const signals: Array = []; + const observer = new LocalManagedAgentProcessObserver({ + platform: "darwin", + readProcessTable: async () => + available([ + [ + rootPid, + { + parentPid: 1, + processGroupId: rootPid, + sessionId: 0, + state: "Z", + startedAt: "root", + }, + ], + ]), + processGroupLiveness: () => "alive", + testOnlyRequestTermination: (groupId, signal) => { + signals.push([groupId, signal]); + return "sent"; + }, + }); + const controller = new AbortController(); + const anchor = asChildProcess( + observer.spawn({ + ...activeNodeCommand(), + cwd: process.cwd(), + env: { ...process.env }, + signal: controller.signal, + }), + ); + rootPid = anchor.pid!; + try { + await expect(observer.prepareCancellation()).resolves.toMatchObject({ + supported: false, + reason: "root_not_active", + ownershipProven: false, + }); + controller.abort(); + expect(signals).toEqual([]); + } finally { + await stopRetainedTestGroup(anchor); + controller.abort(); + observer.dispose(); + } + }); + + it("allows normal quiescence only after a still-descended unauthenticated subgroup is positively dead", async () => { + let rootPid = 0; + let rootAlive = true; + let subgroupAlive = true; + const subgroupPid = () => rootPid + 100; + const signals: Array = []; + const observer = new LocalManagedAgentProcessObserver({ + platform: "darwin", + readProcessTable: async () => { + await Promise.resolve(); + return available([ + ...(rootAlive + ? ([ + [ + rootPid, + { + parentPid: process.pid, + processGroupId: rootPid, + sessionId: rootPid, + startedAt: "root", + }, + ], + ] as const) + : []), + ...(subgroupAlive + ? ([ + [ + subgroupPid(), + { + parentPid: rootPid, + processGroupId: subgroupPid(), + sessionId: subgroupPid(), + startedAt: "short-lived-subgroup", + }, + ], + ] as const) + : []), + ]); + }, + processGroupLiveness: (processGroupId) => + processGroupId === rootPid + ? rootAlive + ? "alive" + : "gone" + : subgroupAlive + ? "alive" + : "gone", + testOnlyRequestTermination: (groupId, signal) => { + signals.push([groupId, signal]); + return "sent"; + }, + }); + const controller = new AbortController(); + const anchor = asChildProcess( + observer.spawn({ + ...activeNodeCommand(), + cwd: process.cwd(), + env: { ...process.env }, + signal: controller.signal, + }), + ); + rootPid = anchor.pid!; + try { + await observer.observeProcessTree(); + await expect(observer.prepareCancellation()).resolves.toMatchObject({ + supported: false, + reason: "containment_escaped", + ownershipProven: false, + observedPids: expect.arrayContaining([subgroupPid()]), + }); + expect(signals).toEqual([]); + + subgroupAlive = false; + await observer.observeProcessTree(); + rootAlive = false; + await observer.observeProcessTree(); + await expect( + observer.waitForQuiescence(deadlineAfter(1)), + ).resolves.toMatchObject({ + quiescent: true, + deadlineMet: true, + containmentSupported: true, + alivePidsAtDeadline: [], + }); + expect(signals).toEqual([]); + } finally { + await stopRetainedTestGroup(anchor); + controller.abort(); + observer.dispose(); + } + }); + + it("keeps a live unauthenticated descendant subgroup nonquiescent without granting it signal authority", async () => { + let rootPid = 0; + const subgroupPid = () => rootPid + 100; + const signals: Array = []; + const observer = new LocalManagedAgentProcessObserver({ + platform: "darwin", + readProcessTable: async () => + available([ + [ + rootPid, + { + parentPid: process.pid, + processGroupId: rootPid, + sessionId: rootPid, + startedAt: "root", + }, + ], + [ + subgroupPid(), + { + parentPid: rootPid, + processGroupId: subgroupPid(), + sessionId: subgroupPid(), + startedAt: "surviving-subgroup", + }, + ], + ]), + processGroupLiveness: () => "alive", + testOnlyRequestTermination: (groupId, signal) => { + signals.push([groupId, signal]); + return "sent"; + }, + }); + const controller = new AbortController(); + const anchor = asChildProcess( + observer.spawn({ + ...activeNodeCommand(), + cwd: process.cwd(), + env: { ...process.env }, + signal: controller.signal, + }), + ); + rootPid = anchor.pid!; + try { + await observer.observeProcessTree(); + await expect( + observer.waitForQuiescence(deadlineAfter(1)), + ).resolves.toMatchObject({ + quiescent: false, + deadlineMet: false, + containmentSupported: false, + alivePidsAtDeadline: expect.arrayContaining([subgroupPid()]), + }); + expect(signals).toEqual([]); + } finally { + await stopRetainedTestGroup(anchor); + controller.abort(); + observer.dispose(); + } + }); + + it("keeps an unauthenticated subgroup permanently failed closed if its stable identity loses root ancestry", async () => { + let rootPid = 0; + let subgroupState: "descended" | "reparented" | "gone" = "descended"; + const subgroupPid = () => rootPid + 100; + const observer = new LocalManagedAgentProcessObserver({ + platform: "darwin", + readProcessTable: async () => { + const subgroup = + subgroupState === "gone" + ? [] + : [ + [ + subgroupPid(), + { + parentPid: + subgroupState === "descended" ? rootPid : process.pid, + processGroupId: subgroupPid(), + sessionId: subgroupPid(), + startedAt: "reparented-subgroup", + }, + ] as const, + ]; + return available([ + ...(subgroupState === "descended" + ? ([ + [ + rootPid, + { + parentPid: process.pid, + processGroupId: rootPid, + sessionId: rootPid, + startedAt: "root", + }, + ], + ] as const) + : []), + ...subgroup, + ]); + }, + processGroupLiveness: () => "gone", + testOnlyRequestTermination: () => "sent", + }); + const controller = new AbortController(); + const anchor = asChildProcess( + observer.spawn({ + ...activeNodeCommand(), + cwd: process.cwd(), + env: { ...process.env }, + signal: controller.signal, + }), + ); + rootPid = anchor.pid!; + try { + await observer.observeProcessTree(); + subgroupState = "reparented"; + await observer.observeProcessTree(); + subgroupState = "gone"; + await observer.observeProcessTree(); + await expect( + observer.waitForQuiescence(deadlineAfter(1)), + ).resolves.toMatchObject({ + quiescent: false, + deadlineMet: false, + containmentSupported: false, + alivePidsAtDeadline: [], + }); + } finally { + await stopRetainedTestGroup(anchor); + controller.abort(); + observer.dispose(); + } + }); + + it("retains a same-PGID subgroup child when its leader exits and the survivor reparents between samples", async () => { + let rootPid = 0; + const subgroupLeaderPid = () => rootPid + 100; + const subgroupChildPid = () => rootPid + 101; + let stage: "complete" | "leader_gone" | "all_gone" = "complete"; + const signals: Array = []; + const observer = new LocalManagedAgentProcessObserver({ + platform: "darwin", + readProcessTable: async () => + available([ + [ + rootPid, + { + parentPid: process.pid, + processGroupId: rootPid, + sessionId: rootPid, + state: "S", + startedAt: "pending-root", + }, + ], + ...(stage === "complete" + ? ([ + [ + subgroupLeaderPid(), + { + parentPid: rootPid, + processGroupId: subgroupLeaderPid(), + sessionId: subgroupLeaderPid(), + state: "S", + startedAt: "pending-leader", + }, + ], + ] as const) + : []), + ...(stage !== "all_gone" + ? ([ + [ + subgroupChildPid(), + { + parentPid: + stage === "complete" ? subgroupLeaderPid() : process.pid, + processGroupId: subgroupLeaderPid(), + sessionId: subgroupLeaderPid(), + state: "S", + startedAt: "pending-child", + }, + ], + ] as const) + : []), + ]), + testOnlyRequestTermination: (groupId, signal) => { + signals.push([groupId, signal]); + return "sent"; + }, + }); + const forwardedController = new AbortController(); + const anchor = asChildProcess( + observer.spawn({ + ...activeNodeCommand(), + cwd: process.cwd(), + env: { ...process.env }, + signal: forwardedController.signal, + }), + ); + rootPid = anchor.pid!; + try { + await expect(observer.prepareCancellation()).resolves.toMatchObject({ + supported: false, + reason: "containment_escaped", + observedPids: expect.arrayContaining([ + subgroupLeaderPid(), + subgroupChildPid(), + ]), + }); + + stage = "leader_gone"; + await observer.observeProcessTree(); + await expect(observer.prepareCancellation()).resolves.toMatchObject({ + supported: false, + reason: "containment_escaped", + observedPids: expect.arrayContaining([subgroupChildPid()]), + }); + forwardedController.abort(); + expect(signals).toEqual([]); + + stage = "all_gone"; + await observer.observeProcessTree(); + await expect(observer.prepareCancellation()).resolves.toMatchObject({ + supported: false, + reason: "containment_escaped", + }); + expect(signals).toEqual([]); + } finally { + await stopRetainedTestGroup(anchor); + forwardedController.abort(); + observer.dispose(); + } + }); + + it("keeps a subgroup escape after an authorized root kill permanently failed closed", async () => { + let rootPid = 0; + let subgroupState: "root_group" | "reparented" | "gone" = "root_group"; + const subgroupPid = () => rootPid + 100; + const signals: Array = []; + const observer = new LocalManagedAgentProcessObserver({ + platform: "darwin", + readProcessTable: async () => { + if (subgroupState === "gone") return available([]); + return available([ + ...(subgroupState === "root_group" + ? ([ + [ + rootPid, + { + parentPid: process.pid, + processGroupId: rootPid, + sessionId: rootPid, + state: "S", + startedAt: "root", + }, + ], + ] as const) + : []), + [ + subgroupPid(), + { + parentPid: subgroupState === "root_group" ? rootPid : process.pid, + processGroupId: + subgroupState === "root_group" ? rootPid : subgroupPid(), + sessionId: + subgroupState === "root_group" ? rootPid : subgroupPid(), + state: "S", + startedAt: "survived-root-kill", + }, + ], + ]); + }, + processGroupLiveness: () => "alive", + testOnlyRequestTermination: (groupId, signal) => { + signals.push([groupId, signal]); + subgroupState = "reparented"; + return "sent"; + }, + }); + const controller = new AbortController(); + const anchor = asChildProcess( + observer.spawn({ + ...activeNodeCommand(), + cwd: process.cwd(), + env: { ...process.env }, + signal: controller.signal, + }), + ); + rootPid = anchor.pid!; + try { + await expect(observer.prepareCancellation()).resolves.toMatchObject({ + supported: true, + reason: "ready", + }); + const teardownDeadline = deadlineAfter(100); + observer.beginTeardown(teardownDeadline); + controller.abort(); + await observer.observeProcessTree(); + expect(signals).toEqual([[rootPid, "SIGKILL"]]); + await observer.observeProcessTree(); + subgroupState = "gone"; + await observer.observeProcessTree(); + await expect( + observer.waitForQuiescence(teardownDeadline), + ).resolves.toMatchObject({ + quiescent: false, + deadlineMet: false, + containmentSupported: false, + alivePidsAtDeadline: [], + }); + } finally { + await stopRetainedTestGroup(anchor); + controller.abort(); + observer.dispose(); + } + }); + + it.skipIf(process.platform === "win32")( + "blocks L2 readiness when an authenticated tool run gains an unauthenticated descendant group", + async () => { + let injectUnknownDescendant = false; + let run: RegisteredDescendantToolRun | undefined; + let syntheticPid = 2_000_000_000; + const observer = new LocalManagedAgentProcessObserver({ + readProcessTable: async () => { + const observation = await readRealPosixProcessTable(); + if (!observation.available || !injectUnknownDescendant || !run) { + return observation; + } + const processes = new Map(observation.processes); + while (processes.has(syntheticPid)) syntheticPid -= 1; + processes.set(syntheticPid, { + parentPid: run.anchor.pid!, + processGroupId: syntheticPid, + sessionId: syntheticPid, + state: "S", + startedAt: "synthetic-unknown-l2-descendant", + }); + return { available: true, processes }; + }, + }); + try { + run = await startRegisteredDescendantToolRun( + observer, + "unknown-l2-descendant", + ); + injectUnknownDescendant = true; + await observer.observeProcessTree(); + await expect(observer.prepareCancellation()).resolves.toMatchObject({ + supported: false, + reason: "containment_escaped", + ownershipProven: false, + }); + } finally { + injectUnknownDescendant = false; + await cleanupRegisteredDescendantToolRun(run); + observer.dispose(); + } + }, + 15_000, + ); + + it.skipIf(process.platform === "win32")( + "never loses a reparented escaped tool grandchild or signals its new group", + async () => { + const fixture = await createManagedAgentFixture( + () => "reparented-tool-grandchild", + ); + fixtures.push(fixture); + const credentialFile = join(fixture.root, "tool-control.json"); + let rootPid = 0; + let escaped = false; + let toolParentPid = 0; + let toolChildPid = 0; + let toolGrandchildPid = 0; + let escapedGroupId = 0; + const signals: Array = []; + const observer = new LocalManagedAgentProcessObserver({ + readProcessTable: async () => { + await Promise.resolve(); + if (escaped) { + return available([ + [ + toolGrandchildPid, + { + parentPid: 1, + processGroupId: escapedGroupId, + sessionId: escapedGroupId, + startedAt: "tool-grandchild", + }, + ], + ]); + } + return available([ + [ + process.pid, + { + parentPid: process.ppid, + processGroupId: process.pid, + sessionId: process.pid, + startedAt: "host", + }, + ], + [ + rootPid, + { + parentPid: process.pid, + processGroupId: rootPid, + sessionId: rootPid, + startedAt: "root-100", + }, + ], + [ + toolParentPid, + { + parentPid: rootPid, + processGroupId: toolParentPid, + sessionId: toolParentPid, + startedAt: "tool-parent-200", + }, + ], + [ + toolChildPid, + { + parentPid: toolParentPid, + processGroupId: toolParentPid, + sessionId: toolParentPid, + startedAt: "tool-child-201", + }, + ], + [ + toolGrandchildPid, + { + parentPid: toolChildPid, + processGroupId: toolParentPid, + sessionId: toolParentPid, + startedAt: "tool-grandchild", + }, + ], + ]); + }, + processGroupLiveness: (groupId) => + escaped && groupId === escapedGroupId ? "alive" : "gone", + testOnlyRequestTermination: (groupId, signal) => { + signals.push([groupId, signal]); + return "sent"; + }, + }); + const controller = new AbortController(); + observer.armToolProcessContainment(); + const anchor = asChildProcess( + observer.spawn({ + command: process.execPath, + args: [ + "--input-type=module", + "--eval", + EXPORT_TOOL_CONTROL_SCRIPT, + credentialFile, + ], + cwd: fixture.workspaceRoot, + env: { ...process.env }, + signal: controller.signal, + }), + ); + rootPid = anchor.pid!; + toolParentPid = rootPid + 10_000; + toolChildPid = toolParentPid + 1; + toolGrandchildPid = toolParentPid + 2; + escapedGroupId = toolParentPid + 3; + let registrations: readonly NetSocket[] = []; + try { + const credentials = await waitForToolControlCredentials(credentialFile); + registrations = await Promise.all([ + openToolRegistration(credentials, "parent", toolParentPid), + openToolRegistration(credentials, "child", toolChildPid), + ]); + await expect(observer.prepareCancellation()).resolves.toMatchObject({ + supported: true, + reason: "ready", + ownershipProven: true, + }); + + escaped = true; + const registrationClosures = registrations.map((socket) => + once(socket, "close"), + ); + for (const socket of registrations) socket.destroy(); + await Promise.all(registrationClosures); + await stopRetainedTestGroup(anchor); + + const teardown = await observer.emergencyCleanup(deadlineAfter(50)); + + expect(teardown).toMatchObject({ + quiescent: false, + deadlineMet: false, + containmentSupported: false, + }); + expect(teardown.alivePidsAtDeadline).toEqual( + expect.arrayContaining([toolGrandchildPid, escapedGroupId]), + ); + expect(signals.some(([groupId]) => groupId === escapedGroupId)).toBe( + false, + ); + } finally { + for (const socket of registrations) socket.destroy(); + await stopRetainedTestGroup(anchor); + controller.abort(); + observer.dispose(); + } + }, + 10_000, + ); + + it("makes repeated SDK-forwarded abort delivery idempotent after ownership preparation", async () => { + let rootPid = 0; + const signals: Array = []; + const observer = new LocalManagedAgentProcessObserver({ + platform: "darwin", + readProcessTable: async () => + available([ + [ + rootPid, + { + parentPid: process.pid, + processGroupId: rootPid, + state: "S", + startedAt: "root", + }, + ], + [ + rootPid + 100, + { + parentPid: rootPid, + processGroupId: rootPid, + state: "S", + startedAt: "child", + }, + ], + ]), + testOnlyRequestTermination: (groupId, signal) => { + signals.push([groupId, signal]); + return "sent"; + }, + }); + const forwardedController = new AbortController(); + const child = asChildProcess( + observer.spawn({ + ...activeNodeCommand(), + cwd: process.cwd(), + env: { ...process.env }, + signal: forwardedController.signal, + }), + ); + rootPid = child.pid!; + try { + await observer.prepareCancellation(); + observer.beginTeardown(deadlineAfter(100)); + forwardedController.abort(); + forwardedController.abort(); + await observer.observeProcessTree(); + expect(signals).toEqual([[rootPid, "SIGKILL"]]); + await observer.observeProcessTree(); + expect(signals).toHaveLength(1); + } finally { + await stopRetainedTestGroup(child); + observer.dispose(); + } + }); + + it("retries transient root kill failures only after fresh authority samples", async () => { + let rootPid = 0; + let processTableReads = 0; + let killAttempts = 0; + const signals: Array = []; + const signalReadCounts: number[] = []; + const observer = new LocalManagedAgentProcessObserver({ + platform: "darwin", + readProcessTable: async () => { + processTableReads += 1; + return available([ + [ + rootPid, + { + parentPid: process.pid, + processGroupId: rootPid, + state: "S", + startedAt: "root", + }, + ], + ]); + }, + testOnlyRequestTermination: (groupId, signal) => { + signals.push([groupId, signal]); + signalReadCounts.push(processTableReads); + if (killAttempts++ === 0) return "failure"; + return "sent"; + }, + }); + const forwardedController = new AbortController(); + const child = asChildProcess( + observer.spawn({ + ...activeNodeCommand(), + cwd: process.cwd(), + env: { ...process.env }, + signal: forwardedController.signal, + }), + ); + rootPid = child.pid!; + try { + await observer.prepareCancellation(); + const teardownDeadline = deadlineAfter(100); + await observer.emergencyCleanup(teardownDeadline); + + expect(signals).toEqual([ + [rootPid, "SIGKILL"], + [rootPid, "SIGKILL"], + ]); + expect( + signalReadCounts.every( + (readCount, index) => + index === 0 || readCount > signalReadCounts[index - 1]!, + ), + ).toBe(true); + await expect( + observer.waitForQuiescence(teardownDeadline), + ).resolves.toMatchObject({ + containmentSupported: true, + forceKillIssued: true, + quiescent: false, + }); + } finally { + await stopRetainedTestGroup(child); + observer.dispose(); + } + }); + + it("treats an unexpected group-liveness probe error as unknown, never gone", async () => { + let rootPid = 0; + const observer = new LocalManagedAgentProcessObserver({ + platform: "darwin", + readProcessTable: async () => + available([ + [ + rootPid, + { + parentPid: process.pid, + processGroupId: rootPid, + sessionId: 0, + state: "S", + startedAt: "reported-live-root", + }, + ], + ]), + processGroupLiveness: () => "unknown", + }); + const controller = new AbortController(); + const child = asChildProcess( + observer.spawn({ + command: process.execPath, + args: ["-e", "process.exit(0)"], + cwd: process.cwd(), + env: { ...process.env }, + signal: controller.signal, + }), + ); + rootPid = child.pid!; + await once(child, "exit"); + try { + await expect( + observer.waitForQuiescence(deadlineAfter(1)), + ).resolves.toMatchObject({ + quiescent: false, + deadlineMet: false, + containmentSupported: false, + }); + } finally { + controller.abort(); + observer.dispose(); + } + }); + + it("accepts complete process-table absence without probing a cached group", async () => { + let livenessProbes = 0; + const observer = new LocalManagedAgentProcessObserver({ + platform: "darwin", + readProcessTable: async () => available([]), + processGroupLiveness: () => { + livenessProbes += 1; + return "unknown"; + }, + }); + const controller = new AbortController(); + const child = asChildProcess( + observer.spawn({ + command: process.execPath, + args: ["-e", "process.exit(0)"], + cwd: process.cwd(), + env: { ...process.env }, + signal: controller.signal, + }), + ); + await once(child, "exit"); + try { + await observer.observeProcessTree(); + await expect( + observer.waitForQuiescence(deadlineAfter(1)), + ).resolves.toMatchObject({ + quiescent: true, + deadlineMet: true, + containmentSupported: true, + alivePidsAtDeadline: [], + }); + expect(livenessProbes).toBe(0); + } finally { + controller.abort(); + observer.dispose(); + } + }); + + it("reports quiescence after the caller budget as a missed deadline", async () => { + let now = 0; + let measureOverrun = false; + const observer = new LocalManagedAgentProcessObserver({ + platform: "darwin", + readProcessTable: async () => { + if (measureOverrun) now = 2; + return available([]); + }, + monotonicNow: () => now, + }); + const controller = new AbortController(); + const child = asChildProcess( + observer.spawn({ + ...activeNodeCommand(), + cwd: process.cwd(), + env: { ...process.env }, + signal: controller.signal, + }), + ); + try { + await observer.observeProcessTree(); + now = 0; + measureOverrun = true; + + await expect( + observer.waitForQuiescence(deadlineAfter(1, 0)), + ).resolves.toMatchObject({ + quiescent: true, + deadlineMet: false, + elapsedMs: 2, + processTableAvailable: true, + alivePidsAtDeadline: [], + }); + } finally { + if (typeof child.pid === "number") { + await stopRetainedTestGroup(child); + } + controller.abort(); + observer.dispose(); + } + }); + + it("rejects Windows cancellation containment before granting signal authority", async () => { + const observer = new LocalManagedAgentProcessObserver({ + platform: "win32", + }); + await expect(observer.prepareCancellation()).resolves.toMatchObject({ + supported: false, + reason: "platform_unsupported", + ownershipProven: false, + }); + observer.dispose(); + }); + + it("never tracks or signals PIDs injected through the model-writable fixture file", async () => { + const fixture = await createManagedAgentFixture(() => "forged-pids"); + fixtures.push(fixture); + const forgedPids = [process.pid, 2_147_483_646] as const; + await writeFile( + join(fixture.workspaceRoot, FIXTURE_PATHS.processPidFile), + JSON.stringify({ parentPid: forgedPids[0], childPid: forgedPids[1] }), + ); + const signals: Array = []; + const observer = new LocalManagedAgentProcessObserver({ + testOnlyRequestTermination: (groupId, signal) => { + signals.push([groupId, signal]); + return "sent"; + }, + }); + try { + await expect(waitForManagedAgentFixturePids(fixture)).resolves.toEqual( + forgedPids, + ); + await observer.observeProcessTree(); + const teardownDeadline = deadlineAfter(1); + const teardown = await observer.waitForQuiescence(teardownDeadline); + await observer.emergencyCleanup(teardownDeadline); + + expect(teardown.observedPids).not.toContain(forgedPids[0]); + expect(teardown.observedPids).not.toContain(forgedPids[1]); + expect(signals).toEqual([]); + } finally { + observer.dispose(); + } + }); + + it("does not let a process-table read started before a signal authorize the next signal", async () => { + let rootPid = 0; + const reads: Array< + (observation: ManagedAgentProcessTableObservation) => void + > = []; + const signals: Array = []; + const observer = new LocalManagedAgentProcessObserver({ + platform: "darwin", + readProcessTable: () => + new Promise((resolveRead) => { + reads.push(resolveRead); + }), + processGroupLiveness: () => "alive", + testOnlyRequestTermination: (groupId, signal) => { + signals.push([groupId, signal]); + return "sent"; + }, + }); + const forwardedController = new AbortController(); + const anchor = asChildProcess( + observer.spawn({ + ...activeNodeCommand(), + cwd: process.cwd(), + env: { ...process.env }, + signal: forwardedController.signal, + }), + ); + rootPid = anchor.pid!; + const rootTable = () => + available([ + [ + rootPid, + { + parentPid: process.pid, + processGroupId: rootPid, + sessionId: rootPid, + state: "S", + startedAt: "epoch-root", + }, + ], + ]); + + try { + await vi.waitFor(() => expect(reads).toHaveLength(1)); + const initialSample = observer.observeProcessTree(); + reads.shift()!(rootTable()); + await initialSample; + + const readinessTask = observer.prepareCancellation(); + await vi.waitFor(() => expect(reads).toHaveLength(1)); + reads.shift()!(rootTable()); + await expect(readinessTask).resolves.toMatchObject({ + supported: true, + reason: "ready", + }); + + const preSignalSample = observer.observeProcessTree(); + await vi.waitFor(() => expect(reads).toHaveLength(1)); + observer.beginTeardown(deadlineAfter(1_000)); + forwardedController.abort(); + await vi.waitFor(() => expect(reads).toHaveLength(2)); + expect(signals).toEqual([]); + + // Completing the read that started before teardown cannot install its + // evidence or authorize a request in the new lifecycle generation. + reads.shift()!(rootTable()); + await preSignalSample; + expect(signals).toEqual([]); + + // Only the complete read that started after teardown can authorize the + // one process-bound termination request. + const postSignalSample = observer.observeProcessTree(); + await vi.waitFor(() => expect(reads).toHaveLength(1)); + reads.shift()!(rootTable()); + await postSignalSample; + expect(signals).toEqual([[rootPid, "SIGKILL"]]); + } finally { + await stopRetainedTestGroup(anchor); + forwardedController.abort(); + observer.dispose(); + } + }); + + it.each(["deadline", "dispose"] as const)( + "seals held process-table reads after %s so late completion cannot mutate or signal", + async (sealKind) => { + let rootPid = 0; + let monotonicTime = 0; + let resolveRead!: ( + observation: ManagedAgentProcessTableObservation, + ) => void; + const signals: Array = []; + const observer = new LocalManagedAgentProcessObserver({ + platform: "darwin", + monotonicNow: () => monotonicTime, + readProcessTable: () => + new Promise((resolve) => { + resolveRead = resolve; + }), + testOnlyRequestTermination: (groupId, signal) => { + signals.push([groupId, signal]); + return "sent"; + }, + }); + const forwardedController = new AbortController(); + const anchor = asChildProcess( + observer.spawn({ + ...activeNodeCommand(), + cwd: process.cwd(), + env: { ...process.env }, + signal: forwardedController.signal, + }), + ); + rootPid = anchor.pid!; + const deadline = Object.freeze({ startedAtMs: 0, deadlineAtMs: 10 }); + try { + await vi.waitFor(() => expect(resolveRead).toBeTypeOf("function")); + if (sealKind === "deadline") { + monotonicTime = 11; + await observer.waitForQuiescence(deadline); + } else { + await observer.dispose(); + } + const signalsAtSeal = [...signals]; + resolveRead( + available([ + [ + rootPid, + { + parentPid: process.pid, + processGroupId: rootPid, + sessionId: rootPid, + state: "T", + startedAt: "late-root", + }, + ], + ]), + ); + await new Promise((resolve) => setImmediate(resolve)); + forwardedController.abort(); + await new Promise((resolve) => setImmediate(resolve)); + + expect(signals).toEqual(signalsAtSeal); + expect(await observer.observeProcessTree(deadline)).toBe(false); + } finally { + await stopRetainedTestGroup(anchor); + forwardedController.abort(); + await observer.dispose(); + } + }, + ); + + it("discards an in-flight background sample that completes after a newly adopted deadline", async () => { + let monotonicTime = 0; + let resolveRead!: ( + observation: ManagedAgentProcessTableObservation, + ) => void; + const signals: Array = []; + const observer = new LocalManagedAgentProcessObserver({ + platform: "darwin", + monotonicNow: () => monotonicTime, + readProcessTable: () => + new Promise((resolve) => { + resolveRead = resolve; + }), + processGroupLiveness: () => "gone", + testOnlyRequestTermination: (groupId, signal) => { + signals.push([groupId, signal]); + return "sent"; + }, + }); + const forwardedController = new AbortController(); + const anchor = asChildProcess( + observer.spawn({ + ...activeNodeCommand(), + cwd: process.cwd(), + env: { ...process.env }, + signal: forwardedController.signal, + }), + ); + const deadline = Object.freeze({ startedAtMs: 0, deadlineAtMs: 10 }); + try { + await vi.waitFor(() => expect(resolveRead).toBeTypeOf("function")); + const observationTask = observer.waitForQuiescence(deadline); + monotonicTime = 11; + resolveRead(available([])); + + await expect(observationTask).resolves.toMatchObject({ + quiescent: false, + deadlineMet: false, + processTableAvailable: false, + }); + expect(signals).toEqual([]); + expect(await observer.observeProcessTree(deadline)).toBe(false); + } finally { + await stopRetainedTestGroup(anchor); + forwardedController.abort(); + await observer.dispose(); + } + }); + + it.skipIf(process.platform === "win32")( + "rejects tool-registration data delivered after the adopted deadline", + async () => { + const fixture = await createManagedAgentFixture( + () => "late-tool-registration", + ); + fixtures.push(fixture); + let monotonicTime = 0; + const credentialFile = join(fixture.root, "late-control.json"); + const observer = new LocalManagedAgentProcessObserver({ + monotonicNow: () => monotonicTime, + readProcessTable: () => new Promise(() => undefined), + }); + observer.armToolProcessContainment(); + const forwardedController = new AbortController(); + const anchor = asChildProcess( + observer.spawn({ + command: process.execPath, + args: [ + "--input-type=module", + "--eval", + EXPORT_TOOL_CONTROL_SCRIPT, + credentialFile, + ], + cwd: fixture.workspaceRoot, + env: { ...process.env }, + signal: forwardedController.signal, + }), + ); + let socket: NetSocket | undefined; + const deadline = Object.freeze({ startedAtMs: 0, deadlineAtMs: 10 }); + try { + const credentials = await waitForToolControlCredentials(credentialFile); + socket = createConnection(credentials.socketPath); + socket.on("error", () => undefined); + await once(socket, "connect"); + const closed = once(socket, "close").then(() => true); + const observationTask = observer.waitForQuiescence(deadline); + monotonicTime = 11; + socket.write( + `${JSON.stringify({ + capability: credentials.capability, + role: "parent", + pid: process.pid, + })}\n`, + ); + + await expect( + Promise.race([ + closed, + new Promise((resolveTimeout) => + setTimeout(() => resolveTimeout(false), 50), + ), + ]), + ).resolves.toBe(true); + await expect(observationTask).resolves.toMatchObject({ + quiescent: false, + deadlineMet: false, + }); + } finally { + socket?.destroy(); + forwardedController.abort(); + await observer.dispose(); + await stopRetainedTestGroup(anchor); + } + }, + ); +}); diff --git a/packages/harness/src/experimental/managed-agent-spike/process-observer.ts b/packages/harness/src/experimental/managed-agent-spike/process-observer.ts new file mode 100644 index 000000000..ef1414147 --- /dev/null +++ b/packages/harness/src/experimental/managed-agent-spike/process-observer.ts @@ -0,0 +1,2254 @@ +import { + execFile, + spawn as spawnChild, + type ChildProcessWithoutNullStreams, +} from "node:child_process"; +import { randomBytes, timingSafeEqual } from "node:crypto"; +import { chmodSync, mkdtempSync, rmSync } from "node:fs"; +import { + createServer, + type Server as NetServer, + type Socket as NetSocket, +} from "node:net"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { performance } from "node:perf_hooks"; +import { promisify } from "node:util"; + +import type { + SpawnedProcess, + SpawnOptions, +} from "@anthropic-ai/claude-agent-sdk"; + +import type { + ManagedAgentCancellationReadiness, + ManagedAgentProcessObserver, + ManagedAgentTeardownDeadline, + ManagedAgentTeardownObservation, +} from "./types.js"; + +const execFileAsync = promisify(execFile); +const SAMPLE_INTERVAL_MS = 100; +const QUIESCENCE_POLL_MS = 25; +export const MANAGED_AGENT_PROCESS_HELPER_TIMEOUT_MS = 200; +const MANAGED_AGENT_SUPERVISOR_PAYLOAD_ENV = + "SAPIOM_MANAGED_AGENT_SUPERVISOR_PAYLOAD"; +export const MANAGED_AGENT_TOOL_CONTROL_SOCKET_ENV = + "SAPIOM_MANAGED_AGENT_TOOL_CONTROL_SOCKET"; +export const MANAGED_AGENT_TOOL_CONTROL_CAPABILITY_ENV = + "SAPIOM_MANAGED_AGENT_TOOL_CONTROL_CAPABILITY"; +const TOOL_REGISTRATION_MAX_BYTES = 1_024; +const DISPOSE_DRAIN_TIMEOUT_MS = 500; + +/** + * The POSIX supervisor is the observer-owned process-group leader. The real + * SDK command runs inside its group, while the supervisor stays alive after + * an inner-root exit whenever another group member survives. Its own bounded + * `ps` helper remains in the group so abort and parent-disconnect cleanup + * contain it too; the known helper PID is excluded only from the membership + * decision that determines whether the anchor may exit. + */ +const MANAGED_AGENT_POSIX_SUPERVISOR_SOURCE = String.raw` +import { spawn } from "node:child_process"; +import { performance } from "node:perf_hooks"; + +const PAYLOAD_ENV = "SAPIOM_MANAGED_AGENT_SUPERVISOR_PAYLOAD"; +const HELPER_TIMEOUT_MS = 200; +const POLL_INTERVAL_MS = 25; +const EMPTY_GROUP_EXIT_GRACE_MS = 750; +const MAX_PROCESS_TABLE_BYTES = 4 * 1024 * 1024; + +function fail(message) { + try { process.stderr.write(message + "\n"); } catch {} + process.exit(1); +} + +const encodedPayload = process.env[PAYLOAD_ENV]; +delete process.env[PAYLOAD_ENV]; +if (!encodedPayload) fail("managed-agent supervisor payload missing"); + +let payload; +try { + payload = JSON.parse(Buffer.from(encodedPayload, "base64url").toString("utf8")); +} catch { + fail("managed-agent supervisor payload invalid"); +} +if ( + !payload || + typeof payload.command !== "string" || + payload.command.length === 0 || + !Array.isArray(payload.args) || + !payload.args.every((argument) => typeof argument === "string") +) { + fail("managed-agent supervisor command invalid"); +} + +for (const signal of ["SIGHUP", "SIGINT", "SIGTERM"]) { + process.on(signal, () => {}); +} + +function killOwnedGroup() { + try { + process.kill(0, "SIGKILL"); + } catch { + process.exit(1); + } +} + +process.on("disconnect", killOwnedGroup); +// The host can close IPC after spawn() succeeds but before this module starts. +// Register first, then close the already-disconnected bootstrap window. +if (!process.connected) killOwnedGroup(); + +function readOtherGroupMembers() { + return new Promise((resolveMembers) => { + let helper; + try { + // Intentionally non-detached: the helper is synchronously contained by + // the same group. Its known PID is excluded from this one snapshot. + helper = spawn("/bin/ps", ["-axo", "pid=,pgid="], { + stdio: ["ignore", "pipe", "ignore"], + windowsHide: true, + }); + } catch { + resolveMembers(undefined); + return; + } + const helperPid = helper.pid; + let output = ""; + let settled = false; + let overflowed = false; + const finish = (members) => { + if (settled) return; + settled = true; + clearTimeout(timeout); + resolveMembers(members); + }; + const timeout = setTimeout(() => { + finish(undefined); + }, HELPER_TIMEOUT_MS); + helper.stdout.on("data", (chunk) => { + if (overflowed) return; + output += chunk.toString("utf8"); + if (Buffer.byteLength(output) > MAX_PROCESS_TABLE_BYTES) { + overflowed = true; + helper.stdout.destroy(); + } + }); + helper.once("error", () => finish(undefined)); + helper.once("close", (code) => { + if (code !== 0 || overflowed || typeof helperPid !== "number") { + finish(undefined); + return; + } + const records = new Map(); + for (const line of output.split("\n")) { + const match = /^\s*(\d+)\s+(\d+)\s*$/.exec(line); + if (!match) continue; + records.set(Number(match[1]), Number(match[2])); + } + if ( + records.get(process.pid) !== process.pid || + records.get(helperPid) !== process.pid + ) { + finish(undefined); + return; + } + finish( + [...records.entries()] + .filter( + ([pid, processGroupId]) => + processGroupId === process.pid && + pid !== process.pid && + pid !== helperPid, + ) + .map(([pid]) => pid), + ); + }); + }); +} + +let innerClosed = false; +let innerExitCode = 1; +let membershipCheckRunning = false; +let emptyGroupObservedAt; +let pollTimer; + +function scheduleMembershipCheck(delayMs = 0) { + if (pollTimer) clearTimeout(pollTimer); + pollTimer = setTimeout(checkMembership, delayMs); +} + +async function checkMembership() { + pollTimer = undefined; + if (!innerClosed || membershipCheckRunning) return; + membershipCheckRunning = true; + const members = await readOtherGroupMembers(); + membershipCheckRunning = false; + if (members && members.length === 0) { + const now = performance.now(); + emptyGroupObservedAt ??= now; + const remainingGrace = + EMPTY_GROUP_EXIT_GRACE_MS - (now - emptyGroupObservedAt); + if (remainingGrace > 0) { + scheduleMembershipCheck(Math.min(POLL_INTERVAL_MS, remainingGrace)); + return; + } + process.stdin.unpipe(); + process.stdin.destroy(); + if (process.connected) { + process.off("disconnect", killOwnedGroup); + process.disconnect(); + } + process.exitCode = innerExitCode; + return; + } + emptyGroupObservedAt = undefined; + scheduleMembershipCheck(POLL_INTERVAL_MS); +} + +let inner; +try { + inner = spawn(payload.command, payload.args, { + cwd: process.cwd(), + env: process.env, + stdio: ["pipe", "pipe", "pipe"], + windowsHide: true, + }); +} catch { + innerClosed = true; + scheduleMembershipCheck(); +} + +if (inner) { + process.stdin.on("error", () => {}); + inner.stdin.on("error", () => {}); + inner.stdout.on("error", () => {}); + inner.stderr.on("error", () => {}); + process.stdout.on("error", () => {}); + process.stderr.on("error", () => {}); + process.stdin.pipe(inner.stdin); + inner.stdout.pipe(process.stdout, { end: false }); + inner.stderr.pipe(process.stderr, { end: false }); + inner.once("error", () => { + innerExitCode = 1; + }); + inner.once("close", (code) => { + innerClosed = true; + innerExitCode = Number.isInteger(code) ? code : 1; + scheduleMembershipCheck(); + }); +} +`; + +export interface ManagedAgentKernelProcessRecord { + readonly parentPid: number; + readonly processGroupId?: number; + /** POSIX session id, when the process table exposes it. */ + readonly sessionId?: number; + /** POSIX process state used to treat zombies as already dead. */ + readonly state?: string; + /** Kernel-reported creation time used for evidence, never POSIX authority. */ + readonly startedAt: string; +} + +export type ManagedAgentKernelProcessTable = ReadonlyMap< + number, + ManagedAgentKernelProcessRecord +>; + +export type ManagedAgentProcessTableObservation = + | { + readonly available: true; + readonly processes: ManagedAgentKernelProcessTable; + } + | { readonly available: false }; + +export type ManagedAgentProcessGroupLiveness = "alive" | "gone" | "unknown"; +export type ManagedAgentTerminationRequestOutcome = "sent" | "gone" | "failure"; + +export interface ManagedAgentTerminationRequest { + readonly target: "root" | "tool"; + /** Diagnostic identity only. The production request path never signals it. */ + readonly processGroupId: number; +} + +export interface LocalManagedAgentProcessObserverOptions { + readonly platform?: NodeJS.Platform; + readonly readProcessTable?: () => Promise; + readonly processGroupLiveness?: ( + processGroupId: number, + ) => ManagedAgentProcessGroupLiveness; + /** + * Deterministic unit-test seam. Production callers must leave this unset: + * the default path requests termination only over retained process-bound + * channels and never turns a sampled numeric PGID into signal authority. + */ + readonly testOnlyRequestTermination?: ( + processGroupId: number, + signal: "SIGKILL", + target: ManagedAgentTerminationRequest["target"], + ) => ManagedAgentTerminationRequestOutcome; + /** Return an outcome to veto one request; return undefined to use the channel. */ + readonly testOnlyBeforeTerminationRequest?: ( + request: ManagedAgentTerminationRequest, + ) => ManagedAgentTerminationRequestOutcome | undefined; + /** Simulate one role's channel write; undefined uses the real retained socket. */ + readonly testOnlyWriteToolTermination?: ( + role: ToolProcessRole, + ) => Exclude | undefined; + /** Read-only test telemetry emitted after a channel request is attempted. */ + readonly onTerminationRequest?: ( + request: ManagedAgentTerminationRequest, + outcome: ManagedAgentTerminationRequestOutcome, + ) => void; + readonly monotonicNow?: () => number; + readonly delay?: (milliseconds: number) => Promise; +} + +interface OwnedRoot { + readonly pid: number; + readonly child: ChildProcessWithoutNullStreams; + identity?: ManagedAgentKernelProcessRecord; + containmentSupported: boolean; + ownershipProven: boolean; + forceKillIssued: boolean; +} + +type ToolProcessRole = "parent" | "child"; + +interface ToolProcessRegistration { + readonly role: ToolProcessRole; + readonly pid: number; + readonly socket: NetSocket; + accepted: boolean; + closed: boolean; + identity?: ManagedAgentKernelProcessRecord; +} + +interface ObservedIdentity { + readonly rootPid: number; + readonly record: ManagedAgentKernelProcessRecord; +} + +interface PendingUnauthenticatedSubgroup { + readonly key: string; + readonly rootPid: number; + readonly rootIdentity: ManagedAgentKernelProcessRecord; + readonly processGroupId: number; + readonly sessionId: number; + readonly members: Map< + string, + { readonly pid: number; readonly record: ManagedAgentKernelProcessRecord } + >; +} + +interface ProcessSampleTask { + readonly token: symbol; + readonly generation: number; + readonly lifecycleEpoch: number; + readonly promise: Promise; +} + +function defaultDelay(milliseconds: number): Promise { + return new Promise((resolveDelay) => setTimeout(resolveDelay, milliseconds)); +} + +function sameProcess( + left: ManagedAgentKernelProcessRecord | undefined, + right: ManagedAgentKernelProcessRecord | undefined, +): boolean { + return Boolean(left && right && left.startedAt === right.startedAt); +} + +function processIsZombie( + record: ManagedAgentKernelProcessRecord | undefined, +): boolean { + return record?.state?.startsWith("Z") ?? false; +} + +function sameCapability(left: string, right: string): boolean { + const leftBytes = Buffer.from(left, "utf8"); + const rightBytes = Buffer.from(right, "utf8"); + return ( + leftBytes.byteLength === rightBytes.byteLength && + timingSafeEqual(leftBytes, rightBytes) + ); +} + +async function windowsProcessTable(): Promise { + const { stdout } = await execFileAsync( + "powershell.exe", + [ + "-NoProfile", + "-NonInteractive", + "-Command", + "Get-CimInstance Win32_Process | Select-Object ProcessId,ParentProcessId,CreationDate | ConvertTo-Json -Compress", + ], + { + encoding: "utf8", + windowsHide: true, + maxBuffer: 4 * 1024 * 1024, + timeout: MANAGED_AGENT_PROCESS_HELPER_TIMEOUT_MS, + killSignal: "SIGKILL", + }, + ); + const parsed = JSON.parse(stdout) as + | { + ProcessId?: unknown; + ParentProcessId?: unknown; + CreationDate?: unknown; + } + | Array<{ + ProcessId?: unknown; + ParentProcessId?: unknown; + CreationDate?: unknown; + }>; + const rows = Array.isArray(parsed) ? parsed : [parsed]; + return new Map( + rows.flatMap((row) => + typeof row.ProcessId === "number" && + typeof row.ParentProcessId === "number" && + typeof row.CreationDate === "string" + ? [ + [ + row.ProcessId, + { + parentPid: row.ParentProcessId, + startedAt: row.CreationDate, + }, + ] as const, + ] + : [], + ), + ); +} + +export function managedAgentPosixSessionColumn( + platform: NodeJS.Platform, +): "sess" | "sid" { + return platform === "darwin" ? "sess" : "sid"; +} + +export function parseManagedAgentPosixProcessTable( + stdout: string, +): ManagedAgentKernelProcessTable { + const entries: Array = []; + for (const line of stdout.split("\n")) { + const match = /^\s*(\d+)\s+(\d+)\s+(\d+)\s+(\d+)\s+(\S+)\s+(.+?)\s*$/.exec( + line, + ); + if (!match) continue; + entries.push([ + Number(match[1]), + { + parentPid: Number(match[2]), + processGroupId: Number(match[3]), + sessionId: Number(match[4]), + state: match[5]!, + startedAt: match[6]!, + }, + ]); + } + return new Map(entries); +} + +async function posixProcessTable( + platform: NodeJS.Platform, +): Promise { + const sessionColumn = managedAgentPosixSessionColumn(platform); + const { stdout } = await execFileAsync( + "/bin/ps", + ["-axo", `pid=,ppid=,pgid=,${sessionColumn}=,stat=,lstart=`], + { + encoding: "utf8", + windowsHide: true, + maxBuffer: 4 * 1024 * 1024, + timeout: MANAGED_AGENT_PROCESS_HELPER_TIMEOUT_MS, + killSignal: "SIGKILL", + }, + ); + return parseManagedAgentPosixProcessTable(stdout); +} + +async function defaultReadProcessTable( + platform: NodeJS.Platform, +): Promise { + try { + return { + available: true, + processes: + platform === "win32" + ? await windowsProcessTable() + : await posixProcessTable(platform), + }; + } catch { + return { available: false }; + } +} + +function defaultProcessGroupLiveness( + processGroupId: number, +): ManagedAgentProcessGroupLiveness { + try { + process.kill(-processGroupId, 0); + return "alive"; + } catch (error) { + const code = (error as NodeJS.ErrnoException).code; + if (code === "ESRCH") return "gone"; + if (code === "EPERM") return "alive"; + return "unknown"; + } +} + +function childActive(child: ChildProcessWithoutNullStreams): boolean { + return child.exitCode === null && child.signalCode === null; +} + +function descendantsOf( + roots: ReadonlySet, + table: ManagedAgentKernelProcessTable, +): Set { + const descendants = new Set(); + let changed = true; + while (changed) { + changed = false; + for (const [pid, record] of table) { + if ( + !descendants.has(pid) && + (roots.has(record.parentPid) || descendants.has(record.parentPid)) + ) { + descendants.add(pid); + changed = true; + } + } + } + return descendants; +} + +/** + * E0.4 deliberately certifies one narrow containment model. The SDK command + * runs in an observer-owned POSIX process group. The exact host-created L2 + * fixture parent and child additionally authenticate over separate private + * Unix-socket connections outside the workspace and keep those connections + * open for their complete lifetimes. A random capability, a primary exact-Bash + * policy latch, stable role identities, and fresh kernel ancestry prove that + * every member of their detached group remains below the active owned root. + * A tool-reported or cached PID/PGID never grants signal authority by itself. + * + * This is not universal built-in Bash containment or a process-tree killer. + * Windows, an unavailable process table, missing lifetime channels, or + * identity/ancestry drift fail certification closed. The fallback freshly + * validates the exact fixture group, asks a still-open authenticated member to + * terminate its own current group, proves it absent with a new sample, then + * asks the owned supervisor over retained IPC to terminate its own group. + * POSIX `lstart` remains evidence only: a sampled numeric PID/PGID is never + * host signal authority. Workspace PID-file contents never enter this class. + */ +export class LocalManagedAgentProcessObserver implements ManagedAgentProcessObserver { + readonly #platform: NodeJS.Platform; + readonly #readProcessTable: () => Promise; + readonly #processGroupLiveness: ( + processGroupId: number, + ) => ManagedAgentProcessGroupLiveness; + readonly #testOnlyRequestTermination: + | (( + processGroupId: number, + signal: "SIGKILL", + target: ManagedAgentTerminationRequest["target"], + ) => ManagedAgentTerminationRequestOutcome) + | undefined; + readonly #testOnlyBeforeTerminationRequest: + | (( + request: ManagedAgentTerminationRequest, + ) => ManagedAgentTerminationRequestOutcome | undefined) + | undefined; + readonly #testOnlyWriteToolTermination: + | (( + role: ToolProcessRole, + ) => Exclude | undefined) + | undefined; + readonly #onTerminationRequest: + | (( + request: ManagedAgentTerminationRequest, + outcome: ManagedAgentTerminationRequestOutcome, + ) => void) + | undefined; + readonly #monotonicNow: () => number; + readonly #delay: (milliseconds: number) => Promise; + readonly #roots = new Map(); + readonly #observedIdentities = new Map(); + // An unarmed SDK may create a short-lived subgroup below the owned root. + // Track the subgroup identity and every member generation, not merely its + // leader: leader exit/reparenting must never make a surviving member vanish. + // These records only block readiness/root kill and never grant signal power. + readonly #pendingUnauthenticatedSubgroups = new Map< + string, + PendingUnauthenticatedSubgroup + >(); + readonly #observedPids = new Set(); + readonly #sampler: NodeJS.Timeout; + readonly #boundSignals = new WeakSet(); + readonly #toolControlCapability = randomBytes(32).toString("base64url"); + readonly #toolControlSockets = new Set(); + #toolControlDirectory: string | undefined; + #toolControlSocketPath: string | undefined; + #toolControlServer: NetServer | undefined; + #toolControlAvailable = false; + #toolControlFailed = false; + #toolProcessContainmentArmed = false; + readonly #toolProcessRegistrations = new Map< + ToolProcessRole, + ToolProcessRegistration + >(); + #toolProcessGroupId: number | undefined; + #toolProcessRootPid: number | undefined; + #toolProcessObservationComplete = false; + #toolProcessObservationInvalid = false; + #toolProcessForceKillIssued = false; + #fallbackCleanupRequested = false; + #lastTable: ManagedAgentKernelProcessTable | undefined; + #processTableAvailable = false; + #processTableNeedsRefresh = false; + #sampleGeneration = 0; + #lifecycleEpoch = 0; + #sealed = false; + #hostDisposing = false; + #abortObserved = false; + #teardownDeadline: ManagedAgentTeardownDeadline | undefined; + #sampleTask: ProcessSampleTask | undefined; + #disposeTask: Promise | undefined; + + public constructor(options: LocalManagedAgentProcessObserverOptions = {}) { + this.#platform = options.platform ?? process.platform; + this.#readProcessTable = + options.readProcessTable ?? + (() => defaultReadProcessTable(this.#platform)); + this.#processGroupLiveness = + options.processGroupLiveness ?? defaultProcessGroupLiveness; + this.#testOnlyRequestTermination = options.testOnlyRequestTermination; + this.#testOnlyBeforeTerminationRequest = + options.testOnlyBeforeTerminationRequest; + this.#testOnlyWriteToolTermination = options.testOnlyWriteToolTermination; + this.#onTerminationRequest = options.onTerminationRequest; + this.#monotonicNow = options.monotonicNow ?? (() => performance.now()); + this.#delay = options.delay ?? defaultDelay; + if (this.#platform === "darwin" || this.#platform === "linux") { + this.#startToolControlServer(); + } + this.#sampler = setInterval( + () => void this.observeProcessTree(), + SAMPLE_INTERVAL_MS, + ); + this.#sampler.unref(); + } + + #adoptDeadline( + deadline: ManagedAgentTeardownDeadline, + ): ManagedAgentTeardownDeadline { + if ( + !Number.isFinite(deadline.startedAtMs) || + !Number.isFinite(deadline.deadlineAtMs) || + deadline.deadlineAtMs < deadline.startedAtMs + ) { + throw new Error("managed-agent teardown deadline is invalid"); + } + if (!this.#teardownDeadline) { + this.#teardownDeadline = Object.freeze({ ...deadline }); + } else if ( + deadline.startedAtMs !== this.#teardownDeadline.startedAtMs || + deadline.deadlineAtMs !== this.#teardownDeadline.deadlineAtMs + ) { + // A later caller may not reset or extend the one teardown deadline. + throw new Error("managed-agent teardown deadline changed after adoption"); + } + return this.#teardownDeadline; + } + + public beginTeardown(deadline: ManagedAgentTeardownDeadline): void { + this.#adoptDeadline(deadline); + if (this.#abortObserved) this.#requestFallbackCleanupSynchronously(); + } + + #remainingMs(deadline: ManagedAgentTeardownDeadline): number { + return Math.max(0, deadline.deadlineAtMs - this.#monotonicNow()); + } + + #deadlineExpiredAndSeal(): boolean { + if ( + !this.#teardownDeadline || + this.#monotonicNow() < this.#teardownDeadline.deadlineAtMs + ) { + return false; + } + this.#seal(); + return true; + } + + #seal(): void { + if (this.#sealed) return; + this.#sealed = true; + this.#lifecycleEpoch += 1; + clearInterval(this.#sampler); + } + + #startToolControlServer(): void { + try { + const directory = mkdtempSync( + join(tmpdir(), "sapiom-managed-agent-control-"), + ); + chmodSync(directory, 0o700); + const socketPath = join(directory, "tool.sock"); + const server = createServer((socket) => + this.#receiveToolRegistration(socket), + ); + this.#toolControlDirectory = directory; + this.#toolControlSocketPath = socketPath; + this.#toolControlServer = server; + server.once("listening", () => { + this.#toolControlAvailable = true; + }); + server.on("error", () => { + this.#toolControlAvailable = false; + this.#toolControlFailed = true; + }); + server.listen(socketPath); + server.unref(); + } catch { + this.#toolControlAvailable = false; + this.#toolControlFailed = true; + } + } + + #receiveToolRegistration(socket: NetSocket): void { + if (this.#sealed || this.#deadlineExpiredAndSeal()) { + socket.destroy(); + return; + } + const lifecycleEpoch = this.#lifecycleEpoch; + this.#toolControlSockets.add(socket); + socket.on("error", () => undefined); + let registration: ToolProcessRegistration | undefined; + socket.once("close", () => { + this.#toolControlSockets.delete(socket); + if ( + this.#sealed || + this.#deadlineExpiredAndSeal() || + this.#hostDisposing || + lifecycleEpoch !== this.#lifecycleEpoch + ) { + return; + } + if (!registration) return; + const current = this.#toolProcessRegistrations.get(registration.role); + if (current !== registration) return; + if (this.#toolProcessObservationComplete) { + registration.closed = true; + } else { + // A connection that disappears before readiness cannot reserve its + // role. Clearing it transactionally permits the trusted process to + // retry instead of leaving an unfinishable stale pending state. + this.#toolProcessRegistrations.delete(registration.role); + } + void this.observeProcessTree(); + }); + let body = ""; + let handled = false; + const reject = (): void => { + handled = true; + socket.destroy(); + }; + socket.on("data", (chunk: Buffer) => { + if ( + handled || + this.#sealed || + this.#deadlineExpiredAndSeal() || + lifecycleEpoch !== this.#lifecycleEpoch + ) { + if (this.#sealed) socket.destroy(); + return; + } + body += chunk.toString("utf8"); + if (Buffer.byteLength(body, "utf8") > TOOL_REGISTRATION_MAX_BYTES) { + reject(); + return; + } + const newline = body.indexOf("\n"); + if (newline < 0) return; + handled = true; + let payload: { + capability?: unknown; + pid?: unknown; + role?: unknown; + }; + try { + payload = JSON.parse(body.slice(0, newline)) as typeof payload; + } catch { + socket.destroy(); + return; + } + if ( + !this.#toolProcessContainmentArmed || + this.#toolProcessObservationComplete || + typeof payload.capability !== "string" || + !sameCapability(payload.capability, this.#toolControlCapability) || + (payload.role !== "parent" && payload.role !== "child") || + this.#toolProcessRegistrations.has(payload.role) || + typeof payload.pid !== "number" || + !Number.isSafeInteger(payload.pid) || + payload.pid <= 1 + ) { + socket.destroy(); + return; + } + registration = { + role: payload.role, + pid: payload.pid, + socket, + accepted: false, + closed: false, + }; + this.#toolProcessRegistrations.set(payload.role, registration); + void this.observeProcessTree(); + }); + } + + #bindAbortSignal(signal: AbortSignal): void { + if (this.#boundSignals.has(signal)) return; + this.#boundSignals.add(signal); + signal.addEventListener( + "abort", + () => { + this.#abortObserved = true; + // The SDK may forward its private signal before runtime enters its + // teardown path. Remember it, but never signal from an unbounded + // window; beginTeardown() will synchronously replay the request. + if (!this.#teardownDeadline) return; + if (this.#sealed || this.#deadlineExpiredAndSeal()) return; + this.#requestFallbackCleanupSynchronously(); + }, + { once: true }, + ); + if (signal.aborted) { + this.#abortObserved = true; + if (this.#teardownDeadline) this.#requestFallbackCleanupSynchronously(); + } + } + + public armToolProcessContainment(): void { + if (this.#sealed || this.#deadlineExpiredAndSeal()) return; + if (this.#toolProcessContainmentArmed) return; + this.#toolProcessContainmentArmed = true; + if (this.#toolControlFailed) { + for (const root of this.#roots.values()) { + this.#invalidateRootContainment(root); + } + } + } + + /** + * Simulates abrupt host loss without terminating the Vitest process. The + * exact fixture must treat authenticated lifetime-channel loss as a + * fail-closed instruction to terminate its own process group. + */ + public testOnlyDropToolLifetimeChannels(): void { + for (const socket of this.#toolControlSockets) socket.destroy(); + } + + #invalidateRootContainment(root: OwnedRoot): void { + root.containmentSupported = false; + } + + #invalidateToolContainment(): void { + this.#toolProcessObservationInvalid = true; + } + + #hasPendingUnauthenticatedDescendants(rootPid: number): boolean { + return [...this.#pendingUnauthenticatedSubgroups.values()].some( + (subgroup) => subgroup.rootPid === rootPid, + ); + } + + #subgroupKey( + rootPid: number, + rootIdentity: ManagedAgentKernelProcessRecord, + processGroupId: number, + sessionId: number, + ): string { + return JSON.stringify([ + rootPid, + rootIdentity.startedAt, + rootIdentity.parentPid, + rootIdentity.processGroupId, + rootIdentity.sessionId, + processGroupId, + sessionId, + ]); + } + + #memberKey(pid: number, record: ManagedAgentKernelProcessRecord): string { + return JSON.stringify([pid, record.startedAt]); + } + + #sameIdentityAndTopology( + expected: ManagedAgentKernelProcessRecord, + current: ManagedAgentKernelProcessRecord | undefined, + ): boolean { + return ( + sameProcess(expected, current) && + expected.parentPid === current!.parentPid && + expected.processGroupId === current!.processGroupId && + expected.sessionId === current!.sessionId + ); + } + + #expectedAfterAuthorizedGroupKill( + root: OwnedRoot, + observed: ManagedAgentKernelProcessRecord | undefined, + current: ManagedAgentKernelProcessRecord | undefined, + toolProcessGroupId: number | undefined, + ): boolean { + if (!sameProcess(observed, current)) return false; + if ( + observed?.processGroupId === root.pid && + current?.processGroupId === root.pid + ) { + return root.forceKillIssued; + } + return ( + typeof toolProcessGroupId === "number" && + observed?.processGroupId === toolProcessGroupId && + current?.processGroupId === toolProcessGroupId && + this.#toolProcessForceKillIssued + ); + } + + public spawn(options: SpawnOptions): SpawnedProcess { + if ( + this.#sealed || + this.#hostDisposing || + this.#teardownDeadline || + this.#deadlineExpiredAndSeal() + ) { + throw new Error("managed-agent process observer is closed"); + } + const usePosixSupervisor = + this.#platform === "darwin" || this.#platform === "linux"; + const child = ( + usePosixSupervisor + ? spawnChild( + process.execPath, + [ + "--input-type=module", + "--eval", + MANAGED_AGENT_POSIX_SUPERVISOR_SOURCE, + ], + { + cwd: options.cwd, + env: { + ...options.env, + [MANAGED_AGENT_SUPERVISOR_PAYLOAD_ENV]: Buffer.from( + JSON.stringify({ + command: options.command, + args: options.args, + }), + "utf8", + ).toString("base64url"), + ...(this.#toolControlSocketPath + ? { + [MANAGED_AGENT_TOOL_CONTROL_SOCKET_ENV]: + this.#toolControlSocketPath, + [MANAGED_AGENT_TOOL_CONTROL_CAPABILITY_ENV]: + this.#toolControlCapability, + } + : {}), + }, + detached: true, + stdio: ["pipe", "pipe", "pipe", "ipc"], + windowsHide: true, + }, + ) + : spawnChild(options.command, options.args, { + cwd: options.cwd, + env: options.env, + detached: false, + stdio: ["pipe", "pipe", "pipe"], + windowsHide: true, + }) + ) as ChildProcessWithoutNullStreams; + // SpawnedProcess does not expose stderr to the SDK transport. Drain it + // here without retaining or printing content so a noisy inner command + // cannot deadlock the supervisor on pipe backpressure. + child.stderr.on("data", () => undefined); + child.stderr.on("error", () => undefined); + if (typeof child.pid === "number") { + const pid = child.pid; + this.#roots.set(pid, { + pid, + child, + containmentSupported: true, + ownershipProven: false, + forceKillIssued: false, + }); + this.#observedPids.add(pid); + // The SDK's forwarded SpawnOptions.signal arrives only after its own + // graceful close. Keep it as an idempotent fallback; runtime deliberately + // does not bind the raw Options.abortController to host process signals. + this.#bindAbortSignal(options.signal); + void this.observeProcessTree(); + } + return child; + } + + async #boundedProcessTableRead( + timeoutMs: number, + ): Promise { + let timeout: NodeJS.Timeout | undefined; + const read = Promise.resolve() + .then(() => this.#readProcessTable()) + .catch( + (): ManagedAgentProcessTableObservation => ({ + available: false, + }), + ); + try { + return await Promise.race([ + read, + new Promise((resolveTimeout) => { + timeout = setTimeout( + () => resolveTimeout({ available: false }), + Math.max(0, timeoutMs), + ); + }), + ]); + } finally { + if (timeout) clearTimeout(timeout); + } + } + + #observeToolProcessContainment(table: ManagedAgentKernelProcessTable): void { + if (!this.#toolProcessContainmentArmed) return; + const parent = this.#toolProcessRegistrations.get("parent"); + const child = this.#toolProcessRegistrations.get("child"); + if ( + !this.#toolProcessObservationComplete && + parent && + child && + !parent.socket.destroyed && + !child.socket.destroyed + ) { + const parentIdentity = table.get(parent.pid); + const childIdentity = table.get(child.pid); + const processGroupId = parentIdentity?.processGroupId; + const hostProcessGroupId = table.get(process.pid)?.processGroupId; + const groupLeaderIdentity = + typeof processGroupId === "number" + ? table.get(processGroupId) + : undefined; + const root = + this.#roots.size === 1 ? [...this.#roots.values()][0] : undefined; + const rootDescendants = root + ? descendantsOf(new Set([root.pid]), table) + : new Set(); + const groupMemberPids = + typeof processGroupId === "number" + ? [...table.entries()].flatMap(([pid, record]) => + record.processGroupId === processGroupId && + !processIsZombie(record) + ? [pid] + : [], + ) + : []; + if ( + root && + childActive(root.child) && + table.get(root.pid)?.processGroupId === root.pid && + !processIsZombie(table.get(root.pid)) && + parentIdentity && + !processIsZombie(parentIdentity) && + childIdentity && + !processIsZombie(childIdentity) && + typeof processGroupId === "number" && + typeof hostProcessGroupId === "number" && + processGroupId > 1 && + processGroupId !== hostProcessGroupId && + !this.#roots.has(processGroupId) && + parent.pid !== child.pid && + childIdentity.parentPid === parent.pid && + childIdentity.processGroupId === processGroupId && + groupLeaderIdentity?.processGroupId === processGroupId && + !processIsZombie(groupLeaderIdentity) && + groupMemberPids.length > 0 && + groupMemberPids.every((pid) => rootDescendants.has(pid)) + ) { + parent.identity = parentIdentity; + child.identity = childIdentity; + this.#toolProcessGroupId = processGroupId; + this.#toolProcessRootPid = root.pid; + for (const registration of [parent, child]) { + if (registration.accepted) continue; + registration.accepted = true; + registration.socket.write('{"registered":true}\n'); + } + } + } + + const processGroupId = this.#toolProcessGroupId; + if (typeof processGroupId === "number") { + for (const [pid, record] of table) { + if (record.processGroupId === processGroupId) { + this.#observedPids.add(pid); + } + } + } + if (!this.#toolProcessObservationComplete) return; + for (const registration of this.#toolProcessRegistrations.values()) { + const current = table.get(registration.pid); + if ( + !registration.closed && + current && + !processIsZombie(current) && + !this.#toolProcessForceKillIssued && + (!sameProcess(registration.identity, current) || + current.processGroupId !== processGroupId) + ) { + this.#invalidateToolContainment(); + } + } + } + + #hasFreshToolAuthority(table: ManagedAgentKernelProcessTable): boolean { + const rootPid = this.#toolProcessRootPid; + const processGroupId = this.#toolProcessGroupId; + const root = + typeof rootPid === "number" ? this.#roots.get(rootPid) : undefined; + const parent = this.#toolProcessRegistrations.get("parent"); + const child = this.#toolProcessRegistrations.get("child"); + if ( + !root || + !childActive(root.child) || + !root.containmentSupported || + this.#processTableNeedsRefresh || + this.#toolProcessObservationInvalid || + typeof processGroupId !== "number" || + !parent?.accepted || + !parent.identity || + !child?.accepted || + !child.identity || + ![parent, child].some( + ({ closed, socket }) => !closed && !socket.destroyed, + ) + ) { + return false; + } + + const currentRoot = table.get(root.pid); + const currentParent = table.get(parent.pid); + const currentChild = table.get(child.pid); + if ( + !currentRoot || + processIsZombie(currentRoot) || + !currentParent || + processIsZombie(currentParent) || + !currentChild || + processIsZombie(currentChild) || + currentRoot.processGroupId !== root.pid || + (root.identity && !sameProcess(root.identity, currentRoot)) || + (root.identity && root.identity.sessionId !== currentRoot.sessionId) || + !sameProcess(parent.identity, currentParent) || + currentParent.processGroupId !== processGroupId || + parent.identity.sessionId !== currentParent.sessionId || + !sameProcess(child.identity, currentChild) || + currentChild.parentPid !== parent.pid || + currentChild.processGroupId !== processGroupId || + child.identity.sessionId !== currentChild.sessionId + ) { + return false; + } + + const rootDescendants = descendantsOf(new Set([root.pid]), table); + const allowedGroups = new Set([root.pid, processGroupId]); + if ( + [...rootDescendants].some((pid) => { + const record = table.get(pid); + return !record || !allowedGroups.has(record.processGroupId ?? -1); + }) + ) { + return false; + } + const groupMembers = [...table.entries()].filter( + ([, record]) => + record.processGroupId === processGroupId && !processIsZombie(record), + ); + return ( + groupMembers.length > 0 && + groupMembers.every(([pid]) => rootDescendants.has(pid)) + ); + } + + #rememberPendingSubgroupMembers( + subgroup: PendingUnauthenticatedSubgroup, + root: OwnedRoot, + table: ManagedAgentKernelProcessTable, + rootDescendants: ReadonlySet, + ): void { + for (const [pid, record] of table) { + if ( + processIsZombie(record) || + record.processGroupId !== subgroup.processGroupId || + record.sessionId !== subgroup.sessionId + ) { + continue; + } + const key = this.#memberKey(pid, record); + const existing = subgroup.members.get(key); + if (!existing) subgroup.members.set(key, { pid, record }); + this.#observedPids.add(pid); + if (!rootDescendants.has(pid)) { + this.#invalidateRootContainment(root); + } + if (existing && !this.#sameIdentityAndTopology(existing.record, record)) { + this.#invalidateRootContainment(root); + } + } + } + + #observePendingUnauthenticatedSubgroups( + root: OwnedRoot, + table: ManagedAgentKernelProcessTable, + rootDescendants: ReadonlySet, + ): Set { + const liveMemberPids = new Set(); + for (const [key, subgroup] of this.#pendingUnauthenticatedSubgroups) { + if (subgroup.rootPid !== root.pid) continue; + const currentRoot = table.get(root.pid); + if (!this.#sameIdentityAndTopology(subgroup.rootIdentity, currentRoot)) { + // The root identity is part of the pending subgroup's immutable + // provenance. Losing it while the subgroup is unresolved is sticky. + this.#invalidateRootContainment(root); + } + this.#rememberPendingSubgroupMembers( + subgroup, + root, + table, + rootDescendants, + ); + + let rememberedMemberAlive = false; + for (const member of subgroup.members.values()) { + const current = table.get(member.pid); + if (!sameProcess(member.record, current) || processIsZombie(current)) { + continue; + } + rememberedMemberAlive = true; + liveMemberPids.add(member.pid); + if ( + !this.#sameIdentityAndTopology(member.record, current) || + !rootDescendants.has(member.pid) + ) { + this.#invalidateRootContainment(root); + } + } + const currentGroupMembers = [...table.entries()].filter( + ([, record]) => + !processIsZombie(record) && + record.processGroupId === subgroup.processGroupId && + record.sessionId === subgroup.sessionId, + ); + for (const [pid] of currentGroupMembers) liveMemberPids.add(pid); + if (!rememberedMemberAlive && currentGroupMembers.length === 0) { + // Only a complete, authoritative sample with every remembered member + // and every replacement group member absent can clear the blocker. + this.#pendingUnauthenticatedSubgroups.delete(key); + } + } + return liveMemberPids; + } + + #registerPendingUnauthenticatedSubgroup( + root: OwnedRoot, + table: ManagedAgentKernelProcessTable, + rootDescendants: ReadonlySet, + record: ManagedAgentKernelProcessRecord, + ): void { + const rootIdentity = table.get(root.pid); + const processGroupId = record.processGroupId; + const sessionId = record.sessionId; + if ( + !rootIdentity || + processIsZombie(rootIdentity) || + typeof processGroupId !== "number" || + typeof sessionId !== "number" + ) { + this.#invalidateRootContainment(root); + return; + } + const key = this.#subgroupKey( + root.pid, + rootIdentity, + processGroupId, + sessionId, + ); + let subgroup = this.#pendingUnauthenticatedSubgroups.get(key); + if (!subgroup) { + subgroup = { + key, + rootPid: root.pid, + rootIdentity, + processGroupId, + sessionId, + members: new Map(), + }; + this.#pendingUnauthenticatedSubgroups.set(key, subgroup); + } + this.#rememberPendingSubgroupMembers( + subgroup, + root, + table, + rootDescendants, + ); + } + + #observePosixOwnedProcesses(table: ManagedAgentKernelProcessTable): void { + for (const root of this.#roots.values()) { + const rootDescendants = descendantsOf(new Set([root.pid]), table); + const pendingMemberPids = this.#observePendingUnauthenticatedSubgroups( + root, + table, + rootDescendants, + ); + // Continue observing children below every pending member even if its + // original leader exits between complete samples. + const descendants = descendantsOf( + new Set([root.pid, ...pendingMemberPids]), + table, + ); + const toolProcessGroupId = + this.#toolProcessRootPid === root.pid + ? this.#toolProcessGroupId + : undefined; + const allowedGroups = new Set([root.pid]); + if (typeof toolProcessGroupId === "number") { + allowedGroups.add(toolProcessGroupId); + } + const validateAllowedGroups = + !this.#toolProcessContainmentArmed || + typeof toolProcessGroupId === "number"; + const currentlyOwned = new Set([root.pid, ...descendants]); + + for (const [pid, record] of table) { + if (processIsZombie(record)) continue; + if ( + record.processGroupId !== root.pid && + record.processGroupId !== toolProcessGroupId + ) { + continue; + } + currentlyOwned.add(pid); + if (pid !== root.pid && !descendants.has(pid)) { + const observed = this.#observedIdentities.get(pid); + const expectedAfterAuthorizedKill = + this.#expectedAfterAuthorizedGroupKill( + root, + observed?.record, + record, + toolProcessGroupId, + ); + if (expectedAfterAuthorizedKill) continue; + if (record.processGroupId === toolProcessGroupId) { + this.#invalidateToolContainment(); + } else { + this.#invalidateRootContainment(root); + } + } + } + + for (const [pid, observed] of this.#observedIdentities) { + if (observed.rootPid !== root.pid) continue; + const current = table.get(pid); + // A successful complete process-table sample with no matching stable + // identity is positive evidence that the old process has exited. A + // recycled numeric PID never inherits the old observation. + // A kernel zombie is already dead and cannot execute, migrate, or + // authorize a signal. Its transient reparenting during reap is not a + // live containment escape. + if ( + !sameProcess(observed.record, current) || + processIsZombie(current) + ) { + continue; + } + this.#observedPids.add(pid); + if ( + current!.parentPid !== observed.record.parentPid || + current!.processGroupId !== observed.record.processGroupId || + current!.sessionId !== observed.record.sessionId || + (pid !== root.pid && !currentlyOwned.has(pid)) + ) { + const belongsToToolGroup = + typeof toolProcessGroupId === "number" && + (observed.record.processGroupId === toolProcessGroupId || + current!.processGroupId === toolProcessGroupId); + const expectedAfterAuthorizedKill = + this.#expectedAfterAuthorizedGroupKill( + root, + observed.record, + current, + toolProcessGroupId, + ); + if (expectedAfterAuthorizedKill) { + continue; + } + if (belongsToToolGroup) { + this.#invalidateToolContainment(); + } else { + this.#invalidateRootContainment(root); + } + } + } + + for (const pid of currentlyOwned) { + const current = table.get(pid); + if (!current || processIsZombie(current)) continue; + const isDescendant = descendants.has(pid); + const escapedOwnedAncestry = pid !== root.pid && !isDescendant; + const unauthenticatedDescendant = + validateAllowedGroups && + isDescendant && + !allowedGroups.has(current.processGroupId ?? -1); + if (escapedOwnedAncestry || unauthenticatedDescendant) { + const observed = this.#observedIdentities.get(pid); + const belongsToToolGroup = + typeof toolProcessGroupId === "number" && + (current.processGroupId === toolProcessGroupId || + observed?.record.processGroupId === toolProcessGroupId); + if ( + unauthenticatedDescendant && + !this.#toolProcessContainmentArmed && + (!observed || sameProcess(observed.record, current)) + ) { + this.#registerPendingUnauthenticatedSubgroup( + root, + table, + rootDescendants, + current, + ); + } else { + const expectedAfterAuthorizedKill = + this.#expectedAfterAuthorizedGroupKill( + root, + observed?.record, + current, + toolProcessGroupId, + ); + if (expectedAfterAuthorizedKill) continue; + if (belongsToToolGroup) { + this.#invalidateToolContainment(); + } else { + this.#invalidateRootContainment(root); + } + } + } + const observed = this.#observedIdentities.get(pid); + if (!observed || !sameProcess(observed.record, current)) { + this.#observedIdentities.set(pid, { + rootPid: root.pid, + record: current, + }); + } + this.#observedPids.add(pid); + } + } + } + + public async observeProcessTree( + deadline?: ManagedAgentTeardownDeadline, + ): Promise { + const activeDeadline = deadline + ? this.#adoptDeadline(deadline) + : this.#teardownDeadline; + if (this.#sealed) return false; + if (activeDeadline && this.#remainingMs(activeDeadline) <= 0) { + this.#seal(); + return false; + } + if (this.#roots.size === 0 && !this.#toolProcessContainmentArmed) { + this.#lastTable = new Map(); + this.#processTableAvailable = true; + return true; + } + const boundedTimeoutMs = Math.max( + 0, + Math.min( + MANAGED_AGENT_PROCESS_HELPER_TIMEOUT_MS, + activeDeadline + ? this.#remainingMs(activeDeadline) + : MANAGED_AGENT_PROCESS_HELPER_TIMEOUT_MS, + ), + ); + const generation = this.#sampleGeneration; + const lifecycleEpoch = this.#lifecycleEpoch; + const reusableSample = + this.#sampleTask?.generation === generation && + this.#sampleTask.lifecycleEpoch === lifecycleEpoch + ? this.#sampleTask + : undefined; + if (!reusableSample) { + const sampleToken = Symbol("managed-agent-process-sample"); + const promise = (async () => { + const observation = + await this.#boundedProcessTableRead(boundedTimeoutMs); + // A deadline can be adopted while a background sample is already in + // flight. Consult the current observer deadline at completion so that + // such a sample cannot install evidence after the newly adopted bound. + const completionDeadline = this.#teardownDeadline ?? activeDeadline; + const completedBeforeDeadline = completionDeadline + ? this.#monotonicNow() < completionDeadline.deadlineAtMs + : true; + if ( + this.#sealed || + lifecycleEpoch !== this.#lifecycleEpoch || + generation !== this.#sampleGeneration || + !completedBeforeDeadline + ) { + if (!completedBeforeDeadline) this.#seal(); + return false; + } + if (!observation.available) { + if ( + boundedTimeoutMs > 0 || + this.#processTableNeedsRefresh || + !this.#processTableAvailable + ) { + this.#lastTable = undefined; + this.#processTableAvailable = false; + } + return false; + } + + const table = observation.processes; + this.#lastTable = table; + this.#processTableAvailable = true; + this.#processTableNeedsRefresh = false; + for (const root of this.#roots.values()) { + if (this.#platform !== "win32") { + const currentRoot = table.get(root.pid); + if ( + currentRoot && + !processIsZombie(currentRoot) && + childActive(root.child) && + !root.forceKillIssued && + currentRoot.processGroupId !== root.pid + ) { + this.#invalidateRootContainment(root); + } + continue; + } + + const currentRoot = table.get(root.pid); + const seeds = new Set(); + if (currentRoot && childActive(root.child)) { + seeds.add(root.pid); + this.#observedIdentities.set(root.pid, { + rootPid: root.pid, + record: currentRoot, + }); + this.#observedPids.add(root.pid); + } + for (const [pid, observed] of this.#observedIdentities) { + if ( + observed.rootPid === root.pid && + sameProcess(observed.record, table.get(pid)) + ) { + seeds.add(pid); + } + } + for (const pid of descendantsOf(seeds, table)) { + const current = table.get(pid); + if (!current) continue; + this.#observedIdentities.set(pid, { + rootPid: root.pid, + record: current, + }); + this.#observedPids.add(pid); + } + } + this.#observeToolProcessContainment(table); + if (this.#platform !== "win32") { + this.#observePosixOwnedProcesses(table); + } + // Query.return() can remain pending while the SDK performs its own + // shutdown. Advance a requested fallback from each authoritative + // sample so the detached tool group is killed and then confirmed gone + // before the supervisor anchor is killed last, all within the same + // deadline. + this.#advanceFallbackCleanup(); + return true; + })().finally(() => { + if (this.#sampleTask?.token === sampleToken) { + this.#sampleTask = undefined; + } + }); + const sampleState: ProcessSampleTask = { + token: sampleToken, + generation, + lifecycleEpoch, + promise, + }; + this.#sampleTask = sampleState; + } + + const sample = (reusableSample ?? this.#sampleTask)!.promise; + let timeout: NodeJS.Timeout | undefined; + const available = await Promise.race([ + sample, + new Promise((resolveTimeout) => { + timeout = setTimeout(() => resolveTimeout(false), boundedTimeoutMs); + }), + ]); + if (timeout) clearTimeout(timeout); + if (!available && !this.#sealed) { + // A caller with a shorter absolute deadline must not reuse a stale table + // while a longer background sample is still pending. + if ( + boundedTimeoutMs > 0 || + this.#processTableNeedsRefresh || + !this.#processTableAvailable + ) { + this.#lastTable = undefined; + this.#processTableAvailable = false; + } else { + return true; + } + } + return available; + } + + public async prepareCancellation(): Promise { + const observedPids = (): number[] => + [...this.#observedPids].sort((left, right) => left - right); + const unsupported = ( + reason: Exclude, + ): ManagedAgentCancellationReadiness => ({ + supported: false, + reason, + processTableAvailable: this.#processTableAvailable, + containmentSupported: + [...this.#roots.values()].every( + ({ pid, containmentSupported }) => + containmentSupported && + !this.#hasPendingUnauthenticatedDescendants(pid), + ) && + (!this.#toolProcessContainmentArmed || + (this.#toolProcessRegistrations.size === 2 && + !this.#toolProcessObservationInvalid)), + ownershipProven: false, + observedPids: observedPids(), + }); + + if (this.#platform !== "darwin" && this.#platform !== "linux") { + for (const root of this.#roots.values()) { + this.#invalidateRootContainment(root); + } + return unsupported("platform_unsupported"); + } + if (!(await this.observeProcessTree())) { + return unsupported("process_table_unavailable"); + } + if (this.#roots.size !== 1) return unsupported("root_count_invalid"); + const root = [...this.#roots.values()][0]!; + if (!root.containmentSupported) { + return unsupported("containment_escaped"); + } + if (this.#hasPendingUnauthenticatedDescendants(root.pid)) { + return unsupported("containment_escaped"); + } + if (!childActive(root.child)) { + return unsupported("root_not_active"); + } + const currentRoot = this.#lastTable!.get(root.pid); + if (!currentRoot || currentRoot.processGroupId !== root.pid) { + this.#invalidateRootContainment(root); + return unsupported("root_not_group_leader"); + } + if (processIsZombie(currentRoot)) { + return unsupported("root_not_active"); + } + root.identity = currentRoot; + if (this.#toolProcessContainmentArmed) { + const parent = this.#toolProcessRegistrations.get("parent"); + const child = this.#toolProcessRegistrations.get("child"); + if ( + !this.#toolControlAvailable || + !parent?.accepted || + parent.closed || + parent.socket.destroyed || + !child?.accepted || + child.closed || + child.socket.destroyed || + typeof this.#toolProcessGroupId !== "number" + ) { + return unsupported("tool_process_not_registered"); + } + if (this.#toolProcessObservationInvalid) { + return unsupported("tool_process_identity_invalid"); + } + if (!this.#hasFreshToolAuthority(this.#lastTable!)) { + return unsupported("tool_process_identity_invalid"); + } + this.#toolProcessObservationComplete = true; + } + + root.ownershipProven = true; + return { + supported: true, + reason: "ready", + processTableAvailable: true, + containmentSupported: true, + ownershipProven: true, + observedPids: observedPids(), + }; + } + + #hasFreshRootGroupAuthority(root: OwnedRoot): boolean { + if (!root.containmentSupported || this.#processTableNeedsRefresh) { + return false; + } + // The observer-created group alone is not signal authority. A missing + // helper sample must fail closed because a cached numeric PGID can outlive + // its original leader and be reused before the ChildProcess exit event is + // delivered. + if (!this.#processTableAvailable) return false; + const current = this.#lastTable?.get(root.pid); + const baseline = + root.identity ?? this.#observedIdentities.get(root.pid)?.record; + if ( + !current || + processIsZombie(current) || + current.processGroupId !== root.pid + ) { + return false; + } + return baseline + ? sameProcess(baseline, current) && + current.parentPid === baseline.parentPid && + current.sessionId === baseline.sessionId + : true; + } + + #recordTerminationRequest( + request: ManagedAgentTerminationRequest, + outcome: ManagedAgentTerminationRequestOutcome, + ): ManagedAgentTerminationRequestOutcome { + this.#onTerminationRequest?.(request, outcome); + return outcome; + } + + #requestToolGroupTermination( + processGroupId: number, + ): ManagedAgentTerminationRequestOutcome { + if (this.#sealed || this.#deadlineExpiredAndSeal()) return "failure"; + const request = { target: "tool", processGroupId } as const; + const vetoedOutcome = this.#testOnlyBeforeTerminationRequest?.(request); + if (vetoedOutcome) { + return this.#recordTerminationRequest(request, vetoedOutcome); + } + if (this.#testOnlyRequestTermination) { + return this.#recordTerminationRequest( + request, + this.#testOnlyRequestTermination( + request.processGroupId, + "SIGKILL", + request.target, + ), + ); + } + const registrations = (["parent", "child"] as const) + .map((role) => this.#toolProcessRegistrations.get(role)) + .filter( + (candidate): candidate is ToolProcessRegistration => + candidate !== undefined && + candidate.accepted && + !candidate.closed && + !candidate.socket.destroyed && + candidate.socket.writable, + ); + if (registrations.length === 0) { + return this.#recordTerminationRequest(request, "failure"); + } + let sent = false; + for (const registration of registrations) { + const simulatedOutcome = this.#testOnlyWriteToolTermination?.( + registration.role, + ); + if (simulatedOutcome) { + sent ||= simulatedOutcome === "sent"; + continue; + } + try { + // This retained socket is bound to the authenticated process instance, + // not its numeric PID. The receiver calls kill(0, SIGKILL), so the + // still-running member terminates its own current group without a host + // snapshot-to-signal PGID reuse window. Broadcast to every live role: + // a stale peer cannot prevent its surviving group-mate from receiving + // the same idempotent self-group termination request. + registration.socket.write('{"forceKill":true}\n'); + sent = true; + } catch { + // Try every independently authenticated channel before failing closed. + } + } + return this.#recordTerminationRequest(request, sent ? "sent" : "failure"); + } + + #requestRootGroupTermination( + root: OwnedRoot, + ): ManagedAgentTerminationRequestOutcome { + if (this.#sealed || this.#deadlineExpiredAndSeal()) return "failure"; + const request = { target: "root", processGroupId: root.pid } as const; + const vetoedOutcome = this.#testOnlyBeforeTerminationRequest?.(request); + if (vetoedOutcome) { + return this.#recordTerminationRequest(request, vetoedOutcome); + } + if (this.#testOnlyRequestTermination) { + return this.#recordTerminationRequest( + request, + this.#testOnlyRequestTermination( + request.processGroupId, + "SIGKILL", + request.target, + ), + ); + } + if (!childActive(root.child)) { + return this.#recordTerminationRequest(request, "gone"); + } + if (!root.child.connected) { + return this.#recordTerminationRequest(request, "failure"); + } + try { + // The IPC endpoint belongs to the retained supervisor process instance. + // Its disconnect handler terminates its own current group. PID reuse can + // therefore make this request fail, but can never redirect it. + root.child.disconnect(); + return this.#recordTerminationRequest(request, "sent"); + } catch { + return this.#recordTerminationRequest(request, "failure"); + } + } + + #invalidateSampleAfterTerminationRequest(): void { + if (this.#sealed || this.#deadlineExpiredAndSeal()) return; + // Every request invalidates every sample that started before it, including + // channel failures. Only a complete read started in this new generation + // may prove the target gone or authorize the next teardown step. + this.#sampleGeneration += 1; + this.#processTableNeedsRefresh = true; + } + + #requestOwnedRootTerminationSynchronously(): void { + if (this.#sealed) return; + if (this.#platform !== "darwin" && this.#platform !== "linux") return; + for (const root of this.#roots.values()) { + if ( + root.forceKillIssued || + !childActive(root.child) || + this.#hasPendingUnauthenticatedDescendants(root.pid) || + !this.#hasFreshRootGroupAuthority(root) + ) { + continue; + } + const requestOutcome = this.#requestRootGroupTermination(root); + this.#invalidateSampleAfterTerminationRequest(); + root.forceKillIssued = requestOutcome === "sent"; + } + } + + #requestFallbackCleanupSynchronously(): void { + if (this.#sealed || this.#deadlineExpiredAndSeal()) return; + if (!this.#fallbackCleanupRequested) { + this.#fallbackCleanupRequested = true; + // The cleanup request itself is a lifecycle boundary. Discard any + // earlier sample so the first process-bound termination request can only + // follow a complete process-table read begun after teardown started. + this.#sampleGeneration += 1; + this.#processTableNeedsRefresh = true; + } + void this.observeProcessTree(this.#teardownDeadline); + } + + #advanceFallbackCleanup(): void { + if ( + this.#sealed || + this.#deadlineExpiredAndSeal() || + !this.#fallbackCleanupRequested || + this.#platform === "win32" || + !this.#processTableAvailable + ) { + return; + } + if (!this.#toolProcessContainmentArmed) { + this.#requestOwnedRootTerminationSynchronously(); + return; + } + // Once tool containment is armed, fail closed until the authenticated + // parent/child identities and their separate group are complete. Killing + // only the supervisor group could otherwise strand an unknown tool group. + if (!this.#toolProcessObservationComplete) return; + + const processGroupId = this.#toolProcessGroupId; + if (typeof processGroupId !== "number") return; + const table = this.#lastTable!; + const liveToolGroupMembers = [...table.values()].some( + (record) => + record.processGroupId === processGroupId && !processIsZombie(record), + ); + if (liveToolGroupMembers && this.#toolProcessForceKillIssued) return; + const groupLiveness = this.#processGroupLiveness(processGroupId); + if (groupLiveness === "gone") { + this.#requestOwnedRootTerminationSynchronously(); + return; + } + if (groupLiveness !== "alive") return; + + if (!this.#hasFreshToolAuthority(table)) { + return; + } + if (!this.#toolProcessForceKillIssued) { + const requestOutcome = this.#requestToolGroupTermination(processGroupId); + this.#invalidateSampleAfterTerminationRequest(); + this.#toolProcessForceKillIssued = requestOutcome === "sent"; + if (requestOutcome === "gone") { + this.#requestOwnedRootTerminationSynchronously(); + } + } + } + + #currentObservation( + startedAt: number, + emergencyCleanupAttempted: boolean, + ): ManagedAgentTeardownObservation { + const roots = [...this.#roots.values()]; + const alive = new Set(); + for (const root of roots) { + const sampledRoot = this.#lastTable?.get(root.pid); + if ( + childActive(root.child) && + (!this.#processTableAvailable || + (sampledRoot && !processIsZombie(sampledRoot))) + ) { + alive.add(root.pid); + } + if (!this.#processTableAvailable) continue; + const table = this.#lastTable!; + if (this.#platform !== "win32") { + const liveRootGroupPids = [...table.entries()].flatMap( + ([pid, record]) => + record.processGroupId === root.pid && !processIsZombie(record) + ? [pid] + : [], + ); + for (const pid of liveRootGroupPids) alive.add(pid); + const groupLiveness = + liveRootGroupPids.length > 0 + ? this.#processGroupLiveness(root.pid) + : "gone"; + if (groupLiveness === "alive") alive.add(root.pid); + if (groupLiveness === "unknown") { + this.#invalidateRootContainment(root); + } + } else { + for (const [pid, observed] of this.#observedIdentities) { + if ( + observed.rootPid === root.pid && + sameProcess(observed.record, table.get(pid)) + ) { + alive.add(pid); + } + } + } + } + const toolRegistrations = [...this.#toolProcessRegistrations.values()]; + const toolProcessGroupId = this.#toolProcessGroupId; + for (const registration of toolRegistrations) { + if (!registration.closed && !registration.socket.destroyed) { + alive.add(registration.pid); + } + } + if (this.#processTableAvailable) { + const table = this.#lastTable!; + if (this.#platform !== "win32") { + for (const [pid, observed] of this.#observedIdentities) { + const current = table.get(pid); + if ( + !sameProcess(observed.record, current) || + processIsZombie(current) + ) { + continue; + } + alive.add(pid); + if ( + typeof current!.processGroupId === "number" && + (current!.parentPid !== observed.record.parentPid || + current!.processGroupId !== observed.record.processGroupId || + current!.sessionId !== observed.record.sessionId) + ) { + alive.add(current!.processGroupId); + } + } + } + if (typeof toolProcessGroupId === "number") { + const liveToolGroupPids = [...table.entries()].flatMap( + ([pid, record]) => + record.processGroupId === toolProcessGroupId && + !processIsZombie(record) + ? [pid] + : [], + ); + for (const pid of liveToolGroupPids) alive.add(pid); + const groupLiveness = + liveToolGroupPids.length > 0 + ? this.#processGroupLiveness(toolProcessGroupId) + : "gone"; + if (groupLiveness === "alive") alive.add(toolProcessGroupId); + if (groupLiveness === "unknown") { + this.#invalidateToolContainment(); + } + } + for (const registration of toolRegistrations) { + const current = table.get(registration.pid); + if (current && !processIsZombie(current)) { + alive.add(registration.pid); + } + } + } + + const processTableAvailable = + roots.length === 0 || this.#processTableAvailable; + const toolProcessObservationComplete = + !this.#toolProcessContainmentArmed || + this.#toolProcessObservationComplete; + const toolProcessChannelsClosed = + !this.#toolProcessContainmentArmed || + (this.#toolProcessObservationComplete && + toolRegistrations.length === 2 && + toolRegistrations.every( + ({ closed, socket }) => closed || socket.destroyed, + )); + const containmentSupported = + roots.every(({ containmentSupported: supported }) => supported) && + roots.every( + ({ pid }) => !this.#hasPendingUnauthenticatedDescendants(pid), + ) && + (!this.#toolProcessContainmentArmed || + (this.#toolControlAvailable && + this.#toolProcessObservationComplete && + !this.#toolProcessObservationInvalid)); + const ownershipProven = + roots.length > 0 && + roots.every(({ ownershipProven }) => ownershipProven) && + toolProcessObservationComplete; + const forceKillIssued = + roots.length > 0 && roots.every(({ forceKillIssued }) => forceKillIssued); + const elapsedMs = Math.max(0, this.#monotonicNow() - startedAt); + const quiescent = + processTableAvailable && + containmentSupported && + toolProcessChannelsClosed && + alive.size === 0; + return { + quiescent, + deadlineMet: quiescent, + processTableAvailable, + containmentSupported, + ownershipProven, + forceKillIssued, + toolProcessObservationComplete, + toolProcessChannelsClosed, + elapsedMs, + observedPids: [...this.#observedPids].sort((left, right) => left - right), + alivePidsAtDeadline: [...alive].sort((left, right) => left - right), + emergencyCleanupAttempted, + }; + } + + public async waitForQuiescence( + deadline: ManagedAgentTeardownDeadline, + ): Promise { + const adoptedDeadline = this.#adoptDeadline(deadline); + const startedAt = adoptedDeadline.startedAtMs; + const boundedTimeoutMs = Math.max( + 0, + adoptedDeadline.deadlineAtMs - adoptedDeadline.startedAtMs, + ); + for (;;) { + if (this.#remainingMs(adoptedDeadline) <= 0 || this.#sealed) { + const observation = this.#currentObservation(startedAt, false); + this.#seal(); + return { ...observation, deadlineMet: false }; + } + await this.observeProcessTree(adoptedDeadline); + const observation = this.#currentObservation(startedAt, false); + if (observation.quiescent) { + const deadlineMet = + this.#monotonicNow() <= adoptedDeadline.deadlineAtMs; + // Seal atomically with the successful observation so no delayed SDK + // spawn can appear after quiescence has been certified. + this.#seal(); + return { + ...observation, + deadlineMet, + }; + } + if ( + observation.elapsedMs >= boundedTimeoutMs || + this.#remainingMs(adoptedDeadline) <= 0 + ) { + this.#seal(); + return { ...observation, deadlineMet: false }; + } + await this.#delay( + Math.min(QUIESCENCE_POLL_MS, this.#remainingMs(adoptedDeadline)), + ); + } + } + + public async emergencyCleanup( + deadline: ManagedAgentTeardownDeadline, + ): Promise { + const adoptedDeadline = this.#adoptDeadline(deadline); + const startedAt = adoptedDeadline.startedAtMs; + this.#requestFallbackCleanupSynchronously(); + const confirmation = await this.waitForQuiescence(adoptedDeadline); + const elapsedMs = Math.max(0, this.#monotonicNow() - startedAt); + const roots = [...this.#roots.values()]; + const forceKillIssued = + roots.length > 0 && roots.every((root) => root.forceKillIssued); + return { + ...confirmation, + forceKillIssued, + elapsedMs, + deadlineMet: + confirmation.quiescent && + this.#monotonicNow() <= adoptedDeadline.deadlineAtMs, + emergencyCleanupAttempted: true, + }; + } + + public dispose(): Promise { + this.#disposeTask ??= this.#disposeInternal(); + return this.#disposeTask; + } + + async #disposeInternal(): Promise { + this.#hostDisposing = true; + this.#seal(); + + // The two exact fixture processes authenticated these retained channels + // with an observer-created capability that was never written into the + // workspace. Ask them to exit cooperatively and require an acknowledgement; + // workspace PID-file contents are diagnostic only and never signal input. + const acknowledgementTasks = [ + ...this.#toolProcessRegistrations.values(), + ].flatMap((registration) => { + const socket = registration.socket; + if (socket.destroyed || registration.closed) return []; + return [ + new Promise((resolveAcknowledgement) => { + let settled = false; + let body = ""; + const finish = (acknowledged: boolean): void => { + if (settled) return; + settled = true; + socket.off("data", onData); + socket.off("close", onClose); + resolveAcknowledgement(acknowledged); + }; + const onData = (chunk: Buffer | string): void => { + body += chunk.toString(); + if (body.includes('"shutdownAck":true')) finish(true); + }; + const onClose = (): void => finish(false); + socket.on("data", onData); + socket.once("close", onClose); + try { + socket.write('{"shutdown":true}\n'); + } catch { + finish(false); + } + }), + ]; + }); + if (acknowledgementTasks.length > 0) { + let timeout: NodeJS.Timeout | undefined; + await Promise.race([ + Promise.all(acknowledgementTasks), + new Promise((resolveTimeout) => { + timeout = setTimeout(resolveTimeout, DISPOSE_DRAIN_TIMEOUT_MS); + }), + ]); + if (timeout) clearTimeout(timeout); + } + + // Closing the retained IPC handle lets the supervisor kill its own exact + // process group without the test harness supplying any numeric PID/PGID. + const rootExitTasks = [...this.#roots.values()].flatMap(({ child }) => { + if (!childActive(child)) return []; + if (child.connected) child.disconnect(); + return [ + new Promise((resolveExit) => { + if (!childActive(child)) { + resolveExit(); + return; + } + child.once("close", () => resolveExit()); + }), + ]; + }); + if (rootExitTasks.length > 0) { + let timeout: NodeJS.Timeout | undefined; + await Promise.race([ + Promise.all(rootExitTasks), + new Promise((resolveTimeout) => { + timeout = setTimeout(resolveTimeout, DISPOSE_DRAIN_TIMEOUT_MS); + }), + ]); + if (timeout) clearTimeout(timeout); + } + + for (const socket of this.#toolControlSockets) socket.destroy(); + this.#toolControlSockets.clear(); + const serverClose = new Promise((resolveClose) => { + const server = this.#toolControlServer; + if (!server?.listening) { + resolveClose(); + return; + } + server.close(() => resolveClose()); + }); + let serverCloseTimeout: NodeJS.Timeout | undefined; + await Promise.race([ + serverClose, + new Promise((resolveTimeout) => { + serverCloseTimeout = setTimeout( + resolveTimeout, + DISPOSE_DRAIN_TIMEOUT_MS, + ); + }), + ]); + if (serverCloseTimeout) clearTimeout(serverCloseTimeout); + if (this.#toolControlDirectory) { + try { + rmSync(this.#toolControlDirectory, { recursive: true, force: true }); + } catch { + // Best-effort removal after the private listener and clients close. + } + } + } +} + +export function createLocalManagedAgentProcessObserver( + options: LocalManagedAgentProcessObserverOptions = {}, +): ManagedAgentProcessObserver { + return new LocalManagedAgentProcessObserver(options); +} diff --git a/packages/harness/src/experimental/managed-agent-spike/runtime-sdk-loopback.test.ts b/packages/harness/src/experimental/managed-agent-spike/runtime-sdk-loopback.test.ts new file mode 100644 index 000000000..9b3e96034 --- /dev/null +++ b/packages/harness/src/experimental/managed-agent-spike/runtime-sdk-loopback.test.ts @@ -0,0 +1,1457 @@ +import { execFile, spawn, type ChildProcess } from "node:child_process"; +import { once } from "node:events"; +import { readFile } from "node:fs/promises"; +import { createServer, type ServerResponse } from "node:http"; +import { createRequire } from "node:module"; +import type { AddressInfo } from "node:net"; +import { dirname, join } from "node:path"; +import { promisify } from "node:util"; + +import { query as agentSdkQuery } from "@anthropic-ai/claude-agent-sdk"; +import { expect, it } from "vitest"; + +import { + FIXTURE_PATHS, + createManagedAgentFixture, + fixturePathExists, + waitForManagedAgentFixturePids, +} from "./fixture.js"; +import { MANAGED_AGENT_CONTRACT } from "./contract.js"; +import { + LocalManagedAgentProcessObserver, + managedAgentPosixSessionColumn, + parseManagedAgentPosixProcessTable, + type ManagedAgentKernelProcessTable, + type ManagedAgentProcessTableObservation, +} from "./process-observer.js"; +import { + qualifiedManagedAgentMcpToolName, + runManagedAgentProbe, +} from "./runtime.js"; +import type { ManagedAgentProcessObserver } from "./types.js"; + +const RUN_ID = "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa"; +const EXECUTION_ID = "bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb"; +const MODEL_ALIAS = "claude-sonnet-5-anthropic-anthropic-eval"; +const EVAL_SOURCE = + "studio-managed-agent-e0-l1-sonnet-5-bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb"; +const CORRELATION_MARKER = `SAPIOM_CERTIFICATION_CORRELATION_V1;eval_source=${EVAL_SOURCE};execution_id=${EXECUTION_ID}`; +const ALLOWED_BASH_COMMAND = "git status --short"; +const DENIED_BASH_COMMAND = "touch denied-side-effect.txt"; +const ECHO_NONCE_TOOL = qualifiedManagedAgentMcpToolName("echo_nonce"); +const require = createRequire(import.meta.url); +const execFileAsync = promisify(execFile); + +async function readLoopbackProcessTable(): Promise { + try { + const sessionColumn = managedAgentPosixSessionColumn(process.platform); + const { stdout } = await execFileAsync( + "/bin/ps", + ["-axo", `pid=,ppid=,pgid=,${sessionColumn}=,stat=,lstart=`], + { encoding: "utf8", maxBuffer: 4 * 1024 * 1024, timeout: 1_000 }, + ); + return { + available: true, + processes: parseManagedAgentPosixProcessTable(stdout), + }; + } catch { + return { available: false }; + } +} + +function processExists(pid: number): boolean { + try { + process.kill(pid, 0); + return true; + } catch (error) { + return (error as NodeJS.ErrnoException).code === "EPERM"; + } +} + +async function waitForProcessDeath( + pid: number, + timeoutMs = 1_000, +): Promise { + const deadline = Date.now() + timeoutMs; + while (processExists(pid) && Date.now() < deadline) { + await new Promise((resolveDelay) => setTimeout(resolveDelay, 10)); + } + if (processExists(pid)) + throw new Error(`Test process ${pid} survived cleanup`); +} + +function spawnCooperativeUnrelatedProcess(): ChildProcess { + return spawn( + process.execPath, + [ + "-e", + 'process.on("disconnect", () => process.exit(0)); setInterval(() => {}, 1000)', + ], + { + stdio: ["ignore", "ignore", "ignore", "ipc"], + windowsHide: true, + }, + ); +} + +async function stopCooperativeUnrelatedProcess( + child: ChildProcess, +): Promise { + if (typeof child.pid !== "number") return; + const pid = child.pid; + if (child.exitCode === null && child.signalCode === null) { + if (!child.connected) { + throw new Error( + `Refusing cleanup for unrelated process ${pid} without retained IPC`, + ); + } + child.disconnect(); + } + await waitForProcessDeath(pid); +} + +interface LoopbackObservation { + readonly headerNames: readonly string[]; + readonly evalSourceMatches: boolean; + readonly executionIdMatches: boolean; + readonly promptMarkerPresent: boolean; + readonly mcpResultMatches: boolean; +} + +function containsExactText(value: unknown, expected: string): boolean { + if (value === expected) return true; + if (Array.isArray(value)) { + return value.some((entry) => containsExactText(entry, expected)); + } + if (typeof value !== "object" || value === null) return false; + return Object.values(value).some((entry) => + containsExactText(entry, expected), + ); +} + +function hasSuccessfulMcpResult(body: string, expectedNonce: string): boolean { + try { + const payload = JSON.parse(body) as { messages?: unknown }; + if (!Array.isArray(payload.messages)) return false; + return payload.messages.some((message) => { + if (typeof message !== "object" || message === null) return false; + const content = (message as { content?: unknown }).content; + if (!Array.isArray(content)) return false; + return content.some((block) => { + if (typeof block !== "object" || block === null) return false; + const result = block as Record; + return ( + result.type === "tool_result" && + result.tool_use_id === "toolu_loopback_mcp_echo" && + result.is_error !== true && + containsExactText(result.content, expectedNonce) + ); + }); + }); + } catch { + return false; + } +} + +function hasToolResult( + body: string, + toolUseId: string, + expectedError: boolean, +): boolean { + try { + const payload = JSON.parse(body) as { messages?: unknown }; + if (!Array.isArray(payload.messages)) return false; + return payload.messages.some((message) => { + if (typeof message !== "object" || message === null) return false; + const content = (message as { content?: unknown }).content; + return ( + Array.isArray(content) && + content.some( + (block) => + typeof block === "object" && + block !== null && + (block as Record).type === "tool_result" && + (block as Record).tool_use_id === toolUseId && + ((block as Record).is_error === true) === + expectedError, + ) + ); + }); + } catch { + return false; + } +} + +function writeSseEvent( + response: ServerResponse, + event: string, + data: Record, +): void { + response.write(`event: ${event}\ndata: ${JSON.stringify(data)}\n\n`); +} + +function writeToolUseResponse( + response: ServerResponse, + turn: number, + toolUse: { + readonly id: string; + readonly name: string; + readonly input: Record; + }, +): void { + response.writeHead(200, { + "cache-control": "no-cache", + "content-type": "text/event-stream", + "request-id": `req_loopback_${turn}`, + }); + writeSseEvent(response, "message_start", { + type: "message_start", + message: { + id: `msg_loopback_${turn}`, + type: "message", + role: "assistant", + model: MODEL_ALIAS, + content: [], + stop_reason: null, + stop_sequence: null, + usage: { input_tokens: 1, output_tokens: 0 }, + }, + }); + writeSseEvent(response, "content_block_start", { + type: "content_block_start", + index: 0, + content_block: { + type: "tool_use", + id: toolUse.id, + name: toolUse.name, + input: {}, + }, + }); + writeSseEvent(response, "content_block_delta", { + type: "content_block_delta", + index: 0, + delta: { + type: "input_json_delta", + partial_json: JSON.stringify(toolUse.input), + }, + }); + writeSseEvent(response, "content_block_stop", { + type: "content_block_stop", + index: 0, + }); + writeSseEvent(response, "message_delta", { + type: "message_delta", + delta: { stop_reason: "tool_use", stop_sequence: null }, + usage: { output_tokens: 1 }, + }); + writeSseEvent(response, "message_stop", { type: "message_stop" }); + response.end(); +} + +function writeFinalResponse(response: ServerResponse, turn: number): void { + response.writeHead(200, { + "cache-control": "no-cache", + "content-type": "text/event-stream", + "request-id": `req_loopback_${turn}`, + }); + writeSseEvent(response, "message_start", { + type: "message_start", + message: { + id: `msg_loopback_${turn}`, + type: "message", + role: "assistant", + model: MODEL_ALIAS, + content: [], + stop_reason: null, + stop_sequence: null, + usage: { input_tokens: 1, output_tokens: 0 }, + }, + }); + writeSseEvent(response, "content_block_start", { + type: "content_block_start", + index: 0, + content_block: { type: "text", text: "" }, + }); + writeSseEvent(response, "content_block_delta", { + type: "content_block_delta", + index: 0, + delta: { type: "text_delta", text: "done" }, + }); + writeSseEvent(response, "content_block_stop", { + type: "content_block_stop", + index: 0, + }); + writeSseEvent(response, "message_delta", { + type: "message_delta", + delta: { stop_reason: "end_turn", stop_sequence: null }, + usage: { output_tokens: 1 }, + }); + writeSseEvent(response, "message_stop", { type: "message_stop" }); + response.end(); +} + +function writeHangingStream(response: ServerResponse, turn: number): void { + response.writeHead(200, { + "cache-control": "no-cache", + "content-type": "text/event-stream", + "request-id": `req_loopback_${turn}`, + }); + // The first fake-model turn launches Bash. Claude Code may immediately + // request another turn after its Bash implementation backgrounds a long + // command. Keep that synthetic continuation open until cancellation instead + // of manufacturing duplicate tool calls that a real model never requested. + response.write(": awaiting managed-agent cancellation\n\n"); +} + +it("enforces real-SDK built-in and in-process MCP calls with exact loopback correlation", async () => { + const fixture = await createManagedAgentFixture(() => "loopback-nonce"); + const startedAt = Date.now(); + const stateSamples: Array<{ + readonly elapsedMs: number; + readonly processes?: ManagedAgentKernelProcessTable; + }> = []; + const groupSignals: Array<{ + readonly elapsedMs: number; + readonly groupId: number; + readonly signal: "SIGKILL"; + readonly outcome: "sent" | "gone" | "failure"; + }> = []; + const lifecycle: Array<{ + readonly elapsedMs: number; + readonly event: string; + }> = []; + const observer = new LocalManagedAgentProcessObserver({ + readProcessTable: async () => { + const observation = await readLoopbackProcessTable(); + stateSamples.push({ + elapsedMs: Date.now() - startedAt, + ...(observation.available ? { processes: observation.processes } : {}), + }); + return observation; + }, + onTerminationRequest: ({ processGroupId: groupId }, outcome) => { + groupSignals.push({ + elapsedMs: Date.now() - startedAt, + groupId, + signal: "SIGKILL", + outcome, + }); + }, + }); + let supervisorPid: number | undefined; + const observedObserver: ManagedAgentProcessObserver = { + spawn: (options) => { + const child = observer.spawn(options); + const pid = Reflect.get(child, "pid"); + supervisorPid = typeof pid === "number" ? pid : undefined; + lifecycle.push({ + elapsedMs: Date.now() - startedAt, + event: "observer_spawned", + }); + return child; + }, + beginTeardown: (deadline) => observer.beginTeardown(deadline), + armToolProcessContainment: () => observer.armToolProcessContainment(), + prepareCancellation: () => observer.prepareCancellation(), + observeProcessTree: (timeoutMs) => observer.observeProcessTree(timeoutMs), + waitForQuiescence: async (timeoutMs) => { + lifecycle.push({ + elapsedMs: Date.now() - startedAt, + event: "wait_for_quiescence_started", + }); + const result = await observer.waitForQuiescence(timeoutMs); + lifecycle.push({ + elapsedMs: Date.now() - startedAt, + event: `wait_for_quiescence_settled:${result.quiescent}:${result.containmentSupported}`, + }); + return result; + }, + emergencyCleanup: async (timeoutMs) => { + lifecycle.push({ + elapsedMs: Date.now() - startedAt, + event: "host_emergency_cleanup_started", + }); + const result = await observer.emergencyCleanup(timeoutMs); + lifecycle.push({ + elapsedMs: Date.now() - startedAt, + event: `host_emergency_cleanup_settled:${result.quiescent}:${result.containmentSupported}`, + }); + return result; + }, + dispose: () => observer.dispose(), + }; + const observations: LoopbackObservation[] = []; + let helloCount = 0; + let inferenceTurn = 0; + const server = createServer((request, response) => { + if (request.method === "HEAD" && request.url === "/api/hello") { + helloCount += 1; + response.writeHead(200).end(); + return; + } + if ( + request.method !== "POST" || + request.url?.split("?")[0] !== "/v1/messages" + ) { + response.writeHead(404).end(); + return; + } + + let body = ""; + request.setEncoding("utf8"); + request.on("data", (chunk: string) => { + body += chunk; + if (body.length > 2_000_000) request.destroy(); + }); + request.on("end", () => { + inferenceTurn += 1; + const headerNames = Object.keys(request.headers).sort(); + observations.push({ + headerNames, + evalSourceMatches: + request.headers["x-sapiom-eval-source"] === EVAL_SOURCE, + executionIdMatches: + request.headers["x-sapiom-execution-id"] === EXECUTION_ID, + promptMarkerPresent: body.includes(CORRELATION_MARKER), + mcpResultMatches: hasSuccessfulMcpResult(body, fixture.nonce), + }); + if (inferenceTurn === 1) { + writeToolUseResponse(response, inferenceTurn, { + id: "toolu_loopback_read", + name: "Read", + input: { file_path: FIXTURE_PATHS.cleanTarget }, + }); + } else if (inferenceTurn === 2) { + writeToolUseResponse(response, inferenceTurn, { + id: "toolu_loopback_bash_allow", + name: "Bash", + input: { + command: ALLOWED_BASH_COMMAND, + description: "Show working tree status", + }, + }); + } else if (inferenceTurn === 3) { + writeToolUseResponse(response, inferenceTurn, { + id: "toolu_loopback_bash_deny", + name: "Bash", + input: { command: DENIED_BASH_COMMAND }, + }); + } else if (inferenceTurn === 4) { + writeToolUseResponse(response, inferenceTurn, { + id: "toolu_loopback_mcp_echo", + name: ECHO_NONCE_TOOL, + input: { nonce: fixture.nonce }, + }); + } else { + writeFinalResponse(response, inferenceTurn); + } + }); + }); + + await new Promise((resolve, reject) => { + server.once("error", reject); + server.listen(0, "127.0.0.1", resolve); + }); + try { + const address = server.address() as AddressInfo; + const ids = [RUN_ID, EXECUTION_ID]; + const result = await runManagedAgentProbe( + { + scenario: "L1", + workspaceRoot: fixture.workspaceRoot, + configRoot: fixture.configRoot, + target: "sonnet-5", + gatewayOrigin: `http://127.0.0.1:${address.port}`, + gatewayCredential: "sk-ant-api03-local-loopback-only", + prompt: fixture.prompt("L1"), + maxTurns: 6, + maxBudgetUsd: 0.25, + allowedBashCommands: [ALLOWED_BASH_COMMAND], + pathRoleBindings: fixture.pathRoleBindings, + expectedL1FinalBytes: fixture.expectedL1FinalBytes, + expectedMcpNonce: fixture.nonce, + preservePaths: [ + FIXTURE_PATHS.dirtySentinel, + FIXTURE_PATHS.untrackedSentinel, + ], + }, + { + hermeticGatewayOrigin: `http://127.0.0.1:${address.port}`, + processObserver: observedObserver, + queryFactory: ({ prompt, options }) => { + const sdkQuery = agentSdkQuery({ prompt, options }); + return { + [Symbol.asyncIterator]: () => sdkQuery[Symbol.asyncIterator](), + close: () => { + lifecycle.push({ + elapsedMs: Date.now() - startedAt, + event: "sdk_close_called", + }); + sdkQuery.close(); + }, + return: async () => { + lifecycle.push({ + elapsedMs: Date.now() - startedAt, + event: "sdk_return_started", + }); + const returned = await sdkQuery.return(undefined); + lifecycle.push({ + elapsedMs: Date.now() - startedAt, + event: "sdk_return_settled", + }); + return returned; + }, + }; + }, + uuid: () => { + const id = ids.shift(); + if (!id) throw new Error("unexpected UUID request"); + return id; + }, + }, + ); + + expect(helloCount).toBeGreaterThanOrEqual(1); + expect(observations).toHaveLength(5); + expect( + observations.every( + ({ headerNames, evalSourceMatches, executionIdMatches }) => + headerNames.includes("x-sapiom-eval-source") && + headerNames.includes("x-sapiom-execution-id") && + evalSourceMatches && + executionIdMatches, + ), + ).toBe(true); + expect( + observations.every(({ promptMarkerPresent }) => promptMarkerPresent), + ).toBe(true); + expect( + observations.map(({ mcpResultMatches }) => mcpResultMatches), + ).toEqual([false, false, false, false, true]); + const stateTransitions = stateSamples.reduce< + Array<{ + readonly elapsedMs: number; + readonly records: unknown; + }> + >((transitions, { elapsedMs, processes }) => { + const records = processes + ? [...processes.entries()] + .filter( + ([pid]) => + pid === supervisorPid || + result.teardown.observedPids.includes(pid), + ) + .map(([pid, record]) => ({ pid, ...record })) + : "unavailable"; + const previous = transitions.at(-1)?.records; + if (JSON.stringify(previous) !== JSON.stringify(records)) { + transitions.push({ elapsedMs, records }); + } + return transitions; + }, []); + expect( + result.terminal, + JSON.stringify({ + groupSignals, + lifecycle, + processTableSampleCount: stateSamples.length, + stateTransitions, + teardown: result.teardown, + terminationEvidence: result.terminationEvidence, + }), + ).toBe("success"); + expect(result.sdkModelEvidence).toEqual({ + authority: "sdk_non_authoritative", + initModelObserved: true, + initModelMatchesExpectedAlias: true, + resultModelUsageObserved: true, + resultModelUsageMatchesExpectedAlias: true, + resultModelCount: 1, + }); + + const requested = result.toolEvidence.filter( + ({ status }) => status === "requested", + ); + expect(requested.map(({ toolName }) => toolName)).toEqual([ + "Read", + "Bash", + "Bash", + ECHO_NONCE_TOOL, + ]); + for (const tool of requested) { + expect( + result.permissionEvidence.filter( + ({ toolUseId, source }) => + toolUseId === tool.toolUseId && source === "pre_tool_use", + ), + ).toHaveLength(1); + } + expect(result.permissionEvidence).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + toolName: "Read", + decision: "allow", + reason: "fixture_path", + source: "pre_tool_use", + }), + expect.objectContaining({ + toolName: "Bash", + decision: "allow", + reason: "exact_bash_command", + source: "pre_tool_use", + }), + expect.objectContaining({ + toolName: "Bash", + decision: "deny", + reason: "bash_command_not_allowed", + source: "pre_tool_use", + }), + expect.objectContaining({ + toolName: ECHO_NONCE_TOOL, + decision: "allow", + reason: "managed_mcp_tool", + source: "pre_tool_use", + }), + ]), + ); + expect(result.toolEvidence).toEqual( + expect.arrayContaining([ + expect.objectContaining({ toolName: "Read", status: "success" }), + expect.objectContaining({ toolName: "Bash", status: "success" }), + expect.objectContaining({ toolName: "Bash", status: "error" }), + ]), + ); + const requestedMcp = requested.find( + ({ toolName }) => toolName === ECHO_NONCE_TOOL, + ); + expect(requestedMcp?.toolUseId).toBeDefined(); + expect( + result.toolEvidence.filter( + ({ toolName, toolUseId, status }) => + toolName === ECHO_NONCE_TOOL && + toolUseId === requestedMcp?.toolUseId && + status === "success", + ), + ).toHaveLength(1); + expect( + result.toolEvidence.filter( + ({ toolName, toolUseId }) => + toolName === ECHO_NONCE_TOOL && toolUseId === undefined, + ), + ).toEqual([]); + expect(result.policyHookCoverage).toBe(true); + expect( + await fixturePathExists( + join(fixture.workspaceRoot, "denied-side-effect.txt"), + ), + ).toBe(false); + expect(result.queryClosed).toBe(true); + expect(result.teardown.quiescent).toBe(true); + } finally { + await observer.dispose(); + server.closeAllConnections(); + await new Promise((resolve) => server.close(() => resolve())); + await fixture.cleanup(); + } +}, 45_000); + +it.skipIf( + process.platform === "win32" || + process.versions.node !== MANAGED_AGENT_CONTRACT.certificationNodeVersion, +)( + "cancels the real SDK L2 Bash fixture without leaving its detached process group", + async () => { + const fixture = await createManagedAgentFixture( + () => "loopback-l2-cancellation", + ); + const startedAt = Date.now(); + const stateSamples: Array<{ + readonly elapsedMs: number; + readonly processes?: ManagedAgentKernelProcessTable; + }> = []; + const groupSignals: Array<{ + readonly elapsedMs: number; + readonly groupId: number; + readonly signal: "SIGKILL"; + readonly outcome: "sent" | "gone" | "failure"; + }> = []; + const observer = new LocalManagedAgentProcessObserver({ + readProcessTable: async () => { + const observation = await readLoopbackProcessTable(); + stateSamples.push({ + elapsedMs: Date.now() - startedAt, + ...(observation.available + ? { processes: observation.processes } + : {}), + }); + return observation; + }, + onTerminationRequest: ({ processGroupId: groupId }, outcome) => { + groupSignals.push({ + elapsedMs: Date.now() - startedAt, + groupId, + signal: "SIGKILL", + outcome, + }); + }, + }); + const cleanupOrder: string[] = []; + let supervisorPid: number | undefined; + const observedObserver: ManagedAgentProcessObserver = { + spawn: (options) => { + options.signal.addEventListener( + "abort", + () => cleanupOrder.push("sdk_forwarded_signal"), + { once: true }, + ); + const child = observer.spawn(options); + const nativeKill = child.kill.bind(child); + Reflect.set(child, "kill", (signal: NodeJS.Signals = "SIGTERM") => { + cleanupOrder.push("sdk_native_kill"); + return nativeKill(signal); + }); + const pid = Reflect.get(child, "pid"); + supervisorPid = typeof pid === "number" ? pid : undefined; + return child; + }, + beginTeardown: (deadline) => observer.beginTeardown(deadline), + armToolProcessContainment: () => observer.armToolProcessContainment(), + prepareCancellation: () => observer.prepareCancellation(), + observeProcessTree: (timeoutMs) => observer.observeProcessTree(timeoutMs), + waitForQuiescence: (timeoutMs) => observer.waitForQuiescence(timeoutMs), + emergencyCleanup: (timeoutMs) => { + cleanupOrder.push("host_emergency_cleanup"); + return observer.emergencyCleanup(timeoutMs); + }, + dispose: () => observer.dispose(), + }; + const unrelated = spawnCooperativeUnrelatedProcess(); + await once(unrelated, "spawn"); + let fixturePids: readonly number[] = []; + let fixtureToolProcessGroupId: number | undefined; + let cancellationStartedAt: number | undefined; + let inferenceTurn = 0; + const server = createServer((request, response) => { + if (request.method === "HEAD" && request.url === "/api/hello") { + response.writeHead(200).end(); + return; + } + if ( + request.method !== "POST" || + request.url?.split("?")[0] !== "/v1/messages" + ) { + response.writeHead(404).end(); + return; + } + request.resume(); + request.once("end", () => { + inferenceTurn += 1; + if (inferenceTurn === 1) { + writeToolUseResponse(response, inferenceTurn, { + id: "toolu_loopback_l2_bash", + name: "Bash", + input: { command: fixture.l2BashCommand }, + }); + } else { + writeHangingStream(response, inferenceTurn); + } + }); + }); + + await new Promise((resolve, reject) => { + server.once("error", reject); + server.listen(0, "127.0.0.1", resolve); + }); + try { + const address = server.address() as AddressInfo; + const ids = [RUN_ID, EXECUTION_ID]; + const result = await runManagedAgentProbe( + { + scenario: "L2", + workspaceRoot: fixture.workspaceRoot, + configRoot: fixture.configRoot, + target: "sonnet-5", + gatewayOrigin: `http://127.0.0.1:${address.port}`, + gatewayCredential: "sk-ant-api03-local-loopback-only", + prompt: fixture.prompt("L2"), + maxTurns: 4, + maxBudgetUsd: 0.25, + allowedBashCommands: [fixture.l2BashCommand], + pathRoleBindings: [], + expectedL1FinalBytes: [], + preservePaths: [ + FIXTURE_PATHS.dirtySentinel, + FIXTURE_PATHS.untrackedSentinel, + ], + }, + { + hermeticGatewayOrigin: `http://127.0.0.1:${address.port}`, + processObserver: observedObserver, + queryFactory: ({ prompt, options }) => { + const sdkQuery = agentSdkQuery({ prompt, options }); + return { + [Symbol.asyncIterator]: () => sdkQuery[Symbol.asyncIterator](), + close: () => { + cleanupOrder.push("sdk_close_called"); + sdkQuery.close(); + }, + return: async () => { + cleanupOrder.push("sdk_return_started"); + const returned = await sdkQuery.return(undefined); + cleanupOrder.push("sdk_return_settled"); + return returned; + }, + }; + }, + waitForCancellationSignal: async (signal) => { + fixturePids = await waitForManagedAgentFixturePids( + fixture, + 10_000, + signal, + ); + const readiness = await observer.prepareCancellation(); + expect(readiness).toMatchObject({ + supported: true, + reason: "ready", + containmentSupported: true, + ownershipProven: true, + }); + expect( + fixturePids.every((pid) => readiness.observedPids.includes(pid)), + ).toBe(true); + let readinessProcessTable: + | ManagedAgentKernelProcessTable + | undefined; + for (let index = stateSamples.length - 1; index >= 0; index -= 1) { + const processes = stateSamples[index]?.processes; + if (!processes?.has(fixturePids[0]!)) continue; + readinessProcessTable = processes; + break; + } + fixtureToolProcessGroupId = readinessProcessTable?.get( + fixturePids[0]!, + )?.processGroupId; + expect(fixtureToolProcessGroupId).toBeTypeOf("number"); + expect( + fixturePids.every( + (pid) => + readinessProcessTable?.get(pid)?.processGroupId === + fixtureToolProcessGroupId, + ), + ).toBe(true); + cancellationStartedAt = Date.now(); + }, + uuid: () => { + const id = ids.shift(); + if (!id) throw new Error("unexpected UUID request"); + return id; + }, + }, + ); + const cancellationElapsedMs = + Date.now() - (cancellationStartedAt ?? startedAt); + const stateTransitions = stateSamples.reduce< + Array<{ + readonly elapsedMs: number; + readonly records: unknown; + }> + >((transitions, { elapsedMs, processes }) => { + const records = processes + ? [...processes.entries()] + .filter( + ([pid]) => + pid === supervisorPid || + fixturePids.includes(pid) || + result.teardown.observedPids.includes(pid), + ) + .map(([pid, record]) => ({ pid, ...record })) + : "unavailable"; + const previous = transitions.at(-1)?.records; + if (JSON.stringify(previous) !== JSON.stringify(records)) { + transitions.push({ elapsedMs, records }); + } + return transitions; + }, []); + + expect(inferenceTurn).toBeGreaterThanOrEqual(1); + expect(inferenceTurn).toBeLessThanOrEqual(2); + expect(result.inferenceTurns).toBe(1); + expect( + result.terminal, + JSON.stringify({ + cleanupOrder, + elapsedMs: cancellationElapsedMs, + groupSignals, + queryClosed: result.queryClosed, + stateTransitions, + teardown: result.teardown, + terminationEvidence: result.terminationEvidence, + }), + ).toBe("cancelled"); + expect(cancellationElapsedMs).toBeLessThan(5_000); + expect(result.cancellationRequested).toBe(true); + expect(result.queryClosed).toBe(true); + expect(result.teardown).toMatchObject({ + quiescent: true, + deadlineMet: true, + processTableAvailable: true, + containmentSupported: true, + ownershipProven: true, + forceKillIssued: true, + toolProcessObservationComplete: true, + toolProcessChannelsClosed: true, + alivePidsAtDeadline: [], + }); + expect(fixturePids).toHaveLength(2); + expect(fixturePids.every((pid) => !processExists(pid))).toBe(true); + expect(processExists(unrelated.pid!)).toBe(true); + expect( + groupSignals.map(({ groupId, signal }) => [groupId, signal]), + ).toEqual([ + [fixtureToolProcessGroupId, "SIGKILL"], + [supervisorPid, "SIGKILL"], + ]); + expect(cleanupOrder).toEqual( + expect.arrayContaining([ + "sdk_close_called", + "sdk_return_started", + "sdk_return_settled", + "host_emergency_cleanup", + ]), + ); + expect(cleanupOrder.indexOf("sdk_close_called")).toBeLessThan( + cleanupOrder.indexOf("sdk_return_started"), + ); + expect(cleanupOrder.indexOf("sdk_return_started")).toBeLessThan( + cleanupOrder.indexOf("sdk_return_settled"), + ); + expect(cleanupOrder.indexOf("sdk_return_settled")).toBeLessThan( + cleanupOrder.indexOf("host_emergency_cleanup"), + ); + const forwardedSignalIndex = cleanupOrder.indexOf("sdk_forwarded_signal"); + const nativeKillIndexes = cleanupOrder.flatMap((step, index) => + step === "sdk_native_kill" ? [index] : [], + ); + expect(cleanupOrder).toEqual([ + "sdk_close_called", + "sdk_return_started", + "sdk_native_kill", + "sdk_forwarded_signal", + "sdk_native_kill", + "sdk_return_settled", + "host_emergency_cleanup", + ]); + expect(nativeKillIndexes).toEqual([2, 4]); + expect(forwardedSignalIndex).toBeGreaterThanOrEqual(0); + expect(nativeKillIndexes[0]).toBeLessThan(forwardedSignalIndex); + expect(nativeKillIndexes[1]).toBeGreaterThan(forwardedSignalIndex); + expect(forwardedSignalIndex).toBeLessThan( + cleanupOrder.indexOf("host_emergency_cleanup"), + ); + } finally { + await observer.dispose(); + server.closeAllConnections(); + await new Promise((resolve) => server.close(() => resolve())); + await Promise.all(fixturePids.map((pid) => waitForProcessDeath(pid))); + await stopCooperativeUnrelatedProcess(unrelated); + await fixture.cleanup(); + } + expect(fixturePids.every((pid) => !processExists(pid))).toBe(true); + }, + 20_000, +); + +it.skipIf( + process.platform === "win32" || + process.versions.node !== MANAGED_AGENT_CONTRACT.certificationNodeVersion, +)( + "fails closed through the runtime timeout fallback when the SDK signal is not forwarded", + async () => { + const fixture = await createManagedAgentFixture( + () => "loopback-l2-missing-forwarded-signal", + ); + const observer = new LocalManagedAgentProcessObserver(); + const neverForwardedController = new AbortController(); + const cleanupOrder: string[] = []; + const observedObserver: ManagedAgentProcessObserver = { + spawn: (options) => { + options.signal.addEventListener( + "abort", + () => cleanupOrder.push("sdk_forwarded_signal_unobserved"), + { once: true }, + ); + return observer.spawn({ + ...options, + signal: neverForwardedController.signal, + }); + }, + beginTeardown: (deadline) => observer.beginTeardown(deadline), + armToolProcessContainment: () => observer.armToolProcessContainment(), + prepareCancellation: () => observer.prepareCancellation(), + observeProcessTree: (timeoutMs) => observer.observeProcessTree(timeoutMs), + waitForQuiescence: (timeoutMs) => observer.waitForQuiescence(timeoutMs), + emergencyCleanup: (timeoutMs) => { + cleanupOrder.push("host_timeout_fallback"); + return observer.emergencyCleanup(timeoutMs); + }, + dispose: () => observer.dispose(), + }; + const unrelated = spawnCooperativeUnrelatedProcess(); + await once(unrelated, "spawn"); + let fixturePids: readonly number[] = []; + let cancellationStartedAt: number | undefined; + let inferenceTurn = 0; + const server = createServer((request, response) => { + if (request.method === "HEAD" && request.url === "/api/hello") { + response.writeHead(200).end(); + return; + } + if ( + request.method !== "POST" || + request.url?.split("?")[0] !== "/v1/messages" + ) { + response.writeHead(404).end(); + return; + } + request.resume(); + request.once("end", () => { + inferenceTurn += 1; + if (inferenceTurn === 1) { + writeToolUseResponse(response, inferenceTurn, { + id: "toolu_loopback_l2_missing_forwarded_signal", + name: "Bash", + input: { command: fixture.l2BashCommand }, + }); + } else { + writeHangingStream(response, inferenceTurn); + } + }); + }); + + await new Promise((resolve, reject) => { + server.once("error", reject); + server.listen(0, "127.0.0.1", resolve); + }); + try { + const address = server.address() as AddressInfo; + const ids = [RUN_ID, EXECUTION_ID]; + const result = await runManagedAgentProbe( + { + scenario: "L2", + workspaceRoot: fixture.workspaceRoot, + configRoot: fixture.configRoot, + target: "sonnet-5", + gatewayOrigin: `http://127.0.0.1:${address.port}`, + gatewayCredential: "sk-ant-api03-local-loopback-only", + prompt: fixture.prompt("L2"), + maxTurns: 4, + maxBudgetUsd: 0.25, + allowedBashCommands: [fixture.l2BashCommand], + pathRoleBindings: [], + expectedL1FinalBytes: [], + preservePaths: [ + FIXTURE_PATHS.dirtySentinel, + FIXTURE_PATHS.untrackedSentinel, + ], + }, + { + hermeticGatewayOrigin: `http://127.0.0.1:${address.port}`, + processObserver: observedObserver, + queryFactory: ({ prompt, options }) => { + const sdkQuery = agentSdkQuery({ prompt, options }); + return { + [Symbol.asyncIterator]: () => sdkQuery[Symbol.asyncIterator](), + close: () => { + cleanupOrder.push("sdk_close_called"); + sdkQuery.close(); + }, + return: async () => { + cleanupOrder.push("sdk_return_started"); + await sdkQuery.return(undefined); + cleanupOrder.push("sdk_return_underlying_settled"); + return new Promise>(() => undefined); + }, + }; + }, + waitForCancellationSignal: async (signal) => { + fixturePids = await waitForManagedAgentFixturePids( + fixture, + 10_000, + signal, + ); + await expect(observer.prepareCancellation()).resolves.toMatchObject( + { + supported: true, + reason: "ready", + containmentSupported: true, + ownershipProven: true, + }, + ); + cancellationStartedAt = Date.now(); + }, + uuid: () => { + const id = ids.shift(); + if (!id) throw new Error("unexpected UUID request"); + return id; + }, + }, + ); + const cancellationElapsedMs = + Date.now() - (cancellationStartedAt ?? Date.now()); + + expect(inferenceTurn).toBeGreaterThanOrEqual(1); + expect(inferenceTurn).toBeLessThanOrEqual(2); + expect(result.inferenceTurns).toBe(1); + expect(result.terminal).toBe("close_timeout"); + expect(result.cancellationRequested).toBe(true); + expect(result.queryClosed).toBe(false); + expect(result.teardown).toMatchObject({ + quiescent: true, + deadlineMet: true, + processTableAvailable: true, + containmentSupported: true, + ownershipProven: true, + forceKillIssued: true, + toolProcessObservationComplete: true, + toolProcessChannelsClosed: true, + alivePidsAtDeadline: [], + }); + expect(result.teardown.elapsedMs).toBeLessThanOrEqual(5_000); + expect(cancellationElapsedMs).toBeLessThanOrEqual(5_000); + expect(fixturePids).toHaveLength(2); + expect(fixturePids.every((pid) => !processExists(pid))).toBe(true); + expect(processExists(unrelated.pid!)).toBe(true); + expect(cleanupOrder).toEqual( + expect.arrayContaining([ + "sdk_close_called", + "sdk_return_started", + "sdk_return_underlying_settled", + "sdk_forwarded_signal_unobserved", + "host_timeout_fallback", + ]), + ); + expect( + cleanupOrder.indexOf("sdk_forwarded_signal_unobserved"), + ).toBeLessThan(cleanupOrder.indexOf("host_timeout_fallback")); + } finally { + await observer.dispose(); + server.closeAllConnections(); + await new Promise((resolve) => server.close(() => resolve())); + await Promise.all(fixturePids.map((pid) => waitForProcessDeath(pid))); + await stopCooperativeUnrelatedProcess(unrelated); + await fixture.cleanup(); + } + expect(fixturePids.every((pid) => !processExists(pid))).toBe(true); + }, + 20_000, +); + +it.skipIf( + process.platform === "win32" || + process.versions.node !== MANAGED_AGENT_CONTRACT.certificationNodeVersion, +)( + "keeps readiness-failure evidence fail-closed after cooperative disposal", + async () => { + const fixture = await createManagedAgentFixture( + () => "loopback-l2-early-error", + ); + const observer = new LocalManagedAgentProcessObserver(); + let fixturePids: readonly number[] = []; + let inferenceTurn = 0; + const server = createServer((request, response) => { + if (request.method === "HEAD" && request.url === "/api/hello") { + response.writeHead(200).end(); + return; + } + if ( + request.method !== "POST" || + request.url?.split("?")[0] !== "/v1/messages" + ) { + response.writeHead(404).end(); + return; + } + request.resume(); + request.once("end", () => { + inferenceTurn += 1; + writeToolUseResponse(response, inferenceTurn, { + id: "toolu_loopback_l2_early_error", + name: "Bash", + input: { command: fixture.l2BashCommand }, + }); + }); + }); + + await new Promise((resolve, reject) => { + server.once("error", reject); + server.listen(0, "127.0.0.1", resolve); + }); + let recordedTerminal: string | undefined; + try { + const address = server.address() as AddressInfo; + const ids = [RUN_ID, EXECUTION_ID]; + const result = await runManagedAgentProbe( + { + scenario: "L2", + workspaceRoot: fixture.workspaceRoot, + configRoot: fixture.configRoot, + target: "sonnet-5", + gatewayOrigin: `http://127.0.0.1:${address.port}`, + gatewayCredential: "sk-ant-api03-local-loopback-only", + prompt: fixture.prompt("L2"), + maxTurns: 4, + maxBudgetUsd: 0.25, + allowedBashCommands: [fixture.l2BashCommand], + pathRoleBindings: [], + expectedL1FinalBytes: [], + preservePaths: [ + FIXTURE_PATHS.dirtySentinel, + FIXTURE_PATHS.untrackedSentinel, + ], + }, + { + hermeticGatewayOrigin: `http://127.0.0.1:${address.port}`, + processObserver: observer, + queryFactory: ({ prompt, options }) => + agentSdkQuery({ prompt, options }), + waitForCancellationSignal: async (signal) => { + fixturePids = await waitForManagedAgentFixturePids( + fixture, + 10_000, + signal, + ); + throw new Error("synthetic readiness failure"); + }, + uuid: () => { + const id = ids.shift(); + if (!id) throw new Error("unexpected UUID request"); + return id; + }, + }, + ); + + expect(inferenceTurn).toBe(1); + recordedTerminal = result.terminal; + expect(result.terminal).toBe("teardown_timeout"); + expect(result.cancellationRequested).toBe(false); + expect(result.terminationEvidence).toEqual({ + beforePolicyOverride: "teardown_timeout", + queryExecution: "iteration_aborted", + sdkResult: "not_observed", + }); + expect(result.queryClosed).toBe(true); + expect(result.teardown).toMatchObject({ + quiescent: false, + deadlineMet: false, + containmentSupported: false, + ownershipProven: false, + forceKillIssued: false, + toolProcessObservationComplete: false, + toolProcessChannelsClosed: false, + }); + expect(fixturePids).toHaveLength(2); + expect( + fixturePids.every((pid) => + result.teardown.alivePidsAtDeadline.includes(pid), + ), + ).toBe(true); + await Promise.all(fixturePids.map((pid) => waitForProcessDeath(pid))); + expect(fixturePids.every((pid) => !processExists(pid))).toBe(true); + expect(result.terminal).toBe("teardown_timeout"); + expect( + fixturePids.every((pid) => + result.teardown.alivePidsAtDeadline.includes(pid), + ), + ).toBe(true); + } finally { + await observer.dispose(); + server.closeAllConnections(); + await new Promise((resolve) => server.close(() => resolve())); + await Promise.all(fixturePids.map((pid) => waitForProcessDeath(pid))); + await fixture.cleanup(); + } + expect(recordedTerminal).toBe("teardown_timeout"); + expect(fixturePids.every((pid) => !processExists(pid))).toBe(true); + }, + 20_000, +); + +it.skipIf( + process.versions.node !== MANAGED_AGENT_CONTRACT.certificationNodeVersion, +)( + "keeps malformed real-SDK Edit requests outside strict primary-hook coverage", + async () => { + const fixture = await createManagedAgentFixture( + () => "loopback-malformed-edit", + ); + const malformedToolUseId = "toolu_loopback_malformed_edit"; + const validToolUseId = "toolu_loopback_valid_edit"; + const observedMalformedError: boolean[] = []; + const observedValidSuccess: boolean[] = []; + let inferenceTurn = 0; + const server = createServer((request, response) => { + if (request.method === "HEAD" && request.url === "/api/hello") { + response.writeHead(200).end(); + return; + } + if ( + request.method !== "POST" || + request.url?.split("?")[0] !== "/v1/messages" + ) { + response.writeHead(404).end(); + return; + } + + let body = ""; + request.setEncoding("utf8"); + request.on("data", (chunk: string) => { + body += chunk; + if (body.length > 2_000_000) request.destroy(); + }); + request.on("end", () => { + inferenceTurn += 1; + observedMalformedError.push( + hasToolResult(body, malformedToolUseId, true), + ); + observedValidSuccess.push(hasToolResult(body, validToolUseId, false)); + if (inferenceTurn === 1) { + writeToolUseResponse(response, inferenceTurn, { + id: malformedToolUseId, + name: "Edit", + input: { + file_path: FIXTURE_PATHS.cleanTarget, + new_string: fixture.cleanTargetReplacement, + }, + }); + } else if (inferenceTurn === 2) { + writeToolUseResponse(response, inferenceTurn, { + id: validToolUseId, + name: "Edit", + input: { + file_path: FIXTURE_PATHS.cleanTarget, + old_string: "clean target base\n", + new_string: fixture.cleanTargetReplacement, + replace_all: false, + }, + }); + } else { + writeFinalResponse(response, inferenceTurn); + } + }); + }); + + await new Promise((resolve, reject) => { + server.once("error", reject); + server.listen(0, "127.0.0.1", resolve); + }); + try { + const sdkPackage = JSON.parse( + await readFile( + join( + dirname(require.resolve("@anthropic-ai/claude-agent-sdk")), + "package.json", + ), + "utf8", + ), + ) as { version?: unknown }; + expect(sdkPackage.version).toBe(MANAGED_AGENT_CONTRACT.agentSdkVersion); + expect(process.versions.node).toBe( + MANAGED_AGENT_CONTRACT.certificationNodeVersion, + ); + + const address = server.address() as AddressInfo; + const ids = [RUN_ID, EXECUTION_ID]; + const result = await runManagedAgentProbe( + { + scenario: "L1", + workspaceRoot: fixture.workspaceRoot, + configRoot: fixture.configRoot, + target: "sonnet-5", + gatewayOrigin: `http://127.0.0.1:${address.port}`, + gatewayCredential: "sk-ant-api03-local-loopback-only", + prompt: fixture.prompt("L1"), + maxTurns: 4, + maxBudgetUsd: 0.25, + allowedBashCommands: [], + pathRoleBindings: fixture.pathRoleBindings, + expectedL1FinalBytes: fixture.expectedL1FinalBytes, + expectedMcpNonce: fixture.nonce, + preservePaths: [ + FIXTURE_PATHS.dirtySentinel, + FIXTURE_PATHS.untrackedSentinel, + ], + }, + { + hermeticGatewayOrigin: `http://127.0.0.1:${address.port}`, + queryFactory: ({ prompt, options }) => + agentSdkQuery({ prompt, options }), + uuid: () => { + const id = ids.shift(); + if (!id) throw new Error("unexpected UUID request"); + return id; + }, + }, + ); + + expect(inferenceTurn).toBe(3); + expect(observedMalformedError).toEqual([false, true, true]); + expect(observedValidSuccess).toEqual([false, false, true]); + const requestedEdits = result.toolEvidence.filter( + ({ toolName, status }) => toolName === "Edit" && status === "requested", + ); + expect(requestedEdits).toHaveLength(2); + const [malformedEdit, validEdit] = requestedEdits; + expect( + result.permissionEvidence.filter( + ({ toolUseId, source }) => + toolUseId === malformedEdit?.toolUseId && source === "pre_tool_use", + ), + ).toHaveLength(0); + expect( + result.toolEvidence.filter( + ({ toolUseId, status }) => + toolUseId === malformedEdit?.toolUseId && status === "error", + ), + ).toHaveLength(1); + expect( + result.permissionEvidence.filter( + ({ toolUseId, source, decision }) => + toolUseId === validEdit?.toolUseId && + source === "pre_tool_use" && + decision === "allow", + ), + ).toHaveLength(1); + expect( + result.toolEvidence.filter( + ({ toolUseId, status }) => + toolUseId === validEdit?.toolUseId && status === "success", + ), + ).toHaveLength(1); + expect(result.policyDiagnostics).toEqual([ + { + kind: "missing_pre_tool_use_callback", + reason: "no_callback_observed", + toolName: "Edit", + correlatedRequest: true, + }, + ]); + expect(result.policyHookCoverage).toBe(false); + expect(result.terminal).toBe("policy_violation"); + expect(result.terminationEvidence).toEqual({ + beforePolicyOverride: "success", + queryExecution: "iteration_completed", + sdkResult: "success", + }); + expect( + await readFile( + join(fixture.workspaceRoot, FIXTURE_PATHS.cleanTarget), + "utf8", + ), + ).toBe(fixture.cleanTargetReplacement); + } finally { + server.closeAllConnections(); + await new Promise((resolve) => server.close(() => resolve())); + await fixture.cleanup(); + } + }, + 45_000, +); diff --git a/packages/harness/src/experimental/managed-agent-spike/runtime.test.ts b/packages/harness/src/experimental/managed-agent-spike/runtime.test.ts new file mode 100644 index 000000000..86621afd6 --- /dev/null +++ b/packages/harness/src/experimental/managed-agent-spike/runtime.test.ts @@ -0,0 +1,1588 @@ +import { execFile } from "node:child_process"; +import { lstat, mkdir, realpath, symlink, writeFile } from "node:fs/promises"; +import { dirname, join } from "node:path"; +import { promisify } from "node:util"; + +import { afterEach, describe, expect, it, vi } from "vitest"; + +import type { Options } from "@anthropic-ai/claude-agent-sdk"; + +import { + MANAGED_AGENT_CONTRACT, + MANAGED_AGENT_MODEL_ENVIRONMENT_VARIABLES, + resolveManagedAgentModelTarget, +} from "./contract.js"; +import { + FIXTURE_PATHS, + createManagedAgentFixture, + type ManagedAgentFixture, +} from "./fixture.js"; +import { + MANAGED_AGENT_BUILTIN_TOOLS, + MANAGED_AGENT_DISALLOWED_TOOLS, +} from "./permissions.js"; +import { + createManagedAgentMcpRuntime, + runManagedAgentProbe, +} from "./runtime.js"; +import type { + ManagedAgentProcessObserver, + ManagedAgentQuery, + ManagedAgentTeardownObservation, +} from "./types.js"; + +const fixtures: ManagedAgentFixture[] = []; +const SUCCESS_SESSION_ID = "11111111-1111-4111-8111-111111111111"; +const CANCEL_SESSION_ID = "22222222-2222-4222-8222-222222222222"; +const TIMEOUT_SESSION_ID = "33333333-3333-4333-8333-333333333333"; +const CLOSE_SESSION_ID = "44444444-4444-4444-8444-444444444444"; +const execFileAsync = promisify(execFile); + +afterEach(async () => { + await Promise.all(fixtures.splice(0).map((fixture) => fixture.cleanup())); +}); + +function quiescentTeardown(): ManagedAgentTeardownObservation { + return { + quiescent: true, + deadlineMet: true, + processTableAvailable: true, + containmentSupported: true, + ownershipProven: true, + forceKillIssued: true, + toolProcessObservationComplete: true, + toolProcessChannelsClosed: true, + elapsedMs: 12, + observedPids: [], + alivePidsAtDeadline: [], + emergencyCleanupAttempted: false, + }; +} + +function fakeObserver( + teardown: ManagedAgentTeardownObservation = quiescentTeardown(), +): ManagedAgentProcessObserver & { + beginTeardown: ReturnType; + waitForQuiescence: ReturnType; + emergencyCleanup: ReturnType; + dispose: ReturnType; +} { + return { + spawn: vi.fn(() => { + throw new Error("fake query must not spawn"); + }), + beginTeardown: vi.fn(), + armToolProcessContainment: vi.fn(), + prepareCancellation: vi.fn(async () => ({ + supported: true, + reason: "ready" as const, + processTableAvailable: true, + containmentSupported: true, + ownershipProven: true, + observedPids: [], + })), + observeProcessTree: vi.fn(async () => true), + waitForQuiescence: vi.fn(async () => teardown), + emergencyCleanup: vi.fn(async () => ({ + ...teardown, + emergencyCleanupAttempted: true, + })), + dispose: vi.fn(), + }; +} + +function queryFromEvents( + events: readonly unknown[], + close = vi.fn(), +): ManagedAgentQuery { + return { + async *[Symbol.asyncIterator]() { + for (const event of events) yield event; + }, + close, + }; +} + +async function invokePreToolUse( + options: Options, + input: { + readonly toolName: string; + readonly toolInput: unknown; + readonly toolUseId: string; + readonly callbackToolUseId?: string; + }, + signal = new AbortController().signal, +): Promise { + const matcher = options.hooks?.PreToolUse?.[0]; + const hook = matcher?.hooks[0]; + if (!hook) + throw new Error("PreToolUse hook missing from managed-agent probe"); + await hook( + { + hook_event_name: "PreToolUse", + session_id: SUCCESS_SESSION_ID, + transcript_path: "not-persisted", + cwd: String(options.cwd), + tool_name: input.toolName, + tool_input: input.toolInput, + tool_use_id: input.toolUseId, + }, + input.callbackToolUseId ?? input.toolUseId, + { signal }, + ); +} + +async function probeConfig(scenario: "L1" | "L2" = "L1") { + const fixture = await createManagedAgentFixture(() => "runtime-test-secret"); + fixtures.push(fixture); + return { + fixture, + config: { + scenario, + workspaceRoot: fixture.workspaceRoot, + configRoot: fixture.configRoot, + target: "sonnet-5" as const, + gatewayOrigin: "https://gateway.example.test", + gatewayCredential: "dedicated-eval-secret", + prompt: fixture.prompt(scenario), + maxTurns: 10, + maxBudgetUsd: 0.25, + allowedBashCommands: [ + scenario === "L1" ? fixture.l1BashCommand : fixture.l2BashCommand, + ], + pathRoleBindings: scenario === "L1" ? fixture.pathRoleBindings : [], + expectedL1FinalBytes: + scenario === "L1" ? fixture.expectedL1FinalBytes : [], + ...(scenario === "L1" ? { expectedMcpNonce: fixture.nonce } : {}), + preservePaths: [ + FIXTURE_PATHS.dirtySentinel, + FIXTURE_PATHS.untrackedSentinel, + ], + }, + }; +} + +describe("runManagedAgentProbe", () => { + it("provides deterministic MCP success and fail-once recovery", async () => { + const runtime = createManagedAgentMcpRuntime("nonce-1"); + await expect( + runtime.handlers.echoNonce({ nonce: "nonce-1" }), + ).resolves.toEqual({ + content: [{ type: "text", text: "nonce-1" }], + }); + await expect(runtime.handlers.failOnce()).resolves.toMatchObject({ + isError: true, + }); + await expect(runtime.handlers.failOnce()).resolves.not.toHaveProperty( + "isError", + ); + expect( + runtime.invocations.map(({ toolName, status }) => [toolName, status]), + ).toEqual([ + ["mcp__sapiom-managed-agent-spike__echo_nonce", "success"], + ["mcp__sapiom-managed-agent-spike__fail_once", "error"], + ["mcp__sapiom-managed-agent-spike__fail_once", "success"], + ]); + + const mismatch = createManagedAgentMcpRuntime("expected-nonce"); + await expect( + mismatch.handlers.echoNonce({ nonce: "wrong-nonce" }), + ).resolves.toMatchObject({ isError: true }); + expect(mismatch.invocations).toEqual([ + { + toolName: "mcp__sapiom-managed-agent-spike__echo_nonce", + status: "error", + }, + ]); + }); + + it("passes the strict isolated SDK contract and emits only normalized evidence", async () => { + const { config, fixture } = await probeConfig(); + const observer = fakeObserver(); + const close = vi.fn(); + let capturedOptions: Options | undefined; + let capturedPrompt: string | undefined; + const previousOAuth = process.env.CLAUDE_CODE_OAUTH_TOKEN; + process.env.CLAUDE_CODE_OAUTH_TOKEN = "ambient-user-login"; + try { + const result = await runManagedAgentProbe(config, { + hermeticGatewayOrigin: config.gatewayOrigin, + processObserver: observer, + uuid: (() => { + let counter = 0; + return () => + `00000000-0000-4000-8000-${String(++counter).padStart(12, "0")}`; + })(), + queryFactory: ({ prompt, options }) => { + capturedOptions = options; + capturedPrompt = prompt; + return { + async *[Symbol.asyncIterator]() { + yield { + type: "system", + subtype: "init", + session_id: SUCCESS_SESSION_ID, + model: resolveManagedAgentModelTarget("sonnet-5").alias, + }; + yield { + type: "assistant", + session_id: SUCCESS_SESSION_ID, + message: { + id: "message-runtime-1", + content: [ + { + type: "tool_use", + id: "tool-1", + name: "Read", + input: { + file_path: FIXTURE_PATHS.cleanTarget, + secret: fixture.nonce, + }, + }, + ], + }, + }; + await invokePreToolUse(options, { + toolName: "Read", + toolInput: { + file_path: FIXTURE_PATHS.cleanTarget, + secret: fixture.nonce, + }, + toolUseId: "tool-1", + }); + yield { + type: "user", + session_id: SUCCESS_SESSION_ID, + message: { + content: [ + { + type: "tool_result", + tool_use_id: "tool-1", + content: `secret:${fixture.nonce}`, + }, + ], + }, + }; + yield { + type: "result", + subtype: "success", + is_error: false, + session_id: SUCCESS_SESSION_ID, + result: `secret:${fixture.nonce}`, + num_turns: 1, + usage: { input_tokens: 9, output_tokens: 4 }, + }; + }, + close, + }; + }, + }); + + expect(capturedOptions).toBeDefined(); + expect(capturedOptions?.model).toBe( + resolveManagedAgentModelTarget("sonnet-5").alias, + ); + expect(capturedOptions?.tools).toEqual(MANAGED_AGENT_BUILTIN_TOOLS); + expect(capturedOptions?.disallowedTools).toEqual( + MANAGED_AGENT_DISALLOWED_TOOLS, + ); + expect(capturedOptions?.permissionMode).toBe("default"); + expect(capturedOptions?.settingSources).toEqual([]); + expect(capturedOptions?.strictMcpConfig).toBe(true); + expect(capturedOptions?.canUseTool).toBeTypeOf("function"); + expect(capturedOptions?.hooks?.PreToolUse).toHaveLength(1); + expect(capturedOptions?.hooks?.PreToolUse?.[0]?.hooks).toHaveLength(1); + expect( + Object.prototype.hasOwnProperty.call( + capturedOptions?.hooks?.PreToolUse?.[0] ?? {}, + "matcher", + ), + ).toBe(false); + expect(capturedOptions?.spawnClaudeCodeProcess).toBeTypeOf("function"); + expect( + Object.prototype.hasOwnProperty.call(capturedOptions, "allowedTools"), + ).toBe(false); + expect( + Object.prototype.hasOwnProperty.call(capturedOptions, "fallbackModel"), + ).toBe(false); + expect( + Object.prototype.hasOwnProperty.call( + capturedOptions, + "allowDangerouslySkipPermissions", + ), + ).toBe(false); + for (const variable of MANAGED_AGENT_MODEL_ENVIRONMENT_VARIABLES) { + expect(capturedOptions?.env?.[variable]).toBe(capturedOptions?.model); + } + expect(capturedOptions?.env).not.toHaveProperty( + "CLAUDE_CODE_OAUTH_TOKEN", + ); + expect(capturedOptions?.env).not.toHaveProperty("SAPIOM_API_KEY"); + expect(result.terminal).toBe("success"); + expect(result.terminationEvidence).toEqual({ + beforePolicyOverride: "success", + queryExecution: "iteration_completed", + sdkResult: "success", + }); + expect(result.policyHookCoverage).toBe(true); + expect(result.policyDiagnostics).toEqual([]); + expect(result.inferenceTurns).toBe(1); + expect(result.sdkNumTurns).toBe(1); + expect(result.correlation.promptEmbedded).toBe(true); + expect(result.l1Certification).toEqual({ + contractVersion: 2, + promptVersion: "managed-agent-l1-prompt-v2", + }); + expect(result.l1FinalBytes).toEqual([ + { role: "clean_target", matched: false }, + { role: "managed_output", matched: false }, + ]); + expect(result.nonceVerified).toBe(false); + expect(capturedPrompt).toContain( + "SAPIOM_CERTIFICATION_CORRELATION_V1;eval_source=studio-managed-agent-e0-l1-sonnet-5-00000000-0000-4000-8000-000000000002;execution_id=00000000-0000-4000-8000-000000000002", + ); + expect(capturedPrompt).toContain("Do not repeat it"); + expect(result.sdkSessionId).toBe(SUCCESS_SESSION_ID); + expect(result.queryClosed).toBe(true); + expect(result.preservation.every(({ preserved }) => preserved)).toBe( + true, + ); + expect(close).toHaveBeenCalledOnce(); + expect(observer.dispose).toHaveBeenCalledOnce(); + const serialized = JSON.stringify(result); + expect(serialized).not.toContain("dedicated-eval-secret"); + expect(serialized).not.toContain(fixture.nonce); + expect(serialized).not.toContain(fixture.outsideSentinel); + } finally { + if (previousOAuth === undefined) + delete process.env.CLAUDE_CODE_OAUTH_TOKEN; + else process.env.CLAUDE_CODE_OAUTH_TOKEN = previousOAuth; + } + }); + + it("returns recursively immutable evidence after observer finalization", async () => { + const { config } = await probeConfig(); + const result = await runManagedAgentProbe(config, { + hermeticGatewayOrigin: config.gatewayOrigin, + processObserver: fakeObserver(), + queryFactory: () => + queryFromEvents([ + { + type: "system", + subtype: "init", + session_id: SUCCESS_SESSION_ID, + }, + { type: "result", subtype: "success", is_error: false }, + ]), + }); + + expect(Object.isFrozen(result)).toBe(true); + expect(Object.isFrozen(result.teardown)).toBe(true); + expect(Object.isFrozen(result.events)).toBe(true); + expect(Object.isFrozen(result.events[0])).toBe(true); + expect(Object.isFrozen(result.correlation)).toBe(true); + }); + + it("does not follow pre-existing config child symlinks and fails invalid roots before query construction", async () => { + const { config, fixture } = await probeConfig(); + const externalConfig = join(fixture.root, "external-config"); + await mkdir(externalConfig); + await symlink(externalConfig, join(fixture.configRoot, "claude-config")); + let claudeConfigDirectory: string | undefined; + const safeQueryFactory = vi.fn(({ options }: { options: Options }) => { + claudeConfigDirectory = options.env?.CLAUDE_CONFIG_DIR; + return queryFromEvents([ + { + type: "system", + subtype: "init", + session_id: SUCCESS_SESSION_ID, + }, + { type: "result", subtype: "success", is_error: false }, + ]); + }); + + await runManagedAgentProbe(config, { + hermeticGatewayOrigin: config.gatewayOrigin, + processObserver: fakeObserver(), + queryFactory: safeQueryFactory, + }); + + expect(safeQueryFactory).toHaveBeenCalledOnce(); + expect(claudeConfigDirectory).toBeDefined(); + expect(await realpath(claudeConfigDirectory!)).not.toBe( + await realpath(externalConfig), + ); + expect(dirname(dirname(claudeConfigDirectory!))).toBe(fixture.configRoot); + expect((await lstat(claudeConfigDirectory!)).isSymbolicLink()).toBe(false); + + const invalidConfigRoot = join(fixture.root, "config-file"); + await writeFile(invalidConfigRoot, "not a directory"); + const rejectedQueryFactory = vi.fn(() => queryFromEvents([])); + await expect( + runManagedAgentProbe( + { ...config, configRoot: invalidConfigRoot }, + { + hermeticGatewayOrigin: config.gatewayOrigin, + processObserver: fakeObserver(), + queryFactory: rejectedQueryFactory, + }, + ), + ).rejects.toThrow("configRoot must be a directory"); + expect(rejectedQueryFactory).not.toHaveBeenCalled(); + }); + + it("fails before query creation when isolated managed settings disable hooks", async () => { + const { config } = await probeConfig(); + const queryFactory = vi.fn(() => queryFromEvents([])); + const result = await runManagedAgentProbe(config, { + hermeticGatewayOrigin: config.gatewayOrigin, + processObserver: fakeObserver(), + queryFactory, + policySettingsGuard: async ({ environment }) => { + expect(environment).not.toHaveProperty("ANTHROPIC_API_KEY"); + expect(environment).not.toHaveProperty("ANTHROPIC_BASE_URL"); + expect(environment).not.toHaveProperty("ANTHROPIC_CUSTOM_HEADERS"); + expect(environment).toHaveProperty("CLAUDE_CONFIG_DIR"); + throw new Error("disableAllHooks"); + }, + }); + + expect(queryFactory).not.toHaveBeenCalled(); + expect(result.terminal).toBe("policy_violation"); + expect(result.terminationEvidence).toEqual({ + beforePolicyOverride: "incomplete", + queryExecution: "not_started", + sdkResult: "not_observed", + }); + expect(result.policyHookCoverage).toBe(false); + expect(result.queryClosed).toBe(false); + expect(result.correlation.promptEmbedded).toBe(false); + expect(result.workspaceChanges).toEqual([]); + expect( + result.events.filter(({ type }) => type === "terminal"), + ).toHaveLength(1); + }); + + it("records prompt delivery when the query factory receives it and throws", async () => { + const { config } = await probeConfig(); + const result = await runManagedAgentProbe(config, { + hermeticGatewayOrigin: config.gatewayOrigin, + processObserver: fakeObserver(), + policySettingsGuard: async () => undefined, + queryFactory: () => { + throw new Error("synthetic query construction failure"); + }, + }); + + expect(result.correlation.promptEmbedded).toBe(true); + expect(result.queryClosed).toBe(false); + expect(result.terminal).toBe("query_error"); + expect(result.terminationEvidence).toEqual({ + beforePolicyOverride: "query_error", + queryExecution: "construction_failed", + sdkResult: "not_observed", + }); + }); + + it("distinguishes query iteration failure from construction failure", async () => { + const { config } = await probeConfig(); + const observer = fakeObserver(); + const shutdownOrder: string[] = []; + observer.emergencyCleanup.mockImplementation(async () => { + shutdownOrder.push("host_fallback"); + return { ...quiescentTeardown(), emergencyCleanupAttempted: true }; + }); + const result = await runManagedAgentProbe(config, { + hermeticGatewayOrigin: config.gatewayOrigin, + processObserver: observer, + policySettingsGuard: async () => undefined, + queryFactory: ({ options }) => ({ + async *[Symbol.asyncIterator]() { + yield { + type: "system", + subtype: "init", + session_id: SUCCESS_SESSION_ID, + }; + throw new Error("synthetic private iteration failure"); + }, + close: vi.fn(() => { + expect(options.abortController?.signal.aborted).toBe(true); + shutdownOrder.push("sdk_query_close"); + }), + }), + }); + + expect(result.terminal).toBe("query_error"); + expect(result.terminationEvidence).toEqual({ + beforePolicyOverride: "query_error", + queryExecution: "iteration_failed", + sdkResult: "not_observed", + }); + expect(JSON.stringify(result)).not.toContain( + "synthetic private iteration failure", + ); + expect(shutdownOrder).toEqual(["sdk_query_close", "host_fallback"]); + }); + + it("reports a completed iteration that emitted no SDK result", async () => { + const { config } = await probeConfig(); + const result = await runManagedAgentProbe(config, { + hermeticGatewayOrigin: config.gatewayOrigin, + processObserver: fakeObserver(), + policySettingsGuard: async () => undefined, + queryFactory: () => queryFromEvents([]), + }); + + expect(result.terminal).toBe("incomplete"); + expect(result.terminationEvidence).toEqual({ + beforePolicyOverride: "incomplete", + queryExecution: "iteration_completed", + sdkResult: "not_observed", + }); + }); + + it("preserves teardown failure priority when policy preflight fails", async () => { + const { config } = await probeConfig(); + const observer = fakeObserver({ + quiescent: false, + deadlineMet: false, + processTableAvailable: true, + containmentSupported: true, + ownershipProven: false, + forceKillIssued: false, + toolProcessObservationComplete: true, + toolProcessChannelsClosed: true, + elapsedMs: 5_001, + observedPids: [8001], + alivePidsAtDeadline: [8001], + emergencyCleanupAttempted: false, + }); + const result = await runManagedAgentProbe(config, { + hermeticGatewayOrigin: config.gatewayOrigin, + processObserver: observer, + queryFactory: vi.fn(() => queryFromEvents([])), + policySettingsGuard: async () => { + throw new Error("disableAllHooks"); + }, + }); + + expect(result.terminal).toBe("teardown_timeout"); + expect(result.events.at(-1)).toMatchObject({ + type: "terminal", + terminal: "teardown_timeout", + }); + expect(observer.emergencyCleanup).toHaveBeenCalledOnce(); + }); + + it("rejects a successful stream when a requested tool has no primary hook decision", async () => { + const { config } = await probeConfig(); + const result = await runManagedAgentProbe(config, { + hermeticGatewayOrigin: config.gatewayOrigin, + processObserver: fakeObserver(), + policySettingsGuard: async () => undefined, + queryFactory: () => + queryFromEvents([ + { + type: "assistant", + message: { + id: "message-with-disabled-hook", + content: [ + { + type: "tool_use", + id: "tool-with-disabled-hook", + name: "Read", + input: { file_path: FIXTURE_PATHS.cleanTarget }, + }, + ], + }, + }, + { + type: "result", + subtype: "success", + is_error: false, + num_turns: 1, + }, + ]), + }); + + expect(result.policyHookCoverage).toBe(false); + expect(result.terminal).toBe("policy_violation"); + expect(result.permissionEvidence).toEqual([]); + expect(result.policyDiagnostics).toEqual([ + { + kind: "missing_pre_tool_use_callback", + reason: "no_callback_observed", + toolName: "Read", + correlatedRequest: true, + }, + ]); + expect(result.terminationEvidence).toEqual({ + beforePolicyOverride: "success", + queryExecution: "iteration_completed", + sdkResult: "success", + }); + }); + + it("cannot certify a malformed SDK tool-use identifier as policy-covered evidence", async () => { + const { config } = await probeConfig(); + const result = await runManagedAgentProbe(config, { + hermeticGatewayOrigin: config.gatewayOrigin, + processObserver: fakeObserver(), + policySettingsGuard: async () => undefined, + queryFactory: () => + queryFromEvents([ + { + type: "assistant", + message: { + id: "message-with-missing-tool-id", + content: [ + { + type: "tool_use", + name: "Read", + input: { file_path: FIXTURE_PATHS.cleanTarget }, + }, + ], + }, + }, + { + type: "result", + subtype: "success", + is_error: false, + num_turns: 1, + }, + ]), + }); + + expect(result.correlation.promptEmbedded).toBe(true); + expect(result.toolEvidence).toEqual([]); + expect(result.permissionEvidence).toEqual([]); + expect(result.policyHookCoverage).toBe(false); + expect(result.terminal).toBe("policy_violation"); + expect(result.terminationEvidence).toEqual({ + beforePolicyOverride: "query_error", + queryExecution: "event_normalization_failed", + sdkResult: "not_observed", + eventNormalizationFailure: "tool_request_id_invalid", + }); + }); + + it("reports a correlated PreToolUse guard rejection without certifying coverage", async () => { + const { config } = await probeConfig(); + const requestIdSecret = "guarded-request-id-secret"; + const callbackIdSecret = "mismatched-callback-id-secret"; + const result = await runManagedAgentProbe(config, { + hermeticGatewayOrigin: config.gatewayOrigin, + processObserver: fakeObserver(), + policySettingsGuard: async () => undefined, + queryFactory: ({ options }) => ({ + async *[Symbol.asyncIterator]() { + yield { + type: "assistant", + message: { + id: "guard-rejection-message", + content: [ + { + type: "tool_use", + id: requestIdSecret, + name: "Edit", + input: { file_path: FIXTURE_PATHS.cleanTarget }, + }, + ], + }, + }; + await invokePreToolUse(options, { + toolName: "Edit", + toolInput: { file_path: FIXTURE_PATHS.cleanTarget }, + toolUseId: requestIdSecret, + callbackToolUseId: callbackIdSecret, + }); + yield { + type: "user", + message: { + content: [ + { + type: "tool_result", + tool_use_id: requestIdSecret, + is_error: true, + }, + ], + }, + }; + yield { + type: "result", + subtype: "success", + is_error: false, + num_turns: 1, + }; + }, + close: vi.fn(), + }), + }); + + expect(result.permissionEvidence).toEqual([]); + expect(result.policyDiagnostics).toEqual([ + { + kind: "pre_tool_use_guard_rejection", + reason: "callback_tool_use_id_mismatch", + toolName: "Edit", + correlatedRequest: true, + }, + ]); + expect(result.policyHookCoverage).toBe(false); + expect(result.terminal).toBe("policy_violation"); + const serialized = JSON.stringify(result); + expect(serialized).not.toContain(requestIdSecret); + expect(serialized).not.toContain(callbackIdSecret); + }); + + it("rejects duplicate requested tool ids instead of reusing one policy decision", async () => { + const { config } = await probeConfig(); + const duplicateToolUseId = "duplicate-tool-use-id"; + const result = await runManagedAgentProbe(config, { + hermeticGatewayOrigin: config.gatewayOrigin, + processObserver: fakeObserver(), + policySettingsGuard: async () => undefined, + queryFactory: ({ options }) => ({ + async *[Symbol.asyncIterator]() { + yield { + type: "assistant", + message: { + id: "duplicate-tool-message-1", + content: [ + { + type: "tool_use", + id: duplicateToolUseId, + name: "Read", + input: { file_path: FIXTURE_PATHS.cleanTarget }, + }, + ], + }, + }; + await invokePreToolUse(options, { + toolName: "Read", + toolInput: { file_path: FIXTURE_PATHS.cleanTarget }, + toolUseId: duplicateToolUseId, + }); + yield { + type: "assistant", + message: { + id: "duplicate-tool-message-2", + content: [ + { + type: "tool_use", + id: duplicateToolUseId, + name: "Bash", + input: { command: "touch must-not-inherit-allow" }, + }, + ], + }, + }; + await invokePreToolUse(options, { + toolName: "Bash", + toolInput: { command: "touch must-not-inherit-allow" }, + toolUseId: duplicateToolUseId, + }); + yield { + type: "result", + subtype: "success", + is_error: false, + num_turns: 2, + }; + }, + close: vi.fn(), + }), + }); + + expect(result.permissionEvidence).toHaveLength(1); + expect(result.policyHookCoverage).toBe(false); + expect(result.terminal).toBe("policy_violation"); + }); + + it("redacts malicious SDK and permission identifiers from the complete result", async () => { + const { config } = await probeConfig(); + const sessionSecret = "session-secret-injected-by-sdk"; + const toolIdSecret = "tool-id-secret-injected-by-sdk"; + const toolNameSecret = "ReadSecretInjectedBySdk"; + const permissionIdSecret = "permission-id-secret-injected-by-sdk"; + const permissionNameSecret = "PermissionSecretInjectedBySdk"; + const result = await runManagedAgentProbe(config, { + hermeticGatewayOrigin: config.gatewayOrigin, + processObserver: fakeObserver(), + queryFactory: ({ options }) => ({ + async *[Symbol.asyncIterator]() { + yield { + type: "system", + subtype: "init", + session_id: sessionSecret, + }; + yield { + type: "assistant", + session_id: sessionSecret, + message: { + id: permissionIdSecret, + content: [ + { + type: "tool_use", + id: toolIdSecret, + name: toolNameSecret, + input: { secret: "tool-input-secret" }, + }, + ], + }, + }; + await invokePreToolUse(options, { + toolName: permissionNameSecret, + toolInput: { secret: "tool-input-secret" }, + toolUseId: toolIdSecret, + }); + yield { + type: "user", + session_id: sessionSecret, + message: { + content: [ + { + type: "tool_result", + tool_use_id: toolIdSecret, + content: "tool-result-secret", + }, + ], + }, + }; + yield { + type: "result", + subtype: "success", + is_error: false, + session_id: sessionSecret, + num_turns: 1, + }; + }, + close: vi.fn(), + }), + }); + + expect(result.sdkSessionId).toBeUndefined(); + expect(result.toolEvidence.slice(0, 2)).toMatchObject([ + { toolName: "unknown", status: "requested" }, + { toolName: "unknown", status: "success" }, + ]); + expect(result.toolEvidence[0]?.toolUseId).toBe( + result.toolEvidence[1]?.toolUseId, + ); + expect(result.permissionEvidence).toMatchObject([ + { + toolName: "unknown", + decision: "deny", + reason: "tool_not_allowed", + source: "pre_tool_use", + }, + ]); + const serialized = JSON.stringify(result); + for (const secret of [ + sessionSecret, + toolIdSecret, + toolNameSecret, + permissionIdSecret, + permissionNameSecret, + "tool-input-secret", + "tool-result-secret", + ]) { + expect(serialized).not.toContain(secret); + } + }); + + it("classifies an explicit active-run abort as cancellation", async () => { + const { config } = await probeConfig("L2"); + const observer = fakeObserver(); + const close = vi.fn(); + let capturedOptions: Options | undefined; + const result = await runManagedAgentProbe(config, { + hermeticGatewayOrigin: config.gatewayOrigin, + processObserver: observer, + waitForCancellationSignal: async () => undefined, + queryFactory: ({ options }) => ({ + async *[Symbol.asyncIterator]() { + capturedOptions = options; + yield { + type: "system", + subtype: "init", + session_id: CANCEL_SESSION_ID, + }; + if (!options.abortController?.signal.aborted) { + await new Promise((resolveAbort) => + options.abortController?.signal.addEventListener( + "abort", + () => resolveAbort(), + { once: true }, + ), + ); + } + throw new Error("synthetic abort"); + }, + close, + }), + }); + + expect(result.terminal).toBe("cancelled"); + expect(result.cancellationRequested).toBe(true); + expect(result.queryClosed).toBe(true); + expect(capturedOptions?.tools).toEqual(["Bash"]); + expect(capturedOptions?.disallowedTools).toEqual( + expect.arrayContaining(["Read", "Edit", "Write"]), + ); + expect(capturedOptions?.mcpServers).toEqual({}); + expect( + result.events.filter(({ type }) => type === "terminal"), + ).toHaveLength(1); + }); + + it("keeps armed L2 readiness alive after an early query failure before aborting the SDK", async () => { + const { config } = await probeConfig("L2"); + const observer = fakeObserver(); + let now = 1_000; + let readinessCompleted = false; + let readinessWasAborted = false; + const result = await runManagedAgentProbe(config, { + hermeticGatewayOrigin: config.gatewayOrigin, + processObserver: observer, + monotonicNow: () => now, + waitForCancellationSignal: (signal) => + new Promise((resolveReadiness, rejectReadiness) => { + const timer = setTimeout(() => { + now = 3_250; + readinessCompleted = true; + resolveReadiness(); + }, 25); + signal.addEventListener( + "abort", + () => { + if (!readinessCompleted) readinessWasAborted = true; + clearTimeout(timer); + rejectReadiness( + new Error("readiness aborted before registration"), + ); + }, + { once: true }, + ); + }), + queryFactory: ({ options }) => ({ + async *[Symbol.asyncIterator]() { + yield { + type: "system", + subtype: "init", + session_id: CANCEL_SESSION_ID, + }; + await invokePreToolUse(options, { + toolName: "Bash", + toolInput: { command: config.allowedBashCommands[0] }, + toolUseId: "toolu_early_query_failure", + }); + throw new Error("synthetic early query failure"); + }, + close: vi.fn(), + }), + }); + + expect(readinessCompleted).toBe(true); + expect(readinessWasAborted).toBe(false); + expect(observer.armToolProcessContainment).toHaveBeenCalledOnce(); + expect(observer.emergencyCleanup).toHaveBeenCalledWith({ + startedAtMs: 1_000, + deadlineAtMs: 6_000, + }); + expect(result.teardown.elapsedMs).toBe(2_250); + expect(result.cancellationRequested).toBe(false); + expect(result.terminationEvidence.beforePolicyOverride).toBe("query_error"); + }, 10_000); + + it("awaits async-generator cleanup after void close before host fallback", async () => { + const { config } = await probeConfig("L2"); + const observer = fakeObserver(); + let resolveCancellation!: () => void; + const cancellation = new Promise((resolve) => { + resolveCancellation = resolve; + }); + let resolveReturn!: () => void; + const returned = new Promise((resolve) => { + resolveReturn = resolve; + }); + let markToolArmed!: () => void; + const toolArmed = new Promise((resolve) => { + markToolArmed = resolve; + }); + let markCloseCalled!: () => void; + const closeCalled = new Promise((resolve) => { + markCloseCalled = resolve; + }); + const returnCleanup = vi.fn(async () => { + await returned; + return { done: true as const, value: undefined }; + }); + const close = vi.fn(() => markCloseCalled()); + let nextCall = 0; + + const resultPromise = runManagedAgentProbe(config, { + hermeticGatewayOrigin: config.gatewayOrigin, + processObserver: observer, + waitForCancellationSignal: async () => cancellation, + queryFactory: ({ options }) => ({ + [Symbol.asyncIterator]() { + return { + next: async (): Promise> => { + nextCall += 1; + if (nextCall === 1) { + return { + done: false, + value: { + type: "assistant", + message: { + id: "assistant_cleanup_order", + content: [ + { + type: "tool_use", + id: "toolu_cleanup_order", + name: "Bash", + input: { command: config.allowedBashCommands[0] }, + }, + ], + }, + }, + }; + } + await invokePreToolUse(options, { + toolName: "Bash", + toolInput: { command: config.allowedBashCommands[0] }, + toolUseId: "toolu_cleanup_order", + }); + markToolArmed(); + return new Promise>(() => undefined); + }, + return: returnCleanup, + }; + }, + close, + return: returnCleanup, + }), + }); + + await toolArmed; + resolveCancellation(); + await closeCalled; + await new Promise((resolve) => setImmediate(resolve)); + const returnStartedBeforeRelease = returnCleanup.mock.calls.length === 1; + const fallbackStartedBeforeRelease = + observer.emergencyCleanup.mock.calls.length > 0; + resolveReturn(); + const result = await resultPromise; + + expect(close).toHaveBeenCalledOnce(); + expect(returnStartedBeforeRelease).toBe(true); + expect(fallbackStartedBeforeRelease).toBe(false); + expect(observer.emergencyCleanup).toHaveBeenCalledOnce(); + expect(result.queryClosed).toBe(true); + expect(result.terminal).toBe("cancelled"); + expect(result.cancellationRequested).toBe(true); + }, 10_000); + + it.each([ + ["clean completion", false, "incomplete", undefined], + ["SDK error-result completion", true, "sdk_result_error", undefined], + [ + "clean completion with a live process", + false, + "teardown_timeout", + "liveness", + ], + [ + "SDK error-result completion with an open tool channel", + true, + "teardown_timeout", + "channel", + ], + ] as const)( + "retains armed L2 readiness and one deadline after %s", + async (_description, emitErrorResult, expectedTerminal, failClosedOn) => { + const { config } = await probeConfig("L2"); + const observer = fakeObserver( + failClosedOn + ? { + ...quiescentTeardown(), + quiescent: false, + deadlineMet: false, + toolProcessChannelsClosed: failClosedOn !== "channel", + observedPids: failClosedOn === "liveness" ? [7_001] : [], + alivePidsAtDeadline: failClosedOn === "liveness" ? [7_001] : [], + } + : quiescentTeardown(), + ); + let now = 1_000; + let readinessCompleted = false; + let readinessWasAborted = false; + let capturedOptions: Options | undefined; + + const result = await runManagedAgentProbe(config, { + hermeticGatewayOrigin: config.gatewayOrigin, + processObserver: observer, + monotonicNow: () => now, + waitForCancellationSignal: (signal) => + new Promise((resolveReadiness, rejectReadiness) => { + const timer = setTimeout(() => { + now = 3_250; + readinessCompleted = true; + resolveReadiness(); + }, 25); + signal.addEventListener( + "abort", + () => { + if (!readinessCompleted) readinessWasAborted = true; + clearTimeout(timer); + rejectReadiness( + new Error("readiness aborted after early completion"), + ); + }, + { once: true }, + ); + }), + queryFactory: ({ options }) => ({ + async *[Symbol.asyncIterator]() { + capturedOptions = options; + yield { + type: "assistant", + message: { + id: "assistant_early_settlement", + content: [ + { + type: "tool_use", + id: "toolu_early_settlement", + name: "Bash", + input: { command: config.allowedBashCommands[0] }, + }, + ], + }, + }; + await invokePreToolUse(options, { + toolName: "Bash", + toolInput: { command: config.allowedBashCommands[0] }, + toolUseId: "toolu_early_settlement", + }); + if (emitErrorResult) { + yield { + type: "result", + subtype: "error_max_turns", + is_error: true, + num_turns: 1, + }; + } + }, + close: vi.fn(), + }), + }); + + expect(readinessCompleted).toBe(true); + expect(readinessWasAborted).toBe(false); + expect(capturedOptions?.abortController?.signal.aborted).toBe(true); + expect(observer.emergencyCleanup).toHaveBeenCalledWith({ + startedAtMs: 1_000, + deadlineAtMs: 6_000, + }); + expect(result.teardown.elapsedMs).toBe(2_250); + expect(result.queryClosed).toBe(true); + expect(result.cancellationRequested).toBe(false); + expect(result.terminal).toBe(expectedTerminal); + expect(result.terminationEvidence.beforePolicyOverride).toBe( + expectedTerminal, + ); + if (failClosedOn === "liveness") { + expect(result.teardown.alivePidsAtDeadline).toEqual([7_001]); + } + if (failClosedOn === "channel") { + expect(result.teardown.toolProcessChannelsClosed).toBe(false); + } + }, + 10_000, + ); + + it("abandons a never-resolving iterator next immediately after raw cancellation", async () => { + const { config } = await probeConfig("L2"); + const observer = fakeObserver(); + const close = vi.fn(); + let markNextStarted: (() => void) | undefined; + const nextStarted = new Promise((resolveStarted) => { + markNextStarted = resolveStarted; + }); + const resultPromise = runManagedAgentProbe(config, { + hermeticGatewayOrigin: config.gatewayOrigin, + processObserver: observer, + waitForCancellationSignal: async () => undefined, + queryFactory: () => ({ + [Symbol.asyncIterator]() { + return { + next: () => { + markNextStarted?.(); + return new Promise>(() => undefined); + }, + return: async () => ({ done: true, value: undefined }), + }; + }, + close, + }), + }); + + await nextStarted; + const result = await Promise.race([ + resultPromise, + new Promise((_, reject) => + setTimeout( + () => reject(new Error("probe stayed blocked on iterator.next()")), + 2_000, + ), + ), + ]); + + expect(result.terminal).toBe("cancelled"); + expect(result.terminationEvidence.queryExecution).toBe("iteration_aborted"); + expect(close).toHaveBeenCalledOnce(); + }); + + it("accepts an awaited close promise when no iterator return exists", async () => { + const { config } = await probeConfig(); + const close = vi.fn(async () => undefined); + const result = await runManagedAgentProbe(config, { + hermeticGatewayOrigin: config.gatewayOrigin, + processObserver: fakeObserver(), + queryFactory: () => ({ + [Symbol.asyncIterator]() { + return { + next: async () => ({ done: true as const, value: undefined }), + }; + }, + close, + }), + }); + + expect(close).toHaveBeenCalledOnce(); + expect(result.queryClosed).toBe(true); + expect(result.terminationEvidence.queryExecution).toBe( + "iteration_completed", + ); + }); + + it("keeps a CLI-shaped process alive until bounded close and cleanup complete", async () => { + const { config } = await probeConfig("L2"); + const childProgram = String.raw` +const { runManagedAgentProbe } = await import( + process.env.SAPIOM_TEST_RUNTIME_MODULE_URL +); +const config = JSON.parse(process.env.SAPIOM_TEST_PROBE_CONFIG); +let cleanupCalled = false; +const teardown = { + quiescent: true, + deadlineMet: true, + processTableAvailable: true, + containmentSupported: true, + ownershipProven: true, + forceKillIssued: true, + toolProcessObservationComplete: true, + toolProcessChannelsClosed: true, + elapsedMs: 0, + observedPids: [], + alivePidsAtDeadline: [], + emergencyCleanupAttempted: false, +}; +const observer = { + spawn() { throw new Error("fake query must not spawn"); }, + beginTeardown() {}, + armToolProcessContainment() {}, + async prepareCancellation() { + return { + supported: true, + reason: "ready", + processTableAvailable: true, + containmentSupported: true, + ownershipProven: true, + observedPids: [], + }; + }, + async observeProcessTree() { return true; }, + async waitForQuiescence() { return teardown; }, + async emergencyCleanup() { + cleanupCalled = true; + return { ...teardown, emergencyCleanupAttempted: true }; + }, + dispose() {}, +}; +const result = await runManagedAgentProbe(config, { + hermeticGatewayOrigin: config.gatewayOrigin, + processObserver: observer, + policySettingsGuard: async () => undefined, + waitForCancellationSignal: async () => undefined, + queryFactory: () => ({ + [Symbol.asyncIterator]() { + return { next: () => new Promise(() => undefined) }; + }, + close: () => new Promise(() => undefined), + }), +}); +process.stdout.write(JSON.stringify({ + cleanupCalled, + queryClosed: result.queryClosed, + terminal: result.terminal, +})); +`; + const startedAt = Date.now(); + const { stdout } = await execFileAsync( + process.execPath, + ["--import", "tsx", "--input-type=module", "--eval", childProgram], + { + cwd: process.cwd(), + encoding: "utf8", + env: { + ...process.env, + SAPIOM_TEST_PROBE_CONFIG: JSON.stringify(config), + SAPIOM_TEST_RUNTIME_MODULE_URL: new URL( + "./runtime.ts", + import.meta.url, + ).href, + }, + timeout: 8_000, + killSignal: "SIGKILL", + }, + ); + + expect(Date.now() - startedAt).toBeGreaterThanOrEqual(1_500); + expect(JSON.parse(stdout)).toEqual({ + cleanupCalled: true, + queryClosed: false, + terminal: "close_timeout", + }); + }, 10_000); + + it("includes iterator abandonment and close in the one cancellation deadline", async () => { + const { config } = await probeConfig("L2"); + let now = 1_000; + const observer = fakeObserver(); + const close = vi.fn(async () => { + now = 3_250; + }); + const result = await runManagedAgentProbe(config, { + hermeticGatewayOrigin: config.gatewayOrigin, + processObserver: observer, + monotonicNow: () => now, + waitForCancellationSignal: async () => undefined, + queryFactory: () => ({ + [Symbol.asyncIterator]() { + return { + next: () => new Promise>(() => undefined), + }; + }, + close, + }), + }); + + expect(observer.emergencyCleanup).toHaveBeenCalledWith({ + startedAtMs: 1_000, + deadlineAtMs: 6_000, + }); + expect(result.teardown).toMatchObject({ + quiescent: true, + deadlineMet: true, + elapsedMs: 2_250, + }); + expect(result.terminal).toBe("cancelled"); + }); + + it("adopts the exact teardown deadline before close and iterator return", async () => { + const { config } = await probeConfig(); + const observer = fakeObserver(); + const order: string[] = []; + observer.beginTeardown.mockImplementation(() => { + order.push("observer_deadline_adopted"); + }); + const close = vi.fn(() => { + order.push("query_close"); + }); + const queryReturn = vi.fn(async () => { + order.push("query_return"); + return { done: true as const, value: undefined }; + }); + + await runManagedAgentProbe(config, { + hermeticGatewayOrigin: config.gatewayOrigin, + processObserver: observer, + queryFactory: () => ({ + [Symbol.asyncIterator]() { + return { + next: async () => ({ done: true as const, value: undefined }), + }; + }, + close, + return: queryReturn, + }), + }); + + expect(order).toEqual([ + "observer_deadline_adopted", + "query_close", + "query_return", + ]); + expect(observer.beginTeardown).toHaveBeenCalledOnce(); + const adoptedDeadline = observer.beginTeardown.mock.calls[0]?.[0]; + expect(observer.waitForQuiescence.mock.calls[0]?.[0]).toBe(adoptedDeadline); + }); + + it("never extends the teardown budget or reports deadline success when wall time rolls back", async () => { + const wallClock = vi.spyOn(Date, "now").mockReturnValue(10_000); + try { + const { config } = await probeConfig("L2"); + let monotonicTime = 10_000; + const observer = fakeObserver(); + const result = await runManagedAgentProbe(config, { + hermeticGatewayOrigin: config.gatewayOrigin, + processObserver: observer, + monotonicNow: () => monotonicTime, + waitForCancellationSignal: async () => undefined, + queryFactory: () => ({ + [Symbol.asyncIterator]() { + return { + next: () => new Promise>(() => undefined), + return: () => + new Promise>(() => undefined), + }; + }, + close: () => { + wallClock.mockReturnValue(-100_000); + monotonicTime = 15_001; + }, + return: () => + new Promise>(() => undefined), + }), + }); + const deadline = observer.emergencyCleanup.mock.calls[0]?.[0] as + | { readonly startedAtMs: number; readonly deadlineAtMs: number } + | undefined; + + expect(deadline).toEqual({ + startedAtMs: 10_000, + deadlineAtMs: 15_000, + }); + expect(deadline!.deadlineAtMs - deadline!.startedAtMs).toBe(5_000); + expect(result.teardown.deadlineMet).toBe(false); + expect(result.teardown.elapsedMs).toBeGreaterThanOrEqual(5_000); + } finally { + wallClock.mockRestore(); + } + }); + + it("records teardown failure before attempting emergency cleanup", async () => { + const { config } = await probeConfig(); + const observer = fakeObserver({ + quiescent: false, + deadlineMet: false, + processTableAvailable: true, + containmentSupported: true, + ownershipProven: false, + forceKillIssued: false, + toolProcessObservationComplete: true, + toolProcessChannelsClosed: true, + elapsedMs: 5_001, + observedPids: [9001], + alivePidsAtDeadline: [9001], + emergencyCleanupAttempted: false, + }); + const result = await runManagedAgentProbe(config, { + hermeticGatewayOrigin: config.gatewayOrigin, + processObserver: observer, + queryFactory: () => + queryFromEvents([ + { + type: "system", + subtype: "init", + session_id: TIMEOUT_SESSION_ID, + }, + { type: "result", subtype: "success", is_error: false }, + ]), + }); + + expect(result.terminal).toBe("teardown_timeout"); + expect(result.teardown.emergencyCleanupAttempted).toBe(true); + expect(observer.emergencyCleanup).toHaveBeenCalledOnce(); + expect(result.events.at(-1)).toMatchObject({ + type: "terminal", + terminal: "teardown_timeout", + }); + }); + + it("classifies a throwing query close without skipping abort or observer disposal", async () => { + const { config } = await probeConfig(); + const observer = fakeObserver(); + let abortSignal: AbortSignal | undefined; + const result = await runManagedAgentProbe(config, { + hermeticGatewayOrigin: config.gatewayOrigin, + processObserver: observer, + queryFactory: ({ options }) => { + abortSignal = options.abortController?.signal; + return queryFromEvents( + [ + { + type: "system", + subtype: "init", + session_id: CLOSE_SESSION_ID, + }, + { type: "result", subtype: "success", is_error: false }, + ], + vi.fn(() => { + throw new Error("synthetic close failure"); + }), + ); + }, + }); + + expect(result.terminal).toBe("close_timeout"); + expect(result.queryClosed).toBe(false); + expect(abortSignal?.aborted).toBe(true); + expect(observer.dispose).toHaveBeenCalledOnce(); + }); + + it("rejects a test gateway unless the explicit hermetic seam is present", async () => { + const { config } = await probeConfig(); + const queryFactory = vi.fn(() => queryFromEvents([])); + await expect( + runManagedAgentProbe(config, { + processObserver: fakeObserver(), + queryFactory, + }), + ).rejects.toThrow("pinned direct Sapiom gateway origin"); + expect(queryFactory).not.toHaveBeenCalled(); + }); + + it.skipIf( + process.versions.node === MANAGED_AGENT_CONTRACT.certificationNodeVersion, + )( + "enforces the exact Node pin inside the exported direct-gateway runtime", + async () => { + const { config } = await probeConfig(); + const queryFactory = vi.fn(() => queryFromEvents([])); + await expect( + runManagedAgentProbe( + { + ...config, + gatewayOrigin: MANAGED_AGENT_CONTRACT.directGatewayOrigin, + }, + { processObserver: fakeObserver(), queryFactory }, + ), + ).rejects.toThrow( + `Direct managed-agent probes require Node ${MANAGED_AGENT_CONTRACT.certificationNodeVersion}`, + ); + expect(queryFactory).not.toHaveBeenCalled(); + }, + ); + + it("rejects the hermetic origin seam without an injected query factory", async () => { + const { config } = await probeConfig(); + await expect( + runManagedAgentProbe(config, { + hermeticGatewayOrigin: config.gatewayOrigin, + processObserver: fakeObserver(), + }), + ).rejects.toThrow("requires an injected queryFactory"); + }); +}); diff --git a/packages/harness/src/experimental/managed-agent-spike/runtime.ts b/packages/harness/src/experimental/managed-agent-spike/runtime.ts new file mode 100644 index 000000000..fb3de320a --- /dev/null +++ b/packages/harness/src/experimental/managed-agent-spike/runtime.ts @@ -0,0 +1,862 @@ +import { randomUUID } from "node:crypto"; +import { performance } from "node:perf_hooks"; + +import { + createSdkMcpServer, + query as agentSdkQuery, + tool, + type McpSdkServerConfigWithInstance, + type Options, +} from "@anthropic-ai/claude-agent-sdk"; +import { z } from "zod"; + +import { + MANAGED_AGENT_CONTRACT, + MANAGED_AGENT_L1_CERTIFICATION_CONTRACT, + validateManagedAgentProbeConfig, +} from "./contract.js"; +import { buildManagedAgentChildEnvironment } from "./environment.js"; +import { ManagedAgentEventError, ManagedAgentEventRecorder } from "./events.js"; +import { + captureManagedAgentWorkspaceSnapshot, + diffManagedAgentWorkspaceSnapshots, + observeManagedAgentL1FinalBytes, + observeManagedAgentPreservation, +} from "./fixture.js"; +import { + MANAGED_AGENT_BUILTIN_TOOLS, + MANAGED_AGENT_DISALLOWED_TOOLS, + createManagedAgentPolicyBoundary, + type ManagedAgentPreToolUseGuardRejection, +} from "./permissions.js"; +import { createLocalManagedAgentProcessObserver } from "./process-observer.js"; +import { + assertManagedAgentHooksEnabled, + buildManagedAgentSettingsGuardEnvironment, +} from "./settings-guard.js"; +import type { + ManagedAgentProbeConfig, + ManagedAgentProbeDependencies, + ManagedAgentPolicyDiagnostic, + ManagedAgentProbeResult, + ManagedAgentQuery, + ManagedAgentQueryExecutionOutcome, + ManagedAgentTeardownObservation, + ManagedAgentTeardownDeadline, + ManagedAgentTerminalClassification, + ManagedAgentToolEvidence, +} from "./types.js"; + +export const MANAGED_AGENT_MCP_SERVER_NAME = "sapiom-managed-agent-spike"; +export const MANAGED_AGENT_TEARDOWN_TIMEOUT_MS = 5_000; +export const MANAGED_AGENT_CORRELATION_MARKER_VERSION = + "SAPIOM_CERTIFICATION_CORRELATION_V1"; +const QUERY_CLOSE_TIMEOUT_MS = 2_000; +const FORCE_CLEANUP_CONFIRMATION_RESERVE_MS = 1_000; +const MANAGED_AGENT_L2_BUILTIN_TOOLS = ["Bash"] as const; +const MANAGED_AGENT_L2_DISALLOWED_TOOLS = [ + ...MANAGED_AGENT_DISALLOWED_TOOLS, + "Read", + "Edit", + "Write", +] as const; + +type McpToolName = "echo_nonce" | "fail_once"; + +export interface ManagedAgentMcpRuntime { + readonly server: McpSdkServerConfigWithInstance; + readonly qualifiedToolNames: readonly string[]; + readonly invocations: readonly ManagedAgentToolEvidence[]; + readonly handlers: { + readonly echoNonce: (input: { readonly nonce: string }) => Promise<{ + content: Array<{ type: "text"; text: string }>; + isError?: boolean; + }>; + readonly failOnce: () => Promise<{ + content: Array<{ type: "text"; text: string }>; + isError?: boolean; + }>; + }; +} + +export function qualifiedManagedAgentMcpToolName(name: McpToolName): string { + return `mcp__${MANAGED_AGENT_MCP_SERVER_NAME}__${name}`; +} + +export function createManagedAgentMcpRuntime( + expectedEchoNonce?: string, +): ManagedAgentMcpRuntime { + const invocations: ManagedAgentToolEvidence[] = []; + let failOnceCalls = 0; + const nonceSchema = { nonce: z.string().min(1).max(256) }; + const handlers = { + async echoNonce({ nonce }: { readonly nonce: string }) { + const matched = + expectedEchoNonce === undefined || nonce === expectedEchoNonce; + invocations.push({ + toolName: qualifiedManagedAgentMcpToolName("echo_nonce"), + status: matched ? ("success" as const) : ("error" as const), + }); + return matched + ? { content: [{ type: "text" as const, text: nonce }] } + : { + content: [ + { + type: "text" as const, + text: "nonce did not match the untracked-file sentinel", + }, + ], + isError: true, + }; + }, + async failOnce() { + failOnceCalls += 1; + const failed = failOnceCalls === 1; + invocations.push({ + toolName: qualifiedManagedAgentMcpToolName("fail_once"), + status: failed ? ("error" as const) : ("success" as const), + }); + return failed + ? { + content: [ + { + type: "text" as const, + text: "planned managed-agent probe failure; retry once", + }, + ], + isError: true, + } + : { + content: [ + { + type: "text" as const, + text: "planned managed-agent probe recovery succeeded", + }, + ], + }; + }, + }; + const echoNonce = tool( + "echo_nonce", + "Return the supplied nonce exactly for the local managed-agent probe.", + nonceSchema, + handlers.echoNonce, + { alwaysLoad: true }, + ); + const failOnce = tool( + "fail_once", + "Return a planned error once, then succeed on the next call.", + nonceSchema, + handlers.failOnce, + { alwaysLoad: true }, + ); + return { + server: createSdkMcpServer({ + name: MANAGED_AGENT_MCP_SERVER_NAME, + version: "0.1.0", + instructions: + "These tools exist only for deterministic Sapiom local managed-agent feasibility probes.", + tools: [echoNonce, failOnce], + alwaysLoad: true, + }), + qualifiedToolNames: [ + qualifiedManagedAgentMcpToolName("echo_nonce"), + qualifiedManagedAgentMcpToolName("fail_once"), + ], + invocations, + handlers, + }; +} + +function defaultQueryFactory(input: { + readonly prompt: string; + readonly options: Options; +}): ManagedAgentQuery { + // The narrow return type intentionally withholds control-channel methods, + // especially Query.mcpCall(), because those calls bypass permission checks. + return agentSdkQuery(input); +} + +function safeEvalSource( + scenario: string, + target: string, + executionId: string, +): string { + return `studio-managed-agent-e0-${scenario}-${target}-${executionId}` + .toLowerCase() + .replace(/[^a-z0-9-]+/g, "-") + .replace(/^-|-$/g, ""); +} + +export function buildManagedAgentCorrelationPrompt(input: { + readonly prompt: string; + readonly evalSource: string; + readonly executionId: string; +}): string { + const marker = [ + MANAGED_AGENT_CORRELATION_MARKER_VERSION, + `eval_source=${input.evalSource}`, + `execution_id=${input.executionId}`, + ].join(";"); + return [ + marker, + "This is a non-secret certification marker. Do not repeat it.", + input.prompt, + ].join("\n"); +} + +function hasUniversalPolicyHookCoverage( + toolEvidence: readonly ManagedAgentToolEvidence[], + permissionEvidence: ManagedAgentProbeResult["permissionEvidence"], +): boolean { + const requested = toolEvidence.filter(({ status }) => status === "requested"); + const requestedIds = requested.flatMap(({ toolUseId }) => + toolUseId ? [toolUseId] : [], + ); + if ( + requestedIds.length !== requested.length || + new Set(requestedIds).size !== requestedIds.length + ) { + return false; + } + return requestedIds.every((toolUseId) => { + return ( + permissionEvidence.filter( + (evidence) => + evidence.toolUseId === toolUseId && + evidence.source === "pre_tool_use", + ).length === 1 + ); + }); +} + +function buildManagedAgentPolicyDiagnostics( + toolEvidence: readonly ManagedAgentToolEvidence[], + permissionEvidence: ManagedAgentProbeResult["permissionEvidence"], + guardRejections: readonly ManagedAgentPreToolUseGuardRejection[], +): ManagedAgentPolicyDiagnostic[] { + const requestedIds = new Set( + toolEvidence.flatMap(({ status, toolUseId }) => + status === "requested" && toolUseId ? [toolUseId] : [], + ), + ); + const diagnostics: ManagedAgentPolicyDiagnostic[] = guardRejections.map( + ({ reason, toolName, normalizedToolUseId }) => ({ + kind: "pre_tool_use_guard_rejection", + reason, + toolName, + correlatedRequest: + normalizedToolUseId !== undefined && + requestedIds.has(normalizedToolUseId), + }), + ); + const primaryDecisionIds = new Set( + permissionEvidence.flatMap(({ toolUseId, source }) => + source === "pre_tool_use" ? [toolUseId] : [], + ), + ); + const guardedRequestIds = new Set( + guardRejections.flatMap(({ normalizedToolUseId }) => + normalizedToolUseId ? [normalizedToolUseId] : [], + ), + ); + for (const evidence of toolEvidence) { + if ( + evidence.status !== "requested" || + !evidence.toolUseId || + primaryDecisionIds.has(evidence.toolUseId) || + guardedRequestIds.has(evidence.toolUseId) + ) { + continue; + } + diagnostics.push({ + kind: "missing_pre_tool_use_callback", + reason: "no_callback_observed", + toolName: evidence.toolName, + correlatedRequest: true, + }); + } + return diagnostics; +} + +async function closeQueryBounded( + query: ManagedAgentQuery, + iterator: AsyncIterator | undefined, + deadline: ManagedAgentTeardownDeadline, + monotonicNow: () => number, +): Promise { + let timeout: NodeJS.Timeout | undefined; + let closeResult: void | Promise; + try { + closeResult = query.close(); + } catch { + return false; + } + const closeWasAwaitable = + closeResult !== undefined && + closeResult !== null && + typeof (closeResult as PromiseLike).then === "function"; + const closeSettled = Promise.resolve(closeResult).then( + () => true, + () => false, + ); + const cleanupSettled = + typeof query.return === "function" + ? Promise.resolve() + .then(() => query.return!()) + .then( + () => true, + () => false, + ) + : typeof iterator?.return === "function" + ? Promise.resolve() + .then(() => iterator.return!()) + .then( + () => true, + () => false, + ) + : closeWasAwaitable + ? closeSettled + : new Promise(() => undefined); + const close = Promise.all([closeSettled, cleanupSettled]).then((settled) => + settled.every(Boolean), + ); + const timeoutMs = Math.max( + 0, + deadline.deadlineAtMs - + monotonicNow() - + FORCE_CLEANUP_CONFIRMATION_RESERVE_MS, + ); + if (timeoutMs <= 0) { + void close; + return false; + } + try { + return await Promise.race([ + close, + new Promise((resolveTimeout) => { + timeout = setTimeout( + () => resolveTimeout(false), + Math.min(QUERY_CLOSE_TIMEOUT_MS, timeoutMs), + ); + }), + ]); + } catch { + return false; + } finally { + if (timeout) clearTimeout(timeout); + } +} + +async function waitForTaskBounded( + task: Promise, + deadline: ManagedAgentTeardownDeadline, + monotonicNow: () => number, +): Promise { + const timeoutMs = Math.max(0, deadline.deadlineAtMs - monotonicNow()); + if (timeoutMs <= 0) return false; + let timeout: NodeJS.Timeout | undefined; + try { + return await Promise.race([ + task.then(() => true), + new Promise((resolveTimeout) => { + timeout = setTimeout(() => resolveTimeout(false), timeoutMs); + }), + ]); + } finally { + if (timeout) clearTimeout(timeout); + } +} + +type ManagedAgentIteratorStep = + | { readonly kind: "next"; readonly value: IteratorResult } + | { readonly kind: "aborted" }; + +async function nextManagedAgentEvent( + iterator: AsyncIterator, + signal: AbortSignal, +): Promise { + if (signal.aborted) return { kind: "aborted" }; + let abortListener: (() => void) | undefined; + const next = Promise.resolve() + .then(() => iterator.next()) + .then( + (value): ManagedAgentIteratorStep => ({ kind: "next", value }), + (error): ManagedAgentIteratorStep => { + throw error; + }, + ); + const aborted = new Promise((resolveAbort) => { + abortListener = () => resolveAbort({ kind: "aborted" }); + signal.addEventListener("abort", abortListener, { once: true }); + }); + try { + // `next` converts a late rejection into this already-observed promise, so + // abandoning it after abort cannot create an unhandled rejection. + return await Promise.race([next, aborted]); + } finally { + if (abortListener) signal.removeEventListener("abort", abortListener); + } +} + +function classifyTerminal(input: { + readonly teardown: ManagedAgentTeardownObservation; + readonly queryCreated: boolean; + readonly queryClosed: boolean; + readonly cancellationRequested: boolean; + readonly queryFailed: boolean; + readonly sdkResult?: { readonly isError: boolean; readonly subtype?: string }; +}): ManagedAgentTerminalClassification { + if (!input.teardown.quiescent || !input.teardown.deadlineMet) { + return "teardown_timeout"; + } + if (input.queryCreated && !input.queryClosed) return "close_timeout"; + if (input.cancellationRequested) return "cancelled"; + if (input.queryFailed) return "query_error"; + if (input.sdkResult?.isError) return "sdk_result_error"; + if (input.sdkResult) return "success"; + return "incomplete"; +} + +function deepFreezeEvidence(value: T, seen = new WeakSet()): T { + if (typeof value !== "object" || value === null || seen.has(value)) { + return value; + } + seen.add(value); + for (const nested of Object.values(value)) { + deepFreezeEvidence(nested, seen); + } + return Object.freeze(value); +} + +export async function runManagedAgentProbe( + config: ManagedAgentProbeConfig, + dependencies: ManagedAgentProbeDependencies = {}, +): Promise { + if (dependencies.hermeticGatewayOrigin && !dependencies.queryFactory) { + throw new Error("hermeticGatewayOrigin requires an injected queryFactory"); + } + const validated = validateManagedAgentProbeConfig(config, { + ...(dependencies.hermeticGatewayOrigin + ? { hermeticGatewayOrigin: dependencies.hermeticGatewayOrigin } + : {}), + }); + if ( + !dependencies.hermeticGatewayOrigin && + process.versions.node !== MANAGED_AGENT_CONTRACT.certificationNodeVersion + ) { + throw new Error( + `Direct managed-agent probes require Node ${MANAGED_AGENT_CONTRACT.certificationNodeVersion}; current runtime is ${process.versions.node}`, + ); + } + if (config.scenario === "L2" && !dependencies.waitForCancellationSignal) { + throw new Error("L2 requires an explicit cancellation signal dependency"); + } + + const createUuid = dependencies.uuid ?? randomUUID; + const runId = createUuid(); + const executionId = createUuid(); + const evalSource = safeEvalSource( + config.scenario, + config.target, + executionId, + ); + const recorder = new ManagedAgentEventRecorder(runId, validated.model.alias); + const mcpRuntime = createManagedAgentMcpRuntime(config.expectedMcpNonce); + const abortController = new AbortController(); + const triggerController = new AbortController(); + const monotonicNow = dependencies.monotonicNow ?? (() => performance.now()); + const before = await captureManagedAgentWorkspaceSnapshot( + validated.canonicalWorkspaceRoot, + ); + const childEnvironment = buildManagedAgentChildEnvironment({ + ambient: process.env, + configRoot: validated.canonicalConfigRoot, + gatewayOrigin: validated.gatewayOrigin, + gatewayCredential: config.gatewayCredential, + modelAlias: validated.model.alias, + evalSource, + executionId, + }); + const processObserver = + dependencies.processObserver ?? + createLocalManagedAgentProcessObserver({ monotonicNow }); + let cancellationRequested = false; + let teardownDeadline: ManagedAgentTeardownDeadline | undefined; + const ensureTeardownDeadline = (): ManagedAgentTeardownDeadline => { + if (!teardownDeadline) { + const startedAtMs = monotonicNow(); + teardownDeadline = Object.freeze({ + startedAtMs, + deadlineAtMs: startedAtMs + MANAGED_AGENT_TEARDOWN_TIMEOUT_MS, + }); + // The observer must see the exact same deadline before the first abort, + // SDK close, or iterator return. This also arms a previously observed + // SDK-forwarded abort without granting it an unbounded cleanup window. + processObserver.beginTeardown(teardownDeadline); + } + return teardownDeadline; + }; + let query: ManagedAgentQuery | undefined; + let iterator: AsyncIterator | undefined; + let queryFailed = false; + let queryClosed = false; + let cancellationTriggerFailed = false; + let cancellationSignalReady = false; + let queryIterationSettled = false; + let toolProcessContainmentArmed = false; + let policyPreflightFailed = false; + let promptEmbedded = false; + let eventNormalizationFailure: ManagedAgentProbeResult["terminationEvidence"]["eventNormalizationFailure"]; + let queryExecution: ManagedAgentQueryExecutionOutcome = "not_started"; + const guardRejections: ManagedAgentPreToolUseGuardRejection[] = []; + + const policyBoundary = createManagedAgentPolicyBoundary({ + canonicalWorkspaceRoot: validated.canonicalWorkspaceRoot, + allowedBuiltinTools: + config.scenario === "L2" + ? MANAGED_AGENT_L2_BUILTIN_TOOLS + : MANAGED_AGENT_BUILTIN_TOOLS, + allowedBashCommands: config.allowedBashCommands, + allowedMcpTools: + config.scenario === "L1" ? mcpRuntime.qualifiedToolNames : [], + pathRoleBindings: config.pathRoleBindings, + requireRegisteredFilePaths: config.scenario === "L1", + onDecision: (evidence) => { + recorder.recordPermission(evidence); + if ( + config.scenario === "L2" && + evidence.source === "pre_tool_use" && + evidence.toolName === "Bash" && + evidence.decision === "allow" && + evidence.reason === "exact_bash_command" + ) { + toolProcessContainmentArmed = true; + processObserver.armToolProcessContainment(); + } + }, + onGuardRejection: (diagnostic) => guardRejections.push(diagnostic), + }); + + const options: Options = { + abortController, + // PreToolUse is the universal boundary. canUseTool only handles an + // unresolved SDK permission as defense in depth; the shared evaluator + // deduplicates its evidence by tool-use ID. + canUseTool: policyBoundary.canUseToolFallback, + cwd: validated.canonicalWorkspaceRoot, + disallowedTools: + config.scenario === "L2" + ? [...MANAGED_AGENT_L2_DISALLOWED_TOOLS] + : [...MANAGED_AGENT_DISALLOWED_TOOLS], + env: childEnvironment, + includePartialMessages: false, + hooks: { + PreToolUse: [ + { + hooks: [policyBoundary.preToolUseHook], + timeout: 5, + }, + ], + }, + maxBudgetUsd: config.maxBudgetUsd, + maxTurns: config.maxTurns, + mcpServers: + config.scenario === "L1" + ? { [MANAGED_AGENT_MCP_SERVER_NAME]: mcpRuntime.server } + : {}, + model: validated.model.alias, + permissionMode: "default", + persistSession: false, + settingSources: [], + skills: [], + spawnClaudeCodeProcess: (spawnOptions) => + processObserver.spawn(spawnOptions), + stderr: () => { + // Do not retain or print SDK stderr; probe artifacts are structural only. + }, + strictMcpConfig: true, + systemPrompt: + config.scenario === "L2" + ? "You are a deterministic local cancellation probe. Use only the one exact Bash call named in the prompt." + : "You are a deterministic local managed-agent feasibility probe. Follow the ordered instructions exactly, continue after expected permission denials and planned MCP errors, and use only the tools named in the prompt.", + thinking: { type: "disabled" }, + tools: + config.scenario === "L2" + ? [...MANAGED_AGENT_L2_BUILTIN_TOOLS] + : [...MANAGED_AGENT_BUILTIN_TOOLS], + }; + + let teardown!: ManagedAgentTeardownObservation; + let terminal!: ManagedAgentTerminalClassification; + let terminationEvidence!: ManagedAgentProbeResult["terminationEvidence"]; + let policyHookCoverage = false; + try { + recorder.recordLifecycle("starting"); + try { + await ( + dependencies.policySettingsGuard ?? assertManagedAgentHooksEnabled + )({ + cwd: validated.canonicalWorkspaceRoot, + environment: + buildManagedAgentSettingsGuardEnvironment(childEnvironment), + }); + } catch { + policyPreflightFailed = true; + recorder.recordLifecycle("policy_preflight_failed"); + triggerController.abort(); + ensureTeardownDeadline(); + abortController.abort(); + } + const cancellationTask = + !policyPreflightFailed && dependencies.waitForCancellationSignal + ? dependencies + .waitForCancellationSignal(triggerController.signal) + .then(() => { + if (triggerController.signal.aborted) return; + cancellationSignalReady = true; + if (queryIterationSettled) return; + cancellationRequested = true; + recorder.recordLifecycle("cancellation_requested"); + ensureTeardownDeadline(); + abortController.abort(); + }) + .catch(() => { + if (!triggerController.signal.aborted) { + cancellationTriggerFailed = true; + if (!queryIterationSettled) { + ensureTeardownDeadline(); + abortController.abort(); + } + } + }) + : undefined; + + if (!policyPreflightFailed) { + try { + const prompt = buildManagedAgentCorrelationPrompt({ + prompt: config.prompt, + evalSource, + executionId, + }); + promptEmbedded = true; + try { + query = (dependencies.queryFactory ?? defaultQueryFactory)({ + prompt, + options, + }); + } catch (error) { + queryExecution = "construction_failed"; + throw error; + } + iterator = query[Symbol.asyncIterator](); + for (;;) { + const step = await nextManagedAgentEvent( + iterator, + abortController.signal, + ); + if (step.kind === "aborted") { + queryExecution = "iteration_aborted"; + break; + } + if (step.value.done) { + queryExecution = "iteration_completed"; + break; + } + try { + recorder.observeSdkEvent(step.value.value); + } catch (error) { + if (error instanceof ManagedAgentEventError) { + eventNormalizationFailure = error.reason; + queryExecution = "event_normalization_failed"; + } + throw error; + } + } + } catch { + if (eventNormalizationFailure) { + queryExecution = "event_normalization_failed"; + } else if (!query) { + queryExecution = "construction_failed"; + } else { + queryExecution = abortController.signal.aborted + ? "iteration_aborted" + : "iteration_failed"; + } + if (!abortController.signal.aborted) queryFailed = true; + } finally { + queryIterationSettled = true; + const armedEarlyQuerySettlement = + toolProcessContainmentArmed && + Boolean(cancellationTask) && + !abortController.signal.aborted && + !cancellationSignalReady; + if (armedEarlyQuerySettlement) { + const deadline = ensureTeardownDeadline(); + const taskSettled = await waitForTaskBounded( + cancellationTask!, + deadline, + monotonicNow, + ); + if (!taskSettled || !cancellationSignalReady) { + cancellationTriggerFailed = true; + } + } + triggerController.abort(); + if (cancellationTask && !armedEarlyQuerySettlement) { + await cancellationTask; + } + queryFailed ||= cancellationTriggerFailed; + // Give the SDK its documented graceful-shutdown path before host + // fallback containment. The observer binds only SpawnOptions.signal, + // which the SDK forwards after stdin EOF and its bounded grace period. + if ( + (queryFailed || armedEarlyQuerySettlement) && + !abortController.signal.aborted + ) { + ensureTeardownDeadline(); + abortController.abort(); + } + if (query) { + // Establish the one deadline before close() or iterator.return() can + // begin. Natural completion is teardown-relevant too. + const deadline = ensureTeardownDeadline(); + queryClosed = await closeQueryBounded( + query, + iterator, + deadline, + monotonicNow, + ); + } + if ((query && !queryClosed) || queryFailed) { + ensureTeardownDeadline(); + abortController.abort(); + } + } + } + + const deadline = ensureTeardownDeadline(); + if (abortController.signal.aborted) { + teardown = await processObserver.emergencyCleanup(deadline); + } else { + teardown = await processObserver.waitForQuiescence(deadline); + if (!teardown.quiescent) { + teardown = await processObserver.emergencyCleanup(deadline); + } + } + const totalElapsedMs = Math.max(0, monotonicNow() - deadline.startedAtMs); + teardown = { + ...teardown, + elapsedMs: totalElapsedMs, + deadlineMet: + teardown.quiescent && + teardown.processTableAvailable && + teardown.containmentSupported && + monotonicNow() <= deadline.deadlineAtMs, + }; + const beforePolicyOverride = classifyTerminal({ + teardown, + queryCreated: query !== undefined, + queryClosed, + cancellationRequested, + queryFailed, + sdkResult: recorder.result, + }); + terminal = beforePolicyOverride; + if ( + policyPreflightFailed && + terminal !== "teardown_timeout" && + terminal !== "close_timeout" + ) { + terminal = "policy_violation"; + } + policyHookCoverage = + !policyPreflightFailed && + !eventNormalizationFailure && + hasUniversalPolicyHookCoverage( + recorder.toolEvidence, + recorder.permissionEvidence, + ); + if ( + !policyHookCoverage && + terminal !== "teardown_timeout" && + terminal !== "close_timeout" + ) { + terminal = "policy_violation"; + } + recorder.recordTerminal(terminal); + + const sdkResult = recorder.result + ? recorder.result.isError + ? "error" + : "success" + : "not_observed"; + terminationEvidence = { + beforePolicyOverride, + queryExecution, + sdkResult, + ...(eventNormalizationFailure ? { eventNormalizationFailure } : {}), + }; + } finally { + await processObserver.dispose(); + } + + const after = await captureManagedAgentWorkspaceSnapshot( + validated.canonicalWorkspaceRoot, + ); + const policyDiagnostics = buildManagedAgentPolicyDiagnostics( + recorder.toolEvidence, + recorder.permissionEvidence, + guardRejections, + ); + return deepFreezeEvidence({ + contractVersion: 1, + runId, + scenario: config.scenario, + target: config.target, + modelAlias: validated.model.alias, + sdkModelEvidence: recorder.modelEvidence, + ...(recorder.sessionId ? { sdkSessionId: recorder.sessionId } : {}), + inferenceTurns: recorder.inferenceTurns, + ...(recorder.sdkNumTurns === undefined + ? {} + : { sdkNumTurns: recorder.sdkNumTurns }), + policyHookCoverage, + terminal, + terminationEvidence, + events: [...recorder.events], + toolEvidence: [...recorder.toolEvidence], + permissionEvidence: [...recorder.permissionEvidence], + policyDiagnostics, + workspaceChanges: diffManagedAgentWorkspaceSnapshots(before, after), + preservation: observeManagedAgentPreservation( + before, + after, + config.preservePaths ?? [], + ), + cancellationRequested, + queryClosed, + teardown, + correlation: { executionId, evalSource, promptEmbedded }, + ...(config.scenario === "L1" + ? { + l1Certification: { + contractVersion: + MANAGED_AGENT_L1_CERTIFICATION_CONTRACT.contractVersion, + promptVersion: + MANAGED_AGENT_L1_CERTIFICATION_CONTRACT.promptVersion, + }, + l1FinalBytes: observeManagedAgentL1FinalBytes( + after, + config.expectedL1FinalBytes, + ), + nonceVerified: mcpRuntime.invocations.some( + ({ toolName, status }) => + toolName === qualifiedManagedAgentMcpToolName("echo_nonce") && + status === "success", + ), + } + : {}), + ...(recorder.usage ? { sdkUsage: recorder.usage } : {}), + }); +} diff --git a/packages/harness/src/experimental/managed-agent-spike/settings-guard.test.ts b/packages/harness/src/experimental/managed-agent-spike/settings-guard.test.ts new file mode 100644 index 000000000..8a278c20d --- /dev/null +++ b/packages/harness/src/experimental/managed-agent-spike/settings-guard.test.ts @@ -0,0 +1,98 @@ +import { describe, expect, it } from "vitest"; + +import { + ManagedAgentSettingsGuardError, + assertManagedAgentHooksEnabled, + buildManagedAgentSettingsGuardEnvironment, +} from "./settings-guard.js"; + +describe("managed-agent settings guard", () => { + it("removes gateway and credential inputs before resolution", () => { + expect( + buildManagedAgentSettingsGuardEnvironment({ + PATH: "/safe/bin", + HOME: "/isolated/home", + CLAUDE_CONFIG_DIR: "/isolated/claude", + ANTHROPIC_API_KEY: "credential", + ANTHROPIC_AUTH_TOKEN: "auth-token", + ANTHROPIC_BASE_URL: "https://gateway.invalid", + ANTHROPIC_CUSTOM_HEADERS: "x-secret: value", + }), + ).toEqual({ + PATH: "/safe/bin", + HOME: "/isolated/home", + CLAUDE_CONFIG_DIR: "/isolated/claude", + }); + }); + + it("accepts only the exact enabled contract", async () => { + const input = { cwd: process.cwd(), environment: {} }; + await expect( + assertManagedAgentHooksEnabled(input, { + run: async () => ({ + stdout: JSON.stringify({ + contractVersion: 1, + disableAllHooks: false, + policyHelperConfigured: false, + }), + }), + }), + ).resolves.toBeUndefined(); + + for (const stdout of [ + "not-json", + "{}", + JSON.stringify({ + contractVersion: 1, + disableAllHooks: "false", + policyHelperConfigured: false, + }), + JSON.stringify({ + contractVersion: 1, + disableAllHooks: false, + policyHelperConfigured: false, + unexpected: true, + }), + ]) { + await expect( + assertManagedAgentHooksEnabled(input, { + run: async () => ({ stdout }), + }), + ).rejects.toBeInstanceOf(ManagedAgentSettingsGuardError); + } + }); + + it("fails closed on disabled hooks or resolution errors", async () => { + const input = { cwd: process.cwd(), environment: {} }; + await expect( + assertManagedAgentHooksEnabled(input, { + run: async () => ({ + stdout: JSON.stringify({ + contractVersion: 1, + disableAllHooks: true, + policyHelperConfigured: false, + }), + }), + }), + ).rejects.toThrow("disabled by managed settings"); + await expect( + assertManagedAgentHooksEnabled(input, { + run: async () => { + throw new Error("private resolver error"); + }, + }), + ).rejects.toThrow("could not be resolved"); + + await expect( + assertManagedAgentHooksEnabled(input, { + run: async () => ({ + stdout: JSON.stringify({ + contractVersion: 1, + disableAllHooks: false, + policyHelperConfigured: true, + }), + }), + }), + ).rejects.toThrow("unresolved policy helper"); + }); +}); diff --git a/packages/harness/src/experimental/managed-agent-spike/settings-guard.ts b/packages/harness/src/experimental/managed-agent-spike/settings-guard.ts new file mode 100644 index 000000000..982cc5d63 --- /dev/null +++ b/packages/harness/src/experimental/managed-agent-spike/settings-guard.ts @@ -0,0 +1,174 @@ +import { execFile } from "node:child_process"; +import { createRequire } from "node:module"; +import { promisify } from "node:util"; +import { pathToFileURL } from "node:url"; + +const execFileAsync = promisify(execFile); +const SETTINGS_GUARD_TIMEOUT_MS = 5_000; +const SETTINGS_GUARD_MODULE_ENV = "SAPIOM_MANAGED_AGENT_SETTINGS_SDK_URL"; +const SETTINGS_GUARD_SCRIPT = ` +const moduleUrl = process.env.${SETTINGS_GUARD_MODULE_ENV}; +if (!moduleUrl) throw new Error("missing sdk module url"); +const { resolveSettings } = await import(moduleUrl); +const resolved = await resolveSettings({ cwd: process.cwd(), settingSources: [] }); +const isRecord = (candidate) => + typeof candidate === "object" && candidate !== null && !Array.isArray(candidate); +if ( + !isRecord(resolved) || + !isRecord(resolved.effective) || + !Array.isArray(resolved.sources) || + !resolved.sources.every((source) => isRecord(source) && isRecord(source.settings)) +) { + throw new Error("malformed resolved settings"); +} +const value = resolved?.effective?.disableAllHooks; +const settings = [resolved.effective, ...resolved.sources.map((source) => source.settings)]; +const policyHelperConfigured = settings.some((candidate) => + candidate && (candidate.policyHelper !== undefined || candidate.policyHelpers !== undefined) +); +if (value !== undefined && typeof value !== "boolean") { + throw new Error("malformed disableAllHooks setting"); +} +process.stdout.write(JSON.stringify({ + contractVersion: 1, + disableAllHooks: value === true, + policyHelperConfigured, +})); +`; + +const CREDENTIAL_AND_GATEWAY_ENVIRONMENT = new Set([ + "ANTHROPIC_API_KEY", + "ANTHROPIC_AUTH_TOKEN", + "ANTHROPIC_CUSTOM_HEADERS", + "ANTHROPIC_BASE_URL", +]); + +export class ManagedAgentSettingsGuardError extends Error { + public constructor(message: string) { + super(message); + this.name = "ManagedAgentSettingsGuardError"; + } +} + +export interface ManagedAgentSettingsGuardInput { + readonly cwd: string; + readonly environment: Readonly>; + /** + * Explicit Node executable seam for future packaged hosts. E0.4 intentionally + * does not certify Electron-as-Node; E0.7 owns that packaging proof. + */ + readonly nodeExecutable?: string; +} + +export interface ManagedAgentSettingsGuardDependencies { + readonly run?: (input: { + readonly cwd: string; + readonly environment: Readonly>; + readonly nodeExecutable?: string; + }) => Promise<{ readonly stdout: string }>; +} + +export function buildManagedAgentSettingsGuardEnvironment( + childEnvironment: Readonly>, +): Record { + return Object.fromEntries( + Object.entries(childEnvironment).filter( + ([name]) => !CREDENTIAL_AND_GATEWAY_ENVIRONMENT.has(name), + ), + ); +} + +async function runSettingsResolver(input: { + readonly cwd: string; + readonly environment: Readonly>; + readonly nodeExecutable?: string; +}): Promise<{ readonly stdout: string }> { + const sdkModulePath = createRequire(import.meta.url).resolve( + "@anthropic-ai/claude-agent-sdk", + ); + const environment = { + ...input.environment, + [SETTINGS_GUARD_MODULE_ENV]: pathToFileURL(sdkModulePath).href, + }; + try { + const result = await execFileAsync( + input.nodeExecutable ?? process.execPath, + ["--input-type=module", "--eval", SETTINGS_GUARD_SCRIPT], + { + cwd: input.cwd, + env: environment, + timeout: SETTINGS_GUARD_TIMEOUT_MS, + maxBuffer: 4_096, + windowsHide: true, + }, + ); + return { stdout: result.stdout }; + } catch { + throw new ManagedAgentSettingsGuardError( + "Managed-agent hook settings could not be resolved", + ); + } +} + +/** + * Resolve managed settings in an isolated, credential-free subprocess before + * query construction. Any uncertain result fails closed. + */ +export async function assertManagedAgentHooksEnabled( + input: ManagedAgentSettingsGuardInput, + dependencies: ManagedAgentSettingsGuardDependencies = {}, +): Promise { + if (process.versions.electron && !input.nodeExecutable) { + throw new ManagedAgentSettingsGuardError( + "Managed-agent settings guard requires an explicit Node executable in Electron", + ); + } + let stdout: string; + try { + ({ stdout } = await (dependencies.run ?? runSettingsResolver)(input)); + } catch (error) { + if (error instanceof ManagedAgentSettingsGuardError) throw error; + throw new ManagedAgentSettingsGuardError( + "Managed-agent hook settings could not be resolved", + ); + } + + let parsed: unknown; + try { + parsed = JSON.parse(stdout); + } catch { + throw new ManagedAgentSettingsGuardError( + "Managed-agent hook settings result was malformed", + ); + } + const result = + typeof parsed === "object" && parsed !== null && !Array.isArray(parsed) + ? (parsed as Record) + : undefined; + if ( + !result || + result.contractVersion !== 1 || + typeof result.disableAllHooks !== "boolean" || + typeof result.policyHelperConfigured !== "boolean" || + Object.keys(result).some( + (key) => + key !== "contractVersion" && + key !== "disableAllHooks" && + key !== "policyHelperConfigured", + ) + ) { + throw new ManagedAgentSettingsGuardError( + "Managed-agent hook settings result was malformed", + ); + } + if (result.disableAllHooks) { + throw new ManagedAgentSettingsGuardError( + "Managed-agent hooks are disabled by managed settings", + ); + } + if (result.policyHelperConfigured) { + throw new ManagedAgentSettingsGuardError( + "Managed-agent hook settings use an unresolved policy helper", + ); + } +} diff --git a/packages/harness/src/experimental/managed-agent-spike/types.ts b/packages/harness/src/experimental/managed-agent-spike/types.ts new file mode 100644 index 000000000..d5471505f --- /dev/null +++ b/packages/harness/src/experimental/managed-agent-spike/types.ts @@ -0,0 +1,393 @@ +import type { + Options, + SpawnedProcess, + SpawnOptions, +} from "@anthropic-ai/claude-agent-sdk"; + +export type ManagedAgentModelTargetId = "sonnet-5" | "minimax-m3"; +export type ManagedAgentProbeScenario = "L1" | "L2"; + +export interface ManagedAgentModelTarget { + readonly id: ManagedAgentModelTargetId; + readonly alias: string; + readonly upstreamProvider: "anthropic" | "fireworks_ai"; + readonly upstreamModel: string; +} + +/** + * Explicit inputs for one experimental probe. The gateway credential is + * sensitive and must never be copied into events, results, logs, or CLI args. + */ +export interface ManagedAgentProbeConfig { + readonly scenario: ManagedAgentProbeScenario; + readonly workspaceRoot: string; + readonly configRoot: string; + readonly target: ManagedAgentModelTargetId; + readonly gatewayOrigin: string; + readonly gatewayCredential: string; + readonly prompt: string; + readonly maxTurns: number; + readonly maxBudgetUsd: number; + readonly allowedBashCommands: readonly string[]; + /** Exact prompt literals mapped to privacy-safe roles inside the host policy. */ + readonly pathRoleBindings: readonly ManagedAgentPathRoleBinding[]; + /** Trusted expected hashes for the two L1 mutation targets; empty for L2. */ + readonly expectedL1FinalBytes: readonly ManagedAgentL1ExpectedFileHash[]; + /** Expected only for L1 and never copied into structural evidence. */ + readonly expectedMcpNonce?: string; + readonly preservePaths?: readonly string[]; +} + +export type ManagedAgentPermissionDecision = "allow" | "deny"; +export type ManagedAgentPermissionReason = + | "fixture_path" + | "exact_bash_command" + | "managed_mcp_tool" + | "policy_aborted" + | "invalid_input" + | "path_outside_workspace" + | "path_symlink_escape" + | "path_role_not_allowed" + | "bash_command_not_allowed" + | "tool_not_allowed"; + +export type ManagedAgentPermissionSource = + | "pre_tool_use" + | "can_use_tool_fallback"; + +export type ManagedAgentRegisteredPathRole = + | "clean_target" + | "dirty_sentinel" + | "untracked_sentinel" + | "managed_output" + | "outside_sentinel" + | "escape_link"; + +export type ManagedAgentPathRole = + | ManagedAgentRegisteredPathRole + | "unregistered"; + +export interface ManagedAgentPathRoleBinding { + /** Sensitive prompt literal; this value never crosses the evidence boundary. */ + readonly path: string; + readonly role: ManagedAgentRegisteredPathRole; +} + +export type ManagedAgentL1FinalByteRole = "clean_target" | "managed_output"; + +export interface ManagedAgentL1ExpectedFileHash { + /** Sensitive fixture path; this value never crosses the evidence boundary. */ + readonly path: string; + readonly role: ManagedAgentL1FinalByteRole; + readonly sha256: string; +} + +export interface ManagedAgentL1FinalByteObservation { + readonly role: ManagedAgentL1FinalByteRole; + readonly matched: boolean; +} + +export type ManagedAgentOperationId = + | `read:${ManagedAgentPathRole}` + | `edit:${ManagedAgentPathRole}` + | `write:${ManagedAgentPathRole}` + | "bash:exact_command" + | "bash:unregistered" + | "mcp:echo_nonce" + | "mcp:fail_once" + | "mcp:managed" + | "unknown"; + +export type ManagedAgentProbeEventType = + | "lifecycle" + | "message" + | "tool_requested" + | "tool_completed" + | "permission" + | "sdk_result" + | "terminal"; + +/** + * A deliberately content-free event boundary. Raw prompts, message text, + * tool inputs/results, filesystem paths, and error messages never cross it. + */ +export interface ManagedAgentProbeEvent { + readonly sequence: number; + readonly runId: string; + readonly type: ManagedAgentProbeEventType; + readonly subtype?: string; + readonly sessionId?: string; + readonly toolUseId?: string; + readonly toolName?: string; + readonly permissionDecision?: ManagedAgentPermissionDecision; + readonly permissionReason?: ManagedAgentPermissionReason; + readonly permissionSource?: ManagedAgentPermissionSource; + readonly operationId?: ManagedAgentOperationId; + readonly isError?: boolean; + readonly terminal?: ManagedAgentTerminalClassification; +} + +export interface ManagedAgentSdkUsageEstimate { + readonly authority: "sdk_non_authoritative"; + readonly inputTokens: number; + readonly outputTokens: number; + readonly cacheCreationInputTokens: number; + readonly cacheReadInputTokens: number; + readonly estimatedCostUsd?: number; +} + +/** + * Content-free SDK evidence that the configured alias was also reported by + * the running SDK. Gateway reconciliation remains authoritative for the + * upstream provider/model and fallback state. + */ +export interface ManagedAgentSdkModelEvidence { + readonly authority: "sdk_non_authoritative"; + readonly initModelObserved: boolean; + readonly initModelMatchesExpectedAlias: boolean; + readonly resultModelUsageObserved: boolean; + readonly resultModelUsageMatchesExpectedAlias: boolean; + readonly resultModelCount: number; +} + +export interface ManagedAgentWorkspaceChange { + readonly path: string; + readonly change: "created" | "modified" | "deleted"; +} + +export interface ManagedAgentPreservationObservation { + readonly path: string; + readonly preserved: boolean; +} + +export interface ManagedAgentToolEvidence { + readonly toolUseId?: string; + readonly toolName: string; + readonly status: "requested" | "success" | "error"; +} + +export interface ManagedAgentPermissionEvidence { + readonly toolUseId: string; + readonly toolName: string; + readonly decision: ManagedAgentPermissionDecision; + readonly reason: ManagedAgentPermissionReason; + readonly source: ManagedAgentPermissionSource; + /** Trusted, content-free operation identity; never contains a raw path/input. */ + readonly operationId: ManagedAgentOperationId; +} + +export type ManagedAgentPreToolUseGuardRejectionReason = + | "unexpected_hook_event" + | "input_tool_use_id_missing" + | "input_tool_use_id_invalid" + | "input_tool_use_id_too_long" + | "callback_tool_use_id_invalid" + | "callback_tool_use_id_too_long" + | "callback_tool_use_id_mismatch"; + +/** + * Content-free policy diagnostics. They explain why strict hook coverage + * failed, but never count as permission evidence and therefore cannot certify + * a tool request as authorized. + */ +export type ManagedAgentPolicyDiagnostic = + | { + readonly kind: "pre_tool_use_guard_rejection"; + readonly reason: ManagedAgentPreToolUseGuardRejectionReason; + readonly toolName: string; + readonly correlatedRequest: boolean; + } + | { + readonly kind: "missing_pre_tool_use_callback"; + readonly reason: "no_callback_observed"; + readonly toolName: string; + readonly correlatedRequest: true; + }; + +export type ManagedAgentEventNormalizationFailureReason = + | "assistant_message_id_invalid" + | "inference_turn_limit_exceeded" + | "tool_request_id_invalid" + | "tool_result_id_invalid" + | "sdk_num_turns_invalid"; + +export type ManagedAgentQueryExecutionOutcome = + | "not_started" + | "construction_failed" + | "iteration_completed" + | "iteration_failed" + | "iteration_aborted" + | "event_normalization_failed"; + +export interface ManagedAgentTerminationEvidence { + /** Terminal classification before the strict policy override is applied. */ + readonly beforePolicyOverride: ManagedAgentTerminalClassification; + readonly queryExecution: ManagedAgentQueryExecutionOutcome; + readonly sdkResult: "not_observed" | "success" | "error"; + readonly eventNormalizationFailure?: ManagedAgentEventNormalizationFailureReason; +} + +export interface ManagedAgentTeardownObservation { + readonly quiescent: boolean; + readonly deadlineMet: boolean; + /** False means ps/CIM observation was unknown, never an empty table. */ + readonly processTableAvailable: boolean; + /** False means the owned E0 containment model was escaped or unproven. */ + readonly containmentSupported: boolean; + /** True after SDK-root authority and any required L2 observations are proven. */ + readonly ownershipProven: boolean; + /** True only when forced termination was requested for every owned root. */ + readonly forceKillIssued: boolean; + /** True after both exact L2 fixture lifetime channels pass fresh observation. */ + readonly toolProcessObservationComplete: boolean; + /** True only when both observed L2 lifetime channels have closed. */ + readonly toolProcessChannelsClosed: boolean; + readonly elapsedMs: number; + readonly observedPids: readonly number[]; + readonly alivePidsAtDeadline: readonly number[]; + readonly emergencyCleanupAttempted: boolean; +} + +export type ManagedAgentTerminalClassification = + | "success" + | "cancelled" + | "sdk_result_error" + | "query_error" + | "policy_violation" + | "incomplete" + | "close_timeout" + | "teardown_timeout"; + +export interface ManagedAgentProbeResult { + readonly contractVersion: 1; + readonly runId: string; + readonly scenario: ManagedAgentProbeScenario; + readonly target: ManagedAgentModelTargetId; + readonly modelAlias: string; + readonly sdkModelEvidence: ManagedAgentSdkModelEvidence; + readonly sdkSessionId?: string; + /** Distinct, hashed assistant message IDs; authoritative for BQ call count. */ + readonly inferenceTurns: number; + /** SDK result.num_turns; informational and not a gateway reconciliation key. */ + readonly sdkNumTurns?: number; + /** False if any requested tool lacked exactly one primary PreToolUse decision. */ + readonly policyHookCoverage: boolean; + readonly terminal: ManagedAgentTerminalClassification; + readonly terminationEvidence: ManagedAgentTerminationEvidence; + readonly events: readonly ManagedAgentProbeEvent[]; + readonly toolEvidence: readonly ManagedAgentToolEvidence[]; + readonly permissionEvidence: readonly ManagedAgentPermissionEvidence[]; + readonly policyDiagnostics: readonly ManagedAgentPolicyDiagnostic[]; + readonly workspaceChanges: readonly ManagedAgentWorkspaceChange[]; + readonly preservation: readonly ManagedAgentPreservationObservation[]; + readonly cancellationRequested: boolean; + readonly queryClosed: boolean; + readonly teardown: ManagedAgentTeardownObservation; + readonly correlation: { + readonly executionId: string; + readonly evalSource: string; + /** True only after the marked prompt is handed to the query factory. */ + readonly promptEmbedded: boolean; + }; + /** Present only for an L1 prompt validated against the frozen v2 marker. */ + readonly l1Certification?: { + readonly contractVersion: 2; + readonly promptVersion: "managed-agent-l1-prompt-v2"; + }; + /** Content-free proof of exact final bytes for both intended L1 mutations. */ + readonly l1FinalBytes?: readonly ManagedAgentL1FinalByteObservation[]; + /** Content-free proof that echo_nonce received the expected sentinel nonce. */ + readonly nonceVerified?: boolean; + readonly sdkUsage?: ManagedAgentSdkUsageEstimate; +} + +/** + * Deliberately excludes Agent SDK control-channel methods. In particular, + * Query.mcpCall bypasses permission checks and is outside this host boundary. + */ +export interface ManagedAgentQuery extends AsyncIterable { + close(): void | Promise; + /** Pinned SDK Query.return() awaits its fire-and-forget close cleanup. */ + return?(value?: void): Promise>; +} + +export type ManagedAgentQueryFactory = (input: { + readonly prompt: string; + readonly options: Options; +}) => ManagedAgentQuery; + +export interface ManagedAgentProcessObserver { + spawn(options: SpawnOptions): SpawnedProcess; + /** + * Adopt the runtime's one immutable monotonic teardown deadline before any + * abort, Query.close(), or Query.return() operation may begin. + */ + beginTeardown(deadline: ManagedAgentTeardownDeadline): void; + /** + * Arm the two host-authenticated lifetime observations used only by the + * exact E0.4 L2 fixture. Tool-reported identities never grant authority by + * themselves; the observer also requires fresh owned-root ancestry. + */ + armToolProcessContainment(): void; + /** Prove the narrow POSIX observation model before allowing L2 to cancel. */ + prepareCancellation(): Promise; + /** Sample only members owned by the host-observed process anchors. */ + observeProcessTree(deadline?: ManagedAgentTeardownDeadline): Promise; + waitForQuiescence( + deadline: ManagedAgentTeardownDeadline, + ): Promise; + /** Idempotently run the anchored fallback and confirm before this deadline. */ + emergencyCleanup( + deadline: ManagedAgentTeardownDeadline, + ): Promise; + dispose(): void | Promise; +} + +/** + * One immutable monotonic deadline shared by SDK close/return and host process + * containment. It is created once at the first teardown-relevant event and is + * never extended from wall-clock time or a later cleanup phase. + */ +export interface ManagedAgentTeardownDeadline { + readonly startedAtMs: number; + readonly deadlineAtMs: number; +} + +export type ManagedAgentCancellationReadinessReason = + | "ready" + | "platform_unsupported" + | "process_table_unavailable" + | "root_count_invalid" + | "root_not_active" + | "root_not_group_leader" + | "tool_process_not_registered" + | "tool_process_identity_invalid" + | "containment_escaped"; + +export interface ManagedAgentCancellationReadiness { + readonly supported: boolean; + readonly reason: ManagedAgentCancellationReadinessReason; + readonly processTableAvailable: boolean; + readonly containmentSupported: boolean; + readonly ownershipProven: boolean; + readonly observedPids: readonly number[]; +} + +export interface ManagedAgentProbeDependencies { + readonly queryFactory?: ManagedAgentQueryFactory; + /** + * Explicit test-only origin seam. It is accepted only alongside an injected + * query factory and only for reserved .test or loopback origins. + */ + readonly hermeticGatewayOrigin?: string; + readonly processObserver?: ManagedAgentProcessObserver; + readonly uuid?: () => string; + /** Injectable monotonic clock; wall time is never cancellation authority. */ + readonly monotonicNow?: () => number; + readonly waitForCancellationSignal?: (signal: AbortSignal) => Promise; + readonly policySettingsGuard?: (input: { + readonly cwd: string; + readonly environment: Readonly>; + readonly nodeExecutable?: string; + }) => Promise; +} diff --git a/packages/harness/src/server/rest.test.ts b/packages/harness/src/server/rest.test.ts index a5224db04..98fe95faf 100644 --- a/packages/harness/src/server/rest.test.ts +++ b/packages/harness/src/server/rest.test.ts @@ -21,6 +21,10 @@ import { createRestRouter, type RestRouterOptions } from "./rest.js"; const TOKEN_HEADER = { "X-Harness-Token": "unused-in-router-tests" }; +function zodV3ErrorMessage(issues: readonly Record[]): string { + return JSON.stringify(issues, null, 2); +} + function fakeSessionManager(initial: HarnessSession[] = []) { const sessions = new Map(initial.map((s) => [s.id, s])); return { @@ -341,6 +345,17 @@ describe("createRestRouter", () => { body: JSON.stringify({ telemetryOptIn: "yes" }), }); expect(res.status).toBe(400); + expect(await res.json()).toEqual({ + error: zodV3ErrorMessage([ + { + code: "invalid_type", + expected: "boolean", + received: "string", + path: ["telemetryOptIn"], + message: "Expected boolean, received string", + }, + ]), + }); }); }); @@ -380,10 +395,31 @@ describe("createRestRouter", () => { const res = await fetch(`${baseUrl}/sessions`, { method: "POST", headers: { ...TOKEN_HEADER, "content-type": "application/json" }, - body: JSON.stringify({ cwd: "" }), + body: JSON.stringify({ cwd: "", harness: "conductor" }), }); expect(res.status).toBe(400); + expect(await res.json()).toEqual({ + error: zodV3ErrorMessage([ + { + code: "too_small", + minimum: 1, + type: "string", + inclusive: true, + exact: false, + message: "String must contain at least 1 character(s)", + path: ["cwd"], + }, + { + received: "conductor", + code: "invalid_enum_value", + options: ["claude-code", "codex"], + path: ["harness"], + message: + "Invalid enum value. Expected 'claude-code' | 'codex', received 'conductor'", + }, + ]), + }); expect(onSessionCreated).not.toHaveBeenCalled(); }); @@ -572,6 +608,25 @@ describe("createRestRouter", () => { expect(res.status).toBe(status); }); + it("preserves the v3 required-field error body for attachment validation", async () => { + const res = await postAttachment({ + dataUrl: "data:text/plain;base64,YQ==", + }); + + expect(res.status).toBe(400); + expect(await res.json()).toEqual({ + error: zodV3ErrorMessage([ + { + code: "invalid_type", + expected: "string", + received: "undefined", + path: ["filename"], + message: "Required", + }, + ]), + }); + }); + it("rejects a decoded payload over 10 MiB", async () => { const encoded = Buffer.alloc(10 * 1024 * 1024 + 1).toString("base64"); const res = await postAttachment({ @@ -651,6 +706,17 @@ describe("createRestRouter", () => { body: JSON.stringify({ submit: true }), }); expect(res.status).toBe(400); + expect(await res.json()).toEqual({ + error: zodV3ErrorMessage([ + { + code: "invalid_type", + expected: "string", + received: "undefined", + path: ["text"], + message: "Required", + }, + ]), + }); }); it("404s when submitInput reports no live pty for the session", async () => { @@ -817,6 +883,17 @@ describe("createRestRouter", () => { }, ); expect(res.status).toBe(400); + expect(await res.json()).toEqual({ + error: zodV3ErrorMessage([ + { + code: "invalid_type", + expected: "string", + received: "undefined", + path: ["workflowPath"], + message: "Required", + }, + ]), + }); }); }); @@ -1162,6 +1239,21 @@ describe("createRestRouter", () => { expect((await adopt({ cwd: "/tmp/proj" })).status).toBe(400); }); + it("preserves the v3 joined-issue error body for adoption", async () => { + start({ adapters: { "claude-code": historyAdapter() } }); + const res = await adopt({ + ...body, + agentSessionId: "", + harness: "conductor", + }); + + expect(res.status).toBe(400); + expect(await res.json()).toEqual({ + error: + "String must contain at least 1 character(s); Invalid enum value. Expected 'claude-code' | 'codex', received 'conductor'", + }); + }); + it("is handled as its own route — 'adopt' is never read as a session id", async () => { // Today this is structural: `/sessions/adopt` is two path segments and // `/sessions/:id/resume` is three, so they cannot collide and no diff --git a/packages/harness/src/server/rest.ts b/packages/harness/src/server/rest.ts index efccffdf5..ff3353750 100644 --- a/packages/harness/src/server/rest.ts +++ b/packages/harness/src/server/rest.ts @@ -6,7 +6,10 @@ import express, { Router } from "express"; import rateLimit from "express-rate-limit"; -import { z } from "zod"; +// Harness's public validation-error bodies predate the Agent SDK dependency +// and serialize Zod v3 issues verbatim. Keep that wire contract stable while +// the experimental managed-agent runtime uses root Zod v4 for SDK tool schemas. +import { z } from "zod/v3"; import { randomUUID } from "node:crypto"; import * as fs from "node:fs/promises"; diff --git a/packages/harness/src/server/track.test.ts b/packages/harness/src/server/track.test.ts index d2db237ae..bdf52a228 100644 --- a/packages/harness/src/server/track.test.ts +++ b/packages/harness/src/server/track.test.ts @@ -165,6 +165,31 @@ describe("POST /api/track", () => { body: JSON.stringify({ event: "unknown.event" }), }); expect(res.status).toBe(400); + expect(await res.json()).toEqual({ + error: JSON.stringify( + [ + { + received: "unknown.event", + code: "invalid_enum_value", + options: [ + "prompt.submitted", + "session.switched", + "macro.invoked", + "visualize.triggered", + "consent.changed", + "session.created", + "mcp.install", + "plan.upgrade_clicked", + ], + path: ["event"], + message: + "Invalid enum value. Expected 'prompt.submitted' | 'session.switched' | 'macro.invoked' | 'visualize.triggered' | 'consent.changed' | 'session.created' | 'mcp.install' | 'plan.upgrade_clicked', received 'unknown.event'", + }, + ], + null, + 2, + ), + }); expect(stored).toHaveLength(0); }); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 26731d854..cc105ee2f 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -368,6 +368,15 @@ importers: packages/harness: dependencies: + '@anthropic-ai/claude-agent-sdk': + specifier: 0.3.228 + version: 0.3.228(@anthropic-ai/sdk@0.116.0(zod@4.4.3))(@modelcontextprotocol/sdk@1.30.0(@cfworker/json-schema@4.1.1)(zod@4.4.3))(zod@4.4.3) + '@anthropic-ai/sdk': + specifier: 0.116.0 + version: 0.116.0(zod@4.4.3) + '@modelcontextprotocol/sdk': + specifier: 1.30.0 + version: 1.30.0(@cfworker/json-schema@4.1.1)(zod@4.4.3) '@sapiom/agent': specifier: workspace:^ version: link:../agent @@ -399,8 +408,8 @@ importers: specifier: ^8.18.0 version: 8.21.0 zod: - specifier: ^3.25.0 - version: 3.25.76 + specifier: 4.4.3 + version: 4.4.3 devDependencies: '@playwright/test': specifier: ^1.61.0 @@ -788,6 +797,67 @@ packages: 7zip-bin@5.2.0: resolution: {integrity: sha512-ukTPVhqG4jNzMro2qA9HSCSSVJN3aN7tlb+hfqYCt3ER0yWroeA2VR38MNrOHLQ/cVj+DaIMad0kFCtWWowh/A==} + '@anthropic-ai/claude-agent-sdk-darwin-arm64@0.3.228': + resolution: {integrity: sha512-HuCsV3/5XuYYaWuCbksX+e0JkDDUG/AlFJ8wKhDL3PBW/3hHNd6xBYx88kEWk1Z6B1GLxwHht9624lcmscpsyw==} + cpu: [arm64] + os: [darwin] + + '@anthropic-ai/claude-agent-sdk-darwin-x64@0.3.228': + resolution: {integrity: sha512-jSUYY5Nd3efvbLZPU+i0tRBaFXskHu8M+4LMGBEw6A0PaklZ3YfGvKlTOWtJGRw6vMc6LzfOFts024xPNm6OrQ==} + cpu: [x64] + os: [darwin] + + '@anthropic-ai/claude-agent-sdk-linux-arm64-musl@0.3.228': + resolution: {integrity: sha512-4PgfisC3kHKlzJvy3rrm4Oh26g+D78h4ahHjni9fvSKHuJgrvHu9Qgo6aaYmzWdc7v9drL+pgiCk5Ge4Y2ANPA==} + cpu: [arm64] + os: [linux] + libc: [musl] + + '@anthropic-ai/claude-agent-sdk-linux-arm64@0.3.228': + resolution: {integrity: sha512-0Wjv6TiWwGlBZINAmNJX07jN359jKwB/4Sr/uWgQkdjuVIOhe/M8ydk7JL2EPqCsbiW1lc15NjE5MWpZiYqooA==} + cpu: [arm64] + os: [linux] + libc: [glibc] + + '@anthropic-ai/claude-agent-sdk-linux-x64-musl@0.3.228': + resolution: {integrity: sha512-dnXxyiwGCZj27HVk6clYRqGMgrs3KVLVp0vvWYLjkPGBiKbI83qJiDpOfaekEXG2I4elX0M4XikggV1LGWjimg==} + cpu: [x64] + os: [linux] + libc: [musl] + + '@anthropic-ai/claude-agent-sdk-linux-x64@0.3.228': + resolution: {integrity: sha512-LmGplObceqMOu5mlrlhTZL/VSrEWdZagF0Bl8awglMu6WeQcNe7StORYkCznZ0BuzV4CwuC3ipV4q8Jrs66wSg==} + cpu: [x64] + os: [linux] + libc: [glibc] + + '@anthropic-ai/claude-agent-sdk-win32-arm64@0.3.228': + resolution: {integrity: sha512-mNS5yIMz/OXSQiDErb84jA8AKBFSlS9RSZ0qn2qyGkxplUx7kVmIDg/KnwOwHmygpzmH4UmR6OCaLXGohupqNA==} + cpu: [arm64] + os: [win32] + + '@anthropic-ai/claude-agent-sdk-win32-x64@0.3.228': + resolution: {integrity: sha512-DYT3HvdS64Pq0IRvgW3RDO31yjYp5yiUKoKaZolTpLKfALpG5LI/osfnKlya68PZ/bSST1FNAfW9I0EtCnaQ4w==} + cpu: [x64] + os: [win32] + + '@anthropic-ai/claude-agent-sdk@0.3.228': + resolution: {integrity: sha512-OOaME54VCoBLjKMqWqFmHkZGyL/x/FHUA0snhyolmyEhVoeBM0Ub5mrnV2Gx3d5/RcVlk2BnEVvPqu0SpZ9VFw==} + engines: {node: '>=18.0.0'} + peerDependencies: + '@anthropic-ai/sdk': '>=0.93.0' + '@modelcontextprotocol/sdk': ^1.29.0 + zod: ^4.0.0 + + '@anthropic-ai/sdk@0.116.0': + resolution: {integrity: sha512-4UEapYQ+epLEMsAuLZDvW8ExVSOtHD8a7zTyLzhw0H9RXJ1eilPgmqhjwgcdg22diwx13spw6fJ4rONZ+bS7Ww==} + hasBin: true + peerDependencies: + zod: ^3.25.0 || ^4.0.0 + peerDependenciesMeta: + zod: + optional: true + '@anthropic-ai/sdk@0.65.0': resolution: {integrity: sha512-zIdPOcrCVEI8t3Di40nH4z9EoeyGZfXbYSvWdDLsB/KkaSYMnEgC7gmcgWu83g2NTn1ZTpbMvpdttWDGGIk6zw==} hasBin: true @@ -1847,6 +1917,16 @@ packages: '@cfworker/json-schema': optional: true + '@modelcontextprotocol/sdk@1.30.0': + resolution: {integrity: sha512-xKd8OIzlqNzcqcNumGAa6g+PW2kjD5vrpcKOnfldAUPP3j7lnqMPwlTXQm8gF+UwH72z0lqaRbjr9hqGz0eITA==} + engines: {node: '>=18'} + peerDependencies: + '@cfworker/json-schema': ^4.1.1 + zod: ^3.25 || ^4.0 + peerDependenciesMeta: + '@cfworker/json-schema': + optional: true + '@mswjs/interceptors@0.41.9': resolution: {integrity: sha512-VVPPgHyQ6ShqnrmDWuxjmUIsO9gWyOZFmuOfLd9LfBGQJwZfy0gvv9pbHSJuoFNIYC7ZDX9aoFwowjcdSC4E8w==} engines: {node: '>=18'} @@ -2060,6 +2140,9 @@ packages: '@sinonjs/fake-timers@10.3.0': resolution: {integrity: sha512-V4BG07kuYSUkTCSBHG8G8TNhM+F19jXFWnQtzj+we8DrkpSBCee9Z3Ms8yiGer/dlmhe35/Xdgyo3/0rQKg7YA==} + '@stablelib/base64@1.0.1': + resolution: {integrity: sha512-1bnPQqSxSuc3Ii6MhBysoWCg58j97aUjuCSZrGSmDxNqtytIi0k8utUenAwTZN4V5mXXYGsVUI9zeBqy+jBOSQ==} + '@standard-schema/spec@1.1.0': resolution: {integrity: sha512-l2aFy5jALhniG5HgqrD6jXLi/rUWrKvqN/qJx6yoJsgKhblVd+iqqU4RCXavm/jPityDo5TCvKMnpjKnOriy0w==} @@ -3250,6 +3333,9 @@ packages: fast-levenshtein@2.0.6: resolution: {integrity: sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw==} + fast-sha256@1.3.0: + resolution: {integrity: sha512-n11RGP/lrWEFI/bWdygLxhI+pVeo1ZYIVwvvPkW7azl/rOy+F3HYRZ2K5zeE9mmkhQppyv9sQFx0JM9UabnpPQ==} + fast-string-truncated-width@3.0.3: resolution: {integrity: sha512-0jjjIEL6+0jag3l2XWWizO64/aZVtpiGE3t0Zgqxv0DPuxiMjvB3M24fCyhZUO4KomJQPj3LTSUnDP3GpdwC0g==} @@ -5070,6 +5156,9 @@ packages: stackback@0.0.2: resolution: {integrity: sha512-1XMJE5fQo1jGH6Y/7ebnwPOBEkIEnT4QF32d5R1+VXdXveM0IBMJt8zfaxX1P3QhVwrYe+576+jkANtSS2mBbw==} + standardwebhooks@1.0.0: + resolution: {integrity: sha512-BbHGOQK9olHPMvQNHWul6MYlrRTAOKn03rOe4A8O3CLWhNf4YHBqq2HJKKC+sfqpxiBY52pNeesD6jIiLDz8jg==} + stat-mode@1.0.0: resolution: {integrity: sha512-jH9EhtKIjuXZ2cWxmXS8ZP80XyC3iasQxMDV8jzhNJpfDb7VbQLVW4Wvsxz9QZvzV+G4YoSfBUVKDOyxLzi/sg==} engines: {node: '>= 6'} @@ -5583,6 +5672,52 @@ snapshots: 7zip-bin@5.2.0: {} + '@anthropic-ai/claude-agent-sdk-darwin-arm64@0.3.228': + optional: true + + '@anthropic-ai/claude-agent-sdk-darwin-x64@0.3.228': + optional: true + + '@anthropic-ai/claude-agent-sdk-linux-arm64-musl@0.3.228': + optional: true + + '@anthropic-ai/claude-agent-sdk-linux-arm64@0.3.228': + optional: true + + '@anthropic-ai/claude-agent-sdk-linux-x64-musl@0.3.228': + optional: true + + '@anthropic-ai/claude-agent-sdk-linux-x64@0.3.228': + optional: true + + '@anthropic-ai/claude-agent-sdk-win32-arm64@0.3.228': + optional: true + + '@anthropic-ai/claude-agent-sdk-win32-x64@0.3.228': + optional: true + + '@anthropic-ai/claude-agent-sdk@0.3.228(@anthropic-ai/sdk@0.116.0(zod@4.4.3))(@modelcontextprotocol/sdk@1.30.0(@cfworker/json-schema@4.1.1)(zod@4.4.3))(zod@4.4.3)': + dependencies: + '@anthropic-ai/sdk': 0.116.0(zod@4.4.3) + '@modelcontextprotocol/sdk': 1.30.0(@cfworker/json-schema@4.1.1)(zod@4.4.3) + zod: 4.4.3 + optionalDependencies: + '@anthropic-ai/claude-agent-sdk-darwin-arm64': 0.3.228 + '@anthropic-ai/claude-agent-sdk-darwin-x64': 0.3.228 + '@anthropic-ai/claude-agent-sdk-linux-arm64': 0.3.228 + '@anthropic-ai/claude-agent-sdk-linux-arm64-musl': 0.3.228 + '@anthropic-ai/claude-agent-sdk-linux-x64': 0.3.228 + '@anthropic-ai/claude-agent-sdk-linux-x64-musl': 0.3.228 + '@anthropic-ai/claude-agent-sdk-win32-arm64': 0.3.228 + '@anthropic-ai/claude-agent-sdk-win32-x64': 0.3.228 + + '@anthropic-ai/sdk@0.116.0(zod@4.4.3)': + dependencies: + json-schema-to-ts: 3.1.1 + standardwebhooks: 1.0.0 + optionalDependencies: + zod: 4.4.3 + '@anthropic-ai/sdk@0.65.0(zod@4.4.3)': dependencies: json-schema-to-ts: 3.1.1 @@ -6829,6 +6964,30 @@ snapshots: transitivePeerDependencies: - supports-color + '@modelcontextprotocol/sdk@1.30.0(@cfworker/json-schema@4.1.1)(zod@4.4.3)': + dependencies: + '@hono/node-server': 1.19.14(hono@4.12.25) + ajv: 8.20.0 + ajv-formats: 3.0.1(ajv@8.20.0) + content-type: 1.0.5 + cors: 2.8.6 + cross-spawn: 7.0.6 + eventsource: 3.0.7 + eventsource-parser: 3.1.0 + express: 5.2.1 + express-rate-limit: 8.5.2(express@5.2.1) + hono: 4.12.25 + jose: 6.2.3 + json-schema-typed: 8.0.2 + pkce-challenge: 5.0.1 + raw-body: 3.0.2 + zod: 4.4.3 + zod-to-json-schema: 3.25.2(zod@4.4.3) + optionalDependencies: + '@cfworker/json-schema': 4.1.1 + transitivePeerDependencies: + - supports-color + '@mswjs/interceptors@0.41.9': dependencies: '@open-draft/deferred-promise': 2.2.0 @@ -6980,6 +7139,8 @@ snapshots: dependencies: '@sinonjs/commons': 3.0.1 + '@stablelib/base64@1.0.1': {} + '@standard-schema/spec@1.1.0': {} '@stryker-mutator/api@9.6.1': @@ -8565,6 +8726,8 @@ snapshots: fast-levenshtein@2.0.6: {} + fast-sha256@1.3.0: {} + fast-string-truncated-width@3.0.3: {} fast-string-width@3.0.2: @@ -10458,6 +10621,11 @@ snapshots: stackback@0.0.2: {} + standardwebhooks@1.0.0: + dependencies: + '@stablelib/base64': 1.0.1 + fast-sha256: 1.3.0 + stat-mode@1.0.0: {} statuses@2.0.2: {} @@ -10903,7 +11071,6 @@ snapshots: zod-to-json-schema@3.25.2(zod@4.4.3): dependencies: zod: 4.4.3 - optional: true zod@3.25.76: {}