feat(harness): add managed Agent SDK feasibility probe

.changeset/managed-agent-spike.md

@@ -0,0 +1,5 @@
+---
+"@sapiom/harness": minor
+---
+
+Add an experimental, programmatic managed-agent probe for validating isolated Agent SDK tools, permissions, cancellation, and workspace preservation.

.github/workflows/harness.yml

@@ -56,6 +56,32 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
+  # Real SDK transport sentinel. This exercises the pinned bundled runtime and
+  # must not silently skip when the repository's floating Node matrix moves.
+  managed-agent-sdk-sentinel:
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    steps:
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
+        with:
+          persist-credentials: false
+
+      - uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v4.4.0
+
+      - name: Use certification Node.js
+        uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
+        with:
+          node-version: "22.23.2"
+          cache: "pnpm"
+
+      - name: Install dependencies
+        run: pnpm install --frozen-lockfile
+
+      - name: Run real Agent SDK loopback sentinels
+        run: >-
+          pnpm --filter @sapiom/harness exec vitest run
+          src/experimental/managed-agent-spike/runtime-sdk-loopback.test.ts
+
   # Playwright mock-mode tier — harness web/e2e specs (VITE_MOCK=1, chromium only).
   # The live/real-pty tiers (e2e:live, sim) are opt-in local only; nothing here
   # invokes them — they require real agent binaries and credentials not in CI.

packages/harness/package.json

@@ -29,6 +29,10 @@
       "types": "./dist/index.d.ts",
       "import": "./dist/index.js"
     },
+    "./experimental/managed-agent-spike": {
+      "types": "./dist/experimental/managed-agent-spike/index.d.ts",
+      "import": "./dist/experimental/managed-agent-spike/index.js"
+    },
     "./package.json": "./package.json"
   },
   "bin": {
@@ -54,6 +58,7 @@
     "test:mutation": "stryker run",
     "test:ui": "playwright test --config web/e2e/playwright.config.ts",
     "test:canvas": "playwright test --config e2e/playwright.config.ts",
+    "probe:managed-agent": "tsx src/experimental/managed-agent-spike/probe-cli.ts",
     "typecheck": "tsc --noEmit && tsc --noEmit -p web/tsconfig.json",
     "lint": "eslint src --ext .ts",
     "prepublishOnly": "pnpm build",
@@ -63,6 +68,9 @@
     "e2e:live": "tsx scripts/e2e-live.ts"
   },
   "dependencies": {
+    "@anthropic-ai/claude-agent-sdk": "0.3.228",
+    "@anthropic-ai/sdk": "0.116.0",
+    "@modelcontextprotocol/sdk": "1.30.0",
     "@sapiom/agent": "workspace:^",
     "@sapiom/agent-core": "workspace:^",
     "@sapiom/analytics-core": "workspace:^",
@@ -73,7 +81,7 @@
     "node-pty": "^1.1.0",
     "open": "^10.1.0",
     "ws": "^8.18.0",
-    "zod": "^3.25.0"
+    "zod": "4.4.3"
   },
   "devDependencies": {
     "@playwright/test": "^1.61.0",

packages/harness/src/experimental/managed-agent-spike/contract.test.ts

@@ -0,0 +1,262 @@
+import { mkdir, mkdtemp, realpath, rm } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+
+import { afterEach, describe, expect, it } from "vitest";
+
+import {
+  MANAGED_AGENT_CONTRACT,
+  MANAGED_AGENT_L1_CERTIFICATION_CONTRACT,
+  MANAGED_AGENT_L1_FINAL_BYTE_ROLES,
+  MANAGED_AGENT_L1_REGISTERED_PATH_ROLES,
+  MANAGED_AGENT_MODEL_TARGETS,
+  ManagedAgentConfigurationError,
+  assertManagedAgentDirectGatewayOrigin,
+  normalizeManagedAgentGatewayOrigin,
+  normalizeManagedAgentHermeticGatewayOrigin,
+  resolveManagedAgentModelTarget,
+  validateManagedAgentProbeConfig,
+} from "./contract.js";
+import type { ManagedAgentProbeConfig } from "./types.js";
+
+const roots: string[] = [];
+
+async function config(): Promise<ManagedAgentProbeConfig> {
+  const root = await mkdtemp(join(tmpdir(), "managed-agent-contract-"));
+  roots.push(root);
+  const workspaceRoot = join(root, "workspace");
+  const configRoot = join(root, "config");
+  await Promise.all([mkdir(workspaceRoot), mkdir(configRoot)]);
+  return {
+    scenario: "L1",
+    workspaceRoot,
+    configRoot,
+    target: "sonnet-5",
+    gatewayOrigin: MANAGED_AGENT_CONTRACT.directGatewayOrigin,
+    gatewayCredential: "dedicated-eval-key",
+    prompt: `${MANAGED_AGENT_L1_CERTIFICATION_CONTRACT.promptMarker}\nprobe`,
+    maxTurns: 10,
+    maxBudgetUsd: 0.25,
+    allowedBashCommands: ["git status --short"],
+    pathRoleBindings: [
+      { path: "clean.txt", role: "clean_target" },
+      { path: "dirty.txt", role: "dirty_sentinel" },
+      { path: "untracked.txt", role: "untracked_sentinel" },
+      { path: "created.txt", role: "managed_output" },
+      { path: "../outside.txt", role: "outside_sentinel" },
+      { path: "escape.txt", role: "escape_link" },
+    ],
+    expectedL1FinalBytes: [
+      { path: "clean.txt", role: "clean_target", sha256: "a".repeat(64) },
+      { path: "created.txt", role: "managed_output", sha256: "b".repeat(64) },
+    ],
+    expectedMcpNonce: "probe-nonce",
+  };
+}
+
+afterEach(async () => {
+  await Promise.all(
+    roots.splice(0).map((root) => rm(root, { recursive: true, force: true })),
+  );
+});
+
+describe("managed-agent contract", () => {
+  it("freezes the versioned L1 prompt and evaluator contract", () => {
+    expect(MANAGED_AGENT_L1_CERTIFICATION_CONTRACT).toEqual({
+      contractVersion: 2,
+      promptVersion: "managed-agent-l1-prompt-v2",
+      promptMarker: "SAPIOM_MANAGED_AGENT_L1_PROMPT_V2",
+      evaluatorVersion: "managed-agent-l1-evaluator-v2",
+    });
+    expect(Object.isFrozen(MANAGED_AGENT_L1_CERTIFICATION_CONTRACT)).toBe(true);
+    expect(Object.isFrozen(MANAGED_AGENT_L1_REGISTERED_PATH_ROLES)).toBe(true);
+    expect(Object.isFrozen(MANAGED_AGENT_L1_FINAL_BYTE_ROLES)).toBe(true);
+  });
+
+  it("pins the certified SDK/runtime and exact two-model allowlist", () => {
+    expect(MANAGED_AGENT_CONTRACT).toMatchObject({
+      agentSdkVersion: "0.3.228",
+      claudeCodeRuntimeVersion: "2.1.228",
+      certificationNodeVersion: "22.23.2",
+      directGatewayOrigin: "https://litellm.services.sapiom.ai",
+    });
+    expect(MANAGED_AGENT_MODEL_TARGETS).toEqual({
+      "sonnet-5": expect.objectContaining({
+        alias: "claude-sonnet-5-anthropic-anthropic-eval",
+      }),
+      "minimax-m3": expect.objectContaining({
+        alias: "minimax-m3-fireworks-sapiom-fireworks_ai-eval",
+      }),
+    });
+  });
+
+  it("rejects arbitrary models instead of accepting a gateway label", () => {
+    expect(() =>
+      resolveManagedAgentModelTarget("claude-anything" as "sonnet-5"),
+    ).toThrow(ManagedAgentConfigurationError);
+  });
+
+  it("accepts only a credential-free HTTP(S) origin", () => {
+    expect(
+      normalizeManagedAgentGatewayOrigin("https://gateway.example.test/"),
+    ).toBe("https://gateway.example.test");
+    for (const value of [
+      "file:///tmp/gateway",
+      "https://user:pass@gateway.example.test",
+      "https://gateway.example.test/v1",
+      "https://gateway.example.test?token=x",
+    ]) {
+      expect(() => normalizeManagedAgentGatewayOrigin(value)).toThrow(
+        ManagedAgentConfigurationError,
+      );
+    }
+  });
+
+  it("pins live traffic to the certified direct gateway origin", () => {
+    expect(
+      assertManagedAgentDirectGatewayOrigin(
+        "https://litellm.services.sapiom.ai/",
+      ),
+    ).toBe(MANAGED_AGENT_CONTRACT.directGatewayOrigin);
+    expect(() =>
+      assertManagedAgentDirectGatewayOrigin(
+        "https://llm.services.proxy.sapiom.ai",
+      ),
+    ).toThrow("pinned direct Sapiom gateway origin");
+  });
+
+  it("limits the explicit hermetic origin seam to .test and loopback", () => {
+    for (const value of [
+      "https://gateway.example.test",
+      "http://localhost:4312",
+      "http://agent.localhost:4312",
+      "http://127.0.0.1:4312",
+      "http://[::1]:4312",
+    ]) {
+      expect(normalizeManagedAgentHermeticGatewayOrigin(value)).toBe(
+        normalizeManagedAgentGatewayOrigin(value),
+      );
+    }
+    for (const value of [
+      MANAGED_AGENT_CONTRACT.directGatewayOrigin,
+      "https://gateway.example.com",
+    ]) {
+      expect(() => normalizeManagedAgentHermeticGatewayOrigin(value)).toThrow(
+        "reserved .test or loopback",
+      );
+    }
+  });
+
+  it("canonicalizes disjoint roots and bounds turns and budget", async () => {
+    const valid = await config();
+    const checked = validateManagedAgentProbeConfig(valid);
+    expect(checked.canonicalWorkspaceRoot).toBe(
+      await realpath(valid.workspaceRoot),
+    );
+    expect(checked.model.id).toBe("sonnet-5");
+    expect(() =>
+      validateManagedAgentProbeConfig({ ...valid, maxBudgetUsd: 1.01 }),
+    ).toThrow("maxBudgetUsd");
+    expect(() =>
+      validateManagedAgentProbeConfig({ ...valid, maxTurns: 21 }),
+    ).toThrow("maxTurns");
+    expect(() =>
+      validateManagedAgentProbeConfig({
+        ...valid,
+        configRoot: valid.workspaceRoot,
+      }),
+    ).toThrow("disjoint");
+    expect(() =>
+      validateManagedAgentProbeConfig({
+        ...valid,
+        expectedMcpNonce: undefined,
+      }),
+    ).toThrow("expectedMcpNonce");
+  });
+
+  it("requires the exact L1 v2 marker and all six unique path roles", async () => {
+    const valid = await config();
+    expect(() =>
+      validateManagedAgentProbeConfig({
+        ...valid,
+        prompt: "SAPIOM_MANAGED_AGENT_L1_PROMPT_V1\nprobe",
+      }),
+    ).toThrow("managed-agent-l1-prompt-v2 marker");
+    expect(() =>
+      validateManagedAgentProbeConfig({
+        ...valid,
+        pathRoleBindings: valid.pathRoleBindings.slice(0, -1),
+      }),
+    ).toThrow("each frozen fixture role exactly once");
+    expect(() =>
+      validateManagedAgentProbeConfig({
+        ...valid,
+        pathRoleBindings: valid.pathRoleBindings.map((binding, index) =>
+          index === 1 ? { ...binding, path: "clean.txt" } : binding,
+        ),
+      }),
+    ).toThrow("each frozen fixture role exactly once");
+  });
+
+  it("requires exact trusted hashes for both intended L1 mutation roles", async () => {
+    const valid = await config();
+    for (const expectedL1FinalBytes of [
+      valid.expectedL1FinalBytes.slice(0, 1),
+      valid.expectedL1FinalBytes.map((expectation, index) =>
+        index === 0 ? { ...expectation, sha256: "not-a-hash" } : expectation,
+      ),
+      valid.expectedL1FinalBytes.map((expectation, index) =>
+        index === 0 ? { ...expectation, path: "dirty.txt" } : expectation,
+      ),
+    ]) {
+      expect(() =>
+        validateManagedAgentProbeConfig({
+          ...valid,
+          expectedL1FinalBytes,
+        }),
+      ).toThrow("exact hashes");
+    }
+  });
+
+  it("keeps L2 free of L1 path-role and final-byte configuration", async () => {
+    const valid = await config();
+    const l2: ManagedAgentProbeConfig = {
+      ...valid,
+      scenario: "L2",
+      prompt: "run exact Bash",
+      pathRoleBindings: [],
+      expectedL1FinalBytes: [],
+      expectedMcpNonce: undefined,
+    };
+    expect(() => validateManagedAgentProbeConfig(l2)).not.toThrow();
+    expect(() =>
+      validateManagedAgentProbeConfig({
+        ...l2,
+        pathRoleBindings: valid.pathRoleBindings,
+      }),
+    ).toThrow("L2 must not configure");
+    expect(() =>
+      validateManagedAgentProbeConfig({
+        ...l2,
+        expectedL1FinalBytes: valid.expectedL1FinalBytes,
+      }),
+    ).toThrow("L2 must not configure");
+  });
+
+  it("requires exact agreement with an explicitly selected hermetic origin", async () => {
+    const valid = await config();
+    const gatewayOrigin = "https://gateway.example.test";
+    expect(
+      validateManagedAgentProbeConfig(
+        { ...valid, gatewayOrigin },
+        { hermeticGatewayOrigin: gatewayOrigin },
+      ).gatewayOrigin,
+    ).toBe(gatewayOrigin);
+    expect(() =>
+      validateManagedAgentProbeConfig(
+        { ...valid, gatewayOrigin },
+        { hermeticGatewayOrigin: "https://other.example.test" },
+      ),
+    ).toThrow("explicit hermetic gateway origin");
+  });
+});