From b19d5171961004dd677646b8f677b0ab0d0c180f Mon Sep 17 00:00:00 2001 From: Rhys Sullivan <39114868+RhysSullivan@users.noreply.github.com> Date: Mon, 24 Aug 2026 19:07:43 -0700 Subject: [PATCH] Capture executed code and result sizes in cloud telemetry --- .claude/skills/prod-telemetry/SKILL.md | 10 ++++- apps/cloud/src/mcp/telemetry.test.ts | 55 ++++++++++++++++++++++++++ apps/cloud/src/mcp/telemetry.ts | 33 +++++++++++++++- packages/core/execution/src/engine.ts | 38 ++++++++++++++---- 4 files changed, 125 insertions(+), 11 deletions(-) diff --git a/.claude/skills/prod-telemetry/SKILL.md b/.claude/skills/prod-telemetry/SKILL.md index 33dae496d5..0128c9ecd0 100644 --- a/.claude/skills/prod-telemetry/SKILL.md +++ b/.claude/skills/prod-telemetry/SKILL.md @@ -38,6 +38,11 @@ join the same traces via traceparent). Sandbox script failures ride the MCP success channel, so `status.code` stays OK — filter on these attributes, not span status. Spans from before the attributes shipped carry neither; absence is not success. + Also `mcp.execute.result_chars` (compact-JSON size of the returned + value, pre-truncation; -1 = unmeasurable), `mcp.execute.log_chars`, + `mcp.execute.emitted` — the dump-vs-narrow signal (the model preview + truncates at 30k chars, so `result_chars > 30000` means the model tried + to pull a truncated blob into context). - `executor.tool.execute` — `mcp.tool.name` (full address), and since PR #992: `executor.tool.outcome` (`ok`/`fail`), `executor.tool.error_code`, `executor.tool.error_status`, @@ -48,7 +53,10 @@ join the same traces via traceparent). `base_url`, and since PR #992 `http.status_code`. - `mcp.request` (outer) — `mcp.auth.organization_id`, `mcp.auth.account_id`, `mcp.tool.name`, CF edge fields (`cf.country`…), - MCP client fingerprint (`mcp.client.name`…). + MCP client fingerprint (`mcp.client.name`…), and on managed-cloud + `execute`/`execute-action` calls `mcp.execute.code` (the script itself, + capped at 10k chars — cloud-only content capture; local/self-host + telemetry never records content). **Recipe — error signatures by class (the daily-digest query):** diff --git a/apps/cloud/src/mcp/telemetry.test.ts b/apps/cloud/src/mcp/telemetry.test.ts index 173241f998..df150bc9ed 100644 --- a/apps/cloud/src/mcp/telemetry.test.ts +++ b/apps/cloud/src/mcp/telemetry.test.ts @@ -103,3 +103,58 @@ describe("annotateMcpRequest — cancellation join keys", () => { }).pipe(Effect.withSpan("mcp.request"), Effect.withTracer(tracer)); }); }); + +describe("annotateMcpRequest — executed-code capture", () => { + it.effect("stamps the script on execute calls", () => { + const { tracer, attributesOf } = makeRecordingTracer(); + return Effect.gen(function* () { + yield* annotate( + postRequest({ + jsonrpc: "2.0", + id: 8, + method: "tools/call", + params: { name: "execute", arguments: { code: "return 6 * 7;" } }, + }), + ); + const attributes = attributesOf("mcp.request"); + expectDefined(attributes); + expect(attributes.get("mcp.execute.code")).toBe("return 6 * 7;"); + }).pipe(Effect.withSpan("mcp.request"), Effect.withTracer(tracer)); + }); + + it.effect("caps oversized scripts with a truncation marker", () => { + const { tracer, attributesOf } = makeRecordingTracer(); + return Effect.gen(function* () { + const code = "x".repeat(12_000); + yield* annotate( + postRequest({ + jsonrpc: "2.0", + id: 9, + method: "tools/call", + params: { name: "execute", arguments: { code } }, + }), + ); + const attributes = attributesOf("mcp.request"); + expectDefined(attributes); + const captured = attributes.get("mcp.execute.code"); + expect(captured).toBe(`${"x".repeat(10_000)}\n… [truncated 2000 chars]`); + }).pipe(Effect.withSpan("mcp.request"), Effect.withTracer(tracer)); + }); + + it.effect("does not capture arguments of other tools", () => { + const { tracer, attributesOf } = makeRecordingTracer(); + return Effect.gen(function* () { + yield* annotate( + postRequest({ + jsonrpc: "2.0", + id: 10, + method: "tools/call", + params: { name: "resume", arguments: { code: "not an execute call" } }, + }), + ); + const attributes = attributesOf("mcp.request"); + expectDefined(attributes); + expect(attributes.get("mcp.execute.code")).toBeUndefined(); + }).pipe(Effect.withSpan("mcp.request"), Effect.withTracer(tracer)); + }); +}); diff --git a/apps/cloud/src/mcp/telemetry.ts b/apps/cloud/src/mcp/telemetry.ts index ec81548884..3f0c300256 100644 --- a/apps/cloud/src/mcp/telemetry.ts +++ b/apps/cloud/src/mcp/telemetry.ts @@ -102,7 +102,10 @@ const InitializeParams = Schema.Struct({ capabilities: Schema.optional(UnknownRecord), }); -const NamedParams = Schema.Struct({ name: Schema.optional(Schema.String) }); +const NamedParams = Schema.Struct({ + name: Schema.optional(Schema.String), + arguments: Schema.optional(UnknownRecord), +}); const UriParams = Schema.Struct({ uri: Schema.optional(Schema.String) }); // `notifications/cancelled` carries no JSON-RPC id of its own, but its params @@ -134,6 +137,29 @@ const readJsonRpcEnvelope = (request: Request): Effect.Effect | undefined, +): Record => { + if (name !== "execute" && name !== "execute-action") return {}; + const code = args?.["code"]; + if (typeof code !== "string") return {}; + return { + "mcp.execute.code": + code.length > MAX_CODE_ATTR_CHARS + ? `${code.slice(0, MAX_CODE_ATTR_CHARS)}\n… [truncated ${code.length - MAX_CODE_ATTR_CHARS} chars]` + : code, + }; +}; + const methodAttrs = (envelope: JsonRpcEnvelope): Record => { const params = envelope.params ?? {}; return Match.value(envelope.method).pipe( @@ -154,7 +180,10 @@ const methodAttrs = (envelope: JsonRpcEnvelope): Record => { Match.when("tools/call", () => Option.match(decodeNamedParams(params), { onNone: () => ({}) as Record, - onSome: ({ name }) => (name ? { "mcp.tool.name": name } : {}), + onSome: ({ name, arguments: args }) => ({ + ...(name ? { "mcp.tool.name": name } : {}), + ...executeCodeAttrs(name, args), + }), }), ), Match.whenOr("resources/read", "resources/subscribe", () => diff --git a/packages/core/execution/src/engine.ts b/packages/core/execution/src/engine.ts index fe10b3d7a9..4e08e3188f 100644 --- a/packages/core/execution/src/engine.ts +++ b/packages/core/execution/src/engine.ts @@ -62,19 +62,41 @@ export type ResumeResponse = { // caller is itself the human approver (the operator-facing Run/Test panel). const acceptAllHandler: ElicitationHandler = () => Effect.succeed({ action: "accept" }); +/** + * Approximate size of the value a script returned, before any preview + * truncation. This is the "did the model narrow in code or dump the raw + * payload" metric: a compact JSON length, not the pretty-printed preview the + * model receives, so treat it as magnitude. -1 means unmeasurable (a value + * `JSON.stringify` rejects, e.g. a BigInt) — unknown size, not zero. + */ +const measureResultChars = (value: unknown): number => { + if (value == null) return 0; + if (typeof value === "string") return value.length; + // oxlint-disable-next-line executor/no-try-catch-or-throw -- boundary: best-effort size probe over an arbitrary sandbox value; a stringify rejection must not fail the execution path + try { + return JSON.stringify(value)?.length ?? 0; + } catch { + return -1; + } +}; + /** * Stamp the current `mcp.execute` / `mcp.execute.resume` span with how the - * execution ended. Sandbox failures ride the success channel as - * `ExecuteResult.error`, so without this the span reads OK and the failure - * class is unqueryable. Attributes stay enumerable identifiers — never the - * error message itself. + * execution ended and how much data it sent back toward model context. + * Sandbox failures ride the success channel as `ExecuteResult.error`, so + * without this the span reads OK and the failure class is unqueryable. + * Attributes stay enumerable identifiers and sizes — never the error message + * or result content itself. */ const annotateExecuteOutcome = (result: ExecuteResult) => - Effect.annotateCurrentSpan( - result.error + Effect.annotateCurrentSpan({ + "mcp.execute.result_chars": measureResultChars(result.result), + "mcp.execute.log_chars": result.logs?.reduce((total, line) => total + line.length, 0) ?? 0, + "mcp.execute.emitted": result.output?.length ?? 0, + ...(result.error ? { "mcp.execute.outcome": "fail", "mcp.execute.error_kind": result.errorKind ?? "unknown" } - : { "mcp.execute.outcome": "ok" }, - ); + : { "mcp.execute.outcome": "ok" }), + }); const annotateExecutionOutcome = (execution: ExecutionResult) => execution.status === "paused"