diff --git a/apps/desktop/build/entitlements.mac.plist b/apps/desktop/build/entitlements.mac.plist deleted file mode 100644 index 043ecf496e..0000000000 --- a/apps/desktop/build/entitlements.mac.plist +++ /dev/null @@ -1,21 +0,0 @@ - - - - - - com.apple.security.cs.allow-jit - - com.apple.security.cs.allow-unsigned-executable-memory - - - com.apple.security.cs.allow-dyld-environment-variables - - - com.apple.security.cs.disable-library-validation - - - diff --git a/apps/desktop/build/icon.png b/apps/desktop/build/icon.png deleted file mode 100644 index 48dadbfc58..0000000000 Binary files a/apps/desktop/build/icon.png and /dev/null differ diff --git a/e2e/scenarios/shape-memory.test.ts b/e2e/scenarios/shape-memory.test.ts new file mode 100644 index 0000000000..a1d2f5e123 --- /dev/null +++ b/e2e/scenarios/shape-memory.test.ts @@ -0,0 +1,178 @@ +// Cross-target: muscle memory — runtime-observed output shapes. Most OpenAPI +// operations declare no response schema, so `tools.describe.tool()` used to +// render `data: unknown` forever and the model had to guess response shapes. +// This journey proves the warm path end to end through public surfaces only: +// a schemaless tool describes as `unknown`, one real invocation against a live +// upstream teaches the shape, and the very next describe serves a real +// TypeScript type marked as observed. +import { randomBytes } from "node:crypto"; +import { createServer } from "node:http"; + +import { expect } from "@effect/vitest"; +import { Effect } from "effect"; +import { composePluginApi } from "@executor-js/api/server"; +import { openApiHttpPlugin } from "@executor-js/plugin-openapi/api"; +import { AuthTemplateSlug, ConnectionName, IntegrationSlug } from "@executor-js/sdk/shared"; + +import { scenario } from "../src/scenario"; +import { Api, Target } from "../src/services"; + +const api = composePluginApi([openApiHttpPlugin()] as const); + +/** One GET operation whose 200 declares no response schema — the shape the + * model would otherwise have to guess. */ +const issuesSpec = JSON.stringify({ + openapi: "3.0.3", + info: { title: "Issues API", version: "1.0.0" }, + paths: { + "/issues": { + get: { + operationId: "listIssues", + summary: "List issues", + responses: { "200": { description: "issues" } }, + }, + }, + }, +}); + +/** A live upstream for the single invocation that teaches the shape. */ +const serveIssuesFixture = Effect.acquireRelease( + Effect.callback<{ readonly url: string; readonly close: () => void }>((resume) => { + const server = createServer((_request, response) => { + response.writeHead(200, { "content-type": "application/json" }); + response.end( + JSON.stringify({ + issues: [ + { id: 1, title: "first", open: true }, + { id: 2, title: "second", open: false }, + ], + total: 2, + }), + ); + }); + server.listen(0, "127.0.0.1", () => { + const addressInfo = server.address(); + const port = typeof addressInfo === "object" && addressInfo !== null ? addressInfo.port : 0; + resume( + Effect.succeed({ + url: `http://127.0.0.1:${port}`, + close: () => { + server.close(); + server.closeAllConnections(); + }, + }), + ); + }); + }), + (fixture) => Effect.sync(fixture.close), +); + +const describeCode = (slug: string) => ` +const details = await tools.describe.tool({ path: "${slug}.org.main.issues.listIssues" }); +return { + outputTypeScript: details.outputTypeScript ?? null, + note: details.outputTypeScriptNote ?? null, + error: details.error ?? null, +}; +`; + +type DescribeOutcome = { + readonly outputTypeScript: string | null; + readonly note: string | null; + readonly error: unknown; +}; + +scenario( + "Muscle memory · a schemaless tool's observed output shape reaches describe", + {}, + Effect.scoped( + Effect.gen(function* () { + const target = yield* Target; + const { client: makeApiClient } = yield* Api; + const identity = yield* target.newIdentity(); + const client = yield* makeApiClient(api, identity); + const slug = IntegrationSlug.make(`shape_memory_${randomBytes(4).toString("hex")}`); + const upstream = yield* serveIssuesFixture; + + yield* Effect.ensuring( + Effect.gen(function* () { + yield* client.openapi.addSpec({ + payload: { + spec: { kind: "blob", value: issuesSpec }, + slug, + baseUrl: upstream.url, + authenticationTemplate: [ + { + slug: "apiKey", + type: "apiKey", + headers: { authorization: ["Bearer ", { type: "variable", name: "token" }] }, + }, + ], + }, + }); + yield* client.connections.create({ + payload: { + owner: "org", + name: ConnectionName.make("main"), + integration: slug, + template: AuthTemplateSlug.make("apiKey"), + value: `key_${randomBytes(8).toString("hex")}`, + }, + }); + + const describe = Effect.gen(function* () { + const executed = yield* client.executions.execute({ + payload: { code: describeCode(String(slug)), autoApprove: true }, + }); + expect(executed.status, executed.text).toBe("completed"); + return JSON.parse(executed.text) as DescribeOutcome; + }); + + // Cold: no declared response schema — the model sees unknown. + const cold = yield* describe; + expect(cold.error, "the tool resolves").toBeNull(); + expect(cold.outputTypeScript, "cold describe has no shape").toContain("data: unknown;"); + expect(cold.note, "cold describe carries no provenance note").toBeNull(); + + // One real call against the live upstream teaches the shape. + const invoked = yield* client.executions.execute({ + payload: { + code: ` +const result = await tools.${slug}.org.main.issues.listIssues({}); +return { ok: result.ok }; +`, + autoApprove: true, + }, + }); + expect(invoked.status, invoked.text).toBe("completed"); + expect(JSON.parse(invoked.text), "the teaching call succeeded").toEqual({ ok: true }); + + // Warm: the observed shape is served, marked as observed. + const warm = yield* describe; + expect(warm.outputTypeScript, "warm describe serves the observed shape").toContain( + "issues", + ); + expect(warm.outputTypeScript, "field types come from the live payload").toContain( + "total", + ); + expect(warm.outputTypeScript, "the shape no longer collapses").not.toContain( + "data: unknown;", + ); + expect(warm.note, "provenance is explicit").toContain("observed from 1 live response"); + }), + Effect.gen(function* () { + yield* client.connections + .remove({ + params: { + owner: "org", + integration: slug, + name: ConnectionName.make("main"), + }, + }) + .pipe(Effect.ignore); + yield* client.openapi.removeSpec({ params: { slug } }).pipe(Effect.ignore); + }), + ); + }), + ), +); diff --git a/packages/core/execution/src/skills.ts b/packages/core/execution/src/skills.ts index f503b26983..deeb31cb3b 100644 --- a/packages/core/execution/src/skills.ts +++ b/packages/core/execution/src/skills.ts @@ -58,6 +58,7 @@ const EXECUTE_SKILL_BODY = [ "- The `tools` object is a lazy proxy — enumerating it (`Object.keys(tools)`, spread, `for...in`) throws. Use `tools.search()` or `tools.executor.coreTools.connections.list({})` instead.", '- Pass an object to system tools, e.g. `tools.search({ query: "..." })`, `tools.executor.coreTools.connections.list({})`, and `tools.describe.tool({ path })`.', '- `tools.describe.tool()` returns compact TypeScript shapes. Use `inputTypeScript`, `outputTypeScript`, and `typeScriptDefinitions`. If the path doesn\'t resolve, the result carries `error: { code: "tool_not_found", suggestions }` — use a suggestion instead of retrying the same path.', + "- When `outputTypeScriptNote` is present, the `data` type was observed from live responses rather than declared by the provider: the listed fields are reliable, but the shape may be incomplete — prefer optional access for anything not listed.", "- For tools that return large collections (e.g. `getStates`, `getAll`), filter results in code rather than calling per-item tools.", "- Do not use `fetch` — all API calls go through `tools.*`.", "- If execution pauses for interaction, resume it with the returned `resumePayload`.", diff --git a/packages/core/execution/src/tool-invoker.test.ts b/packages/core/execution/src/tool-invoker.test.ts index 4d7d6681aa..536fa31f69 100644 --- a/packages/core/execution/src/tool-invoker.test.ts +++ b/packages/core/execution/src/tool-invoker.test.ts @@ -877,6 +877,33 @@ describe("tool discovery", () => { }), ); + it.effect("serves an observed shape with a provenance note once a schemaless tool runs", () => + Effect.gen(function* () { + const executor = yield* makeSearchExecutor(); + const invoker = makeExecutorToolInvoker(executor, { + invokeOptions: { onElicitation: acceptAll }, + }); + + // Cold: no declared output schema — data renders as unknown, no note. + const cold = yield* describeTool(executor, "github.org.main.listRepositoryIssues"); + expect(cold.outputTypeScript).toContain("data: unknown;"); + expect(cold.outputTypeScriptNote).toBeUndefined(); + + yield* invoker.invoke({ + path: "github.org.main.listRepositoryIssues", + args: { owner: "executor", repo: "executor" }, + }); + + // Warm: the live `[]` payload becomes the served type, marked observed + // both inline and via the note. + const warm = yield* describeTool(executor, "github.org.main.listRepositoryIssues"); + expect(warm.outputTypeScript).toBe( + "{ ok: true; data: unknown[] /* observed; may be incomplete */; http?: ToolHttpMeta } | { ok: false; error: ToolError }", + ); + expect(warm.outputTypeScriptNote).toContain("observed from 1 live response"); + }), + ); + it.effect("describes a return type that accepts the sandbox invocation result", () => Effect.gen(function* () { const executor = yield* makeSearchExecutor(); diff --git a/packages/core/execution/src/tool-invoker.ts b/packages/core/execution/src/tool-invoker.ts index 4da9251767..2df47644ed 100644 --- a/packages/core/execution/src/tool-invoker.ts +++ b/packages/core/execution/src/tool-invoker.ts @@ -31,8 +31,12 @@ const TOOL_HTTP_META_TYPESCRIPT = "{ status: number; headers: { [k: string]: str const TOOL_FILE_TYPESCRIPT = '{ _tag: "ToolFile"; name?: string; mimeType: string; encoding: "base64"; data: string; byteLength: number; }'; -const wrapOutputTypeScript = (outputTypeScript?: string): string => - `{ ok: true; data: ${outputTypeScript ?? "unknown"}; http?: ToolHttpMeta } | { ok: false; error: ToolError }`; +const wrapOutputTypeScript = (outputTypeScript?: string, marker?: string): string => + `{ ok: true; data: ${outputTypeScript ?? "unknown"}${marker ?? ""}; http?: ToolHttpMeta } | { ok: false; error: ToolError }`; + +/** Inline provenance for observed types — a model that copies only the type + * string still sees the hint, since the compact render drops descriptions. */ +const OBSERVED_TYPE_MARKER = " /* observed; may be incomplete */"; const withToolResultDefinitions = ( definitions?: Record, @@ -76,6 +80,7 @@ type DescribedTool = { readonly description?: string; readonly inputTypeScript?: string; readonly outputTypeScript?: string; + readonly outputTypeScriptNote?: string; readonly typeScriptDefinitions?: Record; /** Set when the path resolves to no tool — mirrors invoke's tool_not_found. */ readonly error?: { @@ -865,7 +870,18 @@ export const describeTool = Effect.fn("executor.tools.describe")(function* ( name: schema.name ?? path, description: schema.description, inputTypeScript: schema.inputTypeScript, - outputTypeScript: wrapOutputTypeScript(schema.outputTypeScript), + outputTypeScript: wrapOutputTypeScript( + schema.outputTypeScript, + schema.outputSchemaSource === "observed" ? OBSERVED_TYPE_MARKER : undefined, + ), + // The compact TS render drops the schema's provenance description, so an + // observed (runtime-inferred) shape gets an explicit note: the model + // should treat the fields as reliable but not exhaustive. + ...(schema.outputSchemaSource === "observed" + ? { + outputTypeScriptNote: `data type observed from ${schema.outputSchemaObservations ?? 1} live response(s), not declared by the provider; fields may be incomplete.`, + } + : {}), typeScriptDefinitions: withToolResultDefinitions(schema.typeScriptDefinitions), }; return described; diff --git a/packages/core/sdk/src/executor.test.ts b/packages/core/sdk/src/executor.test.ts index 00b878fd64..19319061a5 100644 --- a/packages/core/sdk/src/executor.test.ts +++ b/packages/core/sdk/src/executor.test.ts @@ -673,3 +673,60 @@ describe("createExecutor", () => { }), ); }); + +describe("muscle memory (observed output shapes)", () => { + const provisioned = Effect.fn(function* () { + const executor = yield* makeTestExecutor({ + plugins: [demoPlugin] as const, + coreTools: { webBaseUrl: "http://localhost:3000" }, + }); + yield* executor.demo.seed(); + yield* executor.execute(ToolAddress.make("executor.coreTools.connections.create"), { + owner: "org", + name: String(CONN), + integration: String(INTEG), + template: String(TEMPLATE), + identityLabel: "Demo", + from: { provider: "memory", id: "secret-token" }, + }); + return executor; + }); + + it.effect("serves an observed output shape once a schemaless tool has run", () => + Effect.gen(function* () { + const executor = yield* provisioned(); + + // Cold: `run` declares no output schema, nothing observed yet. + const cold = yield* executor.tools.schema(addr("run")); + expect(cold?.outputSchema).toBeUndefined(); + expect(cold?.outputTypeScript).toBeUndefined(); + + yield* executor.execute(addr("run"), {}); + + // Warm: the live payload `{ ran: "run" }` becomes the served shape, + // with provenance marked on the schema. + const warm = yield* executor.tools.schema(addr("run")); + expect(warm?.outputSchema).toMatchObject({ + type: "object", + properties: { ran: { type: "string" } }, + required: ["ran"], + description: "Observed from 1 live response; fields may be incomplete.", + }); + expect(warm?.outputTypeScript).toContain("ran"); + expect(warm?.outputTypeScript).not.toBe("unknown"); + }), + ); + + it.effect("never overrides a declared output schema with observations", () => + Effect.gen(function* () { + const executor = yield* provisioned(); + + // `inspect` declares `outputSchema: { $ref: "#/$defs/Owner" }`; running + // it observes `{ ran: "inspect" }`, which must not displace the + // declared schema. + yield* executor.execute(addr("inspect"), { pet: { lives: 9 } }); + const schema = yield* executor.tools.schema(addr("inspect")); + expect(schema?.outputSchema).toEqual({ $ref: "#/$defs/Owner" }); + }), + ); +}); diff --git a/packages/core/sdk/src/executor.ts b/packages/core/sdk/src/executor.ts index f1b9443477..2ce9a39227 100644 --- a/packages/core/sdk/src/executor.ts +++ b/packages/core/sdk/src/executor.ts @@ -176,7 +176,8 @@ import { type EnterpriseManagedRollout, } from "./oauth-ema"; import { connectionIdentifier } from "./connection-name-identifier"; -import { annotateToolResultOutcome } from "./tool-result"; +import { annotateToolResultOutcome, isToolResult } from "./tool-result"; +import { makeShapeMemory, observedShapeToJsonSchema, SHAPE_MEMORY_PLUGIN_ID } from "./shape-memory"; import { isUnauthorizedToolFailure } from "./auth-tool-failure"; const PLUGIN_STORAGE_DELETE_KEY_BATCH_SIZE = 90; @@ -1664,6 +1665,13 @@ export const createExecutor = (effect: Effect.Effect) => fuma.transaction(effect); + // Runtime-observed output shapes ("muscle memory"): learned on the + // execute success path, served by tools.schema when a tool declares no + // output schema. Backed by plugin_storage under a reserved system id. + const shapeMemory = makeShapeMemory( + makePluginStorageFacade({ core, pluginId: SHAPE_MEMORY_PLUGIN_ID, owner: ownerBinding }), + ); + // Populated once, never mutated after startup. const staticTools = new Map(); const runtimes = new Map(); @@ -4058,6 +4066,21 @@ export const createExecutor = b.and( @@ -4069,12 +4092,12 @@ export const createExecutor = (); for (const def of definitionRows) defs.set(def.name, decodeJsonColumn(def.schema)); - const referenced = collectReferencedDefinitions([inputSchema, outputSchema], defs); + const referenced = collectReferencedDefinitions([inputSchema, effectiveOutputSchema], defs); const preview = yield* Effect.tryPromise({ try: () => buildToolTypeScriptPreview({ inputSchema, - outputSchema, + outputSchema: effectiveOutputSchema, defs, }), catch: (cause) => @@ -4087,7 +4110,13 @@ export const createExecutor = 0 ? (referenced as Record) @@ -4762,6 +4791,20 @@ export const createExecutor = { + if (staticTools.has(String(address))) return Effect.void; + const parsed = parseToolAddress(String(address)); + if (!parsed) return Effect.void; + const data = isToolResult(result) ? (result.ok ? result.data : undefined) : result; + if (data === undefined) return Effect.void; + return shapeMemory.observe(String(address), parsed.owner, data); + }), Effect.withSpan("executor.tool.execute", { attributes: { "mcp.tool.name": String(address), diff --git a/packages/core/sdk/src/shape-inference.test.ts b/packages/core/sdk/src/shape-inference.test.ts new file mode 100644 index 0000000000..784005e082 --- /dev/null +++ b/packages/core/sdk/src/shape-inference.test.ts @@ -0,0 +1,136 @@ +import { describe, expect, it } from "@effect/vitest"; + +import { inferShape, mergeShapes, observeShape, type InferredShape } from "./shape-inference"; + +describe("inferShape", () => { + it("infers primitives without recording values", () => { + expect(inferShape("secret token")).toEqual({ type: "string" }); + expect(inferShape(42)).toEqual({ type: "number" }); + expect(inferShape(true)).toEqual({ type: "boolean" }); + expect(inferShape(null)).toEqual({ type: "null" }); + expect(inferShape(undefined)).toEqual({ type: "null" }); + }); + + it("infers object structure with all keys required", () => { + expect(inferShape({ id: "abc", count: 3 })).toEqual({ + type: "object", + properties: { id: { type: "string" }, count: { type: "number" } }, + required: ["count", "id"], + }); + }); + + it("merges sampled array elements into one item shape", () => { + expect(inferShape([{ id: 1 }, { id: 2, label: "x" }])).toEqual({ + type: "array", + items: { + type: "object", + properties: { id: { type: "number" }, label: { type: "string" } }, + required: ["id"], + }, + }); + }); + + it("keeps an empty array itemless", () => { + expect(inferShape([])).toEqual({ type: "array" }); + }); + + it("collapses wide objects to a map so data-bearing keys never persist", () => { + const byEmail = Object.fromEntries( + Array.from({ length: 40 }, (_, i) => [`user${i}@example.com`, { active: true }]), + ); + const shape = inferShape(byEmail); + expect(shape.properties).toBeUndefined(); + expect(shape.additionalProperties).toEqual({ + type: "object", + properties: { active: { type: "boolean" } }, + required: ["active"], + }); + }); + + it("degrades to unknown past the depth bound", () => { + let value: unknown = "leaf"; + for (let i = 0; i < 10; i++) value = { child: value }; + const json = JSON.stringify(inferShape(value)); + expect(json).toContain("{}"); + }); +}); + +describe("mergeShapes", () => { + it("makes fields missing from one observation optional", () => { + const merged = mergeShapes(inferShape({ id: "a", email: "x@y.z" }), inferShape({ id: "b" })); + expect(merged).toEqual({ + type: "object", + properties: { id: { type: "string" }, email: { type: "string" } }, + required: ["id"], + }); + }); + + it("unions differing primitive types", () => { + expect(mergeShapes({ type: "string" }, { type: "number" })).toEqual({ + anyOf: [{ type: "string" }, { type: "number" }], + }); + }); + + it("merges same-typed union branches instead of duplicating them", () => { + const union = mergeShapes({ type: "string" }, { type: "null" }); + const widened = mergeShapes(union, inferShape({ id: 1 })); + const again = mergeShapes(widened, inferShape({ id: 2, extra: true })); + expect(again.anyOf).toHaveLength(3); + const objectBranch = again.anyOf?.find((branch) => branch.type === "object"); + expect(objectBranch?.required).toEqual(["id"]); + }); + + it("degrades to unknown when the union grows past its cap", () => { + const wide = [ + { type: "string" } as const, + { type: "number" } as const, + { type: "boolean" } as const, + { type: "null" } as const, + { type: "array" } as const, + ].reduce((left, right) => mergeShapes(left, right), { type: "string" }); + expect(wide).toEqual({}); + }); + + it("merges array item shapes across calls", () => { + const merged = mergeShapes(inferShape([{ id: 1 }]), inferShape([{ id: 2, done: false }])); + expect(merged).toEqual({ + type: "array", + items: { + type: "object", + properties: { id: { type: "number" }, done: { type: "boolean" } }, + required: ["id"], + }, + }); + }); +}); + +describe("observeShape", () => { + it("starts a record from the first observation", () => { + const record = observeShape(null, { ok: true }, 1000); + expect(record.observations).toBe(1); + expect(record.updatedAt).toBe(1000); + expect(record.schema.type).toBe("object"); + }); + + it("accumulates observations by merging", () => { + const first = observeShape(null, { id: "a", email: "x@y.z" }, 1000); + const second = observeShape(first, { id: "b" }, 2000); + expect(second.observations).toBe(2); + expect(second.updatedAt).toBe(2000); + expect(second.schema.required).toEqual(["id"]); + expect(Object.keys(second.schema.properties ?? {}).sort()).toEqual(["email", "id"]); + }); + + it("stays under the serialized size bound for adversarial payloads", () => { + const wide = Object.fromEntries( + Array.from({ length: 24 }, (_, i) => [ + `field_with_a_rather_long_name_${i}`, + Object.fromEntries( + Array.from({ length: 24 }, (_, j) => [`nested_property_name_${j}`, { deep: { x: 1 } }]), + ), + ]), + ); + const record = observeShape(null, wide, 1000); + expect(JSON.stringify(record.schema).length).toBeLessThanOrEqual(16_000); + }); +}); diff --git a/packages/core/sdk/src/shape-inference.ts b/packages/core/sdk/src/shape-inference.ts new file mode 100644 index 0000000000..3cd0e5d33d --- /dev/null +++ b/packages/core/sdk/src/shape-inference.ts @@ -0,0 +1,219 @@ +/** + * Runtime output-shape inference — the "muscle memory" half of code mode. + * + * Most tools declare no output schema, so `tools.describe.tool()` renders + * `data: unknown` and the model guesses response shapes. This module infers a + * lightweight shape from real tool results at dispatch time: field names and + * broad types only, never values. Shapes merge across calls, so unions and + * optional fields converge instead of thrashing. + * + * The output is a small JSON Schema subset (`type`, `properties`, `required`, + * `items`, `additionalProperties`, `anyOf`) so it can be stored as plain JSON + * and rendered through the same schema → TypeScript compiler that declared + * schemas use. + * + * Every dimension is bounded — depth, object width, array sampling, union + * width, serialized size — because inference runs on the hot dispatch path + * against arbitrary upstream payloads. + */ + +export type InferredShape = { + readonly type?: "null" | "boolean" | "number" | "string" | "object" | "array"; + readonly properties?: Readonly>; + readonly required?: readonly string[]; + readonly items?: InferredShape; + readonly additionalProperties?: InferredShape; + readonly anyOf?: readonly InferredShape[]; +}; + +/** `{}` — matches anything; the "we know nothing beyond this point" shape. */ +const UNKNOWN: InferredShape = {}; + +const MAX_DEPTH = 6; +/** Array elements sampled per call; later calls keep widening the merge. */ +const MAX_ARRAY_SAMPLE = 5; +/** + * An object with more own keys than this is treated as a map keyed by data + * (ids, emails, dates) rather than a struct. Collapsing to + * `additionalProperties` keeps data-bearing keys out of the shape — field + * names of a struct are API surface, but map keys are values. + */ +const MAX_OBJECT_KEYS = 24; +/** Union width cap; beyond this the shape degrades to unknown. */ +const MAX_ANYOF = 4; + +const isUnknown = (shape: InferredShape): boolean => + shape.type === undefined && shape.anyOf === undefined; + +/** Infer the shape of one observed value. Reads structure only, never values. */ +export const inferShape = (value: unknown, depth = 0): InferredShape => { + if (value === null || value === undefined) return { type: "null" }; + if (typeof value === "boolean") return { type: "boolean" }; + if (typeof value === "number") return { type: "number" }; + if (typeof value === "string") return { type: "string" }; + if (depth >= MAX_DEPTH) return UNKNOWN; + + if (Array.isArray(value)) { + if (value.length === 0) return { type: "array" }; + const sampled = value + .slice(0, MAX_ARRAY_SAMPLE) + .map((item) => inferShape(item, depth + 1)) + .reduce((left, right) => mergeShapes(left, right)); + return { type: "array", items: sampled }; + } + + if (typeof value === "object") { + const entries = Object.entries(value as Record); + if (entries.length > MAX_OBJECT_KEYS) { + const merged = entries + .slice(0, MAX_ARRAY_SAMPLE) + .map(([, item]) => inferShape(item, depth + 1)) + .reduce((left, right) => mergeShapes(left, right)); + return { type: "object", additionalProperties: merged }; + } + const properties: Record = {}; + for (const [key, item] of entries) { + properties[key] = inferShape(item, depth + 1); + } + return { type: "object", properties, required: entries.map(([key]) => key).sort() }; + } + + // function / symbol / bigint — nothing useful to say structurally. + return UNKNOWN; +}; + +const mergeObjectShapes = (left: InferredShape, right: InferredShape): InferredShape => { + // A map-shaped observation absorbs struct-shaped ones: once keys look like + // data, later struct keys are data too. + if (left.additionalProperties !== undefined || right.additionalProperties !== undefined) { + const values = [ + left.additionalProperties, + right.additionalProperties, + ...Object.values(left.properties ?? {}), + ...Object.values(right.properties ?? {}), + ].filter((shape): shape is InferredShape => shape !== undefined); + return { + type: "object", + additionalProperties: + values.length === 0 ? UNKNOWN : values.reduce((a, b) => mergeShapes(a, b)), + }; + } + + const leftProps = left.properties ?? {}; + const rightProps = right.properties ?? {}; + const keys = [...new Set([...Object.keys(leftProps), ...Object.keys(rightProps)])].sort(); + if (keys.length > MAX_OBJECT_KEYS) { + const values = keys + .slice(0, MAX_ARRAY_SAMPLE) + .map((key) => leftProps[key] ?? rightProps[key]) + .filter((shape): shape is InferredShape => shape !== undefined); + return { + type: "object", + additionalProperties: + values.length === 0 ? UNKNOWN : values.reduce((a, b) => mergeShapes(a, b)), + }; + } + + const properties: Record = {}; + for (const key of keys) { + const a = leftProps[key]; + const b = rightProps[key]; + properties[key] = a !== undefined && b !== undefined ? mergeShapes(a, b) : (a ?? b ?? UNKNOWN); + } + const leftRequired = new Set(left.required ?? []); + const required = (right.required ?? []).filter((key) => leftRequired.has(key)).sort(); + return { type: "object", properties, required }; +}; + +/** + * Merge two observed shapes into the narrowest shape matching both. + * Same-typed shapes merge structurally; differently-typed shapes union into + * `anyOf`, degrading to unknown past `MAX_ANYOF` branches. + */ +export const mergeShapes = (left: InferredShape, right: InferredShape): InferredShape => { + if (isUnknown(left) || isUnknown(right)) return UNKNOWN; + + if (left.anyOf !== undefined || right.anyOf !== undefined) { + const branches = [...(left.anyOf ?? [left]), ...(right.anyOf ?? [right])]; + return branches.reduce((merged, branch) => addUnionBranch(merged, branch), { + anyOf: [], + } as InferredShape); + } + + if (left.type !== right.type) return addUnionBranch({ anyOf: [left] }, right); + + if (left.type === "object") return mergeObjectShapes(left, right); + if (left.type === "array") { + if (left.items === undefined) return right; + if (right.items === undefined) return left; + return { type: "array", items: mergeShapes(left.items, right.items) }; + } + return left; +}; + +const addUnionBranch = (union: InferredShape, branch: InferredShape): InferredShape => { + if (isUnknown(union) || isUnknown(branch)) return UNKNOWN; + const branches = [...(union.anyOf ?? [union])]; + const index = branches.findIndex((existing) => existing.type === branch.type); + const next = + index === -1 + ? [...branches, branch] + : branches.map((existing, i) => (i === index ? mergeShapes(existing, branch) : existing)); + if (next.length > MAX_ANYOF) return UNKNOWN; + return next.length === 1 ? (next[0] ?? UNKNOWN) : { anyOf: next }; +}; + +/** + * One tool's accumulated muscle memory: the merged shape plus enough + * bookkeeping to judge freshness. Stored as plain JSON. + */ +export type ObservedShape = { + readonly schema: InferredShape; + readonly observations: number; + readonly updatedAt: number; +}; + +/** Serialized-size ceiling per tool; a shape past this degrades to unknown + * children rather than growing without bound. */ +const MAX_SHAPE_JSON_CHARS = 16_000; + +const shrink = (shape: InferredShape, depth: number): InferredShape => { + if (depth <= 0) return UNKNOWN; + if (shape.anyOf) return { anyOf: shape.anyOf.map((branch) => shrink(branch, depth - 1)) }; + if (shape.type === "array" && shape.items) { + return { type: "array", items: shrink(shape.items, depth - 1) }; + } + if (shape.type === "object" && shape.additionalProperties) { + return { type: "object", additionalProperties: shrink(shape.additionalProperties, depth - 1) }; + } + if (shape.type === "object" && shape.properties) { + const properties: Record = {}; + for (const [key, child] of Object.entries(shape.properties)) { + properties[key] = shrink(child, depth - 1); + } + return { ...shape, properties }; + } + return shape; +}; + +/** Fold a new observation into an existing record, keeping the result bounded. */ +export const observeShape = ( + previous: ObservedShape | null, + value: unknown, + now: number, +): ObservedShape => { + const observed = inferShape(value); + let schema = previous === null ? observed : mergeShapes(previous.schema, observed); + for ( + let depth = MAX_DEPTH; + depth > 0 && JSON.stringify(schema).length > MAX_SHAPE_JSON_CHARS; + depth-- + ) { + schema = shrink(schema, depth); + } + return { + schema, + observations: (previous?.observations ?? 0) + 1, + updatedAt: now, + }; +}; diff --git a/packages/core/sdk/src/shape-memory.ts b/packages/core/sdk/src/shape-memory.ts new file mode 100644 index 0000000000..d5ab9f1d22 --- /dev/null +++ b/packages/core/sdk/src/shape-memory.ts @@ -0,0 +1,90 @@ +/** + * Muscle memory for tool outputs — the persistence half of runtime shape + * inference (see shape-inference.ts for the algorithm). + * + * Observed shapes live in the already-migrated `plugin_storage` table under a + * reserved system plugin id: owner-scoped, tenant-partitioned, and untouched + * by tool-catalog refresh (which deletes and recreates `tool` rows, so the + * tool row itself is not a viable home). An in-memory read-through cache + * keeps the hot path off the database: within one executor instance a tool's + * shape is loaded at most once, and a write happens only when a new + * observation actually changes the merged shape — after a few calls a stable + * API stops producing writes entirely. + * + * `observe` never fails and is intended to be forked off the dispatch path; + * `recall` degrades to "no memory" on any storage failure. + */ + +import { Clock, Effect } from "effect"; + +import type { Owner } from "./ids"; +import type { PluginStorageFacade } from "./plugin-storage"; +import { observeShape, type ObservedShape } from "./shape-inference"; + +/** Reserved system namespace inside `plugin_storage`; not a real plugin. */ +export const SHAPE_MEMORY_PLUGIN_ID = "executor.shape-memory"; +const COLLECTION = "observed-output-shapes"; + +export type ShapeMemory = { + /** + * Fold one successful tool payload into the tool's remembered shape. + * Structure only — values never leave this call. Never fails. + */ + readonly observe: (address: string, owner: Owner, value: unknown) => Effect.Effect; + /** The remembered shape for an address, or null when nothing is known. */ + readonly recall: (address: string, owner: Owner) => Effect.Effect; +}; + +export const makeShapeMemory = (storage: PluginStorageFacade): ShapeMemory => { + const cache = new Map(); + const persisted = new Map(); + + const cacheKey = (owner: Owner, address: string) => `${owner}:${address}`; + + const load = (address: string, owner: Owner): Effect.Effect => + Effect.gen(function* () { + const key = cacheKey(owner, address); + const hit = cache.get(key); + if (hit !== undefined) return hit; + const entry = yield* storage + .getForOwner({ owner, collection: COLLECTION, key: address }) + .pipe(Effect.catch(() => Effect.succeed(null))); + const record = entry?.data ?? null; + cache.set(key, record); + if (record !== null) persisted.set(key, JSON.stringify(record.schema)); + return record; + }); + + const observe = (address: string, owner: Owner, value: unknown): Effect.Effect => + Effect.gen(function* () { + const key = cacheKey(owner, address); + const prior = yield* load(address, owner); + const now = yield* Clock.currentTimeMillis; + const next = observeShape(prior, value, now); + cache.set(key, next); + // Write only when the merged shape actually changed — observation + // counters alone are bookkeeping, not worth a row write per call. + const schemaJson = JSON.stringify(next.schema); + if (persisted.get(key) === schemaJson) return; + yield* storage + .put({ owner, collection: COLLECTION, key: address, data: next }) + .pipe(Effect.catch(() => Effect.succeed(null))); + persisted.set(key, schemaJson); + }).pipe(Effect.catchCause(() => Effect.void)); + + return { + observe, + recall: (address, owner) => + load(address, owner).pipe(Effect.catchCause(() => Effect.succeed(null))), + }; +}; + +/** + * Render a remembered shape as the JSON Schema served in place of a missing + * declared output schema. The description marks provenance so a reader (and + * the schema view) can tell an observed shape from an author-declared one. + */ +export const observedShapeToJsonSchema = (record: ObservedShape): unknown => ({ + ...record.schema, + description: `Observed from ${record.observations} live response${record.observations === 1 ? "" : "s"}; fields may be incomplete.`, +}); diff --git a/packages/core/sdk/src/types.ts b/packages/core/sdk/src/types.ts index 852d409dbd..ad5d2bb0d4 100644 --- a/packages/core/sdk/src/types.ts +++ b/packages/core/sdk/src/types.ts @@ -21,6 +21,10 @@ export const ToolSchemaView = Schema.Struct({ description: Schema.optional(Schema.String), inputSchema: Schema.optional(Schema.Unknown), outputSchema: Schema.optional(Schema.Unknown), + // "observed" = runtime-inferred from live responses (muscle memory), served + // because the plugin declared no output schema. Absent = declared as-is. + outputSchemaSource: Schema.optional(Schema.Literals(["observed"])), + outputSchemaObservations: Schema.optional(Schema.Number), schemaDefinitions: Schema.optional(Schema.Record(Schema.String, Schema.Unknown)), inputTypeScript: Schema.optional(Schema.String), outputTypeScript: Schema.optional(Schema.String),