From 01dc5ab643aa64b450d9b08a8c85cccf2e4c5c67 Mon Sep 17 00:00:00 2001 From: Rhys Sullivan <39114868+RhysSullivan@users.noreply.github.com> Date: Tue, 25 Aug 2026 21:40:48 -0700 Subject: [PATCH 1/3] Guard whole-document OpenAPI parses by parsed-tree size --- .../plugins/openapi/src/sdk/parse.test.ts | 86 ++++++++++++++++++- packages/plugins/openapi/src/sdk/parse.ts | 77 +++++++++++++++++ 2 files changed, 161 insertions(+), 2 deletions(-) diff --git a/packages/plugins/openapi/src/sdk/parse.test.ts b/packages/plugins/openapi/src/sdk/parse.test.ts index 9990c1ce5f..039f51534a 100644 --- a/packages/plugins/openapi/src/sdk/parse.test.ts +++ b/packages/plugins/openapi/src/sdk/parse.test.ts @@ -1,8 +1,15 @@ import { describe, expect, it } from "@effect/vitest"; -import { Effect } from "effect"; +import { Effect, Layer } from "effect"; +import { HttpClient, HttpClientRequest, HttpClientResponse } from "effect/unstable/http"; import { OpenApiParseError } from "./errors"; -import { parse } from "./parse"; +import { + MAX_JSON_SPEC_CHARS, + MAX_SPEC_TEXT_CHARS, + MAX_YAML_SPEC_LINES, + fetchSpecText, + parse, +} from "./parse"; describe("OpenAPI parse", () => { it.effect("parses JSON OpenAPI documents", () => @@ -82,4 +89,79 @@ paths: {} expect(doc.info.description).toHaveLength(largeDescription.length); }), ); + + it.effect("rejects structure-dense YAML documents above the line cap", () => + Effect.gen(function* () { + // Three lines per path-item; well past the cap while only a few MB of + // text — the shape (not the size) is what a whole parse cannot survive. + const pathItems = " /a:\n get: {}\n".repeat(Math.ceil(MAX_YAML_SPEC_LINES / 2)); + const error = yield* parse( + `openapi: 3.0.0\ninfo:\n title: Dense\n version: 1.0.0\npaths:\n${pathItems}`, + ).pipe(Effect.flip); + + expect(error).toBeInstanceOf(OpenApiParseError); + expect(error).toHaveProperty("message", expect.stringMatching(/too large to parse whole/)); + expect(error).toHaveProperty("message", expect.stringMatching(/lines/)); + }), + ); + + it.effect("rejects JSON documents above the JSON size cap", () => + Effect.gen(function* () { + const padded = `{"openapi":"3.1.0","x-pad":"${"x".repeat(MAX_JSON_SPEC_CHARS)}"}`; + const error = yield* parse(padded).pipe(Effect.flip); + + expect(error).toBeInstanceOf(OpenApiParseError); + expect(error).toHaveProperty("message", expect.stringMatching(/too large to parse/)); + }), + ); + + it.effect("rejects any document above the text ceiling", () => + Effect.gen(function* () { + const padded = `openapi: 3.0.0\ninfo:\n description: "${"x".repeat(MAX_SPEC_TEXT_CHARS)}"\n`; + const error = yield* parse(padded).pipe(Effect.flip); + + expect(error).toBeInstanceOf(OpenApiParseError); + expect(error).toHaveProperty("message", expect.stringMatching(/too large to parse/)); + }), + ); +}); + +describe("OpenAPI fetchSpecText", () => { + const specUrl = "https://example.com/openapi.yaml"; + + const layerWithResponse = (response: Response) => + Layer.succeed(HttpClient.HttpClient)( + HttpClient.make((request: HttpClientRequest.HttpClientRequest) => + Effect.succeed(HttpClientResponse.fromWeb(request, response)), + ), + ); + + it.effect("rejects a document whose declared length is above the text ceiling", () => + Effect.gen(function* () { + const error = yield* fetchSpecText(specUrl).pipe( + Effect.provide( + layerWithResponse( + new Response("openapi: 3.0.0", { + status: 200, + headers: { "content-length": String(MAX_SPEC_TEXT_CHARS + 1) }, + }), + ), + ), + Effect.flip, + ); + + expect(error).toBeInstanceOf(OpenApiParseError); + expect(error).toHaveProperty("message", expect.stringMatching(/too large to parse/)); + }), + ); + + it.effect("fetches a document with an in-range declared length", () => + Effect.gen(function* () { + const specText = yield* fetchSpecText(specUrl).pipe( + Effect.provide(layerWithResponse(new Response("openapi: 3.0.0", { status: 200 }))), + ); + + expect(specText).toBe("openapi: 3.0.0"); + }), + ); }); diff --git a/packages/plugins/openapi/src/sdk/parse.ts b/packages/plugins/openapi/src/sdk/parse.ts index 77b788dae9..8f2557dcb0 100644 --- a/packages/plugins/openapi/src/sdk/parse.ts +++ b/packages/plugins/openapi/src/sdk/parse.ts @@ -7,6 +7,53 @@ import { OpenApiExtractionError, OpenApiParseError } from "./errors"; export type ParsedDocument = OpenAPIV3.Document | OpenAPIV3_1.Document; +const MiB = 1024 * 1024; + +/** + * Whole-document parse guards. What kills a 128MB Cloudflare Workers isolate is + * the parsed TREE, not the text: the 43MB / ~1.6M-line Microsoft Graph YAML + * builds a ~300MB tree and dies mid-request with an empty 503 (measured + * 2026-08), while a same-order text whose bulk is one flat scalar parses fine + * (see "parses Graph-sized YAML" in parse.test.ts). So the guards measure tree + * size by proxy, per input shape, and turn the isolate death into an + * actionable error: + * + * - Any text above `MAX_SPEC_TEXT_CHARS` is rejected outright — the string + * plus any parse output cannot fit regardless of shape. + * - Block YAML builds roughly one node per line (~190 bytes of tree per line + * measured on Graph), so YAML is capped by newline count. + * - JSON (and flow-style YAML, same sniff) concentrates structure without + * newlines, so it is capped by text size; the 16MB Cloudflare JSON spec is + * known-good and must stay under the cap. + * + * Provider adapters that stream via `structuralSplit` (Microsoft Graph) never + * enter this path and are not capped. + */ +export const MAX_SPEC_TEXT_CHARS = 48 * MiB; +export const MAX_JSON_SPEC_CHARS = 32 * MiB; +export const MAX_YAML_SPEC_LINES = 400_000; + +const formatMiB = (chars: number): string => `${Math.ceil((chars / MiB) * 10) / 10}MB`; + +const specGuidance = + "Filter the spec to the operations you need before adding it, or use a curated " + + "provider preset that selects a workload server-side."; + +const specTooLargeMessage = (size: number, limit: number): string => + `OpenAPI document is too large to parse (${formatMiB(size)}, limit ${formatMiB(limit)}). ` + + specGuidance; + +const specTooDenseMessage = (lines: number): string => + `OpenAPI document is too large to parse whole (${lines.toLocaleString("en-US")} lines, ` + + `limit ${MAX_YAML_SPEC_LINES.toLocaleString("en-US")}). ` + + specGuidance; + +const countLines = (text: string): number => { + let count = 1; + for (let pos = text.indexOf("\n"); pos !== -1; pos = text.indexOf("\n", pos + 1)) count += 1; + return count; +}; + export interface SpecFetchCredentials { readonly headers?: Record; readonly queryParams?: Record; @@ -50,6 +97,16 @@ export const fetchSpecText = Effect.fn("OpenApi.fetchSpecText")(function* ( message: `Failed to fetch OpenAPI document: HTTP ${response.status}`, }); } + // Reject documents the whole-parse path can never handle before downloading + // them. The declared byte length bounds the decoded text from above only for + // the coarse any-shape cap, so this never rejects a spec `parseSpecObject` + // would have accepted; the precise per-shape check still runs there. + const declaredLength = Number(response.headers["content-length"]); + if (Number.isFinite(declaredLength) && declaredLength > MAX_SPEC_TEXT_CHARS) { + return yield* new OpenApiParseError({ + message: specTooLargeMessage(declaredLength, MAX_SPEC_TEXT_CHARS), + }); + } const specText = yield* response.text.pipe( Effect.mapError( (_cause) => @@ -108,6 +165,26 @@ export const parseSpecObject = (text: string): Effect.Effect MAX_SPEC_TEXT_CHARS) { + return yield* new OpenApiParseError({ + message: specTooLargeMessage(trimmed.length, MAX_SPEC_TEXT_CHARS), + }); + } + if (trimmed.startsWith("{") || trimmed.startsWith("[")) { + if (trimmed.length > MAX_JSON_SPEC_CHARS) { + return yield* new OpenApiParseError({ + message: specTooLargeMessage(trimmed.length, MAX_JSON_SPEC_CHARS), + }); + } + } else { + const lines = countLines(trimmed); + if (lines > MAX_YAML_SPEC_LINES) { + return yield* new OpenApiParseError({ + message: specTooDenseMessage(lines), + }); + } + } + const parsed = yield* parseJsonLike(trimmed).pipe( Effect.mapError( () => From 0f56998db114f1b46a06ca7eaa722b1745e58fbe Mon Sep 17 00:00:00 2001 From: Rhys Sullivan <39114868+RhysSullivan@users.noreply.github.com> Date: Tue, 25 Aug 2026 21:40:49 -0700 Subject: [PATCH 2/3] Stream the OpenAPI preview for spec-format selections --- .changeset/openapi-streaming-preview.md | 5 + .../microsoft/spec-format-adapter.test.ts | 20 ++ packages/plugins/openapi/src/sdk/extract.ts | 187 +++++++++++++++++- packages/plugins/openapi/src/sdk/plugin.ts | 34 +++- .../openapi/src/sdk/preview-streaming.test.ts | 183 +++++++++++++++++ packages/plugins/openapi/src/sdk/preview.ts | 148 +++++++++++++- 6 files changed, 557 insertions(+), 20 deletions(-) create mode 100644 .changeset/openapi-streaming-preview.md create mode 100644 packages/plugins/openapi/src/sdk/preview-streaming.test.ts diff --git a/.changeset/openapi-streaming-preview.md b/.changeset/openapi-streaming-preview.md new file mode 100644 index 0000000000..fb7d142fb6 --- /dev/null +++ b/.changeset/openapi-streaming-preview.md @@ -0,0 +1,5 @@ +--- +"@executor-js/plugin-openapi": patch +--- + +Preview OpenAPI spec-format selections (Microsoft Graph) through the streaming structural-split path instead of a whole-document parse, and guard generic whole-document parses by parsed-tree size (line count for block YAML, text size for JSON). Previewing a Graph preset URL previously parsed the 43MB source whole and killed the 128MB Workers isolate mid-request, surfacing as an empty 503; it now streams within budget, and oversized generic specs fail with an actionable error instead of taking down the isolate. diff --git a/packages/plugins/openapi/src/providers/microsoft/spec-format-adapter.test.ts b/packages/plugins/openapi/src/providers/microsoft/spec-format-adapter.test.ts index 1dbdfd6ba6..d3da64cd65 100644 --- a/packages/plugins/openapi/src/providers/microsoft/spec-format-adapter.test.ts +++ b/packages/plugins/openapi/src/providers/microsoft/spec-format-adapter.test.ts @@ -3,6 +3,7 @@ import { Effect, Layer } from "effect"; import { HttpClient, HttpClientRequest, HttpClientResponse } from "effect/unstable/http"; import { structuralSplit } from "@executor-js/plugin-openapi"; +import { previewSpecTextStreaming } from "../../sdk/preview"; import { microsoftGraphAdapter } from "./spec-format-adapter"; import { MICROSOFT_GRAPH_OPENAPI_URL } from "./presets"; @@ -90,3 +91,22 @@ it.effect("uses catalog URL fragments to select one Graph workload", () => expect(keepPathItem("/irrelevant", { get: { operationId: "irrelevant.Get" } })).toBeNull(); }), ); + +it.effect("stream-previews a Graph selection without a whole-document parse", () => + Effect.gen(function* () { + const converted = yield* microsoftGraphAdapter.fetch({ + urls: [`${MICROSOFT_GRAPH_OPENAPI_URL}#preset=profile`], + httpClientLayer: graphHttpClientLayer, + }); + const preview = yield* previewSpecTextStreaming(converted.specText, converted.keepPathItem); + + expect(preview.operationCount).toBe(1); + expect(preview.operations.map((operation) => operation.operationId)).toEqual(["me.GetUser"]); + expect(preview.healthCheckCandidates).toHaveLength(1); + expect(preview.healthCheckCandidates[0]?.method).toBe("get"); + expect(preview.oauth2Presets).toHaveLength(1); + expect(preview.servers.map((server) => server.url)).toEqual([ + "https://graph.microsoft.com/v1.0", + ]); + }), +); diff --git a/packages/plugins/openapi/src/sdk/extract.ts b/packages/plugins/openapi/src/sdk/extract.ts index 956e923e81..77a2202607 100644 --- a/packages/plugins/openapi/src/sdk/extract.ts +++ b/packages/plugins/openapi/src/sdk/extract.ts @@ -742,6 +742,23 @@ export const streamOperationBindings = ( const isPathItemValue = (value: unknown): value is Record => value !== null && typeof value === "object" && !Array.isArray(value); +/** Parse one path-item range to its kept (optionally trimmed) value. Applied + * identically wherever a structure is walked more than once, so per-operation + * indexes stay aligned across passes. */ +const parseKeptPathItem = ( + structure: SpecStructure, + range: ByteRange, + keepPathItem: KeepPathItem | undefined, +): readonly [string, PathItemObject] | null => { + const entry = parseEntry(structure.text, range, 2); + if (!entry) return null; + const [path, rawValue] = entry; + if (!isPathItemValue(rawValue)) return null; + if (!keepPathItem) return [path, rawValue as PathItemObject]; + const kept = keepPathItem(path, rawValue); + return kept ? [path, kept as PathItemObject] : null; +}; + /** * Stream invocation bindings straight from a `SpecStructure` (the structural * split of a large spec) without ever materializing the whole-document tree. @@ -775,15 +792,8 @@ export const streamOperationBindingsFromStructure = ( // Parse one path-item range to its kept (optionally trimmed) value, applying // `keepPathItem` identically in both passes so the operation index aligns. - const keptPathItem = (range: ByteRange): readonly [string, PathItemObject] | null => { - const entry = parseEntry(structure.text, range, 2); - if (!entry) return null; - const [path, rawValue] = entry; - if (!isPathItemValue(rawValue)) return null; - if (!keepPathItem) return [path, rawValue as PathItemObject]; - const kept = keepPathItem(path, rawValue); - return kept ? [path, kept as PathItemObject] : null; - }; + const keptPathItem = (range: ByteRange): readonly [string, PathItemObject] | null => + parseKeptPathItem(structure, range, keepPathItem); // Pass 1 (light): collect schema-free tool-path planning metadata in // document order. No bindings, no schemas; one path-item resident at a time. @@ -874,3 +884,162 @@ export const streamOperationBindingsFromStructure = ( return { toolCount: plans.length, toolNames: plans.map((plan) => plan.toolPath) }; }).pipe(Effect.withSpan("OpenApi.streamOperationBindingsFromStructure")); + +// --------------------------------------------------------------------------- +// Streaming preview extraction +// --------------------------------------------------------------------------- + +export interface StreamedPreviewParameter { + readonly name: string; + readonly location: ParameterLocation; + readonly required: boolean; + readonly description?: string; +} + +/** Schema-free per-operation metadata for the preview path: everything the + * add screen's operation list and health-check candidate ranking need, and + * nothing that scales with schema size. */ +export interface StreamedPreviewOperation { + readonly operationId: string; + /** Tool path planned over the full kept operation set, so preview candidates + * match the names registration will assign. */ + readonly toolPath: string; + readonly method: HttpMethod; + readonly pathTemplate: string; + readonly summary: string | undefined; + readonly description: string | undefined; + readonly tags: readonly string[]; + readonly deprecated: boolean; + readonly parameters: readonly StreamedPreviewParameter[]; + /** Position in kept-document order; key for `streamOutputSchemas`. */ + readonly operationIndex: number; +} + +export interface StreamedPreviewExtraction { + /** Parsed document head (openapi, info, servers, tags, security, ...). */ + readonly head: Record; + /** Schema-free components (parameters / requestBodies / responses / + * securitySchemes / ...) for `$ref` resolution and auth extraction. */ + readonly components: Record; + readonly servers: readonly ServerInfo[]; + readonly operations: readonly StreamedPreviewOperation[]; +} + +/** + * Streaming twin of `extract` for the preview path: walk a `SpecStructure` + * path-item by path-item (each parsed in isolation and discarded) and keep only + * schema-free per-operation metadata, so previewing a Graph-sized spec never + * materializes the whole-document tree that OOMs a 128MB Workers isolate. + * `keepPathItem` applies the same selection filter as the streaming compile, so + * the preview describes exactly the operation set registration would persist. + */ +export const streamPreviewOperations = ( + structure: SpecStructure, + keepPathItem?: KeepPathItem, +): StreamedPreviewExtraction => { + const head = parseHead(structure); + const components = parseSmallComponents(structure); + // oxlint-disable-next-line executor/no-double-cast -- boundary: same schema-free resolver doc as `streamOperationBindingsFromStructure` (head + small components, empty paths), read only for .servers and `$ref` resolution. + const resolverDoc = { ...head, paths: {}, components } as unknown as ParsedDocument; + const r = new DocResolver(resolverDoc); + const docServers = extractServers(resolverDoc); + + const inputs: OperationPathInput[] = []; + const metas: Omit[] = []; + for (const range of structure.pathItems) { + const kept = parseKeptPathItem(structure, range, keepPathItem); + if (!kept) continue; + const [path, pathItem] = kept; + for (const method of HTTP_METHODS) { + const operation = pathItem[method]; + if (!operation) continue; + const resolvedPathTemplate = explicitPathTemplate(operation) ?? path; + const tags = (operation.tags ?? []).filter((t) => t.trim().length > 0); + const operationId = deriveOperationId(method, path, operation); + inputs.push({ + operationId, + explicitToolPath: explicitToolPath(operation), + method, + pathTemplate: resolvedPathTemplate, + tag0: tags[0], + }); + const parameters = extractParameters(pathItem, operation, r).map( + (parameter): StreamedPreviewParameter => ({ + name: parameter.name, + location: parameter.location, + required: parameter.required, + ...(Option.isSome(parameter.description) + ? { description: parameter.description.value } + : {}), + }), + ); + metas.push({ + operationId, + method, + pathTemplate: resolvedPathTemplate, + summary: operation.summary, + description: operation.description, + tags, + deprecated: operation.deprecated === true, + parameters, + operationIndex: metas.length, + }); + } + } + + const plans = planToolPaths(inputs); + const toolPathByOpIndex: (string | undefined)[] = new Array(inputs.length); + for (const plan of plans) toolPathByOpIndex[plan.operationIndex] = plan.toolPath; + + return { + head, + components, + servers: docServers, + operations: metas.flatMap((meta) => { + const toolPath = toolPathByOpIndex[meta.operationIndex]; + return toolPath === undefined ? [] : [{ ...meta, toolPath }]; + }), + }; +}; + +/** + * Re-walk the structure and build the raw output schema (component `$ref`s + * intact) for just the operations in `wanted` — the bounded response-schema + * walk behind the preview's typed identity picker. Iteration order and + * `keepPathItem` application match `streamPreviewOperations`, so the indexes + * line up. + */ +export const streamOutputSchemas = ( + structure: SpecStructure, + wanted: ReadonlySet, + keepPathItem?: KeepPathItem, +): ReadonlyMap => { + const result = new Map(); + if (wanted.size === 0) return result; + // oxlint-disable-next-line executor/no-double-cast -- boundary: same schema-free resolver doc as `streamPreviewOperations`. + const resolverDoc = { + ...parseHead(structure), + paths: {}, + components: parseSmallComponents(structure), + } as unknown as ParsedDocument; + const r = new DocResolver(resolverDoc); + + let opIndex = 0; + for (const range of structure.pathItems) { + if (result.size >= wanted.size) break; + const kept = parseKeptPathItem(structure, range, keepPathItem); + if (!kept) continue; + const [, pathItem] = kept; + for (const method of HTTP_METHODS) { + const operation = pathItem[method]; + if (!operation) continue; + const index = opIndex; + opIndex += 1; + if (!wanted.has(index)) continue; + const responseBody = extractResponseBody(operation, r); + const outputSchema = responseBody ? outputSchemaFromResponseBody(responseBody) : undefined; + if (outputSchema !== undefined) result.set(index, outputSchema); + } + } + return result; +}; diff --git a/packages/plugins/openapi/src/sdk/plugin.ts b/packages/plugins/openapi/src/sdk/plugin.ts index c2c4223886..fab5c3b1a7 100644 --- a/packages/plugins/openapi/src/sdk/plugin.ts +++ b/packages/plugins/openapi/src/sdk/plugin.ts @@ -37,6 +37,7 @@ import { OAuth2Preset, SecurityScheme, previewSpecText, + previewSpecTextStreaming, type SpecPreview, } from "./preview"; import { deriveAuthenticationTemplateFromPreview, firstBaseUrlForPreview } from "./derive-auth"; @@ -807,18 +808,25 @@ export const openApiPlugin = definePlugin< const explicitBaseUrl = config.baseUrl ?? resolved.baseUrl; const needsDerivedBaseUrl = explicitBaseUrl == null; const needsDerivedAuth = config.authenticationTemplate == null; + // Spec-format selections (resolved.keepPathItem) preview via the + // streaming path: the whole-document parse of a Graph-sized source is + // the measured isolate OOM. The OAuth-discovery enrich re-parses the + // full text for the same reason, and an adapter spec declares its + // auth (or the adapter supplies the template), so it is skipped. const preview = needsDerivedBaseUrl || needsDerivedAuth - ? yield* previewSpecText(resolved.specText).pipe( - Effect.flatMap((rawPreview) => - enrichPreviewWithDiscoveredOAuth({ - specText: resolved.specText, - preview: rawPreview, - specUrl: resolved.specUrl ?? specInputToSpecUrl(config.spec), - baseUrl: explicitBaseUrl, - }), - ), - ) + ? resolved.keepPathItem + ? yield* previewSpecTextStreaming(resolved.specText, resolved.keepPathItem) + : yield* previewSpecText(resolved.specText).pipe( + Effect.flatMap((rawPreview) => + enrichPreviewWithDiscoveredOAuth({ + specText: resolved.specText, + preview: rawPreview, + specUrl: resolved.specUrl ?? specInputToSpecUrl(config.spec), + baseUrl: explicitBaseUrl, + }), + ), + ) : undefined; const derivedBaseUrl = needsDerivedBaseUrl && preview ? firstBaseUrlForPreview(preview) : undefined; @@ -1101,6 +1109,12 @@ export const openApiPlugin = definePlugin< }, httpClientLayer, ); + // Spec-format selections stream (whole-parse of a Graph-sized + // source OOMs the isolate) and skip the OAuth-discovery enrich — + // same rationale as the addSpec derived preview above. + if (resolved.keepPathItem) { + return yield* previewSpecTextStreaming(resolved.specText, resolved.keepPathItem); + } const preview = yield* previewSpecText(resolved.specText); return yield* enrichPreviewWithDiscoveredOAuth({ specText: resolved.specText, diff --git a/packages/plugins/openapi/src/sdk/preview-streaming.test.ts b/packages/plugins/openapi/src/sdk/preview-streaming.test.ts new file mode 100644 index 0000000000..e85a3ee3c3 --- /dev/null +++ b/packages/plugins/openapi/src/sdk/preview-streaming.test.ts @@ -0,0 +1,183 @@ +import { describe, expect, it } from "@effect/vitest"; +import { Effect } from "effect"; + +import { OpenApiExtractionError } from "./errors"; +import { previewSpecText, previewSpecTextStreaming } from "./preview"; +import type { SpecPreview } from "./preview"; + +// Streamable block-YAML fixture: parameter/schema `$ref`s, a path-level +// parameter, a deprecated operation, top-level security, and a transitive +// schema chain (WidgetList → Widget → Owner) for the response-field walk. +const fixture = `openapi: 3.0.4 +info: + title: Streamed Fixture + version: 1.2.3 + description: Fixture for streaming preview parity +servers: + - url: https://api.example.com/v1 +security: + - appAuth: [] +paths: + /widgets: + get: + operationId: widgets.list + summary: List widgets + tags: + - widgets + parameters: + - $ref: '#/components/parameters/PageSize' + responses: + "200": + description: OK + content: + application/json: + schema: + $ref: '#/components/schemas/WidgetList' + post: + operationId: widgets.create + summary: Create a widget + tags: + - widgets + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/Widget' + responses: + "201": + description: Created + content: + application/json: + schema: + $ref: '#/components/schemas/Widget' + /widgets/{widget-id}: + parameters: + - name: widget-id + in: path + required: true + schema: + type: string + get: + operationId: widgets.get + deprecated: true + tags: + - widgets + - detail + responses: + "200": + description: OK + content: + application/json: + schema: + $ref: '#/components/schemas/Widget' +components: + parameters: + PageSize: + name: pageSize + in: query + required: false + description: Page size + schema: + type: integer + securitySchemes: + appAuth: + type: oauth2 + flows: + authorizationCode: + authorizationUrl: https://auth.example.com/authorize + tokenUrl: https://auth.example.com/token + scopes: + widgets.read: Read widgets + schemas: + WidgetList: + type: object + properties: + items: + type: array + items: + $ref: '#/components/schemas/Widget' + Widget: + type: object + properties: + id: + type: string + owner: + $ref: '#/components/schemas/Owner' + Owner: + type: object + properties: + name: + type: string +`; + +const sortedOperations = (preview: SpecPreview) => + [...preview.operations].sort((a, b) => + `${a.operationId}:${a.method}`.localeCompare(`${b.operationId}:${b.method}`), + ); + +describe("previewSpecTextStreaming", () => { + it.effect("matches the whole-document preview on a streamable spec", () => + Effect.gen(function* () { + const whole = yield* previewSpecText(fixture); + const streamed = yield* previewSpecTextStreaming(fixture); + + expect(streamed.title).toEqual(whole.title); + expect(streamed.description).toEqual(whole.description); + expect(streamed.version).toEqual(whole.version); + expect(streamed.servers).toEqual(whole.servers); + expect(streamed.operationCount).toBe(whole.operationCount); + expect(streamed.tags).toEqual(whole.tags); + expect(streamed.securitySchemes).toEqual(whole.securitySchemes); + expect(streamed.authStrategies).toEqual(whole.authStrategies); + expect(streamed.headerPresets).toEqual(whole.headerPresets); + expect(streamed.oauth2Presets).toEqual(whole.oauth2Presets); + expect(streamed.healthCheckCandidates).toEqual(whole.healthCheckCandidates); + expect(sortedOperations(streamed)).toEqual(sortedOperations(whole)); + }), + ); + + it.effect("projects response fields through the transitive schema closure", () => + Effect.gen(function* () { + const streamed = yield* previewSpecTextStreaming(fixture); + + const listCandidate = streamed.healthCheckCandidates.find( + (candidate) => candidate.method === "get" && candidate.operation.endsWith("list"), + ); + expect(listCandidate).toBeDefined(); + expect(listCandidate!.responseFields ?? []).not.toHaveLength(0); + }), + ); + + it.effect("applies the keep filter to counts, tags, and candidates", () => + Effect.gen(function* () { + const streamed = yield* previewSpecTextStreaming(fixture, (path, pathItem) => + path === "/widgets/{widget-id}" ? null : pathItem, + ); + + expect(streamed.operationCount).toBe(2); + expect(streamed.tags).toEqual(["widgets"]); + expect( + streamed.operations.every((operation) => operation.operationId !== "widgets.get"), + ).toBe(true); + expect( + streamed.healthCheckCandidates.every((candidate) => candidate.method !== "delete"), + ).toBe(true); + expect(streamed.healthCheckCandidates).toHaveLength(2); + }), + ); + + it.effect("fails cleanly on a spec outside the streamable profile", () => + Effect.gen(function* () { + const error = yield* previewSpecTextStreaming( + JSON.stringify({ + openapi: "3.1.0", + info: { title: "Inline", version: "1.0.0" }, + paths: {}, + }), + ).pipe(Effect.flip); + + expect(error).toBeInstanceOf(OpenApiExtractionError); + expect(error).toHaveProperty("message", expect.stringMatching(/streamable/)); + }), + ); +}); diff --git a/packages/plugins/openapi/src/sdk/preview.ts b/packages/plugins/openapi/src/sdk/preview.ts index 56fb3e7038..4266b02347 100644 --- a/packages/plugins/openapi/src/sdk/preview.ts +++ b/packages/plugins/openapi/src/sdk/preview.ts @@ -8,10 +8,22 @@ import { } from "@executor-js/sdk/core"; import { parse, resolveSpecText, type ParsedDocument } from "./parse"; -import { extract } from "./extract"; +import { + extract, + streamOutputSchemas, + streamPreviewOperations, + type StreamedPreviewOperation, +} from "./extract"; import { compileToolDefinitions } from "./definitions"; import { normalizeOpenApiRefs } from "./backing"; +import { OpenApiExtractionError } from "./errors"; import { DocResolver } from "./openapi-utils"; +import { + collectReferencedSchemas, + indexSchemas, + structuralSplit, + type KeepPathItem, +} from "./split"; import { HttpMethod, ServerInfo, type ExtractedOperation, type ExtractionResult } from "./types"; // Mutating HTTP methods: mirrors `REQUIRE_APPROVAL` in `./invoke` but kept @@ -565,6 +577,140 @@ export const previewSpecText = Effect.fn("OpenApi.previewSpecText")(function* (s }); }); +// --------------------------------------------------------------------------- +// Streaming preview (spec-format selections over Graph-sized specs) +// --------------------------------------------------------------------------- + +const streamedCandidate = (op: StreamedPreviewOperation): HealthCheckCandidate => { + const method = op.method.toLowerCase(); + return { + operation: op.toolPath, + method, + requiredArgCount: op.parameters.filter((parameter) => parameter.required).length, + destructive: DESTRUCTIVE_METHODS.has(method), + summary: op.summary ?? op.description ?? `${method.toUpperCase()} ${op.pathTemplate}`, + ...(op.parameters.length > 0 ? { parameters: op.parameters } : {}), + }; +}; + +/** + * Streaming twin of `previewSpecText` for spec-format selections (Microsoft + * Graph): never parses the document whole. The whole-document parse of the 43MB + * Graph source builds a ~300MB tree that kills a 128MB Workers isolate — the + * add/update path already streams via `structuralSplit`, and this brings the + * preview path onto the same primitive: head + schema-free components parse + * small; path-items parse one at a time (through `keepPathItem`, so the preview + * matches what registration persists); and only the top-ranked candidates get + * their response schema walked, against the transitive `$ref` closure rather + * than the full schema map. + */ +export const previewSpecTextStreaming = Effect.fn("OpenApi.previewSpecTextStreaming")(function* ( + specText: string, + keepPathItem?: KeepPathItem, +) { + const structure = structuralSplit(specText); + if (!structure) { + return yield* new OpenApiExtractionError({ + message: + "OpenAPI spec is not in the streamable block-YAML profile (no top-level `paths:` block); cannot stream-preview a spec this large in-band.", + }); + } + + const { head, components, servers, operations } = streamPreviewOperations( + structure, + keepPathItem, + ); + + // oxlint-disable-next-line executor/no-double-cast -- boundary: schema-free resolver doc (head + small components, empty paths), read only for `$ref` resolution into components. + const resolverDoc = { ...head, paths: {}, components } as unknown as ParsedDocument; + const resolver = new DocResolver(resolverDoc); + const rawSchemes = + components.securitySchemes && typeof components.securitySchemes === "object" + ? (components.securitySchemes as Record) + : {}; + const securitySchemes = extractSecuritySchemes(rawSchemes, resolver); + + const rawSecurity = (Array.isArray(head.security) ? head.security : []) as Array< + Record + >; + const declaredStrategies = rawSecurity.map((entry) => + AuthStrategy.make({ schemes: Object.keys(entry) }), + ); + const authStrategies = + declaredStrategies.length > 0 + ? declaredStrategies + : securitySchemes.map((scheme) => AuthStrategy.make({ schemes: [scheme.name] })); + + // Rank all kept operations, keep candidate metadata for the top slice, and + // walk response schemas only for the top survivors — same caps and ranking as + // the whole-document path. + const ranked = operations + .map((op) => ({ operationIndex: op.operationIndex, candidate: streamedCandidate(op) })) + .sort((a, b) => compareHealthCheckCandidates(a.candidate, b.candidate)) + .slice(0, MAX_PREVIEW_CANDIDATES); + + const fieldCandidates = ranked.slice(0, MAX_PREVIEW_RESPONSE_FIELD_CANDIDATES); + const outputSchemas = streamOutputSchemas( + structure, + new Set(fieldCandidates.map((entry) => entry.operationIndex)), + keepPathItem, + ); + // Hoisted `$defs` restricted to the transitive `$ref` closure of the walked + // output schemas — `projectResponseFields` resolves within this map, and the + // closure guarantees every reachable ref is present. + const hoistedDefs: Record = {}; + for (const [name, schema] of Object.entries( + collectReferencedSchemas(structure, indexSchemas(structure), [...outputSchemas.values()]), + )) { + hoistedDefs[name] = normalizeOpenApiRefs(schema); + } + const healthCheckCandidates = ranked.map(({ operationIndex, candidate }, index) => { + if (index >= MAX_PREVIEW_RESPONSE_FIELD_CANDIDATES) return candidate; + const outputSchema = outputSchemas.get(operationIndex); + if (outputSchema === undefined) return candidate; + const responseFields = projectResponseFields(normalizeOpenApiRefs(outputSchema), hoistedDefs); + return responseFields.length > 0 ? { ...candidate, responseFields } : candidate; + }); + + const info = + head.info && typeof head.info === "object" && !Array.isArray(head.info) + ? (head.info as Record) + : {}; + const infoString = (key: string): string | undefined => { + const value = info[key]; + return typeof value === "string" ? value : undefined; + }; + + const tagSet = new Set(); + for (const op of operations) { + for (const tag of op.tags) tagSet.add(tag); + } + + return SpecPreview.make({ + title: Option.fromNullishOr(infoString("title")), + description: Option.fromNullishOr(infoString("description")), + version: Option.fromNullishOr(infoString("version")), + servers, + operationCount: operations.length, + operations: operations.map((op) => + PreviewOperation.make({ + operationId: op.operationId, + method: op.method, + path: op.pathTemplate, + summary: Option.fromNullishOr(op.summary), + tags: op.tags, + deprecated: op.deprecated, + }), + ), + tags: [...tagSet].sort(), + securitySchemes, + authStrategies, + headerPresets: buildHeaderPresets(securitySchemes, authStrategies), + oauth2Presets: buildOAuth2Presets(securitySchemes), + healthCheckCandidates, + }); +}); + /** Preview an OpenAPI spec — extract metadata without registering anything. * Accepts either a URL or raw JSON/YAML text. */ export const previewSpec = Effect.fn("OpenApi.previewSpec")(function* (input: string) { From fcbfe503165ab8984b9080738510e8b033507791 Mon Sep 17 00:00:00 2001 From: Rhys Sullivan <39114868+RhysSullivan@users.noreply.github.com> Date: Tue, 25 Aug 2026 21:42:35 -0700 Subject: [PATCH 3/3] Cover the Graph preset preview in the catalog e2e scenario --- e2e/scenarios/microsoft-graph-default.test.ts | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/e2e/scenarios/microsoft-graph-default.test.ts b/e2e/scenarios/microsoft-graph-default.test.ts index 88619e1191..499a25af92 100644 --- a/e2e/scenarios/microsoft-graph-default.test.ts +++ b/e2e/scenarios/microsoft-graph-default.test.ts @@ -55,6 +55,24 @@ scenario( yield* Effect.ensuring( Effect.gen(function* () { + // The add dialog previews before registering. This must stream: the + // whole-document parse of the Graph source OOMs the dev workerd + // isolate the same way it did the production one. + const preview = yield* client.openapi.previewSpec({ + payload: { + spec: `${MICROSOFT_GRAPH_OPENAPI_URL}#preset=${MICROSOFT_FILES_PRESET_ID}`, + specFormat: "microsoft-graph", + }, + }); + expect( + preview.operationCount, + "the preview streams the files selection without a whole-document parse", + ).toBeGreaterThan(10); + expect( + preview.healthCheckCandidates.length, + "the preview carries ranked health-check candidates for the selection", + ).toBeGreaterThan(0); + const added = yield* client.openapi.addSpec({ payload: { spec: {