Stream the OpenAPI preview for Graph-sized spec selections

.changeset/openapi-streaming-preview.md

@@ -0,0 +1,5 @@
+---
+"@executor-js/plugin-openapi": patch
+---
+
+Preview OpenAPI spec-format selections (Microsoft Graph) through the streaming structural-split path instead of a whole-document parse, and guard generic whole-document parses by parsed-tree size (line count for block YAML, text size for JSON). Previewing a Graph preset URL previously parsed the 43MB source whole and killed the 128MB Workers isolate mid-request, surfacing as an empty 503; it now streams within budget, and oversized generic specs fail with an actionable error instead of taking down the isolate.

e2e/scenarios/microsoft-graph-default.test.ts

@@ -55,6 +55,24 @@ scenario(
 
     yield* Effect.ensuring(
       Effect.gen(function* () {
+        // The add dialog previews before registering. This must stream: the
+        // whole-document parse of the Graph source OOMs the dev workerd
+        // isolate the same way it did the production one.
+        const preview = yield* client.openapi.previewSpec({
+          payload: {
+            spec: `${MICROSOFT_GRAPH_OPENAPI_URL}#preset=${MICROSOFT_FILES_PRESET_ID}`,
+            specFormat: "microsoft-graph",
+          },
+        });
+        expect(
+          preview.operationCount,
+          "the preview streams the files selection without a whole-document parse",
+        ).toBeGreaterThan(10);
+        expect(
+          preview.healthCheckCandidates.length,
+          "the preview carries ranked health-check candidates for the selection",
+        ).toBeGreaterThan(0);
+
         const added = yield* client.openapi.addSpec({
           payload: {
             spec: {

packages/plugins/openapi/src/providers/microsoft/spec-format-adapter.test.ts

@@ -3,6 +3,7 @@ import { Effect, Layer } from "effect";
 import { HttpClient, HttpClientRequest, HttpClientResponse } from "effect/unstable/http";
 import { structuralSplit } from "@executor-js/plugin-openapi";
 
+import { previewSpecTextStreaming } from "../../sdk/preview";
 import { microsoftGraphAdapter } from "./spec-format-adapter";
 import { MICROSOFT_GRAPH_OPENAPI_URL } from "./presets";
 
@@ -90,3 +91,22 @@ it.effect("uses catalog URL fragments to select one Graph workload", () =>
     expect(keepPathItem("/irrelevant", { get: { operationId: "irrelevant.Get" } })).toBeNull();
   }),
 );
+
+it.effect("stream-previews a Graph selection without a whole-document parse", () =>
+  Effect.gen(function* () {
+    const converted = yield* microsoftGraphAdapter.fetch({
+      urls: [`${MICROSOFT_GRAPH_OPENAPI_URL}#preset=profile`],
+      httpClientLayer: graphHttpClientLayer,
+    });
+    const preview = yield* previewSpecTextStreaming(converted.specText, converted.keepPathItem);
+
+    expect(preview.operationCount).toBe(1);
+    expect(preview.operations.map((operation) => operation.operationId)).toEqual(["me.GetUser"]);
+    expect(preview.healthCheckCandidates).toHaveLength(1);
+    expect(preview.healthCheckCandidates[0]?.method).toBe("get");
+    expect(preview.oauth2Presets).toHaveLength(1);
+    expect(preview.servers.map((server) => server.url)).toEqual([
+      "https://graph.microsoft.com/v1.0",
+    ]);
+  }),
+);

packages/plugins/openapi/src/sdk/extract.ts

@@ -742,6 +742,23 @@ export const streamOperationBindings = <E, R>(
 const isPathItemValue = (value: unknown): value is Record<string, unknown> =>
   value !== null && typeof value === "object" && !Array.isArray(value);
 
+/** Parse one path-item range to its kept (optionally trimmed) value. Applied
+ *  identically wherever a structure is walked more than once, so per-operation
+ *  indexes stay aligned across passes. */
+const parseKeptPathItem = (
+  structure: SpecStructure,
+  range: ByteRange,
+  keepPathItem: KeepPathItem | undefined,
+): readonly [string, PathItemObject] | null => {
+  const entry = parseEntry(structure.text, range, 2);
+  if (!entry) return null;
+  const [path, rawValue] = entry;
+  if (!isPathItemValue(rawValue)) return null;
+  if (!keepPathItem) return [path, rawValue as PathItemObject];
+  const kept = keepPathItem(path, rawValue);
+  return kept ? [path, kept as PathItemObject] : null;
+};
+
 /**
  * Stream invocation bindings straight from a `SpecStructure` (the structural
  * split of a large spec) without ever materializing the whole-document tree.
@@ -775,15 +792,8 @@ export const streamOperationBindingsFromStructure = <E, R>(
 
     // Parse one path-item range to its kept (optionally trimmed) value, applying
     // `keepPathItem` identically in both passes so the operation index aligns.
-    const keptPathItem = (range: ByteRange): readonly [string, PathItemObject] | null => {
-      const entry = parseEntry(structure.text, range, 2);
-      if (!entry) return null;
-      const [path, rawValue] = entry;
-      if (!isPathItemValue(rawValue)) return null;
-      if (!keepPathItem) return [path, rawValue as PathItemObject];
-      const kept = keepPathItem(path, rawValue);
-      return kept ? [path, kept as PathItemObject] : null;
-    };
+    const keptPathItem = (range: ByteRange): readonly [string, PathItemObject] | null =>
+      parseKeptPathItem(structure, range, keepPathItem);
 
     // Pass 1 (light): collect schema-free tool-path planning metadata in
     // document order. No bindings, no schemas; one path-item resident at a time.
@@ -874,3 +884,162 @@ export const streamOperationBindingsFromStructure = <E, R>(
 
     return { toolCount: plans.length, toolNames: plans.map((plan) => plan.toolPath) };
   }).pipe(Effect.withSpan("OpenApi.streamOperationBindingsFromStructure"));
+
+// ---------------------------------------------------------------------------
+// Streaming preview extraction
+// ---------------------------------------------------------------------------
+
+export interface StreamedPreviewParameter {
+  readonly name: string;
+  readonly location: ParameterLocation;
+  readonly required: boolean;
+  readonly description?: string;
+}
+
+/** Schema-free per-operation metadata for the preview path: everything the
+ *  add screen's operation list and health-check candidate ranking need, and
+ *  nothing that scales with schema size. */
+export interface StreamedPreviewOperation {
+  readonly operationId: string;
+  /** Tool path planned over the full kept operation set, so preview candidates
+   *  match the names registration will assign. */
+  readonly toolPath: string;
+  readonly method: HttpMethod;
+  readonly pathTemplate: string;
+  readonly summary: string | undefined;
+  readonly description: string | undefined;
+  readonly tags: readonly string[];
+  readonly deprecated: boolean;
+  readonly parameters: readonly StreamedPreviewParameter[];
+  /** Position in kept-document order; key for `streamOutputSchemas`. */
+  readonly operationIndex: number;
+}
+
+export interface StreamedPreviewExtraction {
+  /** Parsed document head (openapi, info, servers, tags, security, ...). */
+  readonly head: Record<string, unknown>;
+  /** Schema-free components (parameters / requestBodies / responses /
+   *  securitySchemes / ...) for `$ref` resolution and auth extraction. */
+  readonly components: Record<string, unknown>;
+  readonly servers: readonly ServerInfo[];
+  readonly operations: readonly StreamedPreviewOperation[];
+}
+
+/**
+ * Streaming twin of `extract` for the preview path: walk a `SpecStructure`
+ * path-item by path-item (each parsed in isolation and discarded) and keep only
+ * schema-free per-operation metadata, so previewing a Graph-sized spec never
+ * materializes the whole-document tree that OOMs a 128MB Workers isolate.
+ * `keepPathItem` applies the same selection filter as the streaming compile, so
+ * the preview describes exactly the operation set registration would persist.
+ */
+export const streamPreviewOperations = (
+  structure: SpecStructure,
+  keepPathItem?: KeepPathItem,
+): StreamedPreviewExtraction => {
+  const head = parseHead(structure);
+  const components = parseSmallComponents(structure);
+  // oxlint-disable-next-line executor/no-double-cast -- boundary: same schema-free resolver doc as `streamOperationBindingsFromStructure` (head + small components, empty paths), read only for .servers and `$ref` resolution.
+  const resolverDoc = { ...head, paths: {}, components } as unknown as ParsedDocument;
+  const r = new DocResolver(resolverDoc);
+  const docServers = extractServers(resolverDoc);
+
+  const inputs: OperationPathInput[] = [];
+  const metas: Omit<StreamedPreviewOperation, "toolPath">[] = [];
+  for (const range of structure.pathItems) {
+    const kept = parseKeptPathItem(structure, range, keepPathItem);
+    if (!kept) continue;
+    const [path, pathItem] = kept;
+    for (const method of HTTP_METHODS) {
+      const operation = pathItem[method];
+      if (!operation) continue;
+      const resolvedPathTemplate = explicitPathTemplate(operation) ?? path;
+      const tags = (operation.tags ?? []).filter((t) => t.trim().length > 0);
+      const operationId = deriveOperationId(method, path, operation);
+      inputs.push({
+        operationId,
+        explicitToolPath: explicitToolPath(operation),
+        method,
+        pathTemplate: resolvedPathTemplate,
+        tag0: tags[0],
+      });
+      const parameters = extractParameters(pathItem, operation, r).map(
+        (parameter): StreamedPreviewParameter => ({
+          name: parameter.name,
+          location: parameter.location,
+          required: parameter.required,
+          ...(Option.isSome(parameter.description)
+            ? { description: parameter.description.value }
+            : {}),
+        }),
+      );
+      metas.push({
+        operationId,
+        method,
+        pathTemplate: resolvedPathTemplate,
+        summary: operation.summary,
+        description: operation.description,
+        tags,
+        deprecated: operation.deprecated === true,
+        parameters,
+        operationIndex: metas.length,
+      });
+    }
+  }
+
+  const plans = planToolPaths(inputs);
+  const toolPathByOpIndex: (string | undefined)[] = new Array(inputs.length);
+  for (const plan of plans) toolPathByOpIndex[plan.operationIndex] = plan.toolPath;
+
+  return {
+    head,
+    components,
+    servers: docServers,
+    operations: metas.flatMap((meta) => {
+      const toolPath = toolPathByOpIndex[meta.operationIndex];
+      return toolPath === undefined ? [] : [{ ...meta, toolPath }];
+    }),
+  };
+};
+
+/**
+ * Re-walk the structure and build the raw output schema (component `$ref`s
+ * intact) for just the operations in `wanted` — the bounded response-schema
+ * walk behind the preview's typed identity picker. Iteration order and
+ * `keepPathItem` application match `streamPreviewOperations`, so the indexes
+ * line up.
+ */
+export const streamOutputSchemas = (
+  structure: SpecStructure,
+  wanted: ReadonlySet<number>,
+  keepPathItem?: KeepPathItem,
+): ReadonlyMap<number, unknown> => {
+  const result = new Map<number, unknown>();
+  if (wanted.size === 0) return result;
+  // oxlint-disable-next-line executor/no-double-cast -- boundary: same schema-free resolver doc as `streamPreviewOperations`.
+  const resolverDoc = {
+    ...parseHead(structure),
+    paths: {},
+    components: parseSmallComponents(structure),
+  } as unknown as ParsedDocument;
+  const r = new DocResolver(resolverDoc);
+
+  let opIndex = 0;
+  for (const range of structure.pathItems) {
+    if (result.size >= wanted.size) break;
+    const kept = parseKeptPathItem(structure, range, keepPathItem);
+    if (!kept) continue;
+    const [, pathItem] = kept;
+    for (const method of HTTP_METHODS) {
+      const operation = pathItem[method];
+      if (!operation) continue;
+      const index = opIndex;
+      opIndex += 1;
+      if (!wanted.has(index)) continue;
+      const responseBody = extractResponseBody(operation, r);
+      const outputSchema = responseBody ? outputSchemaFromResponseBody(responseBody) : undefined;
+      if (outputSchema !== undefined) result.set(index, outputSchema);
+    }
+  }
+  return result;
+};

packages/plugins/openapi/src/sdk/parse.test.ts

@@ -1,8 +1,15 @@
 import { describe, expect, it } from "@effect/vitest";
-import { Effect } from "effect";
+import { Effect, Layer } from "effect";
+import { HttpClient, HttpClientRequest, HttpClientResponse } from "effect/unstable/http";
 
 import { OpenApiParseError } from "./errors";
-import { parse } from "./parse";
+import {
+  MAX_JSON_SPEC_CHARS,
+  MAX_SPEC_TEXT_CHARS,
+  MAX_YAML_SPEC_LINES,
+  fetchSpecText,
+  parse,
+} from "./parse";
 
 describe("OpenAPI parse", () => {
   it.effect("parses JSON OpenAPI documents", () =>
@@ -82,4 +89,79 @@ paths: {}
       expect(doc.info.description).toHaveLength(largeDescription.length);
     }),
   );
+
+  it.effect("rejects structure-dense YAML documents above the line cap", () =>
+    Effect.gen(function* () {
+      // Three lines per path-item; well past the cap while only a few MB of
+      // text — the shape (not the size) is what a whole parse cannot survive.
+      const pathItems = "  /a:\n    get: {}\n".repeat(Math.ceil(MAX_YAML_SPEC_LINES / 2));
+      const error = yield* parse(
+        `openapi: 3.0.0\ninfo:\n  title: Dense\n  version: 1.0.0\npaths:\n${pathItems}`,
+      ).pipe(Effect.flip);
+
+      expect(error).toBeInstanceOf(OpenApiParseError);
+      expect(error).toHaveProperty("message", expect.stringMatching(/too large to parse whole/));
+      expect(error).toHaveProperty("message", expect.stringMatching(/lines/));
+    }),
+  );
+
+  it.effect("rejects JSON documents above the JSON size cap", () =>
+    Effect.gen(function* () {
+      const padded = `{"openapi":"3.1.0","x-pad":"${"x".repeat(MAX_JSON_SPEC_CHARS)}"}`;
+      const error = yield* parse(padded).pipe(Effect.flip);
+
+      expect(error).toBeInstanceOf(OpenApiParseError);
+      expect(error).toHaveProperty("message", expect.stringMatching(/too large to parse/));
+    }),
+  );
+
+  it.effect("rejects any document above the text ceiling", () =>
+    Effect.gen(function* () {
+      const padded = `openapi: 3.0.0\ninfo:\n  description: "${"x".repeat(MAX_SPEC_TEXT_CHARS)}"\n`;
+      const error = yield* parse(padded).pipe(Effect.flip);
+
+      expect(error).toBeInstanceOf(OpenApiParseError);
+      expect(error).toHaveProperty("message", expect.stringMatching(/too large to parse/));
+    }),
+  );
+});
+
+describe("OpenAPI fetchSpecText", () => {
+  const specUrl = "https://example.com/openapi.yaml";
+
+  const layerWithResponse = (response: Response) =>
+    Layer.succeed(HttpClient.HttpClient)(
+      HttpClient.make((request: HttpClientRequest.HttpClientRequest) =>
+        Effect.succeed(HttpClientResponse.fromWeb(request, response)),
+      ),
+    );
+
+  it.effect("rejects a document whose declared length is above the text ceiling", () =>
+    Effect.gen(function* () {
+      const error = yield* fetchSpecText(specUrl).pipe(
+        Effect.provide(
+          layerWithResponse(
+            new Response("openapi: 3.0.0", {
+              status: 200,
+              headers: { "content-length": String(MAX_SPEC_TEXT_CHARS + 1) },
+            }),
+          ),
+        ),
+        Effect.flip,
+      );
+
+      expect(error).toBeInstanceOf(OpenApiParseError);
+      expect(error).toHaveProperty("message", expect.stringMatching(/too large to parse/));
+    }),
+  );
+
+  it.effect("fetches a document with an in-range declared length", () =>
+    Effect.gen(function* () {
+      const specText = yield* fetchSpecText(specUrl).pipe(
+        Effect.provide(layerWithResponse(new Response("openapi: 3.0.0", { status: 200 }))),
+      );
+
+      expect(specText).toBe("openapi: 3.0.0");
+    }),
+  );
 });