From aa423aaa035d1bf8fe317ace941460945c2c39fc Mon Sep 17 00:00:00 2001 From: Rhys Sullivan <39114868+RhysSullivan@users.noreply.github.com> Date: Tue, 25 Aug 2026 22:57:38 -0700 Subject: [PATCH] Serve Microsoft Graph selections from precomputed spec slices --- .changeset/graph-spec-slices.md | 5 + .github/workflows/graph-slices.yml | 47 +++++ .oxlintrc.jsonc | 1 + .../openapi/scripts/generate-graph-slices.ts | 109 ++++++++++ .../openapi/src/providers/microsoft/graph.ts | 26 ++- .../providers/microsoft/slice-build.test.ts | 104 ++++++++++ .../src/providers/microsoft/slice-build.ts | 193 ++++++++++++++++++ .../src/providers/microsoft/slices.test.ts | 70 +++++++ .../openapi/src/providers/microsoft/slices.ts | 87 ++++++++ .../microsoft/spec-format-adapter.test.ts | 75 +++++++ 10 files changed, 714 insertions(+), 3 deletions(-) create mode 100644 .changeset/graph-spec-slices.md create mode 100644 .github/workflows/graph-slices.yml create mode 100644 packages/plugins/openapi/scripts/generate-graph-slices.ts create mode 100644 packages/plugins/openapi/src/providers/microsoft/slice-build.test.ts create mode 100644 packages/plugins/openapi/src/providers/microsoft/slice-build.ts create mode 100644 packages/plugins/openapi/src/providers/microsoft/slices.test.ts create mode 100644 packages/plugins/openapi/src/providers/microsoft/slices.ts diff --git a/.changeset/graph-spec-slices.md b/.changeset/graph-spec-slices.md new file mode 100644 index 0000000000..16345d752d --- /dev/null +++ b/.changeset/graph-spec-slices.md @@ -0,0 +1,5 @@ +--- +"@executor-js/plugin-openapi": patch +--- + +Serve Microsoft Graph preset selections from precomputed slice release assets instead of the 43MB upstream monolith. The monolith fetch almost never survives a 128MB Workers isolate (production traces show one completion in 30 days), so covered selections — every catalog preset, plus any combination within the default bundle — now read a 4–19MB filtered document built offline by the graph-slices workflow, with the monolith path kept only as a fallback and for full-graph/custom-scope selections. diff --git a/.github/workflows/graph-slices.yml b/.github/workflows/graph-slices.yml new file mode 100644 index 0000000000..35768a0ac4 --- /dev/null +++ b/.github/workflows/graph-slices.yml @@ -0,0 +1,47 @@ +# Refresh the Microsoft Graph slice release assets. +# +# The Graph OpenAPI monolith (~43MB) cannot be processed inside a Workers +# isolate, so the runtime reads per-selection slices published on the +# `graph-slices` release tag (see packages/plugins/openapi/src/providers/ +# microsoft/slices.ts). This workflow rebuilds the slices from the current +# upstream spec on a schedule and on demand. +name: Graph slices + +on: + schedule: + # Weekly; Microsoft's msgraph-metadata automation lands upstream refreshes + # on a similar cadence. A failed run leaves the previous assets serving. + - cron: "17 6 * * 1" + workflow_dispatch: {} + +permissions: + contents: write + +jobs: + slices: + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - uses: actions/checkout@v4 + + - uses: oven-sh/setup-bun@v2 + with: + bun-version: 1.3.11 + + - name: Install dependencies + run: bun install --frozen-lockfile + + - name: Generate slices + working-directory: packages/plugins/openapi + run: bun scripts/generate-graph-slices.ts --out "$RUNNER_TEMP/graph-slices" + + - name: Publish to the graph-slices release + env: + GH_TOKEN: ${{ github.token }} + run: | + gh release view graph-slices --repo "$GITHUB_REPOSITORY" >/dev/null 2>&1 || \ + gh release create graph-slices --repo "$GITHUB_REPOSITORY" \ + --title "Microsoft Graph slices" --latest=false \ + --notes "Generated per-preset Microsoft Graph OpenAPI slices. Data release consumed by the openapi plugin's Microsoft adapter; refreshed by the graph-slices workflow." + gh release upload graph-slices "$RUNNER_TEMP/graph-slices"/* \ + --repo "$GITHUB_REPOSITORY" --clobber diff --git a/.oxlintrc.jsonc b/.oxlintrc.jsonc index da06c53517..c86ee9dc58 100644 --- a/.oxlintrc.jsonc +++ b/.oxlintrc.jsonc @@ -64,6 +64,7 @@ "apps/desktop/src/main.ts", "scripts/**/*.{ts,js}", "apps/*/scripts/**/*.{ts,js}", + "packages/*/*/scripts/**/*.{ts,js}", "packages/kernel/runtime-*/src/**/*.{ts,tsx,js,mjs}", ], "rules": { diff --git a/packages/plugins/openapi/scripts/generate-graph-slices.ts b/packages/plugins/openapi/scripts/generate-graph-slices.ts new file mode 100644 index 0000000000..a40c72789f --- /dev/null +++ b/packages/plugins/openapi/scripts/generate-graph-slices.ts @@ -0,0 +1,109 @@ +/** + * Generate the Microsoft Graph slice release assets. + * + * bun scripts/generate-graph-slices.ts [--source ] [--out ] + * + * Fetches (or reads) the Graph OpenAPI monolith, builds one slice per catalog + * preset plus the default bundle via `slice-build.ts`, validates every slice + * against the runtime's streamable profile, and writes `.yaml` files + * plus `manifest.json` to the output directory. The graph-slices workflow runs + * this and uploads the output to the `graph-slices` release tag; runtime + * resolution lives in `src/providers/microsoft/slices.ts`. + * + * Offline-only: this whole-parses the 43MB source, which only works where + * memory is free (CI runner / dev machine), never in a Workers isolate. + */ +import { createHash } from "node:crypto"; +import { mkdir, readFile, writeFile } from "node:fs/promises"; +import { join } from "node:path"; + +import { structuralSplit } from "../src/sdk/split"; +import { + MICROSOFT_GRAPH_DEFAULT_PRESET_IDS, + MICROSOFT_GRAPH_OPENAPI_URL, + microsoftGraphScopePresets, +} from "../src/providers/microsoft/presets"; +import { MICROSOFT_GRAPH_DEFAULT_SLICE_ASSET } from "../src/providers/microsoft/slices"; +import { + buildGraphSliceDocument, + parseGraphSourceDocument, +} from "../src/providers/microsoft/slice-build"; + +const argValue = (flag: string): string | undefined => { + const index = process.argv.indexOf(flag); + return index !== -1 ? process.argv[index + 1] : undefined; +}; + +const source = argValue("--source") ?? MICROSOFT_GRAPH_OPENAPI_URL; +const outDir = argValue("--out") ?? "graph-slices-out"; + +const readSource = async (): Promise => { + if (source.startsWith("http://") || source.startsWith("https://")) { + const response = await fetch(source); + if (!response.ok) { + throw new Error(`Failed to fetch Graph source: HTTP ${response.status}`); + } + return response.text(); + } + return readFile(source, "utf8"); +}; + +const sourceText = await readSource(); +const sourceSha256 = createHash("sha256").update(sourceText).digest("hex"); +const doc = parseGraphSourceDocument(sourceText); +if (!doc) { + throw new Error("Microsoft Graph source did not parse to an object"); +} + +const selections: readonly { readonly asset: string; readonly presetIds: readonly string[] }[] = [ + ...microsoftGraphScopePresets.map((preset) => ({ asset: preset.id, presetIds: [preset.id] })), + { + asset: MICROSOFT_GRAPH_DEFAULT_SLICE_ASSET, + presetIds: MICROSOFT_GRAPH_DEFAULT_PRESET_IDS, + }, +]; + +await mkdir(outDir, { recursive: true }); + +const manifestAssets: Record< + string, + { + readonly bytes: number; + readonly paths: number; + readonly operations: number; + readonly schemas: number; + } +> = {}; + +for (const { asset, presetIds } of selections) { + const slice = buildGraphSliceDocument(doc, presetIds); + if (slice.operationCount === 0) { + throw new Error(`Slice "${asset}" kept zero operations — preset filter or source drifted`); + } + const structure = structuralSplit(slice.specText); + if (!structure) { + throw new Error(`Slice "${asset}" is not in the streamable block-YAML profile`); + } + if (structure.pathItems.length !== slice.pathCount) { + throw new Error( + `Slice "${asset}" splitter sees ${structure.pathItems.length} path-items, expected ${slice.pathCount}`, + ); + } + await writeFile(join(outDir, `${asset}.yaml`), slice.specText); + manifestAssets[asset] = { + bytes: Buffer.byteLength(slice.specText), + paths: slice.pathCount, + operations: slice.operationCount, + schemas: slice.schemaCount, + }; + console.log( + `${asset}: ${(Buffer.byteLength(slice.specText) / 1024 / 1024).toFixed(2)}MB, ` + + `${slice.pathCount} paths, ${slice.operationCount} operations, ${slice.schemaCount} schemas`, + ); +} + +await writeFile( + join(outDir, "manifest.json"), + `${JSON.stringify({ source, sourceSha256, generatedAt: new Date().toISOString(), assets: manifestAssets }, null, 2)}\n`, +); +console.log(`wrote ${selections.length} slices + manifest.json to ${outDir}`); diff --git a/packages/plugins/openapi/src/providers/microsoft/graph.ts b/packages/plugins/openapi/src/providers/microsoft/graph.ts index 921f1fa1a9..36ac23dc4f 100644 --- a/packages/plugins/openapi/src/providers/microsoft/graph.ts +++ b/packages/plugins/openapi/src/providers/microsoft/graph.ts @@ -15,6 +15,7 @@ import { } from "../../sdk/split"; import type { Authentication } from "../../sdk/types"; +import { fetchMicrosoftGraphSlice, microsoftGraphSliceAssetForSelection } from "./slices"; import { MICROSOFT_AUTHORIZATION_URL, MICROSOFT_AUTH_TEMPLATE_SLUG, @@ -759,9 +760,28 @@ export const buildMicrosoftGraphOpenApiSpec = ( ): Effect.Effect => Effect.gen(function* () { const selection = yield* validateSelectionUrls(normalizeSelection(input), urlPolicy); - const sourceText = yield* fetchMicrosoftGraphOpenApiSpec(selection.specUrl).pipe( - Effect.provide(httpClientLayer), - ); + // Covered selections read a precomputed slice (sub-MB) instead of the 43MB + // monolith: in production, the monolith fetch alone almost never survives + // the 128MB isolate (once in the 30 days before 2026-08-26). Slices apply + // only to the pinned Microsoft URL — an override (local Graph emulators) + // serves its own document. A missing/failed slice (asset not yet published, + // release unreachable) falls back to the monolith path, which is the prior + // behavior for the selections a slice would have covered. + const sliceAsset = + selection.specUrl === MICROSOFT_GRAPH_OPENAPI_URL + ? microsoftGraphSliceAssetForSelection(selection) + : null; + const sourceText = + sliceAsset !== null + ? yield* fetchMicrosoftGraphSlice(sliceAsset).pipe( + Effect.catchTag("OpenApiParseError", () => + fetchMicrosoftGraphOpenApiSpec(selection.specUrl), + ), + Effect.provide(httpClientLayer), + ) + : yield* fetchMicrosoftGraphOpenApiSpec(selection.specUrl).pipe( + Effect.provide(httpClientLayer), + ); // Structural split is the only entry point: parsing the whole 37MB tree // OOMs the 128MB Workers isolate (measured: HTTP 503). No fallback. A spec diff --git a/packages/plugins/openapi/src/providers/microsoft/slice-build.test.ts b/packages/plugins/openapi/src/providers/microsoft/slice-build.test.ts new file mode 100644 index 0000000000..8425e0be19 --- /dev/null +++ b/packages/plugins/openapi/src/providers/microsoft/slice-build.test.ts @@ -0,0 +1,104 @@ +import { describe, expect, it } from "@effect/vitest"; + +import { structuralSplit } from "../../sdk/split"; +import { buildGraphSliceDocument, parseGraphSourceDocument } from "./slice-build"; + +// Graph-shaped source: a mail path, an unrelated path, and a schema chain +// where only part is reachable from the mail selection. +const source = `openapi: 3.0.4 +info: + title: Microsoft Graph Fixture + version: v1.0 +servers: + - url: https://graph.microsoft.com/v1.0 +paths: + /me/messages: + get: + operationId: me.ListMessages + security: + - azureAdDelegated: + - Mail.ReadWrite + parameters: + - $ref: '#/components/parameters/Top' + responses: + "200": + description: OK + content: + application/json: + schema: + $ref: '#/components/schemas/microsoft.graph.messageCollection' + /irrelevant: + get: + operationId: irrelevant.Get + responses: + "200": + description: OK + content: + application/json: + schema: + $ref: '#/components/schemas/microsoft.graph.unrelated' +components: + parameters: + Top: + name: $top + in: query + schema: + type: integer + securitySchemes: + azureAdDelegated: + type: oauth2 + flows: + authorizationCode: + authorizationUrl: https://login.microsoftonline.com/common/oauth2/v2.0/authorize + tokenUrl: https://login.microsoftonline.com/common/oauth2/v2.0/token + scopes: + Mail.ReadWrite: Read and write mail + schemas: + microsoft.graph.messageCollection: + type: object + properties: + value: + type: array + items: + $ref: '#/components/schemas/microsoft.graph.message' + microsoft.graph.message: + type: object + properties: + id: + type: string + microsoft.graph.unrelated: + type: object + properties: + name: + type: string +`; + +describe("buildGraphSliceDocument", () => { + it("keeps the selection's paths and prunes components to the reachable closure", () => { + const doc = parseGraphSourceDocument(source); + expect(doc).not.toBeNull(); + const slice = buildGraphSliceDocument(doc!, ["mail"]); + + expect(slice.pathCount).toBe(1); + expect(slice.operationCount).toBe(1); + expect(slice.specText).toContain("/me/messages"); + expect(slice.specText).not.toContain("/irrelevant"); + expect(slice.specText).toContain("microsoft.graph.messageCollection"); + expect(slice.specText).toContain("microsoft.graph.message"); + expect(slice.specText).not.toContain("microsoft.graph.unrelated"); + // Referenced small components survive; securitySchemes always survive. + expect(slice.specText).toContain("$top"); + expect(slice.specText).toContain("azureAdDelegated"); + }); + + it("emits the streamable block-YAML profile the runtime splitter accepts", () => { + const doc = parseGraphSourceDocument(source); + expect(doc).not.toBeNull(); + const slice = buildGraphSliceDocument(doc!, ["mail"]); + + const structure = structuralSplit(slice.specText); + expect(structure).not.toBeNull(); + expect(structure!.pathItems).toHaveLength(slice.pathCount); + expect(structure!.schemas).toHaveLength(slice.schemaCount); + }); +}); diff --git a/packages/plugins/openapi/src/providers/microsoft/slice-build.ts b/packages/plugins/openapi/src/providers/microsoft/slice-build.ts new file mode 100644 index 0000000000..8fde215908 --- /dev/null +++ b/packages/plugins/openapi/src/providers/microsoft/slice-build.ts @@ -0,0 +1,193 @@ +import { JSON_SCHEMA, dump as dumpYamlDocument, load as parseYamlDocument } from "js-yaml"; + +import { microsoftGraphKeepPathItem } from "./graph"; +import { + microsoftGraphExactPathsForPresetIds, + microsoftGraphPathPrefixesForPresetIds, + microsoftGraphTagPrefixesForPresetIds, +} from "./presets"; + +/** + * Offline Microsoft Graph slice construction. NOT part of the runtime import + * graph: the 43MB Graph source cannot be held in a 128MB Workers isolate (its + * fetch alone has completed once in the last 30 days of production traces), so + * slices are built where memory is free — the graph-slices workflow / a dev + * machine — published as release assets, and fetched per selection at runtime + * by `slices.ts`. Import this module only from scripts and tests. + */ + +/** `components` subkeys retained whole in a slice — mirrors the runtime + * splitter's `SMALL_COMPONENT_SECTIONS` (`sdk/split.ts`), which keeps these + * resident for `$ref` resolution. `schemas` is pruned to the transitive + * closure instead; `examples` is dropped. */ +const SMALL_COMPONENT_SECTIONS = [ + "parameters", + "requestBodies", + "responses", + "headers", + "links", + "securitySchemes", +] as const; + +const COMPONENT_REF_PREFIX = "#/components/"; + +const isRecord = (value: unknown): value is Record => + value !== null && typeof value === "object" && !Array.isArray(value); + +const decodeRefSegment = (segment: string): string => + segment.replace(/~1/g, "/").replace(/~0/g, "~"); + +interface ComponentRef { + readonly section: string; + readonly name: string; +} + +const collectComponentRefs = (value: unknown, into: (ref: ComponentRef) => void): void => { + if (typeof value === "string") { + if (value.startsWith(COMPONENT_REF_PREFIX)) { + const rest = value.slice(COMPONENT_REF_PREFIX.length); + const slash = rest.indexOf("/"); + if (slash > 0) { + const section = rest.slice(0, slash); + const name = decodeRefSegment(rest.slice(slash + 1)); + if (name.length > 0) into({ section, name }); + } + } + return; + } + if (Array.isArray(value)) { + for (const item of value) collectComponentRefs(item, into); + return; + } + if (isRecord(value)) { + for (const item of Object.values(value)) collectComponentRefs(item, into); + } +}; + +/** + * Transitive `#/components/
/` closure over every component + * section, seeded from `roots` (the kept path-items). Only components actually + * reachable from a kept operation survive — Graph's ~8k schemas and its + * catch-all responses/parameters sections otherwise drag nearly the whole + * component graph into every slice. + */ +const componentClosure = ( + components: Record, + roots: readonly unknown[], +): Record> => { + const kept: Record> = {}; + const queue: ComponentRef[] = []; + const seen = new Set(); + const enqueue = (ref: ComponentRef): void => { + const key = `${ref.section}/${ref.name}`; + if (seen.has(key)) return; + seen.add(key); + queue.push(ref); + }; + + for (const root of roots) collectComponentRefs(root, enqueue); + for (let i = 0; i < queue.length; i += 1) { + const { section, name } = queue[i]!; + const sectionValues = components[section]; + if (!isRecord(sectionValues)) continue; + const component = sectionValues[name]; + if (component === undefined) continue; + (kept[section] ??= {})[name] = component; + collectComponentRefs(component, enqueue); + } + return kept; +}; + +export interface GraphSliceBuild { + /** The slice as streamable block YAML (the same profile `structuralSplit` + * accepts, so the runtime pipeline treats a slice exactly like a source). */ + readonly specText: string; + readonly pathCount: number; + readonly operationCount: number; + readonly schemaCount: number; +} + +const HTTP_METHODS = new Set(["delete", "get", "head", "options", "patch", "post", "put", "trace"]); + +const countOperations = (paths: Record): number => { + let count = 0; + for (const pathItem of Object.values(paths)) { + if (!isRecord(pathItem)) continue; + for (const key of Object.keys(pathItem)) { + if (HTTP_METHODS.has(key.toLowerCase())) count += 1; + } + } + return count; +}; + +/** + * Build one preset selection's slice from the parsed Graph document: keep the + * selection's path-items (via the same `microsoftGraphKeepPathItem` filter the + * runtime applies), retain the small component sections whole, and prune + * `components.schemas` to the transitive `$ref` closure of everything kept. + */ +export const buildGraphSliceDocument = ( + doc: Record, + presetIds: readonly string[], +): GraphSliceBuild => { + const keepPathItem = microsoftGraphKeepPathItem({ + coversFullGraph: false, + presetIds, + customScopes: [], + exactPaths: microsoftGraphExactPathsForPresetIds(presetIds), + pathPrefixes: microsoftGraphPathPrefixesForPresetIds(presetIds), + tagPrefixes: microsoftGraphTagPrefixesForPresetIds(presetIds), + }); + + const sourcePaths = isRecord(doc.paths) ? doc.paths : {}; + const paths: Record = {}; + for (const [path, pathItem] of Object.entries(sourcePaths)) { + if (!isRecord(pathItem)) continue; + const kept = keepPathItem(path, pathItem); + if (kept) paths[path] = kept; + } + + const sourceComponents = isRecord(doc.components) ? doc.components : {}; + const closure = componentClosure(sourceComponents, [paths]); + const components: Record = {}; + for (const section of SMALL_COMPONENT_SECTIONS) { + if (closure[section]) components[section] = closure[section]; + } + // securitySchemes are never `$ref`'d from operations by name in Graph + // (operations reference them via `security` entries), so retain the section + // whole — it is tiny and the runtime reads OAuth endpoints from it. + if (isRecord(sourceComponents.securitySchemes)) { + components.securitySchemes = sourceComponents.securitySchemes; + } + const schemas = closure.schemas ?? {}; + components.schemas = schemas; + + const slice: Record = { + ...(doc.openapi !== undefined ? { openapi: doc.openapi } : {}), + ...(doc.info !== undefined ? { info: doc.info } : {}), + ...(doc.servers !== undefined ? { servers: doc.servers } : {}), + ...(doc.security !== undefined ? { security: doc.security } : {}), + paths, + components, + }; + + // noRefs duplicates shared subtrees instead of emitting YAML anchors, which + // the streamable block profile forbids; lineWidth -1 keeps scalars on one + // line so no wrapped line can be mistaken for structure. + const specText = dumpYamlDocument(slice, { noRefs: true, lineWidth: -1, schema: JSON_SCHEMA }); + + return { + specText, + pathCount: Object.keys(paths).length, + operationCount: countOperations(paths), + schemaCount: Object.keys(schemas).length, + }; +}; + +/** Parse Graph source YAML for slice building, or null when the source does + * not parse to an object. Offline-only: this is the whole-document parse the + * runtime can never do. */ +export const parseGraphSourceDocument = (sourceText: string): Record | null => { + const parsed = parseYamlDocument(sourceText, { json: true, schema: JSON_SCHEMA }); + return isRecord(parsed) ? parsed : null; +}; diff --git a/packages/plugins/openapi/src/providers/microsoft/slices.test.ts b/packages/plugins/openapi/src/providers/microsoft/slices.test.ts new file mode 100644 index 0000000000..6f94a2fceb --- /dev/null +++ b/packages/plugins/openapi/src/providers/microsoft/slices.test.ts @@ -0,0 +1,70 @@ +import { describe, expect, it } from "@effect/vitest"; + +import { MICROSOFT_GRAPH_ALL_PRESET_IDS, MICROSOFT_GRAPH_DEFAULT_PRESET_IDS } from "./presets"; +import { + MICROSOFT_GRAPH_DEFAULT_SLICE_ASSET, + microsoftGraphSliceAssetForSelection, +} from "./slices"; + +describe("microsoftGraphSliceAssetForSelection", () => { + it("maps a single catalog preset to its asset", () => { + expect( + microsoftGraphSliceAssetForSelection({ + coversFullGraph: false, + presetIds: ["mail"], + customScopes: [], + }), + ).toBe("mail"); + }); + + it("maps the default bundle in any order to the default asset", () => { + expect( + microsoftGraphSliceAssetForSelection({ + coversFullGraph: false, + presetIds: [...MICROSOFT_GRAPH_DEFAULT_PRESET_IDS].reverse(), + customScopes: [], + }), + ).toBe(MICROSOFT_GRAPH_DEFAULT_SLICE_ASSET); + }); + + it("serves combinations within the default bundle from the default slice", () => { + expect( + microsoftGraphSliceAssetForSelection({ + coversFullGraph: false, + presetIds: ["mail", "calendar"], + customScopes: [], + }), + ).toBe(MICROSOFT_GRAPH_DEFAULT_SLICE_ASSET); + }); + + it("needs the monolith for full-graph, custom scopes, unknown presets, and combinations outside the default bundle", () => { + expect( + microsoftGraphSliceAssetForSelection({ + coversFullGraph: true, + presetIds: [...MICROSOFT_GRAPH_ALL_PRESET_IDS], + customScopes: [], + }), + ).toBeNull(); + expect( + microsoftGraphSliceAssetForSelection({ + coversFullGraph: false, + presetIds: ["mail"], + customScopes: ["Chat.Read"], + }), + ).toBeNull(); + expect( + microsoftGraphSliceAssetForSelection({ + coversFullGraph: false, + presetIds: ["not-a-preset"], + customScopes: [], + }), + ).toBeNull(); + expect( + microsoftGraphSliceAssetForSelection({ + coversFullGraph: false, + presetIds: ["mail", "users"], + customScopes: [], + }), + ).toBeNull(); + }); +}); diff --git a/packages/plugins/openapi/src/providers/microsoft/slices.ts b/packages/plugins/openapi/src/providers/microsoft/slices.ts new file mode 100644 index 0000000000..f129e36740 --- /dev/null +++ b/packages/plugins/openapi/src/providers/microsoft/slices.ts @@ -0,0 +1,87 @@ +import { Effect } from "effect"; +import { HttpClient, HttpClientRequest } from "effect/unstable/http"; + +import { OpenApiParseError } from "../../sdk/errors"; + +import { MICROSOFT_GRAPH_DEFAULT_PRESET_IDS, microsoftGraphPresetForId } from "./presets"; + +/** + * Runtime access to precomputed Microsoft Graph slices. + * + * The 43MB Graph monolith cannot be processed in a 128MB Workers isolate — in + * production its fetch alone completed once in the 30 days before 2026-08-26; + * every other preview/add died mid-download with an empty 503. Slices are + * built offline (`slice-build.ts`, refreshed by the graph-slices workflow), + * published as release assets, and fetched here per selection: the isolate + * only ever holds the sub-megabyte filtered document for the selection. + */ + +export const MICROSOFT_GRAPH_SLICE_RELEASE_TAG = "graph-slices"; + +export const MICROSOFT_GRAPH_SLICE_BASE_URL = `https://github.com/UsefulSoftwareCo/executor/releases/download/${MICROSOFT_GRAPH_SLICE_RELEASE_TAG}`; + +/** Asset name for the default catalog bundle (`MICROSOFT_GRAPH_DEFAULT_PRESET_IDS`). */ +export const MICROSOFT_GRAPH_DEFAULT_SLICE_ASSET = "default"; + +/** + * The published asset covering a selection, or null when the selection needs + * the monolith. A slice may be a superset of the selection: the runtime always + * applies the selection's `keepPathItem` filter to whatever source it reads, so + * any combination within the default bundle can be served from the default + * slice and narrowed in-band. The monolith remains necessary for full-graph + * coverage, custom scopes (scope matching walks operations outside any + * preset's paths), and combinations reaching outside the default bundle + * (precomputing every combination is combinatorial). + */ +export const microsoftGraphSliceAssetForSelection = (selection: { + readonly coversFullGraph: boolean; + readonly presetIds: readonly string[]; + readonly customScopes: readonly string[]; +}): string | null => { + if (selection.coversFullGraph) return null; + if (selection.customScopes.length > 0) return null; + if (selection.presetIds.length === 0) return null; + if (selection.presetIds.some((presetId) => !microsoftGraphPresetForId(presetId))) return null; + if (selection.presetIds.length === 1) return selection.presetIds[0]!; + const defaultIds = new Set(MICROSOFT_GRAPH_DEFAULT_PRESET_IDS); + if (selection.presetIds.every((presetId) => defaultIds.has(presetId))) { + return MICROSOFT_GRAPH_DEFAULT_SLICE_ASSET; + } + return null; +}; + +export const microsoftGraphSliceUrl = (asset: string): string => + `${MICROSOFT_GRAPH_SLICE_BASE_URL}/${encodeURIComponent(asset)}.yaml`; + +export const fetchMicrosoftGraphSlice = Effect.fn("Microsoft.fetchGraphSlice")(function* ( + asset: string, +) { + const client = yield* HttpClient.HttpClient; + const response = yield* client + .execute( + HttpClientRequest.get(microsoftGraphSliceUrl(asset)).pipe( + HttpClientRequest.setHeader("Accept", "application/yaml, text/yaml, */*"), + ), + ) + .pipe( + Effect.mapError( + () => + new OpenApiParseError({ + message: `Failed to fetch Microsoft Graph slice: ${asset}`, + }), + ), + ); + if (response.status < 200 || response.status >= 300) { + return yield* new OpenApiParseError({ + message: `Failed to fetch Microsoft Graph slice ${asset}: HTTP ${response.status}`, + }); + } + return yield* response.text.pipe( + Effect.mapError( + () => + new OpenApiParseError({ + message: `Failed to read Microsoft Graph slice body: ${asset}`, + }), + ), + ); +}); diff --git a/packages/plugins/openapi/src/providers/microsoft/spec-format-adapter.test.ts b/packages/plugins/openapi/src/providers/microsoft/spec-format-adapter.test.ts index d3da64cd65..44c09c65d9 100644 --- a/packages/plugins/openapi/src/providers/microsoft/spec-format-adapter.test.ts +++ b/packages/plugins/openapi/src/providers/microsoft/spec-format-adapter.test.ts @@ -6,6 +6,7 @@ import { structuralSplit } from "@executor-js/plugin-openapi"; import { previewSpecTextStreaming } from "../../sdk/preview"; import { microsoftGraphAdapter } from "./spec-format-adapter"; import { MICROSOFT_GRAPH_OPENAPI_URL } from "./presets"; +import { microsoftGraphSliceUrl } from "./slices"; const graphFixture = ` openapi: 3.0.4 @@ -92,6 +93,80 @@ it.effect("uses catalog URL fragments to select one Graph workload", () => }), ); +// Distinct content at the slice URL so tests can tell which source was read. +const sliceFixture = `openapi: 3.0.4 +info: + title: Microsoft Graph Slice Fixture + version: v1.0 +servers: + - url: https://graph.microsoft.com/v1.0 +paths: + /me: + get: + operationId: me.GetUser + security: + - azureAdDelegated: + - User.Read + responses: + "200": + description: OK +components: + securitySchemes: + azureAdDelegated: + type: oauth2 + flows: + authorizationCode: + authorizationUrl: https://login.microsoftonline.com/common/oauth2/v2.0/authorize + tokenUrl: https://login.microsoftonline.com/common/oauth2/v2.0/token + scopes: + User.Read: Read user profile +`; + +const sliceAwareHttpClientLayer = Layer.succeed(HttpClient.HttpClient)( + HttpClient.make((request: HttpClientRequest.HttpClientRequest) => { + const body = + request.url === microsoftGraphSliceUrl("profile") + ? sliceFixture + : request.url === MICROSOFT_GRAPH_OPENAPI_URL + ? graphFixture + : null; + return Effect.succeed( + HttpClientResponse.fromWeb( + request, + new Response(body ?? "not found", { status: body === null ? 404 : 200 }), + ), + ); + }), +); + +it.effect("reads the published slice for a covered selection", () => + Effect.gen(function* () { + const converted = yield* microsoftGraphAdapter.fetch({ + urls: [`${MICROSOFT_GRAPH_OPENAPI_URL}#preset=profile`], + httpClientLayer: sliceAwareHttpClientLayer, + }); + + expect(converted.specText).toBe(sliceFixture); + // The catalog URL (with fragment stripped) stays canonical so refresh + // re-resolves through the adapter, not the slice hosting. + expect(converted.specUrl).toBe(MICROSOFT_GRAPH_OPENAPI_URL); + }), +); + +it.effect("falls back to the monolith when the slice asset is unavailable", () => + Effect.gen(function* () { + // graphHttpClientLayer 404s everything except the monolith URL, including + // the slice URL — the existing selection tests above exercise this same + // fallback implicitly. + const converted = yield* microsoftGraphAdapter.fetch({ + urls: [`${MICROSOFT_GRAPH_OPENAPI_URL}#preset=profile`], + httpClientLayer: graphHttpClientLayer, + }); + + expect(converted.specText).toBe(graphFixture); + }), +); + it.effect("stream-previews a Graph selection without a whole-document parse", () => Effect.gen(function* () { const converted = yield* microsoftGraphAdapter.fetch({