diff --git a/.changeset/bright-graphql-results.md b/.changeset/bright-graphql-results.md new file mode 100644 index 000000000..23d46aec0 --- /dev/null +++ b/.changeset/bright-graphql-results.md @@ -0,0 +1,6 @@ +--- +"@executor-js/plugin-graphql": patch +"@executor-js/execution": patch +--- + +Expose bounded GraphQL return shapes and teach agents to request nested rows with explicit `select` clauses. diff --git a/.gitignore b/.gitignore index 4c3f79186..ba8d79278 100644 --- a/.gitignore +++ b/.gitignore @@ -115,3 +115,4 @@ scratch/ # Throwaway UX prototype (not part of the app) ux-demo/ +apps/host-cloudflare/assets/erxes-introspection.json diff --git a/apps/host-cloudflare/README.md b/apps/host-cloudflare/README.md index 970057028..830b2e5b2 100644 --- a/apps/host-cloudflare/README.md +++ b/apps/host-cloudflare/README.md @@ -60,6 +60,11 @@ Now visiting the Worker prompts an Access login; the Worker validates the issued JWT on every request. Unauthenticated requests return 401. MCP clients present an Access JWT or `Cf-Access-Client-Id`/`-Secret` service-token headers. +A Cloudflare OS deployment can also set the same `CLOUDFLARE_OS_AUTH_SECRET` +secret on both Workers. Exempt only `/os/*` from the Access application: those +routes accept short-lived, signed per-user assertions and expose only Erxes +connection provisioning plus MCP. + The Access values are live Worker variables, not values in `wrangler.jsonc`. Wrangler's `keep_vars` option preserves them during later code deploys. Run the command above again whenever you need to change them. diff --git a/apps/host-cloudflare/assets/.gitkeep b/apps/host-cloudflare/assets/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/apps/host-cloudflare/package.json b/apps/host-cloudflare/package.json index b69383427..cc87f7a78 100644 --- a/apps/host-cloudflare/package.json +++ b/apps/host-cloudflare/package.json @@ -3,7 +3,7 @@ "private": true, "type": "module", "scripts": { - "build": "vite build && node scripts/assert-shell-asset.mjs", + "build": "node scripts/fetch-erxes-introspection.mjs && vite build && node scripts/assert-shell-asset.mjs && cp assets/erxes-introspection.json dist/erxes-introspection.json", "deploy": "vite build && node scripts/assert-shell-asset.mjs && wrangler deploy", "dev": "wrangler dev", "dev:web": "vite dev", diff --git a/apps/host-cloudflare/scripts/deploy.sh b/apps/host-cloudflare/scripts/deploy.sh index 130bd5507..2586c73dc 100755 --- a/apps/host-cloudflare/scripts/deploy.sh +++ b/apps/host-cloudflare/scripts/deploy.sh @@ -1,92 +1,129 @@ #!/usr/bin/env bash -# One-shot deploy for the Executor Cloudflare host. +# Deploy one Executor Cloudflare host for a Cloudflare OS instance. # -# Provisions everything a fresh account needs and deploys the Worker: -# 1. verifies wrangler is logged in -# 2. creates (or reuses) the `executor` D1 database and writes its id into -# wrangler.jsonc -# 3. generates + uploads EXECUTOR_SECRET_KEY (the at-rest secret key) if unset -# 4. deploys the Worker -# 5. prints the single manual step: configure the Cloudflare Access application +# The default is the internal erxes demo: +# worker: erxes-os-internal-executor +# domain: executor.os.erxes.io +# D1: erxes-os-internal-executor +# R2: erxes-os-internal-executor-blobs # -# Idempotent — safe to re-run. Run from anywhere: -# bash apps/host-cloudflare/scripts/deploy.sh +# Set INSTANCE_SLUG and EXECUTOR_DOMAIN for another tenant. Every resource name +# follows INSTANCE_SLUG so multiple installations can share one CF account. set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" APP_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" -CONFIG="$APP_DIR/wrangler.jsonc" +SOURCE_CONFIG="$APP_DIR/wrangler.jsonc" +CONFIG="$APP_DIR/wrangler.instance.jsonc" +SECRETS_FILE="${EXECUTOR_SECRETS_FILE:-$APP_DIR/deploy-secrets.json}" cd "$APP_DIR" +EXPECTED_ACCOUNT_ID="7c8392aff8ac4518aa06dfa4b6337ef2" +INSTANCE_SLUG="${INSTANCE_SLUG:-erxes-os-internal}" +EXECUTOR_DOMAIN="${EXECUTOR_DOMAIN:-executor.os.erxes.io}" +WORKER_NAME="${INSTANCE_SLUG}-executor" +DATABASE_NAME="$WORKER_NAME" +BUCKET_NAME="${WORKER_NAME}-blobs" + step() { printf '\n\033[1;36m==> %s\033[0m\n' "$1"; } info() { printf ' %s\n' "$1"; } +die() { printf 'deploy: %s\n' "$1" >&2; exit 1; } + +[ "${CLOUDFLARE_ACCOUNT_ID:-}" = "$EXPECTED_ACCOUNT_ID" ] || die \ + "CLOUDFLARE_ACCOUNT_ID must be pinned to erxes Inc ($EXPECTED_ACCOUNT_ID)" step "Checking wrangler login" -if ! bunx wrangler whoami >/dev/null 2>&1; then - info "Not logged in. Run: bunx wrangler login" - exit 1 +bunx wrangler whoami >/dev/null 2>&1 || die "not logged in; run bunx wrangler login" +info "account: erxes Inc ($EXPECTED_ACCOUNT_ID)" + +step "Provisioning D1 database '$DATABASE_NAME'" +DB_ID="$(bunx wrangler d1 list --json 2>/dev/null \ + | node -e 'let s="";process.stdin.on("data",d=>s+=d).on("end",()=>{try{const r=JSON.parse(s.slice(s.indexOf("["))).find(d=>d.name===process.argv[1]);process.stdout.write(r?r.uuid:"")}catch{}})' "$DATABASE_NAME")" +if [ -z "$DB_ID" ]; then + CREATE_OUT="$(bunx wrangler d1 create "$DATABASE_NAME" 2>&1)" + DB_ID="$(printf '%s' "$CREATE_OUT" | grep -oE '[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}' | head -1)" + info "created: $DB_ID" +else + info "reusing: $DB_ID" fi -info "Logged in." - -step "Provisioning D1 database 'executor'" -# `d1 create` is non-idempotent (errors if it exists), so list first. -EXISTING_ID="$(bunx wrangler d1 list --json 2>/dev/null \ - | node -e 'let s="";process.stdin.on("data",d=>s+=d).on("end",()=>{try{const r=JSON.parse(s).find(d=>d.name==="executor");process.stdout.write(r?r.uuid:"")}catch{}})')" -if [ -n "$EXISTING_ID" ]; then - DB_ID="$EXISTING_ID" - info "Reusing existing database: $DB_ID" +[ -n "$DB_ID" ] || die "failed to resolve D1 database id" + +step "Provisioning R2 bucket '$BUCKET_NAME'" +if bunx wrangler r2 bucket list 2>/dev/null | grep -q "$BUCKET_NAME"; then + info "reusing existing bucket" else - CREATE_OUT="$(bunx wrangler d1 create executor 2>&1)" - DB_ID="$(printf '%s' "$CREATE_OUT" | grep -oE '[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}' | head -1)" - info "Created database: $DB_ID" + bunx wrangler r2 bucket create "$BUCKET_NAME" >/dev/null + info "created" fi -[ -n "$DB_ID" ] || { echo "Failed to resolve D1 database id" >&2; exit 1; } - -step "Writing D1 id into wrangler.jsonc" -# Replace whatever database_id is present (placeholder or a prior id). -node -e ' - const fs=require("fs"),p=process.argv[1],id=process.argv[2]; - let t=fs.readFileSync(p,"utf8"); - t=t.replace(/("database_id":\s*")[^"]*(")/, `$1${id}$2`); - fs.writeFileSync(p,t); -' "$CONFIG" "$DB_ID" -info "wrangler.jsonc -> $DB_ID" - -step "Ensuring EXECUTOR_SECRET_KEY secret" -if bunx wrangler secret list 2>/dev/null | grep -q EXECUTOR_SECRET_KEY; then - info "Secret already set — leaving it." + +step "Generating instance config" +cp "$SOURCE_CONFIG" "$CONFIG" +node - "$CONFIG" "$DB_ID" "$DATABASE_NAME" "$BUCKET_NAME" "$WORKER_NAME" "$EXECUTOR_DOMAIN" <<'NODE' +const fs = require("node:fs"); +const [path, dbId, dbName, bucketName, workerName, domain] = process.argv.slice(2); +let text = fs.readFileSync(path, "utf8"); +text = text.replace(/("name":\s*")[^"]*(")/, `$1${workerName}$2`); +text = text.replace(/("database_name":\s*")[^"]*(")/, `$1${dbName}$2`); +text = text.replace(/("database_id":\s*")[^"]*(")/, `$1${dbId}$2`); +text = text.replace(/("bucket_name":\s*")[^"]*(")/, `$1${bucketName}$2`); +text = text.replace( + /(\s*"main":\s*"src\/worker\.ts",)/, + `$1\n "routes": [{ "pattern": "${domain}", "custom_domain": true }],`, +); +text = text.replace( + /(\s*"vars":\s*{)/, + `$1\n // Direct UI stays fail-closed until an erxes Access app replaces these values.\n` + + ` // Signed /os/* calls from Cloudflare OS verify before Access and work immediately.\n` + + ` "ACCESS_TEAM_DOMAIN": "invalid.cloudflareaccess.com",\n` + + ` "ACCESS_AUD": "not-configured",\n` + + ` "ADMIN_EMAILS": "amaraaamka0404@gmail.com",`, +); +fs.writeFileSync(path, text); +NODE +info "$CONFIG" + +step "Ensuring deployment secrets" +if [ ! -f "$SECRETS_FILE" ]; then + node - "$SECRETS_FILE" <<'NODE' +const { randomBytes } = require("node:crypto"); +const { writeFileSync } = require("node:fs"); +const path = process.argv[2]; +writeFileSync(path, JSON.stringify({ + EXECUTOR_SECRET_KEY: randomBytes(32).toString("hex"), + CLOUDFLARE_OS_AUTH_SECRET: randomBytes(32).toString("hex"), +}, null, 2) + "\n", { mode: 0o600 }); +NODE + info "generated $SECRETS_FILE" else - SECRET="$(node -e 'console.log(require("node:crypto").randomBytes(32).toString("hex"))')" - printf '%s' "$SECRET" | bunx wrangler secret put EXECUTOR_SECRET_KEY >/dev/null - info "Generated + uploaded a fresh 32-byte key." + info "reusing $SECRETS_FILE" fi +chmod 600 "$SECRETS_FILE" + +step "Fetching Erxes introspection snapshot" +node scripts/fetch-erxes-introspection.mjs step "Building the web SPA" bunx vite build +node scripts/assert-shell-asset.mjs +cp assets/erxes-introspection.json dist/erxes-introspection.json +info "bundled erxes-introspection.json into dist/" -step "Deploying Worker" -bunx wrangler deploy - -cat <<'NEXT' - -==> One manual step left: turn on Cloudflare Access (the auth layer) +if [ "${1:-}" = "--dry-run" ]; then + info "dry-run: skipped Worker deploy and secret upload" + exit 0 +fi - The Worker is deployed but is not ready to serve requests until you configure - a Cloudflare Access application. API and MCP requests return 503 and name the - missing variables until configuration is complete. In the Zero Trust - dashboard: +step "Deploying '$WORKER_NAME'" +bunx wrangler deploy -c "$CONFIG" - 1. Access -> Applications -> Add an application -> Self-hosted - 2. Application domain: executor-cloudflare..workers.dev - 3. Add an Access policy (e.g. "Emails ending in @yourcompany.com") - 4. After saving, copy the Application Audience (AUD) tag, then set: - bunx wrangler deploy --var ACCESS_AUD: \ - --var ACCESS_TEAM_DOMAIN:.cloudflareaccess.com \ - --var ADMIN_EMAILS: +step "Uploading secrets" +bunx wrangler secret bulk -c "$CONFIG" < "$SECRETS_FILE" >/dev/null +info "EXECUTOR_SECRET_KEY and CLOUDFLARE_OS_AUTH_SECRET uploaded" - Wrangler preserves these live variables during later code deploys. +cat < { const issuer = `https://${config.accessTeamDomain}`; + const cloudflareOsKey = config.cloudflareOsAuthSecret + ? new TextEncoder().encode(config.cloudflareOsAuthSecret) + : null; // Cached, lazily-fetched team signing keys; jose handles rotation + caching. const jwks = config.enableDevAuth ? null @@ -82,6 +85,38 @@ export const makeAccessVerifier = (config: CloudflareConfig) => { const verify = (request: Request): Effect.Effect => Effect.gen(function* () { + if (cloudflareOsKey) { + const authorization = request.headers.get("Authorization"); + const token = authorization?.startsWith("Bearer ") ? authorization.slice(7) : null; + if (token) { + const verified = yield* Effect.tryPromise({ + try: () => + jwtVerify(token, cloudflareOsKey, { + issuer: "cloudflare-os", + audience: "executor", + algorithms: ["HS256"], + }), + catch: () => "invalid cloudflare os assertion", + }).pipe(Effect.orElseSucceed(() => null)); + const subject = verified?.payload.sub; + const organizationId = verified?.payload.org; + const email = typeof verified?.payload.email === "string" ? verified.payload.email : ""; + if (typeof subject === "string" && typeof organizationId === "string") { + return { + kind: "member", + accountId: subject, + organizationId, + organizationName: organizationId, + organizationSlug: config.organizationSlug, + email, + name: email || null, + avatarUrl: null, + roles: ["member"], + }; + } + } + } + if (config.enableDevAuth) return devPrincipal; if (!jwks) return null; const token = request.headers.get("Cf-Access-Jwt-Assertion"); diff --git a/apps/host-cloudflare/src/config.ts b/apps/host-cloudflare/src/config.ts index c397c4ef8..03ba23860 100644 --- a/apps/host-cloudflare/src/config.ts +++ b/apps/host-cloudflare/src/config.ts @@ -47,6 +47,8 @@ export interface CloudflareEnv { readonly SELF_HOSTED_ORG_SLUG?: string; /** At-rest secret-encryption key (a `wrangler secret`, NOT a var). */ readonly EXECUTOR_SECRET_KEY?: string; + /** Shared secret used to verify short-lived Cloudflare OS user assertions. */ + readonly CLOUDFLARE_OS_AUTH_SECRET?: string; readonly ALLOW_LOCAL_NETWORK?: string; readonly VITE_PUBLIC_SITE_URL?: string; /** @@ -69,6 +71,7 @@ export interface CloudflareConfig { /** URL slug for org-prefixed console paths (`//policies`). */ readonly organizationSlug: string; readonly secretKey: string; + readonly cloudflareOsAuthSecret?: string; readonly allowLocalNetwork: boolean; /** Explicit web base URL (`VITE_PUBLIC_SITE_URL`). Unset on a Worker with no * static URL — the per-request origin is used instead (see RequestWebOrigin). */ @@ -137,6 +140,11 @@ export const loadConfig = (env: CloudflareConfigEnv): CloudflareConfig => { "EXECUTOR_SECRET_KEY must be set (wrangler secret put EXECUTOR_SECRET_KEY) — it encrypts stored secrets at rest in D1", ); } + const cloudflareOsAuthSecret = env.CLOUDFLARE_OS_AUTH_SECRET?.trim(); + if (cloudflareOsAuthSecret && cloudflareOsAuthSecret.length < 32) { + // oxlint-disable-next-line executor/no-try-catch-or-throw, executor/no-error-constructor -- boundary: weak shared auth would let callers forge user identity + throw new Error("CLOUDFLARE_OS_AUTH_SECRET must be at least 32 characters when set"); + } const enableDevAuth = env.ENABLE_DEV_AUTH === "true"; const accessTeamDomain = normalizeAccessTeamDomain(env.ACCESS_TEAM_DOMAIN); const accessAud = (env.ACCESS_AUD ?? "").trim(); @@ -165,6 +173,7 @@ export const loadConfig = (env: CloudflareConfigEnv): CloudflareConfig => { organizationName: env.SELF_HOSTED_ORG_NAME ?? "Default", organizationSlug: resolveOrgSlug(env.SELF_HOSTED_ORG_SLUG), secretKey, + cloudflareOsAuthSecret, allowLocalNetwork: env.ALLOW_LOCAL_NETWORK === "true", // Pinned origin via the shared resolver. A Worker receives no PaaS platform // vars (env: {} — there is nothing to detect), so only the explicit diff --git a/apps/host-cloudflare/src/worker.ts b/apps/host-cloudflare/src/worker.ts index ac9c1b30b..16f2916f2 100644 --- a/apps/host-cloudflare/src/worker.ts +++ b/apps/host-cloudflare/src/worker.ts @@ -40,6 +40,159 @@ const accessConfigErrorResponse = (missingVars: readonly string[]): Response => }, }); +const ERXES_INTEGRATION = "erxes-officenext"; + +const executorRequest = ( + original: Request, + path: string, + method: string, + body?: unknown, +): Request => { + const url = new URL(original.url); + url.pathname = path; + url.search = ""; + const headers = new Headers(); + const authorization = original.headers.get("Authorization"); + if (authorization) headers.set("Authorization", authorization); + if (body !== undefined) headers.set("Content-Type", "application/json"); + return new Request(url, { + method, + headers, + body: body === undefined ? undefined : JSON.stringify(body), + }); +}; + +const ERXES_INTROSPECTION_ASSET = "/erxes-introspection.json"; + +const loadErxesIntrospection = async ( + env: CloudflareEnv, + request: Request, +): Promise => { + const asset = await env.ASSETS.fetch( + new Request(new URL(ERXES_INTROSPECTION_ASSET, request.url)), + ); + if (!asset.ok) return null; + return asset.text(); +}; + +const ensureErxesIntrospection = async ( + request: Request, + app: (request: Request) => Promise, + endpoint: string, + introspectionJson: string, +): Promise => { + const existing = await app( + executorRequest(request, `/api/graphql/integrations/${ERXES_INTEGRATION}`, "GET"), + ); + if (!existing.ok) return existing; + + const integration = await existing.json(); + if (integration === null) { + const created = await app( + executorRequest(request, "/api/graphql/integrations", "POST", { + endpoint, + slug: ERXES_INTEGRATION, + name: "OfficeNext", + description: "OfficeNext Erxes GraphQL API", + introspectionJson, + authenticationTemplate: [ + { + slug: "cookie", + type: "apiKey", + headers: { Cookie: [{ type: "variable", name: "token" }] }, + }, + ], + }), + ); + if (!created.ok && created.status !== 409) return created; + return null; + } + + const attached = await app( + executorRequest( + request, + `/api/graphql/integrations/${ERXES_INTEGRATION}/introspection`, + "POST", + { + introspectionJson, + }, + ), + ); + if (!attached.ok) return attached; + return null; +}; + +const provisionErxes = async ( + request: Request, + app: (request: Request) => Promise, + env: CloudflareEnv, +): Promise => { + if (request.method !== "POST") return new Response("Method Not Allowed", { status: 405 }); + + let input: { endpoint?: unknown; cookie?: unknown }; + // oxlint-disable-next-line executor/no-try-catch-or-throw -- boundary: malformed external JSON becomes a 400 response + try { + input = (await request.json()) as { endpoint?: unknown; cookie?: unknown }; + } catch { + return new Response("Invalid request", { status: 400 }); + } + if ( + typeof input.endpoint !== "string" || + typeof input.cookie !== "string" || + !input.cookie.startsWith("auth-token=") || + input.cookie.includes("\r") || + input.cookie.includes("\n") + ) { + return new Response("Invalid request", { status: 400 }); + } + + const introspectionJson = await loadErxesIntrospection(env, request); + if (introspectionJson != null) { + const integrationError = await ensureErxesIntrospection( + request, + app, + input.endpoint, + introspectionJson, + ); + if (integrationError != null) return integrationError; + } else { + const existing = await app( + executorRequest(request, `/api/graphql/integrations/${ERXES_INTEGRATION}`, "GET"), + ); + if (!existing.ok) return existing; + if ((await existing.json()) === null) { + const created = await app( + executorRequest(request, "/api/graphql/integrations", "POST", { + endpoint: input.endpoint, + slug: ERXES_INTEGRATION, + name: "OfficeNext", + description: "OfficeNext Erxes GraphQL API", + authenticationTemplate: [ + { + slug: "cookie", + type: "apiKey", + headers: { Cookie: [{ type: "variable", name: "token" }] }, + }, + ], + }), + ); + if (!created.ok && created.status !== 409) return created; + } + } + + return app( + executorRequest(request, "/api/connections", "POST", { + owner: "user", + name: ERXES_INTEGRATION, + integration: ERXES_INTEGRATION, + template: "cookie", + value: input.cookie, + identityLabel: "OfficeNext", + description: "Your OfficeNext account", + }), + ); +}; + export default { fetch: async (request: Request, env: CloudflareEnv, ctx: ExecutionContext): Promise => { const missingAccessVars = missingCloudflareAccessVars(env); @@ -48,7 +201,15 @@ export default { } const serve = await resolveHandler(env); - if (new URL(request.url).pathname === "/mcp") { + const url = new URL(request.url); + if (url.pathname === "/os/mcp") { + url.pathname = "/mcp"; + return serve.mcp(new Request(url, request), env, ctx); + } + if (url.pathname === "/os/provision") { + return provisionErxes(request, serve.app, env); + } + if (url.pathname === "/mcp") { return serve.mcp(request, env, ctx); } return serve.app(request); diff --git a/apps/host-cloudflare/wrangler.jsonc b/apps/host-cloudflare/wrangler.jsonc index f97e19507..efff23aa6 100644 --- a/apps/host-cloudflare/wrangler.jsonc +++ b/apps/host-cloudflare/wrangler.jsonc @@ -2,8 +2,11 @@ "$schema": "node_modules/wrangler/config-schema.json", "name": "executor-cloudflare", "compatibility_date": "2025-04-01", - "compatibility_flags": ["nodejs_compat"], + // Hosted integrations must use public routing. Private-origin fetches from an + // Executor custom domain can bypass TLS when the target is in the same zone. + "compatibility_flags": ["nodejs_compat", "global_fetch_strictly_public"], "main": "src/worker.ts", + // routes are injected by scripts/deploy.sh per installation // Access configuration is set per installation after the first deploy. // Preserve those live bindings when this source config does not declare them. "keep_vars": true, @@ -20,7 +23,7 @@ // deployed Worker). "binding": "ASSETS", "not_found_handling": "single-page-application", - "run_worker_first": ["/api/*", "/mcp", "/mcp/*", "/.well-known/*", "/v1", "/v1/*"], + "run_worker_first": ["/api/*", "/mcp", "/mcp/*", "/os/*", "/.well-known/*", "/v1", "/v1/*"], }, // D1 is the app's SQLite store (the DbProvider seam). `wrangler deploy` // auto-provisions it on first deploy; replace database_id after that, or run @@ -29,7 +32,7 @@ { "binding": "DB", "database_name": "executor", - "database_id": "ae748ca1-032c-4427-a1a0-fe39db77d1a9", + "database_id": "8e0f791e-c9fa-4ce0-a3dd-fa36a03ca102", }, ], // Plugin blob seam backend: multi-MB values (resolved OpenAPI specs, @@ -71,12 +74,17 @@ // secret-encryption key) is a SECRET, set it with // `wrangler secret put EXECUTOR_SECRET_KEY`, never in vars. "vars": { + // Keep normal Executor routes fail-closed until a real Access app replaces these values. + // Signed /os/* calls use CLOUDFLARE_OS_AUTH_SECRET instead. + "ACCESS_TEAM_DOMAIN": "darjs.cloudflareaccess.com", + "ACCESS_AUD": "4b0e36b2e0b981ad929a4657fabe04e855deb8bf77636f259f26f73dc822ae88", "ACCESS_NAME_CLAIM": "name", "ACCESS_GROUPS_CLAIM": "groups", + "ADMIN_EMAILS": "darjsavid@gmail.com", // Never preserve a production dev-auth override through keep_vars. "ENABLE_DEV_AUTH": "false", - "SELF_HOSTED_ORG_ID": "default", - "SELF_HOSTED_ORG_NAME": "Default", + "SELF_HOSTED_ORG_ID": "officenext.erxes.io", + "SELF_HOSTED_ORG_NAME": "OfficeNext", // VITE_PUBLIC_SITE_URL is intentionally unset: with no static URL the worker // derives the web base URL from each request's origin (RequestWebOrigin), so // secret/OAuth handoff links match whatever host the user actually reached. diff --git a/packages/core/execution/src/index.ts b/packages/core/execution/src/index.ts index 3e68509cd..ef23ea755 100644 --- a/packages/core/execution/src/index.ts +++ b/packages/core/execution/src/index.ts @@ -14,6 +14,7 @@ export { export { buildExecuteDescription, INTEGRATION_INVENTORY_HEADER } from "./description"; export { EXECUTE_SKILL, + GRAPHQL_SKILL, CREATE_ARTIFACT_SKILL, SKILLS, findSkill, diff --git a/packages/core/execution/src/skills.test.ts b/packages/core/execution/src/skills.test.ts index 54a1c049b..6a49a19dc 100644 --- a/packages/core/execution/src/skills.test.ts +++ b/packages/core/execution/src/skills.test.ts @@ -1,6 +1,13 @@ import { describe, expect, it } from "@effect/vitest"; -import { EXECUTE_SKILL, SKILLS, findSkill, renderSkillsIndex, skillCatalogFor } from "./skills"; +import { + EXECUTE_SKILL, + GRAPHQL_SKILL, + SKILLS, + findSkill, + renderSkillsIndex, + skillCatalogFor, +} from "./skills"; describe("skills registry", () => { it("includes the execute skill with the full how-to body", () => { @@ -15,8 +22,18 @@ describe("skills registry", () => { ); }); + it("teaches execute callers to fetch the GraphQL selection guide", () => { + expect(SKILLS).toContain(GRAPHQL_SKILL); + expect(EXECUTE_SKILL.body).toContain('skills({ name: "graphql" })'); + expect(GRAPHQL_SKILL.body).toContain( + 'select: "list { _id firstName lastName primaryEmail } totalCount"', + ); + expect(GRAPHQL_SKILL.body).toContain("Use the same `select` input in artifact"); + }); + it("finds a skill by exact name and misses unknown names", () => { expect(findSkill("execute")).toBe(EXECUTE_SKILL); + expect(findSkill("graphql")).toBe(GRAPHQL_SKILL); expect(findSkill("Execute")).toBeUndefined(); expect(findSkill("nope")).toBeUndefined(); }); diff --git a/packages/core/execution/src/skills.ts b/packages/core/execution/src/skills.ts index 1036ae697..f2ddde546 100644 --- a/packages/core/execution/src/skills.ts +++ b/packages/core/execution/src/skills.ts @@ -36,8 +36,9 @@ const EXECUTE_SKILL_BODY = [ '2. `const path = matches[0]?.path; if (!path) return "No matching tools found.";`', "3. `const details = await tools.describe.tool({ path });`", "4. Use `details.inputTypeScript` / `details.outputTypeScript` and `details.typeScriptDefinitions` for compact shapes.", - "5. Use `tools.executor.coreTools.connections.list({})` when you need live saved-connection inventory.", - "6. Call the tool: `const result = await tools.(input);`", + '5. If `details.inputTypeScript` includes `select?: string`, fetch `skills({ name: "graphql" })` before calling the tool.', + "6. Use `tools.executor.coreTools.connections.list({})` when you need live saved-connection inventory.", + "7. Call the tool: `const result = await tools.(input);`", "", "## Rules", "", @@ -71,6 +72,52 @@ export const EXECUTE_SKILL: Skill = { body: EXECUTE_SKILL_BODY, }; +const GRAPHQL_SKILL_BODY = [ + "# graphql", + "", + "Call generated GraphQL tools with an explicit return-field selection.", + "", + "## Why `select` matters", + "", + "Generated GraphQL tools select scalar fields on the return type by default, plus one level of item scalars on list-of-object fields. Nested objects and connection fields still need an explicit `select`.", + "", + "`outputTypeScript` describes fields that are available to request. It does not mean every field is returned when `select` is omitted.", + "", + "## Workflow", + "", + "1. Call `tools.describe.tool({ path })`.", + "2. Confirm `inputTypeScript` includes `select?: string`.", + "3. Read the root return shape from `outputTypeScript`.", + "4. Pass the fields you need as GraphQL selection text.", + "5. Read the response under the tool's root field as shown in `outputTypeScript`.", + "", + "```ts", + "const result = await tools[path]({", + " limit: 50,", + ' select: "list { _id firstName lastName primaryEmail } totalCount",', + "});", + "if (!result.ok) return result.error;", + "return result.data.customers;", + "```", + "", + "## Rules", + "", + "- Write fields for the GraphQL return type, not the operation root. Use `list { ... } totalCount`, not `customers { list { ... } }`.", + "- Do not wrap the whole string in braces. Executor adds them.", + "- `select` replaces the default selection. Include scalar fields such as `totalCount` when you still need them.", + "- Give every object or list a sub-selection: `list { _id name }`.", + "- Keep selections small. Ask only for fields the task uses.", + "- Use the same `select` input in artifact `queryOptions(...)` calls so the saved UI receives the rows it renders.", + "- If GraphQL reports `Cannot query field`, compare the selection with `outputTypeScript`; do not guess another field name.", +].join("\n"); + +export const GRAPHQL_SKILL: Skill = { + name: "graphql", + summary: + "How to use `select` with generated GraphQL tools so nested objects and list rows are returned, with the same input in artifacts.", + body: GRAPHQL_SKILL_BODY, +}; + // The `create-artifact` how-to. Same reasoning as `execute`: the discovery-vs-render // protocol, the TanStack rules and the component inventory are a page of prose // that only matters once a model decides to build a UI, so the tool description @@ -615,6 +662,7 @@ export const ARTIFACT_STYLE_SKILL: Skill = { /** The full skill catalog. Hand-curated; keep it small. */ export const SKILLS: readonly Skill[] = [ EXECUTE_SKILL, + GRAPHQL_SKILL, CREATE_ARTIFACT_SKILL, ARTIFACT_STYLE_SKILL, ]; diff --git a/packages/core/execution/src/tool-invoker.test.ts b/packages/core/execution/src/tool-invoker.test.ts index dd25e6168..1409f39c5 100644 --- a/packages/core/execution/src/tool-invoker.test.ts +++ b/packages/core/execution/src/tool-invoker.test.ts @@ -249,6 +249,34 @@ const crmPlugin = makeTestPlugin({ ], }); +const crmAltPlugin = makeTestPlugin({ + pluginId: "crm-alt-test", + integration: "crm_alt", + tools: [ + { + name: "createContact", + description: "Create a contact in the other CRM", + inputJsonSchema: ContactInputJson, + validator: ContactValidator, + handler: () => Effect.succeed({ id: "contact_alt" }), + }, + ], +}); + +const dottedNamePlugin = makeTestPlugin({ + pluginId: "dotted-name-test", + integration: "catalog", + tools: [ + { + name: "query.records", + description: "List records", + inputJsonSchema: EmptyInputJson, + validator: EmptyValidator, + handler: () => Effect.succeed({ totalCount: 1 }), + }, + ], +}); + const errorPlugin = makeTestPlugin({ pluginId: "error-test", integration: "records", @@ -1110,6 +1138,72 @@ describe("tool discovery", () => { }), ); + it.effect("resolves a unique short tool name to the qualified path", () => + Effect.gen(function* () { + const executor = yield* makeSearchExecutor(); + const invoker = makeExecutorToolInvoker(executor, { + invokeOptions: { onElicitation: acceptAll }, + }); + + const result = yield* invoker.invoke({ + path: "createContact", + args: { email: "a@b.com" }, + }); + expect(result).toEqual({ ok: true, data: { id: "contact_1" } }); + + const described = yield* describeTool(executor, "createContact"); + expect(described.path).toBe("crm.org.main.createContact"); + expect(described.error).toBeUndefined(); + }), + ); + + it.effect("resolves a unique dotted short name", () => + Effect.gen(function* () { + const executor = yield* makeExecutorWith([dottedNamePlugin] as const); + yield* provision(executor as never, [ + { pluginId: "dotted-name-test", integration: "catalog" }, + ]); + const invoker = makeExecutorToolInvoker(executor, { + invokeOptions: { onElicitation: acceptAll }, + }); + + const result = yield* invoker.invoke({ path: "query.records", args: {} }); + expect(result).toEqual({ ok: true, data: { totalCount: 1 } }); + }), + ); + + it.effect("does not guess when a short name matches more than one tool", () => + Effect.gen(function* () { + const executor = yield* makeExecutorWith([crmPlugin, crmAltPlugin] as const); + yield* provision(executor as never, [ + { pluginId: "crm-test", integration: "crm" }, + { pluginId: "crm-alt-test", integration: "crm_alt" }, + ]); + const invoker = makeExecutorToolInvoker(executor, { + invokeOptions: { onElicitation: acceptAll }, + }); + + const result = yield* invoker.invoke({ + path: "createContact", + args: { email: "a@b.com" }, + }); + expect(result).toMatchObject({ + ok: false, + error: { + code: "tool_not_found", + details: { + path: "createContact", + }, + }, + }); + const suggestions = (result as { error: { details: { suggestions: string[] } } }).error + .details.suggestions; + expect(suggestions).toEqual( + expect.arrayContaining(["crm.org.main.createContact", "crm_alt.org.main.createContact"]), + ); + }), + ); + it.effect("returns user-actionable typed errors as ToolResult.fail", () => Effect.gen(function* () { const executor = yield* makeExecutorWith([userActionableErrorPlugin] as const); diff --git a/packages/core/execution/src/tool-invoker.ts b/packages/core/execution/src/tool-invoker.ts index 4da925176..24b831274 100644 --- a/packages/core/execution/src/tool-invoker.ts +++ b/packages/core/execution/src/tool-invoker.ts @@ -299,17 +299,74 @@ const extractNamespace = (path: string): string => { * because it would require an `integrations.list()` lookup on every invocation. * Callers that already know the integration kind can annotate at their own span. */ +type ResolvedSandboxPath = + | { readonly kind: "ok"; readonly path: string } + | { readonly kind: "ambiguous"; readonly suggestions: readonly string[] }; + +const matchesRequestedTool = ( + tool: { readonly path: string; readonly name: string }, + requested: string, +): boolean => + tool.name === requested || tool.path === requested || tool.path.endsWith(`.${requested}`); + +/** Fully-qualified sandbox paths parse as `tools....`. + * Short names (`query.records`, `createContact`) do not, and 404 unless rewritten. */ +const resolveSandboxToolPath = Effect.fn("executor.tools.resolvePath")(function* ( + executor: Executor, + path: string, +) { + if (parseToolAddress(String(pathToAddress(path)))) { + return { kind: "ok", path } satisfies ResolvedSandboxPath; + } + + const all = yield* executor.tools.list({ includeAnnotations: false }).pipe( + Effect.mapError( + (cause) => + new ExecutionToolError({ + message: "Failed to list tools for path resolution", + cause, + }), + ), + ); + const matches = all + .map((tool) => ({ path: addressToPath(String(tool.address)), name: String(tool.name) })) + .filter((tool) => matchesRequestedTool(tool, path)); + + if (matches.length === 1) { + return { kind: "ok", path: matches[0]!.path } satisfies ResolvedSandboxPath; + } + if (matches.length > 1) { + return { + kind: "ambiguous", + suggestions: matches.map((tool) => tool.path), + } satisfies ResolvedSandboxPath; + } + return { kind: "ok", path } satisfies ResolvedSandboxPath; +}); + export const makeExecutorToolInvoker = ( executor: Executor, options: { readonly invokeOptions: InvokeOptions }, ): SandboxToolInvoker => ({ invoke: Effect.fn("mcp.tool.dispatch")(function* ({ path, args }) { + const resolved = yield* resolveSandboxToolPath(executor, path); + if (resolved.kind === "ambiguous") { + const result = ToolResult.fail({ + code: "tool_not_found", + message: `Tool not found: ${path}`, + details: { path, suggestions: resolved.suggestions }, + }); + yield* annotateToolResultOutcome(result); + return result; + } + const dispatchPath = resolved.path; + yield* Effect.annotateCurrentSpan({ - "mcp.tool.name": path, - "mcp.tool.integration": extractNamespace(path), + "mcp.tool.name": dispatchPath, + "mcp.tool.integration": extractNamespace(dispatchPath), }); - const address = pathToAddress(path); + const address = pathToAddress(dispatchPath); const result = yield* executor.execute(address, args, options.invokeOptions).pipe( Effect.catchTag("CredentialResolutionError", (err) => Effect.succeed( @@ -345,7 +402,7 @@ export const makeExecutorToolInvoker = ( return Effect.logError("tool dispatch failed", cause).pipe( Effect.annotateLogs({ "executor.correlation_id": correlationId, - "mcp.tool.name": path, + "mcp.tool.name": dispatchPath, }), Effect.flatMap(() => Effect.fail( @@ -825,6 +882,20 @@ export const describeTool = Effect.fn("executor.tools.describe")(function* ( const builtin = BUILTIN_TOOL_DESCRIPTIONS.get(path); if (builtin) return builtin; + const resolved = yield* resolveSandboxToolPath(executor, path); + if (resolved.kind === "ambiguous") { + return { + path, + name: path, + error: { + code: "tool_not_found", + message: `Tool not found: ${path}`, + suggestions: resolved.suggestions, + }, + } satisfies DescribedTool; + } + path = resolved.path; + const address = pathToAddress(path); // Single tools.schema() call — it already fetches the tool row diff --git a/packages/core/fumadb/src/adapters/drizzle/query.ts b/packages/core/fumadb/src/adapters/drizzle/query.ts index 9471ecda0..80a67ce04 100644 --- a/packages/core/fumadb/src/adapters/drizzle/query.ts +++ b/packages/core/fumadb/src/adapters/drizzle/query.ts @@ -21,6 +21,12 @@ type P_DBType = PostgreSQL.PgDatabase< >; const CREATE_MANY_BATCH_SIZE = 500; +// Cloudflare D1 accepts up to 1000 statements per `batch()` call. +const D1_BATCH_STATEMENT_LIMIT = 1000; + +type D1BatchDb = { + batch(queries: T): Promise<{ readonly [K in keyof T]: unknown }>; +}; function buildWhere( toDrizzle: (col: AnyColumn) => ColumnType, @@ -372,6 +378,25 @@ export function fromDrizzle( if (provider === "sqlite" || provider === "postgresql") { const out: { _id: unknown }[] = []; + const d1Batch = (db as Partial).batch; + if (provider === "sqlite" && typeof d1Batch === "function") { + for (let i = 0; i < batches.length; i += D1_BATCH_STATEMENT_LIMIT) { + const slice = batches.slice(i, i + D1_BATCH_STATEMENT_LIMIT); + const queries = slice.map((batch) => + (db as unknown as P_DBType) + .insert(drizzleTable as unknown as P_TableType) + .values(batch) + .returning({ + _id: (drizzleTable as unknown as P_TableType)[idField], + }), + ); + const results = await d1Batch.call(db, queries); + for (const result of results) { + out.push(...(result as { _id: unknown }[])); + } + } + return out; + } for (const batch of batches) { out.push( ...(await (db as unknown as P_DBType) diff --git a/packages/core/sdk/src/connections.test.ts b/packages/core/sdk/src/connections.test.ts index 7ed046f65..dab82d19e 100644 --- a/packages/core/sdk/src/connections.test.ts +++ b/packages/core/sdk/src/connections.test.ts @@ -700,6 +700,37 @@ describe("connections.checkHealth", () => { expect(result.status).toBe("unknown"); }), ); + + it.effect("re-saving a connection with a fresh catalog skips tool reproduction", () => + Effect.gen(function* () { + const executor = yield* setup(); + yield* executor.connections.create({ + owner: "org", + name: ConnectionName.make("main"), + integration: INTEG, + template: TEMPLATE, + value: "secret-token", + }); + const toolsBefore = yield* executor.tools.list({ integration: INTEG }); + expect(toolsBefore.length).toBeGreaterThan(0); + + yield* executor.connections.create({ + owner: "org", + name: ConnectionName.make("main"), + integration: INTEG, + template: TEMPLATE, + value: "rotated-token", + }); + + const toolsAfter = yield* executor.tools.list({ integration: INTEG }); + expect(toolsAfter.map((tool) => String(tool.name)).sort()).toEqual( + toolsBefore.map((tool) => String(tool.name)).sort(), + ); + + const value = yield* executor.demo.resolveValue("org", "main"); + expect(value).toBe("rotated-token"); + }), + ); }); describe("execute over a connection", () => { diff --git a/packages/core/sdk/src/executor.ts b/packages/core/sdk/src/executor.ts index 56cb2e299..e19de3110 100644 --- a/packages/core/sdk/src/executor.ts +++ b/packages/core/sdk/src/executor.ts @@ -2506,6 +2506,20 @@ export const createExecutor = result.incompleteReason ?? "plugin returned an incomplete tool catalog"; + const connectionCatalogIsFresh = ( + connection: ConnectionRow, + integrationRow: IntegrationRow, + ): boolean => { + if (connection.tools_synced_at == null) return false; + const syncedAt = Number(connection.tools_synced_at); + const revisedAt = + integrationRow.config_revised_at == null ? null : Number(integrationRow.config_revised_at); + if (revisedAt !== null && syncedAt < revisedAt) return false; + const health = Option.getOrNull(decodeLastHealth(connection.last_health)); + if (health?.detail?.startsWith(toolSyncHealthDetailPrefix) === true) return false; + return true; + }; + const produceConnectionTools = ( integrationRow: IntegrationRow, ref: ConnectionRef, @@ -2814,13 +2828,14 @@ export const createExecutor = storageFailureFromUnknown("invalid owner", cause), }); const now = new Date(); + const existingBeforeUpsert = yield* findConnectionRow({ + owner: input.owner, + integration: input.integration, + name, + }); yield* transaction( Effect.gen(function* () { - const existing = yield* findConnectionRow({ - owner: input.owner, - integration: input.integration, - name, - }); + const existing = existingBeforeUpsert; const set: Record = { template: String(input.template), provider: providerKey, @@ -2880,10 +2895,26 @@ export const createExecutor = Effect.succeed([] as readonly Tool[])), - ); + const skipToolProduction = + existingBeforeUpsert != null && + connectionCatalogIsFresh(existingBeforeUpsert, integrationRow) && + (yield* core.findMany("tool", { + where: (b: AnyCb) => + b.and( + byOwner(input.owner)(b), + b("integration", "=", String(input.integration)), + b("connection", "=", String(name)), + ), + limit: 1, + })).length > 0; + if (!skipToolProduction) { + // Produce + persist tools for the new connection. + yield* produceConnectionTools(integrationRow, ref).pipe( + Effect.catchTag("IntegrationNotFoundError", () => + Effect.succeed([] as readonly Tool[]), + ), + ); + } const row = yield* findConnectionRow(ref); return row diff --git a/packages/hosts/mcp/src/tool-server.ts b/packages/hosts/mcp/src/tool-server.ts index 41c61bcd0..0e437cf38 100644 --- a/packages/hosts/mcp/src/tool-server.ts +++ b/packages/hosts/mcp/src/tool-server.ts @@ -1491,6 +1491,7 @@ export const createExecutorMcpServer = ( { description, inputSchema: { code: z.string().trim().min(1) }, + annotations: { readOnlyHint: true }, }, ({ code }, extra) => runToolEffect(executeCode(code, extra)), ), @@ -1515,6 +1516,7 @@ export const createExecutorMcpServer = ( .optional() .describe('The skill to fetch, e.g. "execute". Omit to list available skills.'), }, + annotations: { readOnlyHint: true }, }, ({ name }) => runToolEffect(Effect.succeed(skillsResult(name, executeInventory, skillCatalog))), @@ -1548,6 +1550,7 @@ export const createExecutorMcpServer = ( .describe("Optional JSON-encoded response content for form elicitations") .default("{}"), }, + annotations: { readOnlyHint: true }, }, ({ executionId, action, content: rawContent }, extra) => runToolEffect( diff --git a/packages/plugins/graphql/src/api/group.ts b/packages/plugins/graphql/src/api/group.ts index 363038ef5..7e7f6c92f 100644 --- a/packages/plugins/graphql/src/api/group.ts +++ b/packages/plugins/graphql/src/api/group.ts @@ -1,6 +1,10 @@ import { HttpApiEndpoint, HttpApiGroup } from "effect/unstable/httpapi"; import { Schema } from "effect"; -import { InternalError, IntegrationAlreadyExistsError } from "@executor-js/sdk/shared"; +import { + InternalError, + IntegrationAlreadyExistsError, + IntegrationNotFoundError, +} from "@executor-js/sdk/shared"; import { GraphqlIntrospectionError, GraphqlExtractionError } from "../sdk/errors"; import { GraphqlAuthMethod, GraphqlAuthMethodInput } from "../sdk/types"; @@ -64,6 +68,16 @@ const ConfigureResponse = Schema.Struct({ authenticationTemplate: Schema.Array(GraphqlAuthMethod), }); +const AttachIntrospectionPayload = Schema.Struct({ + introspectionJson: Schema.String, +}); + +const AttachIntrospectionResponse = Schema.Struct({ + slug: Schema.String, + name: Schema.String, + toolCount: Schema.Number, +}); + // --------------------------------------------------------------------------- // Errors with HTTP status // --------------------------------------------------------------------------- @@ -87,6 +101,7 @@ const GraphqlErrors = [ IntrospectionError, ExtractionError, IntegrationAlreadyExistsError, + IntegrationNotFoundError, ] as const; export const GraphqlGroup = HttpApiGroup.make("graphql") @@ -118,4 +133,12 @@ export const GraphqlGroup = HttpApiGroup.make("graphql") success: ConfigureResponse, error: GraphqlErrors, }), + ) + .add( + HttpApiEndpoint.post("attachIntrospection", "/graphql/integrations/:slug/introspection", { + params: IntegrationParams, + payload: AttachIntrospectionPayload, + success: AttachIntrospectionResponse, + error: GraphqlErrors, + }), ); diff --git a/packages/plugins/graphql/src/api/handlers.ts b/packages/plugins/graphql/src/api/handlers.ts index 325924f0c..ff25b9d1a 100644 --- a/packages/plugins/graphql/src/api/handlers.ts +++ b/packages/plugins/graphql/src/api/handlers.ts @@ -86,5 +86,13 @@ export const GraphqlHandlers = HttpApiBuilder.group(ExecutorApiWithGraphql, "gra return { authenticationTemplate: [...authenticationTemplate] }; }), ), + ) + .handle("attachIntrospection", ({ params: path, payload }) => + capture( + Effect.gen(function* () { + const ext = yield* GraphqlExtensionService; + return yield* ext.attachIntrospectionSnapshot(path.slug, payload.introspectionJson); + }), + ), ), ); diff --git a/packages/plugins/graphql/src/sdk/invoke.ts b/packages/plugins/graphql/src/sdk/invoke.ts index b3ae55e20..0475484fb 100644 --- a/packages/plugins/graphql/src/sdk/invoke.ts +++ b/packages/plugins/graphql/src/sdk/invoke.ts @@ -35,18 +35,25 @@ const formatTimeout = (timeoutMs: number): string => const invocationTimeoutMessage = (timeoutMs: number): string => `GraphQL upstream did not complete within ${formatTimeout(timeoutMs)}. The request was aborted. Retry the operation or verify that the endpoint is responsive.`; +const unwrapOuterSelectionBraces = (select: string): string => { + const trimmed = select.trim(); + return trimmed.startsWith("{") && trimmed.endsWith("}") ? trimmed.slice(1, -1).trim() : trimmed; +}; + /** The operation string to send for a call. A caller-supplied `select` overrides * the default scalar-leaf selection: it is spliced into the field's selection * set (`field {