diff --git a/.changeset/worker-bundler-staging-contract.md b/.changeset/worker-bundler-staging-contract.md new file mode 100644 index 0000000000..97f656ec97 --- /dev/null +++ b/.changeset/worker-bundler-staging-contract.md @@ -0,0 +1,11 @@ +--- +"executor": patch +--- + +**The packed Worker toolchain is verified at build time, and an incomplete copy is now reported instead of silently ignored** + +`@cloudflare/worker-bundler` cannot live inside the compiled binary: bunfs has no `node_modules`, so a bare specifier is unresolvable there by construction. The build instead copies the package's `dist/` next to the executable and `native-bindings.ts` publishes that path as `EXECUTOR_WORKER_BUNDLER_DIR` for consumers to load from. + +That handoff was described in two places that were free to drift, and did. The build writes `dist/index.bundled.js` — the entry consumers actually load, packed so it has no bare imports of its own — while the runtime check only looked for `dist/index.js` and `dist/esbuild.wasm`. Nothing verified the staged copy after the compile, so a partial staging produced a binary that looked fine on the build machine and failed on the user's, at startup. Worse, the runtime check failed open: when a file was missing it silently declined to set the environment variable, leaving a consumer to fall through to the bare specifier and crash. + +The required file list is now one shared contract used by both sides, so they cannot disagree. The build asserts the staged copy after compiling each target — every required file present, a size floor on the packed entry, and the `\0asm` magic on the wasm so a truncated copy cannot pass — turning a packaging slip into a failed build rather than a broken install. At runtime, a directory that is present but incomplete is reported on stderr naming the missing files, instead of being swallowed. An absent directory stays quiet, since that is the normal non-packaged path. diff --git a/apps/cli/src/build.ts b/apps/cli/src/build.ts index eef1b69583..6c1f701383 100644 --- a/apps/cli/src/build.ts +++ b/apps/cli/src/build.ts @@ -5,6 +5,7 @@ import { resolve, join, dirname } from "node:path"; import { fileURLToPath, pathToFileURL } from "node:url"; import { parseArgs } from "node:util"; import { $ } from "bun"; +import { WORKER_BUNDLER_DIRNAME, missingWorkerBundlerFiles } from "./worker-bundler-artifact"; const repoRoot = resolve(dirname(fileURLToPath(import.meta.url)), "../../.."); const cliRoot = resolve(repoRoot, "apps/cli"); @@ -12,7 +13,6 @@ const webRoot = resolve(repoRoot, "apps/local"); const distDir = resolve(cliRoot, "dist"); const ONEPASSWORD_CORE_WASM_FILENAME = "onepassword-core_bg.wasm"; const WORKERD_VERSION = "1.20260708.1"; -const WORKER_BUNDLER_DIRNAME = "worker-bundler"; const resolveQuickJsWasmPath = (): string => { const req = createRequire(join(repoRoot, "packages/kernel/runtime-quickjs/package.json")); @@ -89,6 +89,54 @@ const createPackedWorkerBundlerSource = async (distPath: string): Promise => { + const stagedDir = join(binDir, WORKER_BUNDLER_DIRNAME); + + const missing = missingWorkerBundlerFiles(stagedDir, existsSync); + if (missing.length > 0) { + throw new Error( + `Packed @cloudflare/worker-bundler is incomplete for ${targetName}: missing ${missing.join(", ")} under ${stagedDir}. ` + + `The compiled binary resolves this package from disk, not from bunfs, so shipping it partial means a runtime failure on the user's machine.`, + ); + } + + const bundledEntry = join(stagedDir, "dist", "index.bundled.js"); + const bundledSize = (await Bun.file(bundledEntry).arrayBuffer()).byteLength; + if (bundledSize < 100_000) { + throw new Error( + `Packed worker-bundler entry for ${targetName} is implausibly small (${bundledSize} bytes at ${bundledEntry}). Expected the esbuild-packed bundle.`, + ); + } + + const wasmPath = join(stagedDir, "dist", "esbuild.wasm"); + const wasmHead = new Uint8Array((await Bun.file(wasmPath).arrayBuffer()).slice(0, 4)); + if ( + wasmHead[0] !== 0x00 || + wasmHead[1] !== 0x61 || + wasmHead[2] !== 0x73 || + wasmHead[3] !== 0x6d + ) { + throw new Error( + `Staged esbuild.wasm for ${targetName} is not a WebAssembly module (bad magic at ${wasmPath}).`, + ); + } +}; + // --------------------------------------------------------------------------- // Metadata // --------------------------------------------------------------------------- @@ -487,6 +535,7 @@ const buildBinaries = async (targets: Target[], mode: BuildMode) => { join(binDir, WORKER_BUNDLER_DIRNAME, "dist", "index.bundled.js"), packedWorkerBundlerSource, ); + await assertPackedWorkerBundler(binDir, name); // Smoke test on current platform if (isCurrentPlatform(target)) { diff --git a/apps/cli/src/native-bindings.ts b/apps/cli/src/native-bindings.ts index 386d43ef69..ed79fb89e2 100644 --- a/apps/cli/src/native-bindings.ts +++ b/apps/cli/src/native-bindings.ts @@ -17,6 +17,8 @@ import { existsSync } from "node:fs"; import { dirname, join } from "node:path"; +import { WORKER_BUNDLER_DIRNAME, missingWorkerBundlerFiles } from "./worker-bundler-artifact"; + const execDir = dirname(process.execPath); // libSQL: our `libsql` patch reads EXECUTOR_LIBSQL_NATIVE_PATH and loads the @@ -48,12 +50,27 @@ if (typeof Bun !== "undefined" && !process.env.EXECUTOR_WORKERD_BIN && existsSyn process.env.EXECUTOR_WORKERD_BIN = workerdOnDisk; } -const workerBundlerOnDisk = join(execDir, "worker-bundler"); +// worker-bundler: the compiled binary cannot resolve `@cloudflare/worker-bundler` +// by name (bunfs has no node_modules), so build.ts stages the package's dist +// beside the executable and we publish its path here. An absent directory is +// normal off the packaged path (dev, `bun run`), so that stays quiet — but a +// directory that is PRESENT AND INCOMPLETE is a broken install, and swallowing +// it is what turns a packaging slip into an unresolvable bare import at +// startup. Report it on stderr instead of failing open. +const workerBundlerOnDisk = join(execDir, WORKER_BUNDLER_DIRNAME); if ( typeof Bun !== "undefined" && !process.env.EXECUTOR_WORKER_BUNDLER_DIR && - existsSync(join(workerBundlerOnDisk, "dist", "index.js")) && - existsSync(join(workerBundlerOnDisk, "dist", "esbuild.wasm")) + existsSync(workerBundlerOnDisk) ) { - process.env.EXECUTOR_WORKER_BUNDLER_DIR = workerBundlerOnDisk; + const missing = missingWorkerBundlerFiles(workerBundlerOnDisk, existsSync); + if (missing.length === 0) { + process.env.EXECUTOR_WORKER_BUNDLER_DIR = workerBundlerOnDisk; + } else { + process.stderr.write( + `executor: the bundled Worker toolchain at ${workerBundlerOnDisk} is incomplete ` + + `(missing ${missing.join(", ")}). Features that build Workers will be unavailable; ` + + `reinstall or update executor to repair it.\n`, + ); + } } diff --git a/apps/cli/src/worker-bundler-artifact.test.ts b/apps/cli/src/worker-bundler-artifact.test.ts new file mode 100644 index 0000000000..1d63070f33 --- /dev/null +++ b/apps/cli/src/worker-bundler-artifact.test.ts @@ -0,0 +1,59 @@ +import { describe, expect, it } from "@effect/vitest"; +import { join } from "node:path"; + +import { + WORKER_BUNDLER_DIRNAME, + WORKER_BUNDLER_RUNTIME_FILES, + missingWorkerBundlerFiles, +} from "./worker-bundler-artifact"; + +const STAGED_DIR = join("/opt/executor/bin", WORKER_BUNDLER_DIRNAME); + +/** An existence probe that reports only the given POSIX-relative paths. */ +const staged = + (...relatives: readonly string[]) => + (path: string): boolean => + relatives.some((relative) => path === join(STAGED_DIR, ...relative.split("/"))); + +describe("missingWorkerBundlerFiles", () => { + it("reports nothing when the full artifact is staged", () => { + expect(missingWorkerBundlerFiles(STAGED_DIR, staged(...WORKER_BUNDLER_RUNTIME_FILES))).toEqual( + [], + ); + }); + + it("reports every file when the directory is empty", () => { + expect(missingWorkerBundlerFiles(STAGED_DIR, () => false)).toEqual([ + ...WORKER_BUNDLER_RUNTIME_FILES, + ]); + }); + + // The regression this whole seam exists for: the packaged daemon reads the + // wasm from disk beside the entry. A copy that brought the JS but dropped the + // wasm used to pass the build and fail on the user's machine at startup. + it("reports the esbuild wasm when only the JS entrypoints are staged", () => { + expect( + missingWorkerBundlerFiles(STAGED_DIR, staged("dist/index.js", "dist/index.bundled.js")), + ).toEqual(["dist/esbuild.wasm"]); + }); + + // `index.bundled.js` is the file consumers load — the package's own + // `dist/index.js` still has bare imports nothing can resolve inside the + // compiled binary. Staging index.js alone is not enough. + it("reports the packed entry when only the unbundled package entry is staged", () => { + expect( + missingWorkerBundlerFiles(STAGED_DIR, staged("dist/index.js", "dist/esbuild.wasm")), + ).toEqual(["dist/index.bundled.js"]); + }); + + it("names files relative to the staged directory so errors are quotable", () => { + for (const relative of WORKER_BUNDLER_RUNTIME_FILES) { + expect(relative.startsWith("dist/")).toBe(true); + } + }); + + it("requires the packed entry and the wasm, not just the package entry", () => { + expect(WORKER_BUNDLER_RUNTIME_FILES).toContain("dist/index.bundled.js"); + expect(WORKER_BUNDLER_RUNTIME_FILES).toContain("dist/esbuild.wasm"); + }); +}); diff --git a/apps/cli/src/worker-bundler-artifact.ts b/apps/cli/src/worker-bundler-artifact.ts new file mode 100644 index 0000000000..b541515084 --- /dev/null +++ b/apps/cli/src/worker-bundler-artifact.ts @@ -0,0 +1,55 @@ +// --------------------------------------------------------------------------- +// The packed `@cloudflare/worker-bundler` artifact that ships NEXT TO the +// compiled binary. +// +// `bun build --compile` cannot embed a package that is only ever addressed by +// bare specifier at runtime: bunfs has no `node_modules`, so an +// `import("@cloudflare/worker-bundler")` evaluated inside the binary resolves +// against the virtual root and throws +// ResolveMessage: Cannot find module '@cloudflare/worker-bundler' +// The build therefore copies the package's `dist/` next to the executable and +// `native-bindings.ts` publishes its absolute path as +// EXECUTOR_WORKER_BUNDLER_DIR. Consumers must load from that directory; a bare +// specifier is unresolvable in the packaged daemon by construction. +// +// This module is the single source of truth for which files that contract +// needs, so the build-time staging check and the runtime lookup cannot drift. +// They previously did: the build wrote `dist/index.bundled.js` (the file +// consumers actually load) while the runtime check only looked for +// `dist/index.js` and `dist/esbuild.wasm`. That drift fails OPEN — an +// incomplete staged copy passes the build, the runtime silently declines to +// publish the env var, and the consumer falls through to the bare specifier. +// --------------------------------------------------------------------------- + +import { join } from "node:path"; + +/** Directory name staged beside the executable, and looked up from execDir. */ +export const WORKER_BUNDLER_DIRNAME = "worker-bundler"; + +/** + * Files the colocated copy must contain to be usable. + * + * `index.bundled.js` is the entrypoint consumers load (the build packs it with + * esbuild so it has no bare-specifier imports of its own); `esbuild.wasm` is + * read at runtime beside it; `index.js` is the package's own entry, kept so the + * staged directory stays a faithful copy rather than a lookalike. + */ +export const WORKER_BUNDLER_RUNTIME_FILES = [ + "dist/index.js", + "dist/index.bundled.js", + "dist/esbuild.wasm", +] as const; + +/** + * Which required files are absent from a staged worker-bundler directory. + * + * Takes the existence probe as a parameter so the build script, the runtime + * bootstrap, and tests all agree on the requirement without touching a real + * filesystem. Returns the POSIX-relative names, which is what error messages + * should quote. + */ +export const missingWorkerBundlerFiles = ( + dir: string, + exists: (path: string) => boolean, +): readonly string[] => + WORKER_BUNDLER_RUNTIME_FILES.filter((relative) => !exists(join(dir, ...relative.split("/"))));