History

Package workerd for local binaries

Rhys Sullivan committed 4 months ago

Commit 4be58bc1 parent da4f37c8 files · +491 −56Browse files

Unified · Split

8 files
  1. apps/cli/scripts/smoke-packed-apps.ts +192 −0
  2. apps/cli/src/build.ts +92 −1
  3. apps/cli/src/native-bindings.ts +19 −5
  4. apps/desktop/src/sidecar/native-bindings.ts +18 −5
  5. packages/kernel/runtime-workerd-subprocess/src/index.test.ts +41 −1
  6. packages/kernel/runtime-workerd-subprocess/src/index.ts +106 −28
  7. packages/plugins/apps/src/executor/workerd-app-tool-executor.ts +2 −2
  8. packages/plugins/apps/src/pipeline/worker-bundler.ts +21 −14
apps/cli/scripts/smoke-packed-apps.ts+192−0
@@ -0,0 +1,192 @@
1/* oxlint-disable executor/no-try-catch-or-throw, executor/no-error-constructor, executor/no-promise-reject, executor/no-promise-catch -- boundary: standalone build smoke harness tears down a spawned packed binary */
2import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises";
3import { tmpdir } from "node:os";
4import { join, resolve } from "node:path";
5import { spawn, type Subprocess } from "bun";
6
7const binary = process.argv[2];
8if (!binary) throw new Error("usage: bun apps/cli/scripts/smoke-packed-apps.ts <packed-binary>");
9
10const AUTH_TOKEN = "packed-apps-smoke-token";
11const READY_TIMEOUT_MS = 45_000;
12
13const fail = (message: string): never => {
14 throw new Error(`[smoke-packed-apps] FAIL: ${message}`);
15};
16
17const requestJson = async <T>(origin: string, path: string, init: RequestInit = {}): Promise<T> => {
18 const response = await fetch(new URL(path, origin), {
19 ...init,
20 headers: {
21 authorization: `Bearer ${AUTH_TOKEN}`,
22 ...(init.body === undefined ? {} : { "content-type": "application/json" }),
23 ...init.headers,
24 },
25 });
26 const text = await response.text();
27 if (!response.ok) fail(`${init.method ?? "GET"} ${path} returned ${response.status}: ${text}`);
28 return (text.length > 0 ? JSON.parse(text) : null) as T;
29};
30
31const waitForReadyPort = async (proc: Subprocess<"ignore", "pipe", "pipe">): Promise<number> =>
32 new Promise((resolveReady, rejectReady) => {
33 const timer = setTimeout(
34 () => rejectReady(new Error("packed binary did not report ready in time")),
35 READY_TIMEOUT_MS,
36 );
37 let stdout = "";
38 let stderr = "";
39 const finish = (result: number | Error): void => {
40 clearTimeout(timer);
41 if (result instanceof Error) rejectReady(result);
42 else resolveReady(result);
43 };
44 void proc.exited.then((code) => {
45 finish(new Error(`packed binary exited before ready with code ${code}: ${stderr}`));
46 });
47 void (async () => {
48 const reader = proc.stdout.getReader();
49 const decoder = new TextDecoder();
50 try {
51 while (true) {
52 const chunk = await reader.read();
53 if (chunk.done) return;
54 stdout += decoder.decode(chunk.value);
55 const match = stdout.match(/EXECUTOR_READY:(\d+)/);
56 if (match) {
57 finish(Number(match[1]));
58 return;
59 }
60 }
61 } catch (cause) {
62 finish(cause instanceof Error ? cause : new Error(String(cause)));
63 }
64 })();
65 void (async () => {
66 const reader = proc.stderr.getReader();
67 const decoder = new TextDecoder();
68 while (true) {
69 const chunk = await reader.read();
70 if (chunk.done) return;
71 stderr += decoder.decode(chunk.value);
72 }
73 })().catch(() => undefined);
74 });
75
76const main = async (): Promise<void> => {
77 const fixture = await mkdtemp(join(tmpdir(), "executor-packed-apps-fixture-"));
78 const dataDir = await mkdtemp(join(tmpdir(), "executor-packed-apps-data-"));
79 const scopeDir = await mkdtemp(join(tmpdir(), "executor-packed-apps-scope-"));
80 let proc: Subprocess<"ignore", "pipe", "pipe"> | null = null;
81 try {
82 await mkdir(join(fixture, "tools"), { recursive: true });
83 await writeFile(
84 join(fixture, "tools", "greet.ts"),
85 `
86 import { z } from "zod";
87 import { defineTool } from "executor:app";
88
89 const Input = z.object({ name: z.string() });
90
91 export default defineTool({
92 description: "Greet",
93 input: Input,
94 async handler(input) {
95 return { greeting: "hello " + input.name };
96 },
97 });
98 `,
99 );
100 await writeFile(
101 join(fixture, "package.json"),
102 `${JSON.stringify({ name: "packed-apps-smoke", dependencies: { zod: "4.3.6" } })}\n`,
103 );
104 await writeFile(join(fixture, "bun.lock"), "");
105
106 proc = spawn({
107 cmd: [
108 resolve(binary),
109 "daemon",
110 "run",
111 "--foreground",
112 "--port",
113 "0",
114 "--hostname",
115 "127.0.0.1",
116 "--auth-token",
117 AUTH_TOKEN,
118 ],
119 env: {
120 ...process.env,
121 EXECUTOR_CLIENT: "desktop",
122 EXECUTOR_DATA_DIR: dataDir,
123 EXECUTOR_SCOPE_DIR: scopeDir,
124 },
125 stdin: "ignore",
126 stdout: "pipe",
127 stderr: "pipe",
128 });
129
130 const port = await waitForReadyPort(proc);
131 const origin = `http://127.0.0.1:${port}`;
132 console.log(`[smoke-packed-apps] ready on ${origin}`);
133
134 await requestJson(origin, "/api/apps/sources", {
135 method: "POST",
136 body: JSON.stringify({
137 kind: "local-directory",
138 slug: "packed-apps-smoke",
139 app: "packed-apps-smoke",
140 path: fixture,
141 }),
142 });
143
144 const sync = await requestJson<{
145 readonly status: string;
146 readonly tools: readonly string[];
147 readonly errors?: readonly unknown[];
148 }>(origin, "/api/apps/sources/packed-apps-smoke/sync", { method: "POST" });
149 if (sync.status !== "published") fail(`sync failed: ${JSON.stringify(sync.errors)}`);
150 if (JSON.stringify(sync.tools) !== JSON.stringify(["greet"])) {
151 fail(`expected greet tool, got ${JSON.stringify(sync.tools)}`);
152 }
153
154 const invoked = await requestJson<{
155 readonly status: string;
156 readonly structured: unknown;
157 readonly isError: boolean;
158 readonly text: string;
159 }>(origin, "/api/executions", {
160 method: "POST",
161 body: JSON.stringify({
162 code: `export default await tools["packed-apps-smoke.org.published.greet"]({ name: "Ada" })`,
163 autoApprove: true,
164 }),
165 });
166 if (invoked.status !== "completed" || invoked.isError) {
167 fail(`execution failed: ${JSON.stringify(invoked)}`);
168 }
169 if (
170 JSON.stringify(invoked.structured) !==
171 JSON.stringify({
172 status: "completed",
173 result: { ok: true, data: { greeting: "hello Ada" } },
174 logs: [],
175 })
176 ) {
177 fail(`unexpected execution result: ${JSON.stringify(invoked)}`);
178 }
179 console.log("[smoke-packed-apps] OK - local-directory app source bundled and invoked");
180 } finally {
181 if (proc) {
182 proc.kill("SIGTERM");
183 await Promise.race([proc.exited, new Promise((resolve) => setTimeout(resolve, 3000))]);
184 proc.kill("SIGKILL");
185 }
186 await rm(fixture, { recursive: true, force: true }).catch(() => undefined);
187 await rm(dataDir, { recursive: true, force: true }).catch(() => undefined);
188 await rm(scopeDir, { recursive: true, force: true }).catch(() => undefined);
189 }
190};
191
192await main();
apps/cli/src/build.ts+92−1
@@ -2,7 +2,7 @@ import { cp, mkdir, rm, writeFile, chmod } from "node:fs/promises"; · 1 unmodified line
2import { existsSync } from "node:fs";2import { existsSync } from "node:fs";
3import { createRequire } from "node:module";3import { createRequire } from "node:module";
4import { resolve, join, dirname } from "node:path";4import { resolve, join, dirname } from "node:path";
5import { fileURLToPath } from "node:url";5import { fileURLToPath, pathToFileURL } from "node:url";
6import { parseArgs } from "node:util";6import { parseArgs } from "node:util";
7import { $ } from "bun";7import { $ } from "bun";
8 8
@@ -11,6 +11,8 @@ const cliRoot = resolve(repoRoot, "apps/cli"); · 2 unmodified lines
11const webRoot = resolve(repoRoot, "apps/local");11const webRoot = resolve(repoRoot, "apps/local");
12const distDir = resolve(cliRoot, "dist");12const distDir = resolve(cliRoot, "dist");
13const ONEPASSWORD_CORE_WASM_FILENAME = "onepassword-core_bg.wasm";13const ONEPASSWORD_CORE_WASM_FILENAME = "onepassword-core_bg.wasm";
14const WORKERD_VERSION = "1.20260708.1";
15const WORKER_BUNDLER_DIRNAME = "worker-bundler";
14 16
15const resolveQuickJsWasmPath = (): string => {17const resolveQuickJsWasmPath = (): string => {
16 const req = createRequire(join(repoRoot, "packages/kernel/runtime-quickjs/package.json"));18 const req = createRequire(join(repoRoot, "packages/kernel/runtime-quickjs/package.json"));
@@ -33,6 +35,34 @@ const resolveOnePasswordCoreWasmPath = (): string => { · 16 unmodified lines
33 return wasmPath;35 return wasmPath;
34};36};
35 37
38const resolveWorkerBundlerDistPath = (): string => {
39 const req = createRequire(join(repoRoot, "packages/plugins/apps/package.json"));
40 const pkgJson = req.resolve("@cloudflare/worker-bundler/package.json");
41 const distPath = join(dirname(pkgJson), "dist");
42 if (!existsSync(join(distPath, "index.js")) || !existsSync(join(distPath, "esbuild.wasm"))) {
43 throw new Error(`@cloudflare/worker-bundler dist files not found at ${distPath}`);
44 }
45 return distPath;
46};
47
48const createPackedWorkerBundlerSource = async (distPath: string): Promise<string> => {
49 const esbuildPath = join(repoRoot, "node_modules/.bun/node_modules/esbuild/lib/main.js");
50 if (!existsSync(esbuildPath)) throw new Error(`esbuild not found at ${esbuildPath}`);
51 const { build } = await import(pathToFileURL(esbuildPath).href);
52 const result = await build({
53 entryPoints: [join(distPath, "index.js")],
54 bundle: true,
55 format: "esm",
56 platform: "browser",
57 external: ["./esbuild.wasm"],
58 logLevel: "silent",
59 write: false,
60 });
61 const source = result.outputFiles[0]?.text;
62 if (source === undefined) throw new Error("failed to bundle @cloudflare/worker-bundler");
63 return source;
64};
65
36// ---------------------------------------------------------------------------66// ---------------------------------------------------------------------------
37// Metadata67// Metadata
38// ---------------------------------------------------------------------------68// ---------------------------------------------------------------------------
@@ -119,6 +149,7 @@ const bunTarget = (t: Target): Bun.Build.CompileTarget => { · 80 unmodified lines
119};149};
120 150
121const binaryName = (t: Target) => (t.os === "win32" ? "executor.exe" : "executor");151const binaryName = (t: Target) => (t.os === "win32" ? "executor.exe" : "executor");
152const workerdBinaryName = (t: Target) => (t.os === "win32" ? "workerd.exe" : "workerd");
122 153
123const isCurrentPlatform = (t: Target) =>154const isCurrentPlatform = (t: Target) =>
124 t.os === process.platform && t.arch === process.arch && !t.abi;155 t.os === process.platform && t.arch === process.arch && !t.abi;
@@ -211,6 +242,41 @@ const resolveLibsqlNative = (t: Target): string | null => { · 86 unmodified lines
211 }242 }
212};243};
213 244
245const resolveWorkerdBinary = (t: Target): string | null => {
246 const platformMap: Record<string, string> = {
247 "darwin-arm64": "@cloudflare/workerd-darwin-arm64",
248 "darwin-x64": "@cloudflare/workerd-darwin-64",
249 "linux-arm64": "@cloudflare/workerd-linux-arm64",
250 "linux-x64": "@cloudflare/workerd-linux-64",
251 "win32-x64": "@cloudflare/workerd-windows-64",
252 };
253 const key = [t.os, t.arch, t.abi].filter(Boolean).join("-");
254 const pkg = platformMap[key];
255 if (!pkg) return null;
256 const binary = workerdBinaryName(t);
257 try {
258 const req = createRequire(
259 join(repoRoot, "packages/kernel/runtime-workerd-subprocess/package.json"),
260 );
261 const pkgJson = req.resolve(`${pkg}/package.json`);
262 for (const candidate of [
263 join(dirname(pkgJson), "bin", binary),
264 join(dirname(pkgJson), binary),
265 ]) {
266 if (existsSync(candidate)) return candidate;
267 }
268 } catch {
269 const packageRoot = join(
270 repoRoot,
271 `node_modules/.bun/${pkg.replace("/", "+")}@${WORKERD_VERSION}/node_modules/${pkg}`,
272 );
273 for (const candidate of [join(packageRoot, "bin", binary), join(packageRoot, binary)]) {
274 if (existsSync(candidate)) return candidate;
275 }
276 }
277 return null;
278};
279
214// ---------------------------------------------------------------------------280// ---------------------------------------------------------------------------
215// Build mode281// Build mode
216// ---------------------------------------------------------------------------282// ---------------------------------------------------------------------------
@@ -333,6 +399,8 @@ const buildBinaries = async (targets: Target[], mode: BuildMode) => { · 116 unmodified lines
333 399
334 const quickJsWasmPath = resolveQuickJsWasmPath();400 const quickJsWasmPath = resolveQuickJsWasmPath();
335 const onePasswordCoreWasmPath = resolveOnePasswordCoreWasmPath();401 const onePasswordCoreWasmPath = resolveOnePasswordCoreWasmPath();
402 const workerBundlerDistPath = resolveWorkerBundlerDistPath();
403 const packedWorkerBundlerSource = await createPackedWorkerBundlerSource(workerBundlerDistPath);
336 404
337 try {405 try {
338 for (const target of targets) {406 for (const target of targets) {
@@ -375,12 +443,35 @@ const buildBinaries = async (targets: Target[], mode: BuildMode) => { · 36 unmodified lines
375 await cp(libsqlNative, join(binDir, "libsql.node"));443 await cp(libsqlNative, join(binDir, "libsql.node"));
376 }444 }
377 445
446 const workerdBinary = resolveWorkerdBinary(target);
447 if (workerdBinary && existsSync(workerdBinary)) {
448 await cp(workerdBinary, join(binDir, workerdBinaryName(target)));
449 if (target.os !== "win32") await chmod(join(binDir, workerdBinaryName(target)), 0o755);
450 }
451
452 await cp(workerBundlerDistPath, join(binDir, WORKER_BUNDLER_DIRNAME, "dist"), {
453 recursive: true,
454 });
455 await writeFile(
456 join(binDir, WORKER_BUNDLER_DIRNAME, "dist", "index.bundled.js"),
457 packedWorkerBundlerSource,
458 );
459
378 // Smoke test on current platform460 // Smoke test on current platform
379 if (isCurrentPlatform(target)) {461 if (isCurrentPlatform(target)) {
380 const bin = join(binDir, binaryName(target));462 const bin = join(binDir, binaryName(target));
381 console.log(` Smoke test: ${bin} --version`);463 console.log(` Smoke test: ${bin} --version`);
382 const version = await $`${bin} --version`.text();464 const version = await $`${bin} --version`.text();
383 console.log(` OK: ${version.trim()}`);465 console.log(` OK: ${version.trim()}`);
466 console.log(` Smoke test: packed apps source sync and invoke`);
467 const smoke = Bun.spawn(
468 ["bun", "run", join(cliRoot, "scripts/smoke-packed-apps.ts"), bin],
469 {
470 cwd: repoRoot,
471 stdio: ["ignore", "inherit", "inherit"],
472 },
473 );
474 if ((await smoke.exited) !== 0) throw new Error("packed apps smoke failed");
384 }475 }
385 476
386 // Variant package.json. All variants publish to the SAME npm package477 // Variant package.json. All variants publish to the SAME npm package
apps/cli/src/native-bindings.ts+19−5
@@ -1,11 +1,10 @@
1// ---------------------------------------------------------------------------1// ---------------------------------------------------------------------------
2// Native-binding bootstrap for the `bun build --compile` binary.2// Native-binding bootstrap for the `bun build --compile` binary.
3//3//
4// `bun --compile` bundles JS into bunfs but does NOT include `.node` native4// `bun --compile` bundles JS into bunfs but does NOT include native binaries,
5// addons, so a dynamic `require('@libsql/<platform>')` / keyring walk inside5// wasm sidecars, or runtime-read package files. build.ts copies those artifacts
6// the binary fails. build.ts copies each platform's `.node` next to the6// next to the executable; here we publish their on-disk paths via env vars the
7// executable (`libsql.node`, `keyring.node`); here we publish their on-disk7// loaders read.
8// paths via env vars the loaders read.
9//8//
10// This MUST be the FIRST import in main.ts. ES modules evaluate every import9// This MUST be the FIRST import in main.ts. ES modules evaluate every import
11// before the importer's own body, and libSQL resolves its native addon EAGERLY10// before the importer's own body, and libSQL resolves its native addon EAGERLY
@@ -43,3 +42,18 @@ if ( · 31 unmodified lines
43) {42) {
44 process.env.EXECUTOR_KEYRING_NATIVE_PATH = keyringNodeOnDisk;43 process.env.EXECUTOR_KEYRING_NATIVE_PATH = keyringNodeOnDisk;
45}44}
45
46const workerdOnDisk = join(execDir, process.platform === "win32" ? "workerd.exe" : "workerd");
47if (typeof Bun !== "undefined" && !process.env.EXECUTOR_WORKERD_BIN && existsSync(workerdOnDisk)) {
48 process.env.EXECUTOR_WORKERD_BIN = workerdOnDisk;
49}
50
51const workerBundlerOnDisk = join(execDir, "worker-bundler");
52if (
53 typeof Bun !== "undefined" &&
54 !process.env.EXECUTOR_WORKER_BUNDLER_DIR &&
55 existsSync(join(workerBundlerOnDisk, "dist", "index.js")) &&
56 existsSync(join(workerBundlerOnDisk, "dist", "esbuild.wasm"))
57) {
58 process.env.EXECUTOR_WORKER_BUNDLER_DIR = workerBundlerOnDisk;
59}
apps/desktop/src/sidecar/native-bindings.ts+18−5
@@ -1,11 +1,9 @@
1// ---------------------------------------------------------------------------1// ---------------------------------------------------------------------------
2// Native-binding bootstrap for manually compiled sidecar binaries.2// Native-binding bootstrap for manually compiled sidecar binaries.
3//3//
4// `bun --compile` bundles JS into bunfs but does NOT include `.node` native4// `bun --compile` bundles JS into bunfs but does NOT include native binaries,
5// addons, so a dynamic `require('@libsql/<platform>')` / keyring walk inside5// wasm sidecars, or runtime-read package files. If this dev sidecar is compiled
6// the binary fails. If this dev sidecar is compiled directly, copy each6// directly, copy those artifacts next to the executable; here we publish their
7// platform's `.node` next to the executable (`libsql.node`, `keyring.node`);
8// here we publish their
9// on-disk paths via env vars the loaders read. Mirrors apps/cli/src/native-bindings.ts.7// on-disk paths via env vars the loaders read. Mirrors apps/cli/src/native-bindings.ts.
10//8//
11// This MUST be the FIRST import in server.ts. ES modules evaluate every import9// This MUST be the FIRST import in server.ts. ES modules evaluate every import
@@ -44,3 +42,18 @@ if ( · 32 unmodified lines
44) {42) {
45 process.env.EXECUTOR_KEYRING_NATIVE_PATH = keyringNodeOnDisk;43 process.env.EXECUTOR_KEYRING_NATIVE_PATH = keyringNodeOnDisk;
46}44}
45
46const workerdOnDisk = join(execDir, process.platform === "win32" ? "workerd.exe" : "workerd");
47if (typeof Bun !== "undefined" && !process.env.EXECUTOR_WORKERD_BIN && existsSync(workerdOnDisk)) {
48 process.env.EXECUTOR_WORKERD_BIN = workerdOnDisk;
49}
50
51const workerBundlerOnDisk = join(execDir, "worker-bundler");
52if (
53 typeof Bun !== "undefined" &&
54 !process.env.EXECUTOR_WORKER_BUNDLER_DIR &&
55 existsSync(join(workerBundlerOnDisk, "dist", "index.js")) &&
56 existsSync(join(workerBundlerOnDisk, "dist", "esbuild.wasm"))
57) {
58 process.env.EXECUTOR_WORKER_BUNDLER_DIR = workerBundlerOnDisk;
59}
packages/kernel/runtime-workerd-subprocess/src/index.test.ts+41−1
@@ -1,5 +1,6 @@
1import { describe, expect, it } from "@effect/vitest";1import { describe, expect, it } from "@effect/vitest";
2import { stat } from "node:fs/promises";2import { chmod, mkdtemp, rm, stat, writeFile } from "node:fs/promises";
3import { tmpdir } from "node:os";
3import { join } from "node:path";4import { join } from "node:path";
4import * as Data from "effect/Data";5import * as Data from "effect/Data";
5import * as Effect from "effect/Effect";6import * as Effect from "effect/Effect";
@@ -9,6 +10,7 @@ import { · 3 unmodified lines
9 createWorkerdModuleRunner,10 createWorkerdModuleRunner,
10 isWorkerdAvailable,11 isWorkerdAvailable,
11 makeWorkerdSubprocessExecutor,12 makeWorkerdSubprocessExecutor,
13 workerdBinaryCandidatesForTest,
12 WORKERD_VERSION,14 WORKERD_VERSION,
13} from "./index";15} from "./index";
14 16
@@ -107,6 +109,44 @@ it("exports the pinned workerd version", () => { · 92 unmodified lines
107 expect(WORKERD_VERSION).toBe("1.20260708.1");109 expect(WORKERD_VERSION).toBe("1.20260708.1");
108});110});
109 111
112describe("workerd binary resolution", () => {
113 it("honors EXECUTOR_WORKERD_BIN before probing", async () => {
114 const tmp = await mkdtemp(join(tmpdir(), "executor-workerd-bin-test-"));
115 const previous = process.env.EXECUTOR_WORKERD_BIN;
116 const bin = join(tmp, "workerd");
117 try {
118 await writeFile(bin, "#!/bin/sh\nexit 0\n", { mode: 0o755 });
119 await chmod(bin, 0o755);
120 process.env.EXECUTOR_WORKERD_BIN = bin;
121 expect(isWorkerdAvailable()).toBe(true);
122 } finally {
123 if (previous === undefined) {
124 delete process.env.EXECUTOR_WORKERD_BIN;
125 } else {
126 process.env.EXECUTOR_WORKERD_BIN = previous;
127 }
128 await rm(tmp, { recursive: true, force: true });
129 }
130 });
131
132 it("orders generic package probes before platform package probes", () => {
133 const roots = ["/app", "/workspace"];
134 expect(workerdBinaryCandidatesForTest(roots, "darwin", "arm64")).toEqual([
135 "/app/node_modules/workerd/bin/workerd",
136 "/workspace/node_modules/workerd/bin/workerd",
137 "/app/node_modules/@cloudflare/workerd-darwin-arm64/bin/workerd",
138 "/app/node_modules/@cloudflare/workerd-darwin-arm64/workerd",
139 "/workspace/node_modules/@cloudflare/workerd-darwin-arm64/bin/workerd",
140 "/workspace/node_modules/@cloudflare/workerd-darwin-arm64/workerd",
141 ]);
142 });
143
144 it("probes workerd.exe in the Windows package root layout", () => {
145 const candidates = workerdBinaryCandidatesForTest(["C:\\app"], "win32", "x64");
146 expect(candidates).toContain("C:\\app/node_modules/@cloudflare/workerd-windows-64/workerd.exe");
147 });
148});
149
110describe.skipIf(!isWorkerdAvailable())("runtime-workerd-subprocess", () => {150describe.skipIf(!isWorkerdAvailable())("runtime-workerd-subprocess", () => {
111 it("boots and executes a trivial module", async () => {151 it("boots and executes a trivial module", async () => {
112 const runner = makeRunner();152 const runner = makeRunner();
packages/kernel/runtime-workerd-subprocess/src/index.ts+106−28
@@ -1,7 +1,9 @@
1/* oxlint-disable executor/no-try-catch-or-throw, executor/no-promise-reject, executor/no-instanceof-tagged-error -- subprocess boundary normalizes JavaScript errors into typed runtime errors */1/* oxlint-disable executor/no-try-catch-or-throw, executor/no-promise-reject, executor/no-instanceof-tagged-error -- subprocess boundary normalizes JavaScript errors into typed runtime errors */
2import { spawn, spawnSync, type ChildProcessByStdio } from "node:child_process";2import { spawn, spawnSync, type ChildProcessByStdio } from "node:child_process";
3import { existsSync, readFileSync } from "node:fs";
3import { access, mkdir, rm, writeFile } from "node:fs/promises";4import { access, mkdir, rm, writeFile } from "node:fs/promises";
4import { createServer, type IncomingMessage, type Server, type ServerResponse } from "node:http";5import { createServer, type IncomingMessage, type Server, type ServerResponse } from "node:http";
6import { createRequire } from "node:module";
5import { createServer as createNetServer, type Server as NetServer } from "node:net";7import { createServer as createNetServer, type Server as NetServer } from "node:net";
6import type { Readable } from "node:stream";8import type { Readable } from "node:stream";
7import { tmpdir } from "node:os";9import { tmpdir } from "node:os";
@@ -77,6 +79,7 @@ type PendingRequest = { · 69 unmodified lines
77};79};
78 80
79const packageDir = dirname(dirname(fileURLToPath(import.meta.url)));81const packageDir = dirname(dirname(fileURLToPath(import.meta.url)));
82const packageRequire = createRequire(import.meta.url);
80 83
81const normalizeError = (cause: unknown): Error =>84const normalizeError = (cause: unknown): Error =>
82 cause instanceof Error ? cause : new Error(String(cause));85 cause instanceof Error ? cause : new Error(String(cause));
@@ -150,16 +153,86 @@ const reserveLocalPort = (): Promise<{ · 67 unmodified lines
150 });153 });
151 });154 });
152 155
153const findWorkerdBin = async (configured?: string): Promise<string> => {156const workerdBinaryName = (platform: NodeJS.Platform = process.platform): string =>
154 if (configured) return configured;157 platform === "win32" ? "workerd.exe" : "workerd";
158
159const detectLinuxMusl = (): boolean => {
160 if (process.platform !== "linux") return false;
161 if (existsSync("/etc/alpine-release")) return true;
162 try {
163 return readFileSync("/usr/bin/ldd", "utf8").toLowerCase().includes("musl");
164 } catch {
165 return false;
166 }
167};
168
169const workerdPlatformPackage = (
170 platform: NodeJS.Platform = process.platform,
171 arch: NodeJS.Architecture = process.arch,
172): string | null => {
173 if (platform === "darwin" && arch === "arm64") return "@cloudflare/workerd-darwin-arm64";
174 if (platform === "darwin" && arch === "x64") return "@cloudflare/workerd-darwin-64";
175 if (platform === "linux" && detectLinuxMusl()) return null;
176 if (platform === "linux" && arch === "arm64") return "@cloudflare/workerd-linux-arm64";
177 if (platform === "linux" && arch === "x64") return "@cloudflare/workerd-linux-64";
178 if (platform === "win32" && arch === "x64") return "@cloudflare/workerd-windows-64";
179 return null;
180};
181
182const workerdPackageDirCandidates = (packageName: string): readonly string[] => {
183 const packageJson = `${packageName}/package.json`;
184 const resolved = new Set<string>();
185 for (const req of [
186 packageRequire,
187 createRequire(join(process.cwd(), "package.json")),
188 createRequire(join(packageDir, "package.json")),
189 ]) {
190 try {
191 resolved.add(dirname(req.resolve(packageJson)));
192 } catch {
193 // Try the next package-manager layout.
194 }
195 }
196 return [...resolved];
197};
198
199export const workerdBinaryCandidatesForTest = (
200 roots: readonly string[],
201 platform: NodeJS.Platform,
202 arch: NodeJS.Architecture,
203): readonly string[] => {
204 const binary = workerdBinaryName(platform);
205 const platformPackage = workerdPlatformPackage(platform, arch);
206 return [
207 ...roots.map((root) => join(root, "node_modules", "workerd", "bin", binary)),
208 ...(platformPackage
209 ? roots.flatMap((root) => [
210 join(root, "node_modules", platformPackage, "bin", binary),
211 join(root, "node_modules", platformPackage, binary),
212 ])
213 : []),
214 ];
215};
155 216
156 const candidates = [217const workerdBinaryCandidates = (): readonly string[] => {
157 join(packageDir, "node_modules", "workerd", "bin", "workerd"),218 const roots = [packageDir, join(packageDir, "..", "..", ".."), process.cwd()];
158 join(packageDir, "..", "..", "..", "node_modules", "workerd", "bin", "workerd"),219 const platformPackage = workerdPlatformPackage();
159 join(process.cwd(), "node_modules", "workerd", "bin", "workerd"),220 return [
221 ...(platformPackage
222 ? workerdPackageDirCandidates(platformPackage).flatMap((dir) => [
223 join(dir, "bin", workerdBinaryName()),
224 join(dir, workerdBinaryName()),
225 ])
226 : []),
227 ...workerdBinaryCandidatesForTest(roots, process.platform, process.arch),
160 ];228 ];
229};
161 230
162 for (const candidate of candidates) {231const findWorkerdBin = async (configured?: string): Promise<string> => {
232 if (configured) return configured;
233 if (process.env.EXECUTOR_WORKERD_BIN) return process.env.EXECUTOR_WORKERD_BIN;
234
235 for (const candidate of workerdBinaryCandidates()) {
163 try {236 try {
164 await access(candidate);237 await access(candidate);
165 return candidate;238 return candidate;
@@ -168,36 +241,41 @@ const findWorkerdBin = async (configured?: string): Promise<string> => { · 2 unmodified lines
168 }241 }
169 }242 }
170 243
171 return "workerd";244 if (workerdPlatformPackage() === null) throw new Error("workerd is unavailable on this platform");
172};245 const pathWorkerd = spawnSync("workerd", ["--version"], {
173
174export const isWorkerdAvailable = (workerdBin?: string): boolean => {
175 const configured = workerdBin ?? "workerd";
176 const result = spawnSync(configured, ["--version"], {
177 stdio: "ignore",246 stdio: "ignore",
178 timeout: 5000,247 timeout: 5000,
179 });248 });
180 if (result.error === undefined && result.status === 0) return true;249 if (pathWorkerd.error !== undefined || pathWorkerd.status !== 0) {
250 throw new Error("workerd is unavailable on this platform");
251 }
252 return "workerd";
253};
181 254
182 const local = spawnSync(255export const isWorkerdAvailable = (workerdBin?: string): boolean => {
183 join(packageDir, "node_modules", "workerd", "bin", "workerd"),256 const configured = workerdBin ?? process.env.EXECUTOR_WORKERD_BIN;
184 ["--version"],257 if (configured) {
185 {258 const result = spawnSync(configured, ["--version"], {
186 stdio: "ignore",259 stdio: "ignore",
187 timeout: 5000,260 timeout: 5000,
188 },261 });
189 );262 return result.error === undefined && result.status === 0;
190 if (local.error === undefined && local.status === 0) return true;263 }
191 264
192 const workspace = spawnSync(265 for (const candidate of workerdBinaryCandidates()) {
193 join(packageDir, "..", "..", "..", "node_modules", "workerd", "bin", "workerd"),266 const result = spawnSync(candidate, ["--version"], {
194 ["--version"],
195 {
196 stdio: "ignore",267 stdio: "ignore",
197 timeout: 5000,268 timeout: 5000,
198 },269 });
199 );270 if (result.error === undefined && result.status === 0) return true;
200 return workspace.error === undefined && workspace.status === 0;271 }
272
273 if (workerdPlatformPackage() === null) return false;
274 const result = spawnSync("workerd", ["--version"], {
275 stdio: "ignore",
276 timeout: 5000,
277 });
278 return result.error === undefined && result.status === 0;
201};279};
202 280
203const capnpString = (value: string): string => JSON.stringify(value);281const capnpString = (value: string): string => JSON.stringify(value);
packages/plugins/apps/src/executor/workerd-app-tool-executor.ts+2−2
@@ -1,4 +1,4 @@
1/* oxlint-disable executor/no-try-catch-or-throw, executor/no-double-cast, executor/no-promise-reject, executor/no-instanceof-tagged-error -- workerd boundary converts worker envelopes into typed AppExecutorError */1/* oxlint-disable executor/no-try-catch-or-throw, executor/no-double-cast, executor/no-promise-reject, executor/no-instanceof-tagged-error, executor/no-instanceof-error, executor/no-unknown-error-message -- workerd boundary converts worker envelopes and subprocess startup failures into typed AppExecutorError */
2import { Effect } from "effect";2import { Effect } from "effect";
3 3
4import {4import {
@@ -47,7 +47,7 @@ const mapCause = (cause: unknown, kind: AppExecutorError["kind"], message: strin · 42 unmodified lines
47 ? cause47 ? cause
48 : new AppExecutorError({48 : new AppExecutorError({
49 kind,49 kind,
50 message,50 message: `${message}: ${cause instanceof Error ? cause.message : String(cause)}`,
51 cause,51 cause,
52 });52 });
53 53
packages/plugins/apps/src/pipeline/worker-bundler.ts+21−14
@@ -1,4 +1,5 @@
1/* oxlint-disable executor/no-try-catch-or-throw, executor/no-unknown-error-message, executor/no-error-constructor, executor/no-json-parse, executor/no-instanceof-tagged-error, executor/no-instanceof-error -- boundary: subprocess-backed bundler validates package JSON and converts worker failures into typed AppExecutorError */1/* oxlint-disable executor/no-try-catch-or-throw, executor/no-unknown-error-message, executor/no-error-constructor, executor/no-json-parse, executor/no-instanceof-tagged-error, executor/no-instanceof-error -- boundary: subprocess-backed bundler validates package JSON and converts worker failures into typed AppExecutorError */
2import { existsSync } from "node:fs";
2import { readFile } from "node:fs/promises";3import { readFile } from "node:fs/promises";
3import { dirname, join } from "node:path";4import { dirname, join } from "node:path";
4import { fileURLToPath } from "node:url";5import { fileURLToPath } from "node:url";
@@ -7,7 +8,6 @@ import { · 2 unmodified lines
7 createWorkerdModuleRunner,8 createWorkerdModuleRunner,
8 WORKERD_VERSION,9 WORKERD_VERSION,
9} from "@executor-js/runtime-workerd-subprocess";10} from "@executor-js/runtime-workerd-subprocess";
10import { build } from "esbuild";
11import { Effect } from "effect";11import { Effect } from "effect";
12 12
13import { AppExecutorError } from "../executor/app-tool-executor";13import { AppExecutorError } from "../executor/app-tool-executor";
@@ -18,7 +18,9 @@ import { WORKER_BUNDLER_ESBUILD_VERSION, WORKER_BUNDLER_VERSION } from "./worker · 4 unmodified lines
18 18
19const textEncoder = new TextEncoder();19const textEncoder = new TextEncoder();
20const packageDir = dirname(dirname(dirname(fileURLToPath(import.meta.url))));20const packageDir = dirname(dirname(dirname(fileURLToPath(import.meta.url))));
21const workerBundlerDir = join(packageDir, "node_modules", "@cloudflare", "worker-bundler");21const workerBundlerDir =
22 process.env.EXECUTOR_WORKER_BUNDLER_DIR ??
23 join(packageDir, "node_modules", "@cloudflare", "worker-bundler");
22 24
23const executorAppSource = `25const executorAppSource = `
24const makeIntegrationDeclaration = (state) => Object.freeze({26const makeIntegrationDeclaration = (state) => Object.freeze({
@@ -97,19 +99,24 @@ const bundledWorkerBundler = async (): Promise<{ · 72 unmodified lines
97 readonly source: string;99 readonly source: string;
98 readonly wasm: Uint8Array;100 readonly wasm: Uint8Array;
99}> => {101}> => {
100 const entry = join(workerBundlerDir, "dist", "index.js");102 const distDir = join(workerBundlerDir, "dist");
101 const [result, wasm] = await Promise.all([103 const bundledEntry = join(distDir, "index.bundled.js");
102 build({104 const [sourceFromDisk, wasm] = await Promise.all([
103 entryPoints: [entry],105 existsSync(bundledEntry) ? readFile(bundledEntry, "utf8") : Promise.resolve(null),
104 bundle: true,106 readFile(join(distDir, "esbuild.wasm")),
105 format: "esm",
106 platform: "browser",
107 write: false,
108 external: ["./esbuild.wasm"],
109 logLevel: "silent",
110 }),
111 readFile(join(workerBundlerDir, "dist", "esbuild.wasm")),
112 ]);107 ]);
108 if (sourceFromDisk !== null) return { source: sourceFromDisk, wasm };
109
110 const { build } = await import("esbuild");
111 const result = await build({
112 entryPoints: [join(distDir, "index.js")],
113 bundle: true,
114 format: "esm",
115 platform: "browser",
116 write: false,
117 external: ["./esbuild.wasm"],
118 logLevel: "silent",
119 });
113 const source = result.outputFiles[0]?.text;120 const source = result.outputFiles[0]?.text;
114 if (source === undefined) throw new Error("failed to bundle @cloudflare/worker-bundler");121 if (source === undefined) throw new Error("failed to bundle @cloudflare/worker-bundler");
115 return { source, wasm };122 return { source, wasm };