History

Simplify self-host Docker runtime packaging

Rhys Sullivan committed 4 months ago

Commit 84df8de1 parent a65590b2 files · +241 −129Browse files

Unified · Split

2 files
  1. apps/host-selfhost/Dockerfile +5 −129
  2. scripts/package-selfhost-runtime.ts +236 −0
apps/host-selfhost/Dockerfile+5−129
@@ -1,132 +1,26 @@
1−# Self-hosted Executor — single container, no external services.
2−# Build context is the REPO ROOT (the bun workspace install needs every member):
1+# Self-hosted Executor, single container, no external services.
2+# Build context is the repo root because Bun workspaces need every member:
33 # docker build -f apps/host-selfhost/Dockerfile -t executor-selfhost .
44 #
5−# Runtime needs nothing but this container + a volume for the data dir:
5+# Runtime needs this container plus a volume for the data dir:
66 # docker run -p 4788:4788 -e BETTER_AUTH_SECRET=$(openssl rand -hex 32) \
77 # -e EXECUTOR_BOOTSTRAP_ADMIN_EMAIL=you@example.com \
88 # -e EXECUTOR_BOOTSTRAP_ADMIN_PASSWORD=... \
99 # -e EXECUTOR_WEB_BASE_URL=https://your.domain \
1010 # -v executor-data:/data executor-selfhost
11−#
12−# SQLite (libSQL, file:/data/...) lives in /data, QuickJS + MCP run in-process —
13−# so there's no postgres/worker/proxy to orchestrate.
1411
15−# ── Production deps: installs only what the runtime image needs ─────────────
1612 FROM oven/bun:1 AS prod-deps
1713 WORKDIR /app
1814 COPY . .
19−# This stage is independent from the SPA build stage, so BuildKit can run the
20−# slow production install in parallel with the dev install and Vite build.
21−# --ignore-scripts: the root `prepare` hook runs a dev-only tool
22−# (effect-language-service); runtime deps are prebuilt JS with no postinstall.
2315 RUN bun install --frozen-lockfile --production --ignore-scripts --filter @executor-js/host-selfhost \
24− && find node_modules/.bun -maxdepth 1 -name '*linux-x64-musl*' -exec rm -rf {} + \
25− && rm -rf \
26− node_modules/.bun/@ant-design+* \
27− node_modules/.bun/@base-ui+* \
28− node_modules/.bun/@cloudflare+workers-types@* \
29− node_modules/.bun/@electric-sql+pglite@* \
30− node_modules/.bun/@emotion+react@* \
31− node_modules/.bun/@emoji-mart+* \
32− node_modules/.bun/@esbuild+* \
33− node_modules/.bun/@floating-ui+react@* \
34− node_modules/.bun/@dnd-kit+* \
35− node_modules/.bun/@hookform+* \
36− node_modules/.bun/@lobehub+* \
37− node_modules/.bun/@mermaid-js+* \
38− node_modules/.bun/@pierre+diffs@* \
39− node_modules/.bun/@pierre+theme@* \
40− node_modules/.bun/@primer+octicons@* \
41− node_modules/.bun/@rc-component+* \
42− node_modules/.bun/@rolldown+binding-* \
43− node_modules/.bun/@reduxjs+* \
44− node_modules/.bun/@shikijs+* \
45− node_modules/.bun/@splinetool+* \
46− node_modules/.bun/@tanstack+start-plugin-core@* \
47− node_modules/.bun/@tanstack+react-router@* \
48− node_modules/.bun/@tanstack+router-core@* \
49− node_modules/.bun/@types+* \
50− node_modules/.bun/ahooks@* \
51− node_modules/.bun/antd@* \
52− node_modules/.bun/better-sqlite3@* \
53− node_modules/.bun/bun-types@* \
54− node_modules/.bun/caniuse-lite@* \
55− node_modules/.bun/chroma-js@* \
56− node_modules/.bun/cytoscape*@* \
57− node_modules/.bun/d3@* \
58− node_modules/.bun/date-fns*@* \
59− node_modules/.bun/drizzle-kit@* \
60− node_modules/.bun/dompurify@* \
61− node_modules/.bun/emoji-mart@* \
62− node_modules/.bun/framer-motion@* \
63− node_modules/.bun/jiti@* \
64− node_modules/.bun/katex@* \
65− node_modules/.bun/layout-base@* \
66− node_modules/.bun/leva@* \
67− node_modules/.bun/lightningcss-* \
68− node_modules/.bun/lit-html@* \
69− node_modules/.bun/lucide-react@* \
70− node_modules/.bun/mermaid@* \
71− node_modules/.bun/motion@* \
72− node_modules/.bun/motion-* \
73− node_modules/.bun/polished@* \
74− node_modules/.bun/prettier@* \
75− node_modules/.bun/react-day-picker@* \
76− node_modules/.bun/react-hook-form@* \
77− node_modules/.bun/react@* \
78− node_modules/.bun/react-dom@* \
79− node_modules/.bun/react-dropzone@* \
80− node_modules/.bun/recharts@* \
81− node_modules/.bun/rolldown@* \
82− node_modules/.bun/shiki@* \
83− node_modules/.bun/solid-js@* \
84− node_modules/.bun/source-map@* \
85− node_modules/.bun/tailwind-merge@* \
86− node_modules/.bun/tsx@* \
87− node_modules/.bun/@upsetjs+venn.js@* \
88− node_modules/.bun/victory-vendor@* \
89− node_modules/.bun/virtua@* \
90− node_modules/.bun/typeorm@* \
91− node_modules/.bun/vite@* \
92− node_modules/.bun/vitest@* \
93− && find node_modules -type f \( -name '*.map' -o -name '*.d.ts' -o -name '*.d.mts' -o -name '*.d.cts' -o -name '*.ts' -o -name '*.tsx' -o -name '*.md' -o -name '*.markdown' \) -delete \
94− && find node_modules -type d \( -iname test -o -iname tests -o -iname __tests__ -o -iname fixtures -o -iname fixture -o -iname examples \) -prune -exec rm -rf {} + \
95− && find packages apps/host-selfhost -type f \( -name '*.map' -o -name '*.d.ts' -o -name '*.test.ts' -o -name '*.test.tsx' -o -name '*.md' -o -name '*.markdown' \) -delete \
96− && rm -rf \
97− packages/app \
98− packages/react \
99− packages/plugins/*/fixtures \
100− packages/plugins/*/src/react \
101− packages/plugins/*/src/**/__snapshots__ \
102− packages/plugins/*/test \
103− packages/plugins/*/testing \
104− packages/*/*/CHANGELOG.md \
105− packages/*/*/tsconfig.json \
106− packages/*/*/tsup.config.ts \
107− packages/*/*/vitest.config.ts \
108− packages/plugins/*/vitest.config.ts \
109− apps/host-selfhost/CHANGELOG.md \
110− apps/host-selfhost/Dockerfile \
111− apps/host-selfhost/docker-compose.yml \
112− apps/host-selfhost/scripts \
113− apps/host-selfhost/vitest.config.ts \
114− apps/host-selfhost/web \
115− apps/host-selfhost/tsconfig.json \
116− apps/host-selfhost/tsr.routes.ts \
117− apps/host-selfhost/vite.config.ts
16+ && bun run scripts/package-selfhost-runtime.ts
11817
119−# ── Build stage: install the workspace + build the SPA ──────────────────────
12018 FROM oven/bun:1 AS build
12119 WORKDIR /app
12220 COPY . .
123−# Full install (dev deps included, vite/turbo/plugins are needed to build).
12421 RUN bun install --frozen-lockfile
125−# Builds @executor-js/vite-plugin (via turbo) then the self-host SPA into
126−# apps/host-selfhost/dist.
12722 RUN cd apps/host-selfhost && bun run build
12823
129−# ── Runtime stage: serve the built app under Bun ────────────────────────────
13024 FROM gcr.io/distroless/cc-debian12 AS runtime
13125 WORKDIR /app
13226 LABEL org.opencontainers.image.source="https://github.com/RhysSullivan/executor" \
@@ -137,29 +31,11 @@ ENV NODE_ENV=production \ · 4 unmodified lines
13731 PORT=4788 \
13832 EXECUTOR_DATA_DIR=/data
13933 COPY --from=prod-deps /usr/local/bin/bun /usr/local/bin/bun
140−COPY --from=prod-deps /app/node_modules /app/node_modules
141−COPY --from=prod-deps /app/packages/core/api /app/packages/core/api
142−COPY --from=prod-deps /app/packages/core/config /app/packages/core/config
143−COPY --from=prod-deps /app/packages/core/execution /app/packages/core/execution
144−COPY --from=prod-deps /app/packages/core/fumadb /app/packages/core/fumadb
145−COPY --from=prod-deps /app/packages/core/sdk /app/packages/core/sdk
146−COPY --from=prod-deps /app/packages/hosts/mcp /app/packages/hosts/mcp
147−COPY --from=prod-deps /app/packages/kernel/core /app/packages/kernel/core
148−COPY --from=prod-deps /app/packages/kernel/runtime-quickjs /app/packages/kernel/runtime-quickjs
149−COPY --from=prod-deps /app/packages/plugins/encrypted-secrets /app/packages/plugins/encrypted-secrets
150−COPY --from=prod-deps /app/packages/plugins/google /app/packages/plugins/google
151−COPY --from=prod-deps /app/packages/plugins/graphql /app/packages/plugins/graphql
152−COPY --from=prod-deps /app/packages/plugins/mcp /app/packages/plugins/mcp
153−COPY --from=prod-deps /app/packages/plugins/microsoft /app/packages/plugins/microsoft
154−COPY --from=prod-deps /app/packages/plugins/openapi /app/packages/plugins/openapi
155−COPY --from=prod-deps /app/apps/host-selfhost /app/apps/host-selfhost
34+COPY --from=prod-deps /app/.selfhost-runtime /app
15635 COPY --from=build /app/apps/host-selfhost/dist /app/apps/host-selfhost/dist
15736 WORKDIR /app/apps/host-selfhost
15837 VOLUME ["/data"]
15938 EXPOSE 4788
160−# Readiness probe against the public /api/health endpoint (a trivial DB ping).
16139 HEALTHCHECK --interval=30s --timeout=5s --start-period=20s --retries=5 \
16240 CMD bun -e "fetch('http://127.0.0.1:4788/api/health').then(r=>process.exit(r.ok?0:1),()=>process.exit(1))"
163−# serve.ts binds the Effect AppLayer (API + /mcp + /api/auth + /docs) and serves
164−# the built SPA from ./dist — one process.
16541 CMD ["bun", "run", "src/serve.ts"]
scripts/package-selfhost-runtime.ts+236−0
@@ -0,0 +1,236 @@
1+import {
2+ copyFileSync,
3+ existsSync,
4+ lstatSync,
5+ mkdirSync,
6+ readdirSync,
7+ readFileSync,
8+ readlinkSync,
9+ rmSync,
10+ symlinkSync,
11+} from "node:fs";
12+import { dirname, join, relative } from "node:path";
13+
14+const root = process.cwd();
15+const out = join(root, ".selfhost-runtime");
16+const appName = "@executor-js/host-selfhost";
17+
18+const packageRoots = ["apps", "packages", "examples"] as const;
19+const packagePruneExtensions = [".map", ".d.ts", ".d.mts", ".d.cts", ".md", ".markdown"];
20+const nodeModulePruneExtensions = [...packagePruneExtensions, ".ts", ".tsx"];
21+const sourcePruneExtensions = [".test.ts", ".test.tsx"];
22+const prunedDirNames = new Set([
23+ ".git",
24+ ".turbo",
25+ "test",
26+ "tests",
27+ "__tests__",
28+ "fixtures",
29+ "fixture",
30+ "examples",
31+ "__snapshots__",
32+]);
33+const packageConfigFiles = new Set([
34+ "tsconfig.json",
35+ "tsup.config.ts",
36+ "vite.config.ts",
37+ "vitest.config.ts",
38+ "CHANGELOG.md",
39+]);
40+
41+const nodeModulesPackagePrunes = [
42+ /^@ant-design\+/,
43+ /^@base-ui\+/,
44+ /^@cloudflare\+workers-types@/,
45+ /^@electric-sql\+pglite@/,
46+ /^@emotion\+react@/,
47+ /^@emoji-mart\+/,
48+ /^@esbuild\+.*linux-x64-musl/,
49+ /^@esbuild\+/,
50+ /^@floating-ui\+react@/,
51+ /^@dnd-kit\+/,
52+ /^@hookform\+/,
53+ /^@lobehub\+/,
54+ /^@mermaid-js\+/,
55+ /^@pierre\+diffs@/,
56+ /^@pierre\+theme@/,
57+ /^@primer\+octicons@/,
58+ /^@rc-component\+/,
59+ /^@rolldown\+binding-/,
60+ /^@reduxjs\+/,
61+ /^@shikijs\+/,
62+ /^@splinetool\+/,
63+ /^@tanstack\+start-plugin-core@/,
64+ /^@tanstack\+react-router@/,
65+ /^@tanstack\+router-core@/,
66+ /^@types\+/,
67+ /^ahooks@/,
68+ /^antd@/,
69+ /^better-sqlite3@/,
70+ /^bun-types@/,
71+ /^caniuse-lite@/,
72+ /^chroma-js@/,
73+ /^cytoscape/,
74+ /^d3@/,
75+ /^date-fns/,
76+ /^drizzle-kit@/,
77+ /^dompurify@/,
78+ /^emoji-mart@/,
79+ /^framer-motion@/,
80+ /^jiti@/,
81+ /^katex@/,
82+ /^layout-base@/,
83+ /^leva@/,
84+ /^lightningcss-/,
85+ /^lit-html@/,
86+ /^lucide-react@/,
87+ /^mermaid@/,
88+ /^motion@/,
89+ /^motion-/,
90+ /^polished@/,
91+ /^prettier@/,
92+ /^react-day-picker@/,
93+ /^react-hook-form@/,
94+ /^react@/,
95+ /^react-dom@/,
96+ /^react-dropzone@/,
97+ /^recharts@/,
98+ /^rolldown@/,
99+ /^shiki@/,
100+ /^solid-js@/,
101+ /^source-map@/,
102+ /^tailwind-merge@/,
103+ /^tsx@/,
104+ /^@upsetjs\+venn\.js@/,
105+ /^victory-vendor@/,
106+ /^virtua@/,
107+ /^typeorm@/,
108+ /^vite@/,
109+ /^vitest@/,
110+];
111+
112+type PackageJson = {
113+ readonly name?: string;
114+ readonly dependencies?: Record<string, string>;
115+ readonly optionalDependencies?: Record<string, string>;
116+};
117+
118+type WorkspacePackage = {
119+ readonly name: string;
120+ readonly dir: string;
121+ readonly packageJson: PackageJson;
122+};
123+
124+const readJson = <T>(path: string): T => JSON.parse(readFileSync(path, "utf8")) as T;
125+
126+const copyRecursive = (
127+ source: string,
128+ target: string,
129+ shouldSkip: (
130+ source: string,
131+ relativePath: string,
132+ stats: ReturnType<typeof lstatSync>,
133+ ) => boolean,
134+) => {
135+ const stats = lstatSync(source);
136+ const rel = relative(root, source);
137+ if (shouldSkip(source, rel, stats)) return;
138+
139+ if (stats.isSymbolicLink()) {
140+ mkdirSync(dirname(target), { recursive: true });
141+ symlinkSync(readlinkSync(source), target);
142+ return;
143+ }
144+
145+ if (stats.isDirectory()) {
146+ mkdirSync(target, { recursive: true });
147+ for (const entry of readdirSync(source)) {
148+ copyRecursive(join(source, entry), join(target, entry), shouldSkip);
149+ }
150+ return;
151+ }
152+
153+ if (stats.isFile()) {
154+ mkdirSync(dirname(target), { recursive: true });
155+ copyFileSync(source, target);
156+ }
157+};
158+
159+const discoverWorkspacePackages = (): Map<string, WorkspacePackage> => {
160+ const packages = new Map<string, WorkspacePackage>();
161+ for (const rootName of packageRoots) {
162+ const rootDir = join(root, rootName);
163+ if (!existsSync(rootDir)) continue;
164+ for (const first of readdirSync(rootDir)) {
165+ const firstDir = join(rootDir, first);
166+ if (!lstatSync(firstDir).isDirectory()) continue;
167+ const candidates = [firstDir, ...readdirSync(firstDir).map((name) => join(firstDir, name))];
168+ for (const candidate of candidates) {
169+ const packageJsonPath = join(candidate, "package.json");
170+ if (!existsSync(packageJsonPath)) continue;
171+ const packageJson = readJson<PackageJson>(packageJsonPath);
172+ if (packageJson.name) {
173+ packages.set(packageJson.name, { name: packageJson.name, dir: candidate, packageJson });
174+ }
175+ }
176+ }
177+ }
178+ return packages;
179+};
180+
181+const workspacePackages = discoverWorkspacePackages();
182+const target = workspacePackages.get(appName);
183+if (!target) throw new Error(`Could not find ${appName}`);
184+
185+const reachable = new Map<string, WorkspacePackage>();
186+const queue = [target];
187+for (const current of queue) {
188+ if (reachable.has(current.name)) continue;
189+ reachable.set(current.name, current);
190+ const dependencies = {
191+ ...current.packageJson.dependencies,
192+ ...current.packageJson.optionalDependencies,
193+ };
194+ for (const dependencyName of Object.keys(dependencies)) {
195+ const workspaceDependency = workspacePackages.get(dependencyName);
196+ if (workspaceDependency) queue.push(workspaceDependency);
197+ }
198+}
199+
200+rmSync(out, { recursive: true, force: true });
201+mkdirSync(out, { recursive: true });
202+
203+copyRecursive(join(root, "package.json"), join(out, "package.json"), () => false);
204+copyRecursive(join(root, "bun.lock"), join(out, "bun.lock"), () => false);
205+
206+copyRecursive(join(root, "node_modules"), join(out, "node_modules"), (source, rel, stats) => {
207+ const base = source.split("/node_modules/.bun/")[1]?.split("/")[0];
208+ if (base?.includes("linux-x64-musl")) return true;
209+ if (base && nodeModulesPackagePrunes.some((pattern) => pattern.test(base))) return true;
210+ const name = source.split("/").at(-1) ?? "";
211+ if (stats.isDirectory() && source !== join(root, "node_modules") && prunedDirNames.has(name))
212+ return true;
213+ if (stats.isFile() && nodeModulePruneExtensions.some((extension) => source.endsWith(extension)))
214+ return true;
215+ return false;
216+});
217+
218+for (const pkg of reachable.values()) {
219+ const destination = join(out, relative(root, pkg.dir));
220+ copyRecursive(pkg.dir, destination, (source, rel, stats) => {
221+ const name = source.split("/").at(-1) ?? "";
222+ if (stats.isDirectory() && prunedDirNames.has(name)) return true;
223+ if (stats.isFile() && packageConfigFiles.has(name)) return true;
224+ if (stats.isFile() && sourcePruneExtensions.some((extension) => source.endsWith(extension)))
225+ return true;
226+ if (stats.isFile() && packagePruneExtensions.some((extension) => source.endsWith(extension)))
227+ return true;
228+ if (rel === "apps/host-selfhost/Dockerfile" || rel === "apps/host-selfhost/docker-compose.yml")
229+ return true;
230+ if (rel.startsWith("apps/host-selfhost/web/")) return true;
231+ if (rel.startsWith("apps/host-selfhost/scripts/")) return true;
232+ return false;
233+ });
234+}
235+
236+console.log(`Packaged ${reachable.size} workspace packages into ${relative(root, out)}`);