History

Simplify self-host Docker runtime packaging

Rhys Sullivan committed 4 months ago

Commit 84df8de1 parent a65590b2 files · +241 −129Browse files

Unified · Split

2 files
  1. apps/host-selfhost/Dockerfile +5 −129
  2. scripts/package-selfhost-runtime.ts +236 −0
apps/host-selfhost/Dockerfile+5−129
@@ -1,132 +1,26 @@
1# Self-hosted Executor — single container, no external services.1# Self-hosted Executor, single container, no external services.
2# Build context is the REPO ROOT (the bun workspace install needs every member):2# Build context is the repo root because Bun workspaces need every member:
3# docker build -f apps/host-selfhost/Dockerfile -t executor-selfhost .3# docker build -f apps/host-selfhost/Dockerfile -t executor-selfhost .
4#4#
5# Runtime needs nothing but this container + a volume for the data dir:5# Runtime needs this container plus a volume for the data dir:
6# docker run -p 4788:4788 -e BETTER_AUTH_SECRET=$(openssl rand -hex 32) \6# docker run -p 4788:4788 -e BETTER_AUTH_SECRET=$(openssl rand -hex 32) \
7# -e EXECUTOR_BOOTSTRAP_ADMIN_EMAIL=you@example.com \7# -e EXECUTOR_BOOTSTRAP_ADMIN_EMAIL=you@example.com \
8# -e EXECUTOR_BOOTSTRAP_ADMIN_PASSWORD=... \8# -e EXECUTOR_BOOTSTRAP_ADMIN_PASSWORD=... \
9# -e EXECUTOR_WEB_BASE_URL=https://your.domain \9# -e EXECUTOR_WEB_BASE_URL=https://your.domain \
10# -v executor-data:/data executor-selfhost10# -v executor-data:/data executor-selfhost
11#
12# SQLite (libSQL, file:/data/...) lives in /data, QuickJS + MCP run in-process —
13# so there's no postgres/worker/proxy to orchestrate.
14 11
15# ── Production deps: installs only what the runtime image needs ─────────────
16FROM oven/bun:1 AS prod-deps12FROM oven/bun:1 AS prod-deps
17WORKDIR /app13WORKDIR /app
18COPY . .14COPY . .
19# This stage is independent from the SPA build stage, so BuildKit can run the
20# slow production install in parallel with the dev install and Vite build.
21# --ignore-scripts: the root `prepare` hook runs a dev-only tool
22# (effect-language-service); runtime deps are prebuilt JS with no postinstall.
23RUN bun install --frozen-lockfile --production --ignore-scripts --filter @executor-js/host-selfhost \15RUN bun install --frozen-lockfile --production --ignore-scripts --filter @executor-js/host-selfhost \
24 && find node_modules/.bun -maxdepth 1 -name '*linux-x64-musl*' -exec rm -rf {} + \16 && bun run scripts/package-selfhost-runtime.ts
25 && rm -rf \
26 node_modules/.bun/@ant-design+* \
27 node_modules/.bun/@base-ui+* \
28 node_modules/.bun/@cloudflare+workers-types@* \
29 node_modules/.bun/@electric-sql+pglite@* \
30 node_modules/.bun/@emotion+react@* \
31 node_modules/.bun/@emoji-mart+* \
32 node_modules/.bun/@esbuild+* \
33 node_modules/.bun/@floating-ui+react@* \
34 node_modules/.bun/@dnd-kit+* \
35 node_modules/.bun/@hookform+* \
36 node_modules/.bun/@lobehub+* \
37 node_modules/.bun/@mermaid-js+* \
38 node_modules/.bun/@pierre+diffs@* \
39 node_modules/.bun/@pierre+theme@* \
40 node_modules/.bun/@primer+octicons@* \
41 node_modules/.bun/@rc-component+* \
42 node_modules/.bun/@rolldown+binding-* \
43 node_modules/.bun/@reduxjs+* \
44 node_modules/.bun/@shikijs+* \
45 node_modules/.bun/@splinetool+* \
46 node_modules/.bun/@tanstack+start-plugin-core@* \
47 node_modules/.bun/@tanstack+react-router@* \
48 node_modules/.bun/@tanstack+router-core@* \
49 node_modules/.bun/@types+* \
50 node_modules/.bun/ahooks@* \
51 node_modules/.bun/antd@* \
52 node_modules/.bun/better-sqlite3@* \
53 node_modules/.bun/bun-types@* \
54 node_modules/.bun/caniuse-lite@* \
55 node_modules/.bun/chroma-js@* \
56 node_modules/.bun/cytoscape*@* \
57 node_modules/.bun/d3@* \
58 node_modules/.bun/date-fns*@* \
59 node_modules/.bun/drizzle-kit@* \
60 node_modules/.bun/dompurify@* \
61 node_modules/.bun/emoji-mart@* \
62 node_modules/.bun/framer-motion@* \
63 node_modules/.bun/jiti@* \
64 node_modules/.bun/katex@* \
65 node_modules/.bun/layout-base@* \
66 node_modules/.bun/leva@* \
67 node_modules/.bun/lightningcss-* \
68 node_modules/.bun/lit-html@* \
69 node_modules/.bun/lucide-react@* \
70 node_modules/.bun/mermaid@* \
71 node_modules/.bun/motion@* \
72 node_modules/.bun/motion-* \
73 node_modules/.bun/polished@* \
74 node_modules/.bun/prettier@* \
75 node_modules/.bun/react-day-picker@* \
76 node_modules/.bun/react-hook-form@* \
77 node_modules/.bun/react@* \
78 node_modules/.bun/react-dom@* \
79 node_modules/.bun/react-dropzone@* \
80 node_modules/.bun/recharts@* \
81 node_modules/.bun/rolldown@* \
82 node_modules/.bun/shiki@* \
83 node_modules/.bun/solid-js@* \
84 node_modules/.bun/source-map@* \
85 node_modules/.bun/tailwind-merge@* \
86 node_modules/.bun/tsx@* \
87 node_modules/.bun/@upsetjs+venn.js@* \
88 node_modules/.bun/victory-vendor@* \
89 node_modules/.bun/virtua@* \
90 node_modules/.bun/typeorm@* \
91 node_modules/.bun/vite@* \
92 node_modules/.bun/vitest@* \
93 && find node_modules -type f \( -name '*.map' -o -name '*.d.ts' -o -name '*.d.mts' -o -name '*.d.cts' -o -name '*.ts' -o -name '*.tsx' -o -name '*.md' -o -name '*.markdown' \) -delete \
94 && find node_modules -type d \( -iname test -o -iname tests -o -iname __tests__ -o -iname fixtures -o -iname fixture -o -iname examples \) -prune -exec rm -rf {} + \
95 && find packages apps/host-selfhost -type f \( -name '*.map' -o -name '*.d.ts' -o -name '*.test.ts' -o -name '*.test.tsx' -o -name '*.md' -o -name '*.markdown' \) -delete \
96 && rm -rf \
97 packages/app \
98 packages/react \
99 packages/plugins/*/fixtures \
100 packages/plugins/*/src/react \
101 packages/plugins/*/src/**/__snapshots__ \
102 packages/plugins/*/test \
103 packages/plugins/*/testing \
104 packages/*/*/CHANGELOG.md \
105 packages/*/*/tsconfig.json \
106 packages/*/*/tsup.config.ts \
107 packages/*/*/vitest.config.ts \
108 packages/plugins/*/vitest.config.ts \
109 apps/host-selfhost/CHANGELOG.md \
110 apps/host-selfhost/Dockerfile \
111 apps/host-selfhost/docker-compose.yml \
112 apps/host-selfhost/scripts \
113 apps/host-selfhost/vitest.config.ts \
114 apps/host-selfhost/web \
115 apps/host-selfhost/tsconfig.json \
116 apps/host-selfhost/tsr.routes.ts \
117 apps/host-selfhost/vite.config.ts
118 17
119# ── Build stage: install the workspace + build the SPA ──────────────────────
120FROM oven/bun:1 AS build18FROM oven/bun:1 AS build
121WORKDIR /app19WORKDIR /app
122COPY . .20COPY . .
123# Full install (dev deps included, vite/turbo/plugins are needed to build).
124RUN bun install --frozen-lockfile21RUN bun install --frozen-lockfile
125# Builds @executor-js/vite-plugin (via turbo) then the self-host SPA into
126# apps/host-selfhost/dist.
127RUN cd apps/host-selfhost && bun run build22RUN cd apps/host-selfhost && bun run build
128 23
129# ── Runtime stage: serve the built app under Bun ────────────────────────────
130FROM gcr.io/distroless/cc-debian12 AS runtime24FROM gcr.io/distroless/cc-debian12 AS runtime
131WORKDIR /app25WORKDIR /app
132LABEL org.opencontainers.image.source="https://github.com/RhysSullivan/executor" \26LABEL org.opencontainers.image.source="https://github.com/RhysSullivan/executor" \
@@ -137,29 +31,11 @@ ENV NODE_ENV=production \ · 4 unmodified lines
137 PORT=4788 \31 PORT=4788 \
138 EXECUTOR_DATA_DIR=/data32 EXECUTOR_DATA_DIR=/data
139COPY --from=prod-deps /usr/local/bin/bun /usr/local/bin/bun33COPY --from=prod-deps /usr/local/bin/bun /usr/local/bin/bun
140COPY --from=prod-deps /app/node_modules /app/node_modules34COPY --from=prod-deps /app/.selfhost-runtime /app
141COPY --from=prod-deps /app/packages/core/api /app/packages/core/api
142COPY --from=prod-deps /app/packages/core/config /app/packages/core/config
143COPY --from=prod-deps /app/packages/core/execution /app/packages/core/execution
144COPY --from=prod-deps /app/packages/core/fumadb /app/packages/core/fumadb
145COPY --from=prod-deps /app/packages/core/sdk /app/packages/core/sdk
146COPY --from=prod-deps /app/packages/hosts/mcp /app/packages/hosts/mcp
147COPY --from=prod-deps /app/packages/kernel/core /app/packages/kernel/core
148COPY --from=prod-deps /app/packages/kernel/runtime-quickjs /app/packages/kernel/runtime-quickjs
149COPY --from=prod-deps /app/packages/plugins/encrypted-secrets /app/packages/plugins/encrypted-secrets
150COPY --from=prod-deps /app/packages/plugins/google /app/packages/plugins/google
151COPY --from=prod-deps /app/packages/plugins/graphql /app/packages/plugins/graphql
152COPY --from=prod-deps /app/packages/plugins/mcp /app/packages/plugins/mcp
153COPY --from=prod-deps /app/packages/plugins/microsoft /app/packages/plugins/microsoft
154COPY --from=prod-deps /app/packages/plugins/openapi /app/packages/plugins/openapi
155COPY --from=prod-deps /app/apps/host-selfhost /app/apps/host-selfhost
156COPY --from=build /app/apps/host-selfhost/dist /app/apps/host-selfhost/dist35COPY --from=build /app/apps/host-selfhost/dist /app/apps/host-selfhost/dist
157WORKDIR /app/apps/host-selfhost36WORKDIR /app/apps/host-selfhost
158VOLUME ["/data"]37VOLUME ["/data"]
159EXPOSE 478838EXPOSE 4788
160# Readiness probe against the public /api/health endpoint (a trivial DB ping).
161HEALTHCHECK --interval=30s --timeout=5s --start-period=20s --retries=5 \39HEALTHCHECK --interval=30s --timeout=5s --start-period=20s --retries=5 \
162 CMD bun -e "fetch('http://127.0.0.1:4788/api/health').then(r=>process.exit(r.ok?0:1),()=>process.exit(1))"40 CMD bun -e "fetch('http://127.0.0.1:4788/api/health').then(r=>process.exit(r.ok?0:1),()=>process.exit(1))"
163# serve.ts binds the Effect AppLayer (API + /mcp + /api/auth + /docs) and serves
164# the built SPA from ./dist — one process.
165CMD ["bun", "run", "src/serve.ts"]41CMD ["bun", "run", "src/serve.ts"]
scripts/package-selfhost-runtime.ts+236−0
@@ -0,0 +1,236 @@
1import {
2 copyFileSync,
3 existsSync,
4 lstatSync,
5 mkdirSync,
6 readdirSync,
7 readFileSync,
8 readlinkSync,
9 rmSync,
10 symlinkSync,
11} from "node:fs";
12import { dirname, join, relative } from "node:path";
13
14const root = process.cwd();
15const out = join(root, ".selfhost-runtime");
16const appName = "@executor-js/host-selfhost";
17
18const packageRoots = ["apps", "packages", "examples"] as const;
19const packagePruneExtensions = [".map", ".d.ts", ".d.mts", ".d.cts", ".md", ".markdown"];
20const nodeModulePruneExtensions = [...packagePruneExtensions, ".ts", ".tsx"];
21const sourcePruneExtensions = [".test.ts", ".test.tsx"];
22const prunedDirNames = new Set([
23 ".git",
24 ".turbo",
25 "test",
26 "tests",
27 "__tests__",
28 "fixtures",
29 "fixture",
30 "examples",
31 "__snapshots__",
32]);
33const packageConfigFiles = new Set([
34 "tsconfig.json",
35 "tsup.config.ts",
36 "vite.config.ts",
37 "vitest.config.ts",
38 "CHANGELOG.md",
39]);
40
41const nodeModulesPackagePrunes = [
42 /^@ant-design\+/,
43 /^@base-ui\+/,
44 /^@cloudflare\+workers-types@/,
45 /^@electric-sql\+pglite@/,
46 /^@emotion\+react@/,
47 /^@emoji-mart\+/,
48 /^@esbuild\+.*linux-x64-musl/,
49 /^@esbuild\+/,
50 /^@floating-ui\+react@/,
51 /^@dnd-kit\+/,
52 /^@hookform\+/,
53 /^@lobehub\+/,
54 /^@mermaid-js\+/,
55 /^@pierre\+diffs@/,
56 /^@pierre\+theme@/,
57 /^@primer\+octicons@/,
58 /^@rc-component\+/,
59 /^@rolldown\+binding-/,
60 /^@reduxjs\+/,
61 /^@shikijs\+/,
62 /^@splinetool\+/,
63 /^@tanstack\+start-plugin-core@/,
64 /^@tanstack\+react-router@/,
65 /^@tanstack\+router-core@/,
66 /^@types\+/,
67 /^ahooks@/,
68 /^antd@/,
69 /^better-sqlite3@/,
70 /^bun-types@/,
71 /^caniuse-lite@/,
72 /^chroma-js@/,
73 /^cytoscape/,
74 /^d3@/,
75 /^date-fns/,
76 /^drizzle-kit@/,
77 /^dompurify@/,
78 /^emoji-mart@/,
79 /^framer-motion@/,
80 /^jiti@/,
81 /^katex@/,
82 /^layout-base@/,
83 /^leva@/,
84 /^lightningcss-/,
85 /^lit-html@/,
86 /^lucide-react@/,
87 /^mermaid@/,
88 /^motion@/,
89 /^motion-/,
90 /^polished@/,
91 /^prettier@/,
92 /^react-day-picker@/,
93 /^react-hook-form@/,
94 /^react@/,
95 /^react-dom@/,
96 /^react-dropzone@/,
97 /^recharts@/,
98 /^rolldown@/,
99 /^shiki@/,
100 /^solid-js@/,
101 /^source-map@/,
102 /^tailwind-merge@/,
103 /^tsx@/,
104 /^@upsetjs\+venn\.js@/,
105 /^victory-vendor@/,
106 /^virtua@/,
107 /^typeorm@/,
108 /^vite@/,
109 /^vitest@/,
110];
111
112type PackageJson = {
113 readonly name?: string;
114 readonly dependencies?: Record<string, string>;
115 readonly optionalDependencies?: Record<string, string>;
116};
117
118type WorkspacePackage = {
119 readonly name: string;
120 readonly dir: string;
121 readonly packageJson: PackageJson;
122};
123
124const readJson = <T>(path: string): T => JSON.parse(readFileSync(path, "utf8")) as T;
125
126const copyRecursive = (
127 source: string,
128 target: string,
129 shouldSkip: (
130 source: string,
131 relativePath: string,
132 stats: ReturnType<typeof lstatSync>,
133 ) => boolean,
134) => {
135 const stats = lstatSync(source);
136 const rel = relative(root, source);
137 if (shouldSkip(source, rel, stats)) return;
138
139 if (stats.isSymbolicLink()) {
140 mkdirSync(dirname(target), { recursive: true });
141 symlinkSync(readlinkSync(source), target);
142 return;
143 }
144
145 if (stats.isDirectory()) {
146 mkdirSync(target, { recursive: true });
147 for (const entry of readdirSync(source)) {
148 copyRecursive(join(source, entry), join(target, entry), shouldSkip);
149 }
150 return;
151 }
152
153 if (stats.isFile()) {
154 mkdirSync(dirname(target), { recursive: true });
155 copyFileSync(source, target);
156 }
157};
158
159const discoverWorkspacePackages = (): Map<string, WorkspacePackage> => {
160 const packages = new Map<string, WorkspacePackage>();
161 for (const rootName of packageRoots) {
162 const rootDir = join(root, rootName);
163 if (!existsSync(rootDir)) continue;
164 for (const first of readdirSync(rootDir)) {
165 const firstDir = join(rootDir, first);
166 if (!lstatSync(firstDir).isDirectory()) continue;
167 const candidates = [firstDir, ...readdirSync(firstDir).map((name) => join(firstDir, name))];
168 for (const candidate of candidates) {
169 const packageJsonPath = join(candidate, "package.json");
170 if (!existsSync(packageJsonPath)) continue;
171 const packageJson = readJson<PackageJson>(packageJsonPath);
172 if (packageJson.name) {
173 packages.set(packageJson.name, { name: packageJson.name, dir: candidate, packageJson });
174 }
175 }
176 }
177 }
178 return packages;
179};
180
181const workspacePackages = discoverWorkspacePackages();
182const target = workspacePackages.get(appName);
183if (!target) throw new Error(`Could not find ${appName}`);
184
185const reachable = new Map<string, WorkspacePackage>();
186const queue = [target];
187for (const current of queue) {
188 if (reachable.has(current.name)) continue;
189 reachable.set(current.name, current);
190 const dependencies = {
191 ...current.packageJson.dependencies,
192 ...current.packageJson.optionalDependencies,
193 };
194 for (const dependencyName of Object.keys(dependencies)) {
195 const workspaceDependency = workspacePackages.get(dependencyName);
196 if (workspaceDependency) queue.push(workspaceDependency);
197 }
198}
199
200rmSync(out, { recursive: true, force: true });
201mkdirSync(out, { recursive: true });
202
203copyRecursive(join(root, "package.json"), join(out, "package.json"), () => false);
204copyRecursive(join(root, "bun.lock"), join(out, "bun.lock"), () => false);
205
206copyRecursive(join(root, "node_modules"), join(out, "node_modules"), (source, rel, stats) => {
207 const base = source.split("/node_modules/.bun/")[1]?.split("/")[0];
208 if (base?.includes("linux-x64-musl")) return true;
209 if (base && nodeModulesPackagePrunes.some((pattern) => pattern.test(base))) return true;
210 const name = source.split("/").at(-1) ?? "";
211 if (stats.isDirectory() && source !== join(root, "node_modules") && prunedDirNames.has(name))
212 return true;
213 if (stats.isFile() && nodeModulePruneExtensions.some((extension) => source.endsWith(extension)))
214 return true;
215 return false;
216});
217
218for (const pkg of reachable.values()) {
219 const destination = join(out, relative(root, pkg.dir));
220 copyRecursive(pkg.dir, destination, (source, rel, stats) => {
221 const name = source.split("/").at(-1) ?? "";
222 if (stats.isDirectory() && prunedDirNames.has(name)) return true;
223 if (stats.isFile() && packageConfigFiles.has(name)) return true;
224 if (stats.isFile() && sourcePruneExtensions.some((extension) => source.endsWith(extension)))
225 return true;
226 if (stats.isFile() && packagePruneExtensions.some((extension) => source.endsWith(extension)))
227 return true;
228 if (rel === "apps/host-selfhost/Dockerfile" || rel === "apps/host-selfhost/docker-compose.yml")
229 return true;
230 if (rel.startsWith("apps/host-selfhost/web/")) return true;
231 if (rel.startsWith("apps/host-selfhost/scripts/")) return true;
232 return false;
233 });
234}
235
236console.log(`Packaged ${reachable.size} workspace packages into ${relative(root, out)}`);