Make installer self-bootstrap Convex functions

Rhys Sullivan committed 8 months ago

Commit 100bd0e1 parent 05dd6594 files · +181 −64Browse files

Unified · Split

4 files

executor/executor.ts +36 −28
executor/install +48 −23
executor/packages/core/src/managed-runtime-bootstrap.ts +85 −2
executor/scripts/e2e/install.e2e.test.ts +12 −11

@@ -116,39 +116,47 @@ function isPidRunning(pid: number): boolean { · 115 unmodified lines
116	116	}
117	117
118	118	async function listListeningPids(port: number): Promise<number[]> {
119		− const lsof = Bun.spawn(["lsof", "-ti", `tcp:${port}`, "-sTCP:LISTEN"], {
120		− stdin: "ignore",
121		− stdout: "pipe",
122		− stderr: "ignore",
123		− });
124		− const lsofExit = await lsof.exited;
125		− if (lsofExit === 0) {
126		− const text = await new Response(lsof.stdout).text();
127		− const fromLsof = text
128		− .split("\n")
129		− .map((line) => Number(line.trim()))
130		− .filter((value) => Number.isInteger(value) && value > 0);
131		− if (fromLsof.length > 0) {
132		− return fromLsof;
	119	+ try {
	120	+ const lsof = Bun.spawn(["lsof", "-ti", `tcp:${port}`, "-sTCP:LISTEN"], {
	121	+ stdin: "ignore",
	122	+ stdout: "pipe",
	123	+ stderr: "ignore",
	124	+ });
	125	+ const lsofExit = await lsof.exited;
	126	+ if (lsofExit === 0) {
	127	+ const text = await new Response(lsof.stdout).text();
	128	+ const fromLsof = text
	129	+ .split("\n")
	130	+ .map((line) => Number(line.trim()))
	131	+ .filter((value) => Number.isInteger(value) && value > 0);
	132	+ if (fromLsof.length > 0) {
	133	+ return fromLsof;
	134	+ }
133	135	}
	136	+ } catch {
	137	+ // lsof may be unavailable in minimal environments
134	138	}
135	139
136		− const ss = Bun.spawn(["ss", "-ltnp", `( sport = :${port} )`], {
137		− stdin: "ignore",
138		− stdout: "pipe",
139		− stderr: "ignore",
140		− });
141		− const ssExit = await ss.exited;
142		− if (ssExit !== 0) {
	140	+ try {
	141	+ const ss = Bun.spawn(["ss", "-ltnp", `( sport = :${port} )`], {
	142	+ stdin: "ignore",
	143	+ stdout: "pipe",
	144	+ stderr: "ignore",
	145	+ });
	146	+ const ssExit = await ss.exited;
	147	+ if (ssExit !== 0) {
	148	+ return [];
	149	+ }
	150	+
	151	+ const ssText = await new Response(ss.stdout).text();
	152	+ const matches = [...ssText.matchAll(/pid=(\d+)/g)];
	153	+ const values = matches
	154	+ .map((match) => Number(match[1]))
	155	+ .filter((value) => Number.isInteger(value) && value > 0);
	156	+ return [...new Set(values)];
	157	+ } catch {
143	158	return [];
144	159	}
145		−
146		− const ssText = await new Response(ss.stdout).text();
147		− const matches = [...ssText.matchAll(/pid=(\d+)/g)];
148		− const values = matches
149		− .map((match) => Number(match[1]))
150		− .filter((value) => Number.isInteger(value) && value > 0);
151		− return [...new Set(values)];
152	160	}
153	161
154	162	async function commandForPid(pid: number): Promise<string> {

executor/install+48−23

@@ -365,13 +365,19 @@ download_bootstrap_project_archive() { · 364 unmodified lines
365	365	}
366	366
367	367	prepare_release_bootstrap_project() {
368		− local bootstrap_dir="$RUNTIME_DIR/bootstrap-project"
369		− local version_file="$bootstrap_dir/.executor-bootstrap-version"
	368	+ local bootstrap_root="$RUNTIME_DIR/bootstrap-project"
	369	+ local version_file="$bootstrap_root/.executor-bootstrap-version"
370	370	local cache_key="${RESOLVED_VERSION:-main}"
371	371
372		− if [[ -f "$version_file" ]] && [[ "$(cat "$version_file" 2>/dev/null \|\| true)" == "$cache_key" ]] && is_convex_project_dir "$bootstrap_dir"; then
373		− printf "%s" "$bootstrap_dir"
374		− return
	372	+ if [[ -f "$version_file" ]] && [[ "$(cat "$version_file" 2>/dev/null \|\| true)" == "$cache_key" ]]; then
	373	+ if is_convex_project_dir "$bootstrap_root"; then
	374	+ printf "%s" "$bootstrap_root"
	375	+ return
	376	+ fi
	377	+ if is_convex_project_dir "$bootstrap_root/executor"; then
	378	+ printf "%s" "$bootstrap_root/executor"
	379	+ return
	380	+ fi
375	381	fi
376	382
377	383	local archive_file
@@ -382,24 +388,29 @@ prepare_release_bootstrap_project() { · 4 unmodified lines
382	388	return
383	389	fi
384	390
385		− rm -rf "$bootstrap_dir"
386		− mkdir -p "$bootstrap_dir"
387		− if ! tar -xzf "$archive_file" -C "$bootstrap_dir" --strip-components=1; then
	391	+ rm -rf "$bootstrap_root"
	392	+ mkdir -p "$bootstrap_root"
	393	+ if ! tar -xzf "$archive_file" -C "$bootstrap_root" --strip-components=1; then
388	394	rm -f "$archive_file"
389		− rm -rf "$bootstrap_dir"
	395	+ rm -rf "$bootstrap_root"
390	396	printf ""
391	397	return
392	398	fi
393	399	rm -f "$archive_file"
394	400
395		− if ! is_convex_project_dir "$bootstrap_dir"; then
396		− rm -rf "$bootstrap_dir"
	401	+ local project_dir="$bootstrap_root"
	402	+ if is_convex_project_dir "$bootstrap_root"; then
	403	+ project_dir="$bootstrap_root"
	404	+ elif is_convex_project_dir "$bootstrap_root/executor"; then
	405	+ project_dir="$bootstrap_root/executor"
	406	+ else
	407	+ rm -rf "$bootstrap_root"
397	408	printf ""
398	409	return
399	410	fi
400	411
401	412	printf "%s\n" "$cache_key" > "$version_file"
402		− printf "%s" "$bootstrap_dir"
	413	+ printf "%s" "$project_dir"
403	414	}
404	415
405	416	resolve_project_dir() {
@@ -447,15 +458,13 @@ start_background_services() { · 41 unmodified lines
447	458
448	459	if wait_for_http "http://127.0.0.1:${BACKEND_PORT}/version" 2; then
449	460	echo -e "${MUTED}Managed backend already running${NC} http://127.0.0.1:${BACKEND_PORT}"
450		− run_executor up >>"$LOG_DIR/backend.log" 2>&1 \|\| true
451	461	elif [[ -f "$BACKEND_PID_FILE" ]]; then
452	462	echo -e "${MUTED}Managed backend already tracked${NC} (pid $(cat "$BACKEND_PID_FILE"))"
453		− run_executor up >>"$LOG_DIR/backend.log" 2>&1 \|\| true
454	463	else
455	464	if [[ -n "$PROJECT_DIR" ]]; then
456		− nohup env EXECUTOR_PROJECT_DIR="$PROJECT_DIR" "$INSTALL_DIR/$APP" up >"$LOG_DIR/backend.log" 2>&1 &
	465	+ nohup env EXECUTOR_PROJECT_DIR="$PROJECT_DIR" "$INSTALL_DIR/$APP" up --disable-beacon >"$LOG_DIR/backend.log" 2>&1 &
457	466	else
458		− nohup "$INSTALL_DIR/$APP" up >"$LOG_DIR/backend.log" 2>&1 &
	467	+ nohup "$INSTALL_DIR/$APP" up --disable-beacon >"$LOG_DIR/backend.log" 2>&1 &
459	468	fi
460	469	echo "$!" > "$BACKEND_PID_FILE"
461	470	if wait_for_http "http://127.0.0.1:${BACKEND_PORT}/version"; then
@@ -496,12 +505,20 @@ ensure_project_bootstrapped() { · 34 unmodified lines
496	505	fi
497	506
498	507	echo -e "${MUTED}Ensuring Convex functions are bootstrapped${NC}"
499		− run_executor up >>"$LOG_DIR/backend.log" 2>&1 \|\| true
500	508
501		− if run_executor doctor >/dev/null 2>&1; then
502		− echo -e "${MUTED}Convex functions bootstrapped${NC}"
503		− return 0
504		− fi
	509	+ local attempt
	510	+ for attempt in 1 2 3; do
	511	+ run_executor up >>"$LOG_DIR/backend.log" 2>&1 \|\| true
	512	+
	513	+ if run_executor doctor >/dev/null 2>&1; then
	514	+ echo -e "${MUTED}Convex functions bootstrapped${NC}"
	515	+ return 0
	516	+ fi
	517	+
	518	+ if (( attempt < 3 )); then
	519	+ sleep 2
	520	+ fi
	521	+ done
505	522
506	523	echo -e "${RED}Convex functions are not fully bootstrapped yet.${NC}"
507	524	echo " Run: EXECUTOR_PROJECT_DIR=$PROJECT_DIR executor up"
@@ -628,8 +645,16 @@ if [[ "$NO_MODIFY_PATH" != "1" ]]; then · 120 unmodified lines
628	645	fi
629	646
630	647	start_background_services
631		−wait_for_executor_ready \|\| true
632		−ensure_project_bootstrapped \|\| true
	648	+runtime_ready=0
	649	+if wait_for_executor_ready; then
	650	+ runtime_ready=1
	651	+fi
	652	+
	653	+if (( runtime_ready == 1 )); then
	654	+ ensure_project_bootstrapped \|\| true
	655	+else
	656	+ echo -e "${RED}Skipping function bootstrap because services are not healthy yet.${NC}"
	657	+fi
633	658
634	659	echo ""
635	660	echo "Installed to: $INSTALL_DIR/$APP"

executor/packages/core/src/managed-runtime-bootstrap.ts+85−2

@@ -7,6 +7,22 @@ import { managedRuntimeVersions, pathExists } from "./managed-runtime-info"; · 6 unmodified lines
7	7	import { ensureConvexCliRuntime } from "./managed-runtime-installation";
8	8	import { runProcess } from "./managed-runtime-process";
9	9
	10	+const BOOTSTRAP_REQUIRED_DEPENDENCIES = [
	11	+ "convex",
	12	+ "@convex-dev/migrations",
	13	+ "@convex-dev/stripe",
	14	+ "@convex-dev/workos-authkit",
	15	+ "@apidevtools/swagger-parser",
	16	+ "@workos-inc/node",
	17	+ "convex-helpers",
	18	+ "jose",
	19	+ "openapi-typescript",
	20	+ "zod",
	21	+ "graphql",
	22	+ "better-result",
	23	+ "@modelcontextprotocol/sdk",
	24	+];
	25	+
10	26	async function generateSelfHostedAdminKey(info: ManagedRuntimeInfo): Promise<string> {
11	27	const response = await fetch("https://api.convex.dev/api/local_deployment/generate_admin_key", {
12	28	method: "POST",
@@ -94,11 +110,13 @@ async function hasConvexProjectConfig(candidate: string): Promise<boolean> { · 81 unmodified lines
94	110	return await hasAnyPath(getConfigCandidates(resolvedFunctionsPath));
95	111	}
96	112
97		−async function findProjectDir(): Promise<string \| null> {
	113	+async function findProjectDir(info?: ManagedRuntimeInfo): Promise<string \| null> {
98	114	const roots = [
99	115	Bun.env.EXECUTOR_PROJECT_DIR,
100	116	process.cwd(),
101	117	path.resolve(import.meta.dir, ".."),
	118	+ info ? path.join(info.rootDir, "bootstrap-project") : null,
	119	+ info ? path.join(info.rootDir, "bootstrap-project", "executor") : null,
102	120	].filter((value): value is string => Boolean(value && value.trim().length > 0));
103	121
104	122	const candidates: string[] = [];
@@ -128,6 +146,69 @@ async function writeBootstrapEnvFile(info: ManagedRuntimeInfo, adminKey: string) · 23 unmodified lines
128	146	return filePath;
129	147	}
130	148
	149	+async function hasBootstrapDependencies(projectDir: string): Promise<boolean> {
	150	+ const nodeModulesDir = path.join(projectDir, "node_modules");
	151	+ if (!(await pathExists(nodeModulesDir))) {
	152	+ return false;
	153	+ }
	154	+
	155	+ for (const dependency of BOOTSTRAP_REQUIRED_DEPENDENCIES) {
	156	+ const dependencyPath = path.join(nodeModulesDir, ...dependency.split("/"));
	157	+ if (!(await pathExists(dependencyPath))) {
	158	+ return false;
	159	+ }
	160	+ }
	161	+
	162	+ return true;
	163	+}
	164	+
	165	+async function ensureProjectDependencies(info: ManagedRuntimeInfo, projectDir: string): Promise<void> {
	166	+ const bootstrapRelative = path.relative(path.join(info.rootDir, "bootstrap-project"), projectDir);
	167	+ const isRuntimeBootstrapProject = bootstrapRelative.length > 0 && !bootstrapRelative.startsWith("..") && !path.isAbsolute(bootstrapRelative);
	168	+
	169	+ if (!isRuntimeBootstrapProject) {
	170	+ return;
	171	+ }
	172	+
	173	+ if (await hasBootstrapDependencies(projectDir)) {
	174	+ return;
	175	+ }
	176	+
	177	+ const env = {
	178	+ ...process.env,
	179	+ PATH: `${path.dirname(info.nodeBin)}:${process.env.PATH ?? ""}`,
	180	+ };
	181	+
	182	+ console.log(`[executor] installing bootstrap dependencies in ${projectDir}`);
	183	+ const install = await runProcess(
	184	+ info.npmBin,
	185	+ [
	186	+ "install",
	187	+ "--prefix",
	188	+ projectDir,
	189	+ "--no-save",
	190	+ "--no-audit",
	191	+ "--no-fund",
	192	+ "--loglevel",
	193	+ "error",
	194	+ "--omit",
	195	+ "dev",
	196	+ "--ignore-scripts",
	197	+ ...BOOTSTRAP_REQUIRED_DEPENDENCIES,
	198	+ ],
	199	+ { env, stdout: "pipe", stderr: "pipe" },
	200	+ );
	201	+
	202	+ if (install.exitCode !== 0) {
	203	+ const detail = install.stderr.trim() \|\| install.stdout.trim() \|\| `exit ${install.exitCode}`;
	204	+ throw new Error(`Failed installing bootstrap dependencies in ${projectDir}: ${detail}`);
	205	+ }
	206	+
	207	+ if (!(await hasBootstrapDependencies(projectDir))) {
	208	+ throw new Error(`Bootstrap dependency install completed but dependencies are still missing in ${projectDir}`);
	209	+ }
	210	+}
	211	+
131	212	async function runManagedConvexCli(
132	213	info: ManagedRuntimeInfo,
133	214	projectDir: string,
@@ -181,7 +262,7 @@ export type BootstrapHealth = { · 47 unmodified lines
181	262	};
182	263
183	264	export async function checkBootstrapHealth(info: ManagedRuntimeInfo): Promise<BootstrapHealth> {
184		− const projectDir = await findProjectDir();
	265	+ const projectDir = await findProjectDir(info);
185	266	if (!projectDir) {
186	267	return { state: "no_project" };
187	268	}
@@ -252,6 +333,8 @@ export async function ensureProjectBootstrapped(info: ManagedRuntimeInfo): Promi · 64 unmodified lines
252	333	throw new Error("Convex bootstrap failed to resolve project directory.");
253	334	}
254	335
	336	+ await ensureProjectDependencies(info, projectDir);
	337	+
255	338	const adminKey = await generateSelfHostedAdminKey(info);
256	339	const envFilePath = await writeBootstrapEnvFile(info, adminKey);
257	340

executor/scripts/e2e/install.e2e.test.ts+12−11

@@ -81,18 +81,12 @@ test("installer e2e: uninstall, install, deploy, and cleanup", async () => { · 80 unmodified lines
81	81	const installScript = path.join(repoRoot, "install");
82	82	const binaryPath = path.join(repoRoot, "dist", "executor");
83	83
84		− if (!(await pathExists(binaryPath))) {
85		− const buildBinary = await runCommand(["bun", "run", "build:binary"], {
86		− cwd: repoRoot,
87		− env: process.env,
88		− timeoutMs: 240_000,
89		− });
90		− assertSuccess(buildBinary, "build binary");
91		− }
92		−
93	84	const tempRoot = await fs.mkdtemp(path.join(os.tmpdir(), "executor-install-e2e-"));
94	85	const installCwd = path.join(tempRoot, "workdir");
95	86	const bootstrapArchive = path.join(tempRoot, "bootstrap-project.tar.gz");
	87	+ const archiveRoot = path.dirname(repoRoot);
	88	+ const archiveParent = path.dirname(archiveRoot);
	89	+ const archiveFolder = path.basename(archiveRoot);
96	90	const homeDir = path.join(tempRoot, "home");
97	91	const executorHome = path.join(homeDir, ".executor");
98	92	const installDir = path.join(executorHome, "bin");
@@ -119,6 +113,13 @@ test("installer e2e: uninstall, install, deploy, and cleanup", async () => { · 20 unmodified lines
119	113	};
120	114
121	115	try {
	116	+ const buildBinary = await runCommand(["bun", "run", "build:binary"], {
	117	+ cwd: repoRoot,
	118	+ env: process.env,
	119	+ timeoutMs: 300_000,
	120	+ });
	121	+ assertSuccess(buildBinary, "build binary");
	122	+
122	123	await fs.mkdir(installCwd, { recursive: true });
123	124
124	125	const archiveSource = await runCommand([
@@ -131,8 +132,8 @@ test("installer e2e: uninstall, install, deploy, and cleanup", async () => { · 6 unmodified lines
131	132	"-czf",
132	133	bootstrapArchive,
133	134	"-C",
134		− path.dirname(repoRoot),
135		− path.basename(repoRoot),
	135	+ archiveParent,
	136	+ archiveFolder,
136	137	], {
137	138	cwd: repoRoot,
138	139	env,