History

Make installer self-bootstrap Convex functions

Rhys Sullivan committed 8 months ago

Commit 100bd0e1 parent 05dd6594 files · +181 −64Browse files

Unified · Split

4 files
  1. executor/executor.ts +36 −28
  2. executor/install +48 −23
  3. executor/packages/core/src/managed-runtime-bootstrap.ts +85 −2
  4. executor/scripts/e2e/install.e2e.test.ts +12 −11
executor/executor.ts+36−28
@@ -116,39 +116,47 @@ function isPidRunning(pid: number): boolean { · 115 unmodified lines
116}116}
117 117
118async function listListeningPids(port: number): Promise<number[]> {118async function listListeningPids(port: number): Promise<number[]> {
119 const lsof = Bun.spawn(["lsof", "-ti", `tcp:${port}`, "-sTCP:LISTEN"], {119 try {
120 stdin: "ignore",120 const lsof = Bun.spawn(["lsof", "-ti", `tcp:${port}`, "-sTCP:LISTEN"], {
121 stdout: "pipe",121 stdin: "ignore",
122 stderr: "ignore",122 stdout: "pipe",
123 });123 stderr: "ignore",
124 const lsofExit = await lsof.exited;124 });
125 if (lsofExit === 0) {125 const lsofExit = await lsof.exited;
126 const text = await new Response(lsof.stdout).text();126 if (lsofExit === 0) {
127 const fromLsof = text127 const text = await new Response(lsof.stdout).text();
128 .split("\n")128 const fromLsof = text
129 .map((line) => Number(line.trim()))129 .split("\n")
130 .filter((value) => Number.isInteger(value) && value > 0);130 .map((line) => Number(line.trim()))
131 if (fromLsof.length > 0) {131 .filter((value) => Number.isInteger(value) && value > 0);
132 return fromLsof;132 if (fromLsof.length > 0) {
133 return fromLsof;
134 }
133 }135 }
136 } catch {
137 // lsof may be unavailable in minimal environments
134 }138 }
135 139
136 const ss = Bun.spawn(["ss", "-ltnp", `( sport = :${port} )`], {140 try {
137 stdin: "ignore",141 const ss = Bun.spawn(["ss", "-ltnp", `( sport = :${port} )`], {
138 stdout: "pipe",142 stdin: "ignore",
139 stderr: "ignore",143 stdout: "pipe",
140 });144 stderr: "ignore",
141 const ssExit = await ss.exited;145 });
142 if (ssExit !== 0) {146 const ssExit = await ss.exited;
147 if (ssExit !== 0) {
148 return [];
149 }
150
151 const ssText = await new Response(ss.stdout).text();
152 const matches = [...ssText.matchAll(/pid=(\d+)/g)];
153 const values = matches
154 .map((match) => Number(match[1]))
155 .filter((value) => Number.isInteger(value) && value > 0);
156 return [...new Set(values)];
157 } catch {
143 return [];158 return [];
144 }159 }
145
146 const ssText = await new Response(ss.stdout).text();
147 const matches = [...ssText.matchAll(/pid=(\d+)/g)];
148 const values = matches
149 .map((match) => Number(match[1]))
150 .filter((value) => Number.isInteger(value) && value > 0);
151 return [...new Set(values)];
152}160}
153 161
154async function commandForPid(pid: number): Promise<string> {162async function commandForPid(pid: number): Promise<string> {
executor/install+48−23
@@ -365,13 +365,19 @@ download_bootstrap_project_archive() { · 364 unmodified lines
365}365}
366 366
367prepare_release_bootstrap_project() {367prepare_release_bootstrap_project() {
368 local bootstrap_dir="$RUNTIME_DIR/bootstrap-project"368 local bootstrap_root="$RUNTIME_DIR/bootstrap-project"
369 local version_file="$bootstrap_dir/.executor-bootstrap-version"369 local version_file="$bootstrap_root/.executor-bootstrap-version"
370 local cache_key="${RESOLVED_VERSION:-main}"370 local cache_key="${RESOLVED_VERSION:-main}"
371 371
372 if [[ -f "$version_file" ]] && [[ "$(cat "$version_file" 2>/dev/null || true)" == "$cache_key" ]] && is_convex_project_dir "$bootstrap_dir"; then372 if [[ -f "$version_file" ]] && [[ "$(cat "$version_file" 2>/dev/null || true)" == "$cache_key" ]]; then
373 printf "%s" "$bootstrap_dir"373 if is_convex_project_dir "$bootstrap_root"; then
374 return374 printf "%s" "$bootstrap_root"
375 return
376 fi
377 if is_convex_project_dir "$bootstrap_root/executor"; then
378 printf "%s" "$bootstrap_root/executor"
379 return
380 fi
375 fi381 fi
376 382
377 local archive_file383 local archive_file
@@ -382,24 +388,29 @@ prepare_release_bootstrap_project() { · 4 unmodified lines
382 return388 return
383 fi389 fi
384 390
385 rm -rf "$bootstrap_dir"391 rm -rf "$bootstrap_root"
386 mkdir -p "$bootstrap_dir"392 mkdir -p "$bootstrap_root"
387 if ! tar -xzf "$archive_file" -C "$bootstrap_dir" --strip-components=1; then393 if ! tar -xzf "$archive_file" -C "$bootstrap_root" --strip-components=1; then
388 rm -f "$archive_file"394 rm -f "$archive_file"
389 rm -rf "$bootstrap_dir"395 rm -rf "$bootstrap_root"
390 printf ""396 printf ""
391 return397 return
392 fi398 fi
393 rm -f "$archive_file"399 rm -f "$archive_file"
394 400
395 if ! is_convex_project_dir "$bootstrap_dir"; then401 local project_dir="$bootstrap_root"
396 rm -rf "$bootstrap_dir"402 if is_convex_project_dir "$bootstrap_root"; then
403 project_dir="$bootstrap_root"
404 elif is_convex_project_dir "$bootstrap_root/executor"; then
405 project_dir="$bootstrap_root/executor"
406 else
407 rm -rf "$bootstrap_root"
397 printf ""408 printf ""
398 return409 return
399 fi410 fi
400 411
401 printf "%s\n" "$cache_key" > "$version_file"412 printf "%s\n" "$cache_key" > "$version_file"
402 printf "%s" "$bootstrap_dir"413 printf "%s" "$project_dir"
403}414}
404 415
405resolve_project_dir() {416resolve_project_dir() {
@@ -447,15 +458,13 @@ start_background_services() { · 41 unmodified lines
447 458
448 if wait_for_http "http://127.0.0.1:${BACKEND_PORT}/version" 2; then459 if wait_for_http "http://127.0.0.1:${BACKEND_PORT}/version" 2; then
449 echo -e "${MUTED}Managed backend already running${NC} http://127.0.0.1:${BACKEND_PORT}"460 echo -e "${MUTED}Managed backend already running${NC} http://127.0.0.1:${BACKEND_PORT}"
450 run_executor up >>"$LOG_DIR/backend.log" 2>&1 || true
451 elif [[ -f "$BACKEND_PID_FILE" ]]; then461 elif [[ -f "$BACKEND_PID_FILE" ]]; then
452 echo -e "${MUTED}Managed backend already tracked${NC} (pid $(cat "$BACKEND_PID_FILE"))"462 echo -e "${MUTED}Managed backend already tracked${NC} (pid $(cat "$BACKEND_PID_FILE"))"
453 run_executor up >>"$LOG_DIR/backend.log" 2>&1 || true
454 else463 else
455 if [[ -n "$PROJECT_DIR" ]]; then464 if [[ -n "$PROJECT_DIR" ]]; then
456 nohup env EXECUTOR_PROJECT_DIR="$PROJECT_DIR" "$INSTALL_DIR/$APP" up >"$LOG_DIR/backend.log" 2>&1 &465 nohup env EXECUTOR_PROJECT_DIR="$PROJECT_DIR" "$INSTALL_DIR/$APP" up --disable-beacon >"$LOG_DIR/backend.log" 2>&1 &
457 else466 else
458 nohup "$INSTALL_DIR/$APP" up >"$LOG_DIR/backend.log" 2>&1 &467 nohup "$INSTALL_DIR/$APP" up --disable-beacon >"$LOG_DIR/backend.log" 2>&1 &
459 fi468 fi
460 echo "$!" > "$BACKEND_PID_FILE"469 echo "$!" > "$BACKEND_PID_FILE"
461 if wait_for_http "http://127.0.0.1:${BACKEND_PORT}/version"; then470 if wait_for_http "http://127.0.0.1:${BACKEND_PORT}/version"; then
@@ -496,12 +505,20 @@ ensure_project_bootstrapped() { · 34 unmodified lines
496 fi505 fi
497 506
498 echo -e "${MUTED}Ensuring Convex functions are bootstrapped${NC}"507 echo -e "${MUTED}Ensuring Convex functions are bootstrapped${NC}"
499 run_executor up >>"$LOG_DIR/backend.log" 2>&1 || true
500 508
501 if run_executor doctor >/dev/null 2>&1; then509 local attempt
502 echo -e "${MUTED}Convex functions bootstrapped${NC}"510 for attempt in 1 2 3; do
503 return 0511 run_executor up >>"$LOG_DIR/backend.log" 2>&1 || true
504 fi512
513 if run_executor doctor >/dev/null 2>&1; then
514 echo -e "${MUTED}Convex functions bootstrapped${NC}"
515 return 0
516 fi
517
518 if (( attempt < 3 )); then
519 sleep 2
520 fi
521 done
505 522
506 echo -e "${RED}Convex functions are not fully bootstrapped yet.${NC}"523 echo -e "${RED}Convex functions are not fully bootstrapped yet.${NC}"
507 echo " Run: EXECUTOR_PROJECT_DIR=$PROJECT_DIR executor up"524 echo " Run: EXECUTOR_PROJECT_DIR=$PROJECT_DIR executor up"
@@ -628,8 +645,16 @@ if [[ "$NO_MODIFY_PATH" != "1" ]]; then · 120 unmodified lines
628fi645fi
629 646
630start_background_services647start_background_services
631wait_for_executor_ready || true648runtime_ready=0
632ensure_project_bootstrapped || true649if wait_for_executor_ready; then
650 runtime_ready=1
651fi
652
653if (( runtime_ready == 1 )); then
654 ensure_project_bootstrapped || true
655else
656 echo -e "${RED}Skipping function bootstrap because services are not healthy yet.${NC}"
657fi
633 658
634echo ""659echo ""
635echo "Installed to: $INSTALL_DIR/$APP"660echo "Installed to: $INSTALL_DIR/$APP"
executor/packages/core/src/managed-runtime-bootstrap.ts+85−2
@@ -7,6 +7,22 @@ import { managedRuntimeVersions, pathExists } from "./managed-runtime-info"; · 6 unmodified lines
7import { ensureConvexCliRuntime } from "./managed-runtime-installation";7import { ensureConvexCliRuntime } from "./managed-runtime-installation";
8import { runProcess } from "./managed-runtime-process";8import { runProcess } from "./managed-runtime-process";
9 9
10const BOOTSTRAP_REQUIRED_DEPENDENCIES = [
11 "convex",
12 "@convex-dev/migrations",
13 "@convex-dev/stripe",
14 "@convex-dev/workos-authkit",
15 "@apidevtools/swagger-parser",
16 "@workos-inc/node",
17 "convex-helpers",
18 "jose",
19 "openapi-typescript",
20 "zod",
21 "graphql",
22 "better-result",
23 "@modelcontextprotocol/sdk",
24];
25
10async function generateSelfHostedAdminKey(info: ManagedRuntimeInfo): Promise<string> {26async function generateSelfHostedAdminKey(info: ManagedRuntimeInfo): Promise<string> {
11 const response = await fetch("https://api.convex.dev/api/local_deployment/generate_admin_key", {27 const response = await fetch("https://api.convex.dev/api/local_deployment/generate_admin_key", {
12 method: "POST",28 method: "POST",
@@ -94,11 +110,13 @@ async function hasConvexProjectConfig(candidate: string): Promise<boolean> { · 81 unmodified lines
94 return await hasAnyPath(getConfigCandidates(resolvedFunctionsPath));110 return await hasAnyPath(getConfigCandidates(resolvedFunctionsPath));
95}111}
96 112
97async function findProjectDir(): Promise<string | null> {113async function findProjectDir(info?: ManagedRuntimeInfo): Promise<string | null> {
98 const roots = [114 const roots = [
99 Bun.env.EXECUTOR_PROJECT_DIR,115 Bun.env.EXECUTOR_PROJECT_DIR,
100 process.cwd(),116 process.cwd(),
101 path.resolve(import.meta.dir, ".."),117 path.resolve(import.meta.dir, ".."),
118 info ? path.join(info.rootDir, "bootstrap-project") : null,
119 info ? path.join(info.rootDir, "bootstrap-project", "executor") : null,
102 ].filter((value): value is string => Boolean(value && value.trim().length > 0));120 ].filter((value): value is string => Boolean(value && value.trim().length > 0));
103 121
104 const candidates: string[] = [];122 const candidates: string[] = [];
@@ -128,6 +146,69 @@ async function writeBootstrapEnvFile(info: ManagedRuntimeInfo, adminKey: string) · 23 unmodified lines
128 return filePath;146 return filePath;
129}147}
130 148
149async function hasBootstrapDependencies(projectDir: string): Promise<boolean> {
150 const nodeModulesDir = path.join(projectDir, "node_modules");
151 if (!(await pathExists(nodeModulesDir))) {
152 return false;
153 }
154
155 for (const dependency of BOOTSTRAP_REQUIRED_DEPENDENCIES) {
156 const dependencyPath = path.join(nodeModulesDir, ...dependency.split("/"));
157 if (!(await pathExists(dependencyPath))) {
158 return false;
159 }
160 }
161
162 return true;
163}
164
165async function ensureProjectDependencies(info: ManagedRuntimeInfo, projectDir: string): Promise<void> {
166 const bootstrapRelative = path.relative(path.join(info.rootDir, "bootstrap-project"), projectDir);
167 const isRuntimeBootstrapProject = bootstrapRelative.length > 0 && !bootstrapRelative.startsWith("..") && !path.isAbsolute(bootstrapRelative);
168
169 if (!isRuntimeBootstrapProject) {
170 return;
171 }
172
173 if (await hasBootstrapDependencies(projectDir)) {
174 return;
175 }
176
177 const env = {
178 ...process.env,
179 PATH: `${path.dirname(info.nodeBin)}:${process.env.PATH ?? ""}`,
180 };
181
182 console.log(`[executor] installing bootstrap dependencies in ${projectDir}`);
183 const install = await runProcess(
184 info.npmBin,
185 [
186 "install",
187 "--prefix",
188 projectDir,
189 "--no-save",
190 "--no-audit",
191 "--no-fund",
192 "--loglevel",
193 "error",
194 "--omit",
195 "dev",
196 "--ignore-scripts",
197 ...BOOTSTRAP_REQUIRED_DEPENDENCIES,
198 ],
199 { env, stdout: "pipe", stderr: "pipe" },
200 );
201
202 if (install.exitCode !== 0) {
203 const detail = install.stderr.trim() || install.stdout.trim() || `exit ${install.exitCode}`;
204 throw new Error(`Failed installing bootstrap dependencies in ${projectDir}: ${detail}`);
205 }
206
207 if (!(await hasBootstrapDependencies(projectDir))) {
208 throw new Error(`Bootstrap dependency install completed but dependencies are still missing in ${projectDir}`);
209 }
210}
211
131async function runManagedConvexCli(212async function runManagedConvexCli(
132 info: ManagedRuntimeInfo,213 info: ManagedRuntimeInfo,
133 projectDir: string,214 projectDir: string,
@@ -181,7 +262,7 @@ export type BootstrapHealth = { · 47 unmodified lines
181};262};
182 263
183export async function checkBootstrapHealth(info: ManagedRuntimeInfo): Promise<BootstrapHealth> {264export async function checkBootstrapHealth(info: ManagedRuntimeInfo): Promise<BootstrapHealth> {
184 const projectDir = await findProjectDir();265 const projectDir = await findProjectDir(info);
185 if (!projectDir) {266 if (!projectDir) {
186 return { state: "no_project" };267 return { state: "no_project" };
187 }268 }
@@ -252,6 +333,8 @@ export async function ensureProjectBootstrapped(info: ManagedRuntimeInfo): Promi · 64 unmodified lines
252 throw new Error("Convex bootstrap failed to resolve project directory.");333 throw new Error("Convex bootstrap failed to resolve project directory.");
253 }334 }
254 335
336 await ensureProjectDependencies(info, projectDir);
337
255 const adminKey = await generateSelfHostedAdminKey(info);338 const adminKey = await generateSelfHostedAdminKey(info);
256 const envFilePath = await writeBootstrapEnvFile(info, adminKey);339 const envFilePath = await writeBootstrapEnvFile(info, adminKey);
257 340
executor/scripts/e2e/install.e2e.test.ts+12−11
@@ -81,18 +81,12 @@ test("installer e2e: uninstall, install, deploy, and cleanup", async () => { · 80 unmodified lines
81 const installScript = path.join(repoRoot, "install");81 const installScript = path.join(repoRoot, "install");
82 const binaryPath = path.join(repoRoot, "dist", "executor");82 const binaryPath = path.join(repoRoot, "dist", "executor");
83 83
84 if (!(await pathExists(binaryPath))) {
85 const buildBinary = await runCommand(["bun", "run", "build:binary"], {
86 cwd: repoRoot,
87 env: process.env,
88 timeoutMs: 240_000,
89 });
90 assertSuccess(buildBinary, "build binary");
91 }
92
93 const tempRoot = await fs.mkdtemp(path.join(os.tmpdir(), "executor-install-e2e-"));84 const tempRoot = await fs.mkdtemp(path.join(os.tmpdir(), "executor-install-e2e-"));
94 const installCwd = path.join(tempRoot, "workdir");85 const installCwd = path.join(tempRoot, "workdir");
95 const bootstrapArchive = path.join(tempRoot, "bootstrap-project.tar.gz");86 const bootstrapArchive = path.join(tempRoot, "bootstrap-project.tar.gz");
87 const archiveRoot = path.dirname(repoRoot);
88 const archiveParent = path.dirname(archiveRoot);
89 const archiveFolder = path.basename(archiveRoot);
96 const homeDir = path.join(tempRoot, "home");90 const homeDir = path.join(tempRoot, "home");
97 const executorHome = path.join(homeDir, ".executor");91 const executorHome = path.join(homeDir, ".executor");
98 const installDir = path.join(executorHome, "bin");92 const installDir = path.join(executorHome, "bin");
@@ -119,6 +113,13 @@ test("installer e2e: uninstall, install, deploy, and cleanup", async () => { · 20 unmodified lines
119 };113 };
120 114
121 try {115 try {
116 const buildBinary = await runCommand(["bun", "run", "build:binary"], {
117 cwd: repoRoot,
118 env: process.env,
119 timeoutMs: 300_000,
120 });
121 assertSuccess(buildBinary, "build binary");
122
122 await fs.mkdir(installCwd, { recursive: true });123 await fs.mkdir(installCwd, { recursive: true });
123 124
124 const archiveSource = await runCommand([125 const archiveSource = await runCommand([
@@ -131,8 +132,8 @@ test("installer e2e: uninstall, install, deploy, and cleanup", async () => { · 6 unmodified lines
131 "-czf",132 "-czf",
132 bootstrapArchive,133 bootstrapArchive,
133 "-C",134 "-C",
134 path.dirname(repoRoot),135 archiveParent,
135 path.basename(repoRoot),136 archiveFolder,
136 ], {137 ], {
137 cwd: repoRoot,138 cwd: repoRoot,
138 env,139 env,