History

harden install and release runtime checks

Validate runtime-injected Convex URL in sandbox install flows, assert seeded anonymous issuer correctness during runtime artifact build, and add a release-gating local install e2e step to catch environment-specific regressions before publish.

Rhys Sullivan committed 8 months ago

Commit 8f20d6b1 parent f04fd457 files · +158 −35Browse files

Unified · Split

7 files
  1. .github/workflows/release.yml +5 −0
  2. executor/packages/core/src/managed-runtime.ts +26 −12
  3. executor/scripts/e2e/install-checks.ts +9 −0
  4. executor/scripts/e2e/install.e2e.test.ts +26 −13
  5. executor/scripts/e2e/install.sandbox.e2e.test.ts +43 −5
  6. executor/scripts/e2e/install.sandbox.session.ts +11 −1
  7. executor/scripts/release/build-release.ts +38 −4
.github/workflows/release.yml+5−0
@@ -114,6 +114,11 @@ jobs: · 113 unmodified lines
114114 - name: Build release artifacts
115115 run: bun run --cwd executor build:release
116116
117+ - name: Validate local install flow against built artifacts
118+ run: |
119+ bun run --cwd executor build:binary
120+ EXECUTOR_INSTALL_E2E_SKIP_BUILD=1 bun test executor/scripts/e2e/install.e2e.test.ts
121+
117122 - name: Verify release artifacts exist
118123 shell: bash
119124 run: |
executor/packages/core/src/managed-runtime.ts+26−12
@@ -20,6 +20,22 @@ function trimEnv(name: string): string | null { · 19 unmodified lines
2020 return value && value.length > 0 ? value : null;
2121 }
2222
23+function resolveManagedConvexEndpoints(info: ManagedRuntimeInfo): {
24+ convexUrl: string;
25+ convexSiteUrl: string;
26+} {
27+ const managedConvexUrl = `http://${info.config.hostInterface}:${info.config.backendPort}`;
28+ const managedConvexSiteUrl = `http://${info.config.hostInterface}:${info.config.siteProxyPort}`;
29+
30+ const overrideConvexUrl = trimEnv("EXECUTOR_WEB_CONVEX_URL");
31+ const overrideConvexSiteUrl = trimEnv("EXECUTOR_WEB_CONVEX_SITE_URL");
32+
33+ return {
34+ convexUrl: overrideConvexUrl ?? managedConvexUrl,
35+ convexSiteUrl: overrideConvexSiteUrl ?? managedConvexSiteUrl,
36+ };
37+}
38+
2339 function normalizePemForEnv(value: string): string {
2440 return value.replace(/\r\n/g, "\n").replace(/\n/g, "\\n").trim();
2541 }
@@ -162,8 +178,7 @@ export async function ensureManagedRuntime(options: { quiet?: boolean } = {}): P · 136 unmodified lines
162178 export async function runManagedBackend(args: string[]): Promise<number> {
163179 const info = await ensureManagedRuntime();
164180 const anonymousAuthEnv = await resolveManagedAnonymousAuthEnv(info);
165− const managedConvexUrl = `http://${info.config.hostInterface}:${info.config.backendPort}`;
166− const managedConvexSiteUrl = `http://${info.config.hostInterface}:${info.config.siteProxyPort}`;
181+ const endpoints = resolveManagedConvexEndpoints(info);
167182
168183 if (args.length === 0) {
169184 try {
@@ -179,9 +194,9 @@ export async function runManagedBackend(args: string[]): Promise<number> { · 9 unmodified lines
179194 ...process.env,
180195 ...anonymousAuthEnv,
181196 PATH: `${path.dirname(info.nodeBin)}:${process.env.PATH ?? ""}`,
182− CONVEX_URL: managedConvexUrl,
183− CONVEX_SITE_URL: managedConvexSiteUrl,
184− ANONYMOUS_AUTH_ISSUER: managedConvexSiteUrl,
197+ CONVEX_URL: endpoints.convexUrl,
198+ CONVEX_SITE_URL: endpoints.convexSiteUrl,
199+ ANONYMOUS_AUTH_ISSUER: endpoints.convexSiteUrl,
185200 };
186201
187202 const proc = Bun.spawn([info.backendBinary, ...backendArgs(info, args)], {
@@ -211,8 +226,7 @@ export async function runManagedWeb(options: { port?: number } = {}): Promise<nu · 23 unmodified lines
211226
212227 const webPort = options.port ?? Number(Bun.env.EXECUTOR_WEB_PORT ?? 5312);
213228 const host = Bun.env.EXECUTOR_WEB_INTERFACE ?? "127.0.0.1";
214− const managedConvexUrl = `http://${info.config.hostInterface}:${info.config.backendPort}`;
215− const managedConvexSiteUrl = `http://${info.config.hostInterface}:${info.config.siteProxyPort}`;
229+ const endpoints = resolveManagedConvexEndpoints(info);
216230
217231 const env = {
218232 ...process.env,
@@ -220,11 +234,11 @@ export async function runManagedWeb(options: { port?: number } = {}): Promise<nu · 1 unmodified line
220234 NODE_ENV: "production",
221235 PORT: String(webPort),
222236 HOSTNAME: host,
223− CONVEX_URL: managedConvexUrl,
224− CONVEX_SITE_URL: managedConvexSiteUrl,
225− EXECUTOR_WEB_CONVEX_URL: managedConvexUrl,
226− EXECUTOR_WEB_CONVEX_SITE_URL: managedConvexSiteUrl,
227− ANONYMOUS_AUTH_ISSUER: managedConvexSiteUrl,
237+ CONVEX_URL: endpoints.convexUrl,
238+ CONVEX_SITE_URL: endpoints.convexSiteUrl,
239+ EXECUTOR_WEB_CONVEX_URL: endpoints.convexUrl,
240+ EXECUTOR_WEB_CONVEX_SITE_URL: endpoints.convexSiteUrl,
241+ ANONYMOUS_AUTH_ISSUER: endpoints.convexSiteUrl,
228242 };
229243
230244 const proc = await runProcess(info.nodeBin, [info.webServerEntry], {
executor/scripts/e2e/install-checks.ts+9−0
@@ -13,3 +13,12 @@ export function anonymousBootstrapCheckScript(ports: { backendPort: number; webP · 12 unmodified lines
1313 "printf '%s' \"$mutation_json\" | \"$node_bin\" -e \"let d='';process.stdin.on('data',c=>d+=c);process.stdin.on('end',()=>{const j=JSON.parse(d);if(j.status!=='success' || !j.value || typeof j.value.workspaceId!=='string' || typeof j.value.sessionId!=='string'){console.error(d);process.exit(3)}console.log(JSON.stringify({workspaceId:j.value.workspaceId,sessionId:j.value.sessionId}))})\"",
1414 ].join("; ");
1515 }
16+
17+export function runtimeConfigCheckScript(options: { webPort: number; expectedConvexUrl: string }): string {
18+ return [
19+ "set -euo pipefail",
20+ `expected_convex_url='${options.expectedConvexUrl}'`,
21+ `html=$(curl -fsS http://127.0.0.1:${options.webPort}/)`,
22+ "printf '%s' \"$html\" | node -e \"const expected=process.argv[1];let d='';process.stdin.on('data',c=>d+=c);process.stdin.on('end',()=>{const m=d.match(/window\\.__EXECUTOR_RUNTIME_CONFIG__ = ([^;]+);/);if(!m){console.error('runtime config script missing');process.exit(2)};const cfg=JSON.parse(m[1]);if(cfg?.convexUrl!==expected){console.error(JSON.stringify({expected,actual:cfg?.convexUrl}));process.exit(3)};if(expected.startsWith('https://')&&!String(cfg?.convexUrl||'').startsWith('https://')){console.error('expected https convex url');process.exit(4)};console.log(JSON.stringify({convexUrl:cfg.convexUrl}))})\" \"$expected_convex_url\"",
23+ ].join("; ");
24+}
executor/scripts/e2e/install.e2e.test.ts+26−13
@@ -77,6 +77,11 @@ function assertSuccess(result: CommandResult, label: string): void { · 76 unmodified lines
7777 );
7878 }
7979
80+function shouldSkipBuildSteps(): boolean {
81+ const raw = process.env.EXECUTOR_INSTALL_E2E_SKIP_BUILD?.trim().toLowerCase();
82+ return raw === "1" || raw === "true" || raw === "yes";
83+}
84+
8085 test("installer e2e: uninstall, install, deploy, and cleanup", async () => {
8186 const repoRoot = repositoryRoot();
8287 const installScript = path.join(repoRoot, "install");
@@ -109,19 +114,27 @@ test("installer e2e: uninstall, install, deploy, and cleanup", async () => { · 26 unmodified lines
109114 };
110115
111116 try {
112− const buildBinary = await runCommand(["bun", "run", "build:binary"], {
113− cwd: repoRoot,
114− env: process.env,
115− timeoutMs: 300_000,
116− });
117− assertSuccess(buildBinary, "build binary");
118−
119− const buildRelease = await runCommand(["bun", "run", "build:release"], {
120− cwd: repoRoot,
121− env: process.env,
122− timeoutMs: 1_200_000,
123− });
124− assertSuccess(buildRelease, "build release");
117+ if (!shouldSkipBuildSteps()) {
118+ const buildBinary = await runCommand(["bun", "run", "build:binary"], {
119+ cwd: repoRoot,
120+ env: process.env,
121+ timeoutMs: 300_000,
122+ });
123+ assertSuccess(buildBinary, "build binary");
124+
125+ const buildRelease = await runCommand(["bun", "run", "build:release"], {
126+ cwd: repoRoot,
127+ env: process.env,
128+ timeoutMs: 1_200_000,
129+ });
130+ assertSuccess(buildRelease, "build release");
131+ }
132+
133+ if (!(await pathExists(binaryPath))) {
134+ throw new Error(
135+ `Missing expected binary at ${binaryPath}. Run 'bun run build:binary' or unset EXECUTOR_INSTALL_E2E_SKIP_BUILD.`,
136+ );
137+ }
125138
126139 await fs.mkdir(installCwd, { recursive: true });
127140
executor/scripts/e2e/install.sandbox.e2e.test.ts+43−5
@@ -1,6 +1,6 @@
11 import { expect, test } from "bun:test";
22 import { Sandbox } from "@vercel/sandbox";
3−import { anonymousBootstrapCheckScript, runtimeDoctorScript } from "./install-checks";
3+import { anonymousBootstrapCheckScript, runtimeConfigCheckScript, runtimeDoctorScript } from "./install-checks";
44
55 type CommandResult = {
66 exitCode: number;
@@ -12,6 +12,7 @@ async function runSandboxBash( · 5 unmodified lines
1212 sandbox: Sandbox,
1313 script: string,
1414 timeoutMs: number,
15+ env?: Record<string, string>,
1516 ): Promise<CommandResult> {
1617 const controller = new AbortController();
1718 const timeout = setTimeout(() => {
@@ -22,6 +23,7 @@ async function runSandboxBash( · 4 unmodified lines
2223 const command = await sandbox.runCommand({
2324 cmd: "bash",
2425 args: ["-lc", script],
26+ env,
2527 signal: controller.signal,
2628 });
2729
@@ -63,6 +65,10 @@ function assertSuccess(result: CommandResult, label: string): void { · 35 unmodified lines
6365 }
6466
6567 test("installer works in fresh Vercel sandbox", async () => {
68+ const backendPort = 5410;
69+ const sitePort = 5411;
70+ const webPort = 5312;
71+
6672 const credentials =
6773 process.env.VERCEL_TOKEN && process.env.VERCEL_TEAM_ID && process.env.VERCEL_PROJECT_ID
6874 ? {
@@ -76,6 +82,7 @@ test("installer works in fresh Vercel sandbox", async () => { · 7 unmodified lines
7682 try {
7783 return await Sandbox.create({
7884 runtime: "node22",
85+ ports: [webPort, backendPort, sitePort],
7986 timeout: 30 * 60 * 1000,
8087 ...credentials,
8188 });
@@ -90,13 +97,34 @@ test("installer works in fresh Vercel sandbox", async () => { · 8 unmodified lines
9097 let sandbox: Sandbox | null = await createSandbox();
9198
9299 try {
100+ const buildSandboxEnv = (activeSandbox: Sandbox) => {
101+ const convexUrl = activeSandbox.domain(backendPort);
102+ const convexSiteUrl = activeSandbox.domain(sitePort);
103+ return {
104+ convexUrl,
105+ env: {
106+ EXECUTOR_BACKEND_INTERFACE: "0.0.0.0",
107+ EXECUTOR_WEB_INTERFACE: "0.0.0.0",
108+ EXECUTOR_BACKEND_PORT: String(backendPort),
109+ EXECUTOR_BACKEND_SITE_PORT: String(sitePort),
110+ EXECUTOR_WEB_PORT: String(webPort),
111+ CONVEX_URL: convexUrl,
112+ CONVEX_SITE_URL: convexSiteUrl,
113+ EXECUTOR_WEB_CONVEX_URL: convexUrl,
114+ EXECUTOR_WEB_CONVEX_SITE_URL: convexSiteUrl,
115+ },
116+ };
117+ };
118+
119+ let sandboxSettings = buildSandboxEnv(sandbox);
120+
93121 const installScript = [
94122 "set -euo pipefail",
95123 "cd ~",
96124 "if [ -x ~/.executor/bin/executor ]; then ~/.executor/bin/executor uninstall --yes || true; fi",
97125 "rm -rf ~/.executor",
98126 "start=$(date +%s)",
99− "curl -fsSL https://executor.sh/install | bash",
127+ "curl -fsSL https://executor.sh/install | bash -s -- --no-modify-path --no-star-prompt",
100128 "end=$(date +%s)",
101129 "echo INSTALL_SECONDS=$((end-start))",
102130 runtimeDoctorScript(),
@@ -104,7 +132,7 @@ test("installer works in fresh Vercel sandbox", async () => { · 1 unmodified line
104132
105133 let install: CommandResult;
106134 try {
107− install = await runSandboxBash(sandbox, installScript, 900_000);
135+ install = await runSandboxBash(sandbox, installScript, 900_000, sandboxSettings.env);
108136 } catch (error) {
109137 if (!isSandboxStoppedError(error)) {
110138 throw error;
@@ -116,16 +144,26 @@ test("installer works in fresh Vercel sandbox", async () => { · 5 unmodified lines
116144 // ignore
117145 }
118146 sandbox = await createSandbox();
119− install = await runSandboxBash(sandbox, installScript, 900_000);
147+ sandboxSettings = buildSandboxEnv(sandbox);
148+ install = await runSandboxBash(sandbox, installScript, 900_000, sandboxSettings.env);
120149 }
121150 assertSuccess(install, "sandbox install + doctor");
122151
123152 const output = `${install.stdout}\n${install.stderr}`;
124153 expect(output).toContain("Executor status: ready");
154+ const runtimeConfigCheck = await runSandboxBash(
155+ sandbox,
156+ runtimeConfigCheckScript({ webPort, expectedConvexUrl: sandboxSettings.convexUrl }),
157+ 300_000,
158+ sandboxSettings.env,
159+ );
160+ assertSuccess(runtimeConfigCheck, "sandbox runtime config convex URL check");
161+
125162 const anonymousCheck = await runSandboxBash(
126163 sandbox,
127− anonymousBootstrapCheckScript({ backendPort: 5410, webPort: 5312 }),
164+ anonymousBootstrapCheckScript({ backendPort, webPort }),
128165 300_000,
166+ sandboxSettings.env,
129167 );
130168 assertSuccess(anonymousCheck, "sandbox anonymous bootstrap check");
131169
executor/scripts/e2e/install.sandbox.session.ts+11−1
@@ -1,6 +1,6 @@
11 import { Sandbox } from "@vercel/sandbox";
22 import { generateKeyPairSync } from "node:crypto";
3−import { anonymousBootstrapCheckScript, runtimeDoctorScript } from "./install-checks";
3+import { anonymousBootstrapCheckScript, runtimeConfigCheckScript, runtimeDoctorScript } from "./install-checks";
44
55 type CommandResult = {
66 exitCode: number;
@@ -238,6 +238,16 @@ try { · 231 unmodified lines
238238 );
239239 assertSuccess(doctor, "sandbox doctor --verbose");
240240
241+ const runtimeConfigCheck = await runSandboxBash(
242+ sandbox,
243+ runtimeConfigCheckScript({ webPort, expectedConvexUrl: convexUrl }),
244+ {
245+ timeoutMs: installTimeoutMs,
246+ env: sandboxEnv,
247+ },
248+ );
249+ assertSuccess(runtimeConfigCheck, "sandbox runtime config convex URL check");
250+
241251 const anonymousCheck = await runSandboxBash(
242252 sandbox,
243253 anonymousBootstrapCheckScript({ backendPort, webPort }),
executor/scripts/release/build-release.ts+38−4
@@ -246,12 +246,48 @@ async function generateSelfHostedAdminKey(config: RuntimeConfig): Promise<string · 245 unmodified lines
246246 return parsed.adminKey;
247247 }
248248
249+async function assertSeededClientConfig(config: RuntimeConfig, expectedAnonymousIssuer: string): Promise<void> {
250+ const response = await fetch(`http://${config.hostInterface}:${config.backendPort}/api/query`, {
251+ method: "POST",
252+ headers: {
253+ "content-type": "application/json",
254+ },
255+ body: JSON.stringify({
256+ path: "app:getClientConfig",
257+ args: {},
258+ }),
259+ });
260+
261+ if (!response.ok) {
262+ const text = await response.text();
263+ throw new Error(`Failed querying app:getClientConfig during runtime seeding: ${text || response.statusText}`);
264+ }
265+
266+ const parsed = await response.json() as {
267+ status?: string;
268+ value?: {
269+ anonymousAuthIssuer?: unknown;
270+ };
271+ };
272+
273+ if (parsed.status !== "success") {
274+ throw new Error(`Runtime seeding query returned unexpected status: ${JSON.stringify(parsed)}`);
275+ }
276+
277+ if (parsed.value?.anonymousAuthIssuer !== expectedAnonymousIssuer) {
278+ throw new Error(
279+ `Runtime seeding issuer mismatch. Expected ${expectedAnonymousIssuer}, got ${String(parsed.value?.anonymousAuthIssuer)}`,
280+ );
281+ }
282+}
283+
249284 async function seedManagedRuntimeState(
250285 rootDir: string,
251286 runtimeDir: string,
252287 config: RuntimeConfig,
253288 anonymousAuth: AnonymousAuthSeed,
254289 ): Promise<void> {
290+ const expectedAnonymousIssuer = `http://${config.hostInterface}:${defaultManagedSitePort}`;
255291 const adminKey = await generateSelfHostedAdminKey(config);
256292 const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), "executor-seed-env-"));
257293 const envFilePath = path.join(tempDir, "selfhost.env");
@@ -265,7 +301,7 @@ async function seedManagedRuntimeState( · 7 unmodified lines
265301
266302 const envEntries: Array<{ name: string; value: string }> = [
267303 { name: "WORKOS_CLIENT_ID", value: "disabled" },
268− { name: "ANONYMOUS_AUTH_ISSUER", value: `http://${config.hostInterface}:${defaultManagedSitePort}` },
304+ { name: "ANONYMOUS_AUTH_ISSUER", value: expectedAnonymousIssuer },
269305 { name: "ANONYMOUS_AUTH_PRIVATE_KEY_PEM", value: anonymousAuth.privateKeyPem },
270306 { name: "ANONYMOUS_AUTH_PUBLIC_KEY_PEM", value: anonymousAuth.publicKeyPem },
271307 { name: "MCP_API_KEY_SECRET", value: anonymousAuth.apiKeySecret },
@@ -282,9 +318,7 @@ async function seedManagedRuntimeState( · 10 unmodified lines
282318 cwd: rootDir,
283319 });
284320
285− await runCommand(["bunx", "convex", "run", "app:getClientConfig", "--env-file", envFilePath], {
286− cwd: rootDir,
287− });
321+ await assertSeededClientConfig(config, expectedAnonymousIssuer);
288322 } finally {
289323 await fs.rm(tempDir, { recursive: true, force: true });
290324 }