History

harden install and release runtime checks

Validate runtime-injected Convex URL in sandbox install flows, assert seeded anonymous issuer correctness during runtime artifact build, and add a release-gating local install e2e step to catch environment-specific regressions before publish.

Rhys Sullivan committed 8 months ago

Commit 8f20d6b1 parent f04fd457 files · +158 −35Browse files

Unified · Split

7 files
  1. .github/workflows/release.yml +5 −0
  2. executor/packages/core/src/managed-runtime.ts +26 −12
  3. executor/scripts/e2e/install-checks.ts +9 −0
  4. executor/scripts/e2e/install.e2e.test.ts +26 −13
  5. executor/scripts/e2e/install.sandbox.e2e.test.ts +43 −5
  6. executor/scripts/e2e/install.sandbox.session.ts +11 −1
  7. executor/scripts/release/build-release.ts +38 −4
.github/workflows/release.yml+5−0
@@ -114,6 +114,11 @@ jobs: · 113 unmodified lines
114 - name: Build release artifacts114 - name: Build release artifacts
115 run: bun run --cwd executor build:release115 run: bun run --cwd executor build:release
116 116
117 - name: Validate local install flow against built artifacts
118 run: |
119 bun run --cwd executor build:binary
120 EXECUTOR_INSTALL_E2E_SKIP_BUILD=1 bun test executor/scripts/e2e/install.e2e.test.ts
121
117 - name: Verify release artifacts exist122 - name: Verify release artifacts exist
118 shell: bash123 shell: bash
119 run: |124 run: |
executor/packages/core/src/managed-runtime.ts+26−12
@@ -20,6 +20,22 @@ function trimEnv(name: string): string | null { · 19 unmodified lines
20 return value && value.length > 0 ? value : null;20 return value && value.length > 0 ? value : null;
21}21}
22 22
23function resolveManagedConvexEndpoints(info: ManagedRuntimeInfo): {
24 convexUrl: string;
25 convexSiteUrl: string;
26} {
27 const managedConvexUrl = `http://${info.config.hostInterface}:${info.config.backendPort}`;
28 const managedConvexSiteUrl = `http://${info.config.hostInterface}:${info.config.siteProxyPort}`;
29
30 const overrideConvexUrl = trimEnv("EXECUTOR_WEB_CONVEX_URL");
31 const overrideConvexSiteUrl = trimEnv("EXECUTOR_WEB_CONVEX_SITE_URL");
32
33 return {
34 convexUrl: overrideConvexUrl ?? managedConvexUrl,
35 convexSiteUrl: overrideConvexSiteUrl ?? managedConvexSiteUrl,
36 };
37}
38
23function normalizePemForEnv(value: string): string {39function normalizePemForEnv(value: string): string {
24 return value.replace(/\r\n/g, "\n").replace(/\n/g, "\\n").trim();40 return value.replace(/\r\n/g, "\n").replace(/\n/g, "\\n").trim();
25}41}
@@ -162,8 +178,7 @@ export async function ensureManagedRuntime(options: { quiet?: boolean } = {}): P · 136 unmodified lines
162export async function runManagedBackend(args: string[]): Promise<number> {178export async function runManagedBackend(args: string[]): Promise<number> {
163 const info = await ensureManagedRuntime();179 const info = await ensureManagedRuntime();
164 const anonymousAuthEnv = await resolveManagedAnonymousAuthEnv(info);180 const anonymousAuthEnv = await resolveManagedAnonymousAuthEnv(info);
165 const managedConvexUrl = `http://${info.config.hostInterface}:${info.config.backendPort}`;181 const endpoints = resolveManagedConvexEndpoints(info);
166 const managedConvexSiteUrl = `http://${info.config.hostInterface}:${info.config.siteProxyPort}`;
167 182
168 if (args.length === 0) {183 if (args.length === 0) {
169 try {184 try {
@@ -179,9 +194,9 @@ export async function runManagedBackend(args: string[]): Promise<number> { · 9 unmodified lines
179 ...process.env,194 ...process.env,
180 ...anonymousAuthEnv,195 ...anonymousAuthEnv,
181 PATH: `${path.dirname(info.nodeBin)}:${process.env.PATH ?? ""}`,196 PATH: `${path.dirname(info.nodeBin)}:${process.env.PATH ?? ""}`,
182 CONVEX_URL: managedConvexUrl,197 CONVEX_URL: endpoints.convexUrl,
183 CONVEX_SITE_URL: managedConvexSiteUrl,198 CONVEX_SITE_URL: endpoints.convexSiteUrl,
184 ANONYMOUS_AUTH_ISSUER: managedConvexSiteUrl,199 ANONYMOUS_AUTH_ISSUER: endpoints.convexSiteUrl,
185 };200 };
186 201
187 const proc = Bun.spawn([info.backendBinary, ...backendArgs(info, args)], {202 const proc = Bun.spawn([info.backendBinary, ...backendArgs(info, args)], {
@@ -211,8 +226,7 @@ export async function runManagedWeb(options: { port?: number } = {}): Promise<nu · 23 unmodified lines
211 226
212 const webPort = options.port ?? Number(Bun.env.EXECUTOR_WEB_PORT ?? 5312);227 const webPort = options.port ?? Number(Bun.env.EXECUTOR_WEB_PORT ?? 5312);
213 const host = Bun.env.EXECUTOR_WEB_INTERFACE ?? "127.0.0.1";228 const host = Bun.env.EXECUTOR_WEB_INTERFACE ?? "127.0.0.1";
214 const managedConvexUrl = `http://${info.config.hostInterface}:${info.config.backendPort}`;229 const endpoints = resolveManagedConvexEndpoints(info);
215 const managedConvexSiteUrl = `http://${info.config.hostInterface}:${info.config.siteProxyPort}`;
216 230
217 const env = {231 const env = {
218 ...process.env,232 ...process.env,
@@ -220,11 +234,11 @@ export async function runManagedWeb(options: { port?: number } = {}): Promise<nu · 1 unmodified line
220 NODE_ENV: "production",234 NODE_ENV: "production",
221 PORT: String(webPort),235 PORT: String(webPort),
222 HOSTNAME: host,236 HOSTNAME: host,
223 CONVEX_URL: managedConvexUrl,237 CONVEX_URL: endpoints.convexUrl,
224 CONVEX_SITE_URL: managedConvexSiteUrl,238 CONVEX_SITE_URL: endpoints.convexSiteUrl,
225 EXECUTOR_WEB_CONVEX_URL: managedConvexUrl,239 EXECUTOR_WEB_CONVEX_URL: endpoints.convexUrl,
226 EXECUTOR_WEB_CONVEX_SITE_URL: managedConvexSiteUrl,240 EXECUTOR_WEB_CONVEX_SITE_URL: endpoints.convexSiteUrl,
227 ANONYMOUS_AUTH_ISSUER: managedConvexSiteUrl,241 ANONYMOUS_AUTH_ISSUER: endpoints.convexSiteUrl,
228 };242 };
229 243
230 const proc = await runProcess(info.nodeBin, [info.webServerEntry], {244 const proc = await runProcess(info.nodeBin, [info.webServerEntry], {
executor/scripts/e2e/install-checks.ts+9−0
@@ -13,3 +13,12 @@ export function anonymousBootstrapCheckScript(ports: { backendPort: number; webP · 12 unmodified lines
13 "printf '%s' \"$mutation_json\" | \"$node_bin\" -e \"let d='';process.stdin.on('data',c=>d+=c);process.stdin.on('end',()=>{const j=JSON.parse(d);if(j.status!=='success' || !j.value || typeof j.value.workspaceId!=='string' || typeof j.value.sessionId!=='string'){console.error(d);process.exit(3)}console.log(JSON.stringify({workspaceId:j.value.workspaceId,sessionId:j.value.sessionId}))})\"",13 "printf '%s' \"$mutation_json\" | \"$node_bin\" -e \"let d='';process.stdin.on('data',c=>d+=c);process.stdin.on('end',()=>{const j=JSON.parse(d);if(j.status!=='success' || !j.value || typeof j.value.workspaceId!=='string' || typeof j.value.sessionId!=='string'){console.error(d);process.exit(3)}console.log(JSON.stringify({workspaceId:j.value.workspaceId,sessionId:j.value.sessionId}))})\"",
14 ].join("; ");14 ].join("; ");
15}15}
16
17export function runtimeConfigCheckScript(options: { webPort: number; expectedConvexUrl: string }): string {
18 return [
19 "set -euo pipefail",
20 `expected_convex_url='${options.expectedConvexUrl}'`,
21 `html=$(curl -fsS http://127.0.0.1:${options.webPort}/)`,
22 "printf '%s' \"$html\" | node -e \"const expected=process.argv[1];let d='';process.stdin.on('data',c=>d+=c);process.stdin.on('end',()=>{const m=d.match(/window\\.__EXECUTOR_RUNTIME_CONFIG__ = ([^;]+);/);if(!m){console.error('runtime config script missing');process.exit(2)};const cfg=JSON.parse(m[1]);if(cfg?.convexUrl!==expected){console.error(JSON.stringify({expected,actual:cfg?.convexUrl}));process.exit(3)};if(expected.startsWith('https://')&&!String(cfg?.convexUrl||'').startsWith('https://')){console.error('expected https convex url');process.exit(4)};console.log(JSON.stringify({convexUrl:cfg.convexUrl}))})\" \"$expected_convex_url\"",
23 ].join("; ");
24}
executor/scripts/e2e/install.e2e.test.ts+26−13
@@ -77,6 +77,11 @@ function assertSuccess(result: CommandResult, label: string): void { · 76 unmodified lines
77 );77 );
78}78}
79 79
80function shouldSkipBuildSteps(): boolean {
81 const raw = process.env.EXECUTOR_INSTALL_E2E_SKIP_BUILD?.trim().toLowerCase();
82 return raw === "1" || raw === "true" || raw === "yes";
83}
84
80test("installer e2e: uninstall, install, deploy, and cleanup", async () => {85test("installer e2e: uninstall, install, deploy, and cleanup", async () => {
81 const repoRoot = repositoryRoot();86 const repoRoot = repositoryRoot();
82 const installScript = path.join(repoRoot, "install");87 const installScript = path.join(repoRoot, "install");
@@ -109,19 +114,27 @@ test("installer e2e: uninstall, install, deploy, and cleanup", async () => { · 26 unmodified lines
109 };114 };
110 115
111 try {116 try {
112 const buildBinary = await runCommand(["bun", "run", "build:binary"], {117 if (!shouldSkipBuildSteps()) {
113 cwd: repoRoot,118 const buildBinary = await runCommand(["bun", "run", "build:binary"], {
114 env: process.env,119 cwd: repoRoot,
115 timeoutMs: 300_000,120 env: process.env,
116 });121 timeoutMs: 300_000,
117 assertSuccess(buildBinary, "build binary");122 });
118 123 assertSuccess(buildBinary, "build binary");
119 const buildRelease = await runCommand(["bun", "run", "build:release"], {124
120 cwd: repoRoot,125 const buildRelease = await runCommand(["bun", "run", "build:release"], {
121 env: process.env,126 cwd: repoRoot,
122 timeoutMs: 1_200_000,127 env: process.env,
123 });128 timeoutMs: 1_200_000,
124 assertSuccess(buildRelease, "build release");129 });
130 assertSuccess(buildRelease, "build release");
131 }
132
133 if (!(await pathExists(binaryPath))) {
134 throw new Error(
135 `Missing expected binary at ${binaryPath}. Run 'bun run build:binary' or unset EXECUTOR_INSTALL_E2E_SKIP_BUILD.`,
136 );
137 }
125 138
126 await fs.mkdir(installCwd, { recursive: true });139 await fs.mkdir(installCwd, { recursive: true });
127 140
executor/scripts/e2e/install.sandbox.e2e.test.ts+43−5
@@ -1,6 +1,6 @@
1import { expect, test } from "bun:test";1import { expect, test } from "bun:test";
2import { Sandbox } from "@vercel/sandbox";2import { Sandbox } from "@vercel/sandbox";
3import { anonymousBootstrapCheckScript, runtimeDoctorScript } from "./install-checks";3import { anonymousBootstrapCheckScript, runtimeConfigCheckScript, runtimeDoctorScript } from "./install-checks";
4 4
5type CommandResult = {5type CommandResult = {
6 exitCode: number;6 exitCode: number;
@@ -12,6 +12,7 @@ async function runSandboxBash( · 5 unmodified lines
12 sandbox: Sandbox,12 sandbox: Sandbox,
13 script: string,13 script: string,
14 timeoutMs: number,14 timeoutMs: number,
15 env?: Record<string, string>,
15): Promise<CommandResult> {16): Promise<CommandResult> {
16 const controller = new AbortController();17 const controller = new AbortController();
17 const timeout = setTimeout(() => {18 const timeout = setTimeout(() => {
@@ -22,6 +23,7 @@ async function runSandboxBash( · 4 unmodified lines
22 const command = await sandbox.runCommand({23 const command = await sandbox.runCommand({
23 cmd: "bash",24 cmd: "bash",
24 args: ["-lc", script],25 args: ["-lc", script],
26 env,
25 signal: controller.signal,27 signal: controller.signal,
26 });28 });
27 29
@@ -63,6 +65,10 @@ function assertSuccess(result: CommandResult, label: string): void { · 35 unmodified lines
63}65}
64 66
65test("installer works in fresh Vercel sandbox", async () => {67test("installer works in fresh Vercel sandbox", async () => {
68 const backendPort = 5410;
69 const sitePort = 5411;
70 const webPort = 5312;
71
66 const credentials =72 const credentials =
67 process.env.VERCEL_TOKEN && process.env.VERCEL_TEAM_ID && process.env.VERCEL_PROJECT_ID73 process.env.VERCEL_TOKEN && process.env.VERCEL_TEAM_ID && process.env.VERCEL_PROJECT_ID
68 ? {74 ? {
@@ -76,6 +82,7 @@ test("installer works in fresh Vercel sandbox", async () => { · 7 unmodified lines
76 try {82 try {
77 return await Sandbox.create({83 return await Sandbox.create({
78 runtime: "node22",84 runtime: "node22",
85 ports: [webPort, backendPort, sitePort],
79 timeout: 30 * 60 * 1000,86 timeout: 30 * 60 * 1000,
80 ...credentials,87 ...credentials,
81 });88 });
@@ -90,13 +97,34 @@ test("installer works in fresh Vercel sandbox", async () => { · 8 unmodified lines
90 let sandbox: Sandbox | null = await createSandbox();97 let sandbox: Sandbox | null = await createSandbox();
91 98
92 try {99 try {
100 const buildSandboxEnv = (activeSandbox: Sandbox) => {
101 const convexUrl = activeSandbox.domain(backendPort);
102 const convexSiteUrl = activeSandbox.domain(sitePort);
103 return {
104 convexUrl,
105 env: {
106 EXECUTOR_BACKEND_INTERFACE: "0.0.0.0",
107 EXECUTOR_WEB_INTERFACE: "0.0.0.0",
108 EXECUTOR_BACKEND_PORT: String(backendPort),
109 EXECUTOR_BACKEND_SITE_PORT: String(sitePort),
110 EXECUTOR_WEB_PORT: String(webPort),
111 CONVEX_URL: convexUrl,
112 CONVEX_SITE_URL: convexSiteUrl,
113 EXECUTOR_WEB_CONVEX_URL: convexUrl,
114 EXECUTOR_WEB_CONVEX_SITE_URL: convexSiteUrl,
115 },
116 };
117 };
118
119 let sandboxSettings = buildSandboxEnv(sandbox);
120
93 const installScript = [121 const installScript = [
94 "set -euo pipefail",122 "set -euo pipefail",
95 "cd ~",123 "cd ~",
96 "if [ -x ~/.executor/bin/executor ]; then ~/.executor/bin/executor uninstall --yes || true; fi",124 "if [ -x ~/.executor/bin/executor ]; then ~/.executor/bin/executor uninstall --yes || true; fi",
97 "rm -rf ~/.executor",125 "rm -rf ~/.executor",
98 "start=$(date +%s)",126 "start=$(date +%s)",
99 "curl -fsSL https://executor.sh/install | bash",127 "curl -fsSL https://executor.sh/install | bash -s -- --no-modify-path --no-star-prompt",
100 "end=$(date +%s)",128 "end=$(date +%s)",
101 "echo INSTALL_SECONDS=$((end-start))",129 "echo INSTALL_SECONDS=$((end-start))",
102 runtimeDoctorScript(),130 runtimeDoctorScript(),
@@ -104,7 +132,7 @@ test("installer works in fresh Vercel sandbox", async () => { · 1 unmodified line
104 132
105 let install: CommandResult;133 let install: CommandResult;
106 try {134 try {
107 install = await runSandboxBash(sandbox, installScript, 900_000);135 install = await runSandboxBash(sandbox, installScript, 900_000, sandboxSettings.env);
108 } catch (error) {136 } catch (error) {
109 if (!isSandboxStoppedError(error)) {137 if (!isSandboxStoppedError(error)) {
110 throw error;138 throw error;
@@ -116,16 +144,26 @@ test("installer works in fresh Vercel sandbox", async () => { · 5 unmodified lines
116 // ignore144 // ignore
117 }145 }
118 sandbox = await createSandbox();146 sandbox = await createSandbox();
119 install = await runSandboxBash(sandbox, installScript, 900_000);147 sandboxSettings = buildSandboxEnv(sandbox);
148 install = await runSandboxBash(sandbox, installScript, 900_000, sandboxSettings.env);
120 }149 }
121 assertSuccess(install, "sandbox install + doctor");150 assertSuccess(install, "sandbox install + doctor");
122 151
123 const output = `${install.stdout}\n${install.stderr}`;152 const output = `${install.stdout}\n${install.stderr}`;
124 expect(output).toContain("Executor status: ready");153 expect(output).toContain("Executor status: ready");
154 const runtimeConfigCheck = await runSandboxBash(
155 sandbox,
156 runtimeConfigCheckScript({ webPort, expectedConvexUrl: sandboxSettings.convexUrl }),
157 300_000,
158 sandboxSettings.env,
159 );
160 assertSuccess(runtimeConfigCheck, "sandbox runtime config convex URL check");
161
125 const anonymousCheck = await runSandboxBash(162 const anonymousCheck = await runSandboxBash(
126 sandbox,163 sandbox,
127 anonymousBootstrapCheckScript({ backendPort: 5410, webPort: 5312 }),164 anonymousBootstrapCheckScript({ backendPort, webPort }),
128 300_000,165 300_000,
166 sandboxSettings.env,
129 );167 );
130 assertSuccess(anonymousCheck, "sandbox anonymous bootstrap check");168 assertSuccess(anonymousCheck, "sandbox anonymous bootstrap check");
131 169
executor/scripts/e2e/install.sandbox.session.ts+11−1
@@ -1,6 +1,6 @@
1import { Sandbox } from "@vercel/sandbox";1import { Sandbox } from "@vercel/sandbox";
2import { generateKeyPairSync } from "node:crypto";2import { generateKeyPairSync } from "node:crypto";
3import { anonymousBootstrapCheckScript, runtimeDoctorScript } from "./install-checks";3import { anonymousBootstrapCheckScript, runtimeConfigCheckScript, runtimeDoctorScript } from "./install-checks";
4 4
5type CommandResult = {5type CommandResult = {
6 exitCode: number;6 exitCode: number;
@@ -238,6 +238,16 @@ try { · 231 unmodified lines
238 );238 );
239 assertSuccess(doctor, "sandbox doctor --verbose");239 assertSuccess(doctor, "sandbox doctor --verbose");
240 240
241 const runtimeConfigCheck = await runSandboxBash(
242 sandbox,
243 runtimeConfigCheckScript({ webPort, expectedConvexUrl: convexUrl }),
244 {
245 timeoutMs: installTimeoutMs,
246 env: sandboxEnv,
247 },
248 );
249 assertSuccess(runtimeConfigCheck, "sandbox runtime config convex URL check");
250
241 const anonymousCheck = await runSandboxBash(251 const anonymousCheck = await runSandboxBash(
242 sandbox,252 sandbox,
243 anonymousBootstrapCheckScript({ backendPort, webPort }),253 anonymousBootstrapCheckScript({ backendPort, webPort }),
executor/scripts/release/build-release.ts+38−4
@@ -246,12 +246,48 @@ async function generateSelfHostedAdminKey(config: RuntimeConfig): Promise<string · 245 unmodified lines
246 return parsed.adminKey;246 return parsed.adminKey;
247}247}
248 248
249async function assertSeededClientConfig(config: RuntimeConfig, expectedAnonymousIssuer: string): Promise<void> {
250 const response = await fetch(`http://${config.hostInterface}:${config.backendPort}/api/query`, {
251 method: "POST",
252 headers: {
253 "content-type": "application/json",
254 },
255 body: JSON.stringify({
256 path: "app:getClientConfig",
257 args: {},
258 }),
259 });
260
261 if (!response.ok) {
262 const text = await response.text();
263 throw new Error(`Failed querying app:getClientConfig during runtime seeding: ${text || response.statusText}`);
264 }
265
266 const parsed = await response.json() as {
267 status?: string;
268 value?: {
269 anonymousAuthIssuer?: unknown;
270 };
271 };
272
273 if (parsed.status !== "success") {
274 throw new Error(`Runtime seeding query returned unexpected status: ${JSON.stringify(parsed)}`);
275 }
276
277 if (parsed.value?.anonymousAuthIssuer !== expectedAnonymousIssuer) {
278 throw new Error(
279 `Runtime seeding issuer mismatch. Expected ${expectedAnonymousIssuer}, got ${String(parsed.value?.anonymousAuthIssuer)}`,
280 );
281 }
282}
283
249async function seedManagedRuntimeState(284async function seedManagedRuntimeState(
250 rootDir: string,285 rootDir: string,
251 runtimeDir: string,286 runtimeDir: string,
252 config: RuntimeConfig,287 config: RuntimeConfig,
253 anonymousAuth: AnonymousAuthSeed,288 anonymousAuth: AnonymousAuthSeed,
254): Promise<void> {289): Promise<void> {
290 const expectedAnonymousIssuer = `http://${config.hostInterface}:${defaultManagedSitePort}`;
255 const adminKey = await generateSelfHostedAdminKey(config);291 const adminKey = await generateSelfHostedAdminKey(config);
256 const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), "executor-seed-env-"));292 const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), "executor-seed-env-"));
257 const envFilePath = path.join(tempDir, "selfhost.env");293 const envFilePath = path.join(tempDir, "selfhost.env");
@@ -265,7 +301,7 @@ async function seedManagedRuntimeState( · 7 unmodified lines
265 301
266 const envEntries: Array<{ name: string; value: string }> = [302 const envEntries: Array<{ name: string; value: string }> = [
267 { name: "WORKOS_CLIENT_ID", value: "disabled" },303 { name: "WORKOS_CLIENT_ID", value: "disabled" },
268 { name: "ANONYMOUS_AUTH_ISSUER", value: `http://${config.hostInterface}:${defaultManagedSitePort}` },304 { name: "ANONYMOUS_AUTH_ISSUER", value: expectedAnonymousIssuer },
269 { name: "ANONYMOUS_AUTH_PRIVATE_KEY_PEM", value: anonymousAuth.privateKeyPem },305 { name: "ANONYMOUS_AUTH_PRIVATE_KEY_PEM", value: anonymousAuth.privateKeyPem },
270 { name: "ANONYMOUS_AUTH_PUBLIC_KEY_PEM", value: anonymousAuth.publicKeyPem },306 { name: "ANONYMOUS_AUTH_PUBLIC_KEY_PEM", value: anonymousAuth.publicKeyPem },
271 { name: "MCP_API_KEY_SECRET", value: anonymousAuth.apiKeySecret },307 { name: "MCP_API_KEY_SECRET", value: anonymousAuth.apiKeySecret },
@@ -282,9 +318,7 @@ async function seedManagedRuntimeState( · 10 unmodified lines
282 cwd: rootDir,318 cwd: rootDir,
283 });319 });
284 320
285 await runCommand(["bunx", "convex", "run", "app:getClientConfig", "--env-file", envFilePath], {321 await assertSeededClientConfig(config, expectedAnonymousIssuer);
286 cwd: rootDir,
287 });
288 } finally {322 } finally {
289 await fs.rm(tempDir, { recursive: true, force: true });323 await fs.rm(tempDir, { recursive: true, force: true });
290 }324 }