History

Speed up and stabilize CI e2e (#1551)

* Speed up selfhost e2e: tunable sandbox deadline + 3-way sharding * Fix the local e2e suite: toolkit MCP DB lock, Bun-only spawn, stale auth selector * Stabilize CI process lifecycle

Rhys Sullivan committed 2 months ago

Commit f64028a1 parent c83cdd226 files · +714 −217Browse files

Unified · Split

26 files
  1. .github/workflows/ci.yml +29 −26
  2. apps/cli/src/main.ts +2 −0
  3. apps/host-selfhost/src/config.ts +25 −0
  4. apps/host-selfhost/src/execution.ts +6 −1
  5. apps/local/src/executor.ts +37 −18
  6. apps/local/src/main.ts +5 −0
  7. apps/local/src/serve.ts +39 −11
  8. e2e/local/auth.test.ts +5 −3
  9. e2e/local/boot-process.test.ts +101 −0
  10. e2e/local/cli-mcp-daemon-attach-stress.test.ts +52 −62
  11. e2e/local/cli-mcp-protocol.test.ts +4 −17
  12. e2e/local/daemon-process.ts +55 −0
  13. e2e/local/local-server.ts +8 −2
  14. e2e/local/vite-dev-routing.test.ts +3 −2
  15. e2e/scenarios/resume-after-sandbox-deadline.test.ts +35 −20
  16. e2e/selfhost/posthog-mcp-oauth.test.ts +43 −20
  17. e2e/setup/boot.ts +122 −13
  18. e2e/setup/cloud.boot.ts +5 −3
  19. e2e/setup/cloud.globalsetup.ts +2 −2
  20. e2e/setup/cloudflare.boot.ts +7 −2
  21. e2e/setup/motel.ts +3 −3
  22. e2e/setup/sandbox-timeout.ts +28 −0
  23. e2e/setup/selfhost.boot.ts +10 −2
  24. e2e/setup/selfhost.globalsetup.ts +8 −2
  25. e2e/src/ports.ts +23 −5
  26. packages/hosts/mcp/src/stdio-integration.test.ts +57 −3
.github/workflows/ci.yml+29−26
@@ -192,14 +192,25 @@ jobs: · 191 unmodified lines
192 fail-fast: false192 fail-fast: false
193 matrix:193 matrix:
194 include:194 include:
195 # Each cloud shard boots its own fresh dev stack. On 4 vCPU runners,195 # PGlite is deliberately single-connection, and under a sustained
196 # four fatter shards keep the longest shard below selfhost while saving196 # multi-minute shard it can stop accepting postgres sockets. Keep
197 # four runner boots and four warm cache restores.197 # every hermetic dev stack short: eight serial shards remove that
198 - { target: cloud, shard: 1/4, shard-name: 1of4 }198 # lifetime-dependent failure and put cloud below the selfhost lane.
199 - { target: cloud, shard: 2/4, shard-name: 2of4 }199 - { target: cloud, shard: 1/8, shard-name: 1of8 }
200 - { target: cloud, shard: 3/4, shard-name: 3of4 }200 - { target: cloud, shard: 2/8, shard-name: 2of8 }
201 - { target: cloud, shard: 4/4, shard-name: 4of4 }201 - { target: cloud, shard: 3/8, shard-name: 3of8 }
202 - target: selfhost202 - { target: cloud, shard: 4/8, shard-name: 4of8 }
203 - { target: cloud, shard: 5/8, shard-name: 5of8 }
204 - { target: cloud, shard: 6/8, shard-name: 6of8 }
205 - { target: cloud, shard: 7/8, shard-name: 7of8 }
206 - { target: cloud, shard: 8/8, shard-name: 8of8 }
207 # Selfhost shards the same way: each shard is its own runner booting
208 # its own fresh instance (own port block + data dir), so the
209 # project's shared-bootstrap-admin assumption stays intact per shard
210 # and `fileParallelism: false` still serializes within a shard.
211 - { target: selfhost, shard: 1/3, shard-name: 1of3 }
212 - { target: selfhost, shard: 2/3, shard-name: 2of3 }
213 - { target: selfhost, shard: 3/3, shard-name: 3of3 }
203 runs-on: blacksmith-4vcpu-ubuntu-2404214 runs-on: blacksmith-4vcpu-ubuntu-2404
204 timeout-minutes: 30215 timeout-minutes: 30
205 steps:216 steps:
@@ -241,20 +252,20 @@ jobs: · 35 unmodified lines
241 252
242 # The globalsetup boots the target's own dev server (ports are claimed253 # The globalsetup boots the target's own dev server (ports are claimed
243 # per checkout, so this is hermetic) and tears it down after the run.254 # per checkout, so this is hermetic) and tears it down after the run.
244 # --retry=2: browser scenarios can still hit isolated waitFor timeouts255 # Do not retry scenarios: retries hide flakes and multiply slow timeout
245 # (single-test waitFor timeouts, not systemic failures); a retry on the256 # failures. The fixtures and process lifecycle are deterministic enough
246 # same booted stack clears them.257 # that the first result is the result.
247 - name: Run cloud scenarios258 - name: Run cloud scenarios
248 if: matrix.target == 'cloud'259 if: matrix.target == 'cloud'
249 env:260 env:
250 MCP_SESSION_TIMEOUT_MS: "3000"261 MCP_SESSION_TIMEOUT_MS: "3000"
251 MCP_PAUSED_SESSION_IDLE_TIMEOUT_MS: "6000"262 MCP_PAUSED_SESSION_IDLE_TIMEOUT_MS: "6000"
252 run: bunx vitest run --project cloud --retry=2 ${{ matrix.shard && format('--shard={0}', matrix.shard) || '' }}263 run: bunx vitest run --project cloud ${{ matrix.shard && format('--shard={0}', matrix.shard) || '' }}
253 working-directory: e2e264 working-directory: e2e
254 265
255 - name: Run selfhost scenarios266 - name: Run selfhost scenarios
256 if: matrix.target == 'selfhost'267 if: matrix.target == 'selfhost'
257 run: bunx vitest run --project selfhost --retry=2268 run: bunx vitest run --project selfhost ${{ matrix.shard && format('--shard={0}', matrix.shard) || '' }}
258 working-directory: e2e269 working-directory: e2e
259 270
260 # Failed runs keep their trace.zip / session.mp4 / step screenshots in271 # Failed runs keep their trace.zip / session.mp4 / step screenshots in
@@ -268,10 +279,7 @@ jobs: · 7 unmodified lines
268 retention-days: 7279 retention-days: 7
269 280
270 e2e-local:281 e2e-local:
271 name: E2E (stdio MCP)282 name: E2E (local)
272 # Skipped on pull_request: the local scenario boots a real `executor web`
273 # plus a browser and is currently flaky on PRs. Still runs on push to main.
274 if: github.event_name != 'pull_request'
275 runs-on: blacksmith-4vcpu-ubuntu-2404283 runs-on: blacksmith-4vcpu-ubuntu-2404
276 timeout-minutes: 20284 timeout-minutes: 20
277 steps:285 steps:
@@ -314,15 +322,10 @@ jobs: · 36 unmodified lines
314 run: bunx playwright install --with-deps chromium chromium-headless-shell322 run: bunx playwright install --with-deps chromium chromium-headless-shell
315 working-directory: e2e323 working-directory: e2e
316 324
317 # The `local` project is excluded from the default `test` chain (each325 # Each scenario owns its server, browser, data directory, and descendants;
318 # scenario boots its own `executor web`). Run just the stdio MCP scenario326 # run the complete hermetic suite on PRs without scenario retries.
319 # here: it is the auto-connect / env-as-secret regression guard, and327 - name: Run local scenarios
320 # running it alone avoids the boot-resource accumulation and the328 run: bunx vitest run --project local
321 # pre-existing browser flakiness of the rest of the local suite. Expanding
322 # to the full `local` project (bun run test:local) is a follow-up once
323 # those are stabilized.
324 - name: Run the stdio MCP scenario
325 run: bunx vitest run --project local local/stdio-mcp.test.ts
326 working-directory: e2e329 working-directory: e2e
327 330
328 desktop-smoke:331 desktop-smoke:
apps/cli/src/main.ts+2−0
@@ -215,9 +215,11 @@ const waitForShutdownSignal = () => · 214 unmodified lines
215 const shutdown = () => resume(Effect.void);215 const shutdown = () => resume(Effect.void);
216 process.once("SIGINT", shutdown);216 process.once("SIGINT", shutdown);
217 process.once("SIGTERM", shutdown);217 process.once("SIGTERM", shutdown);
218 process.once("SIGHUP", shutdown);
218 return Effect.sync(() => {219 return Effect.sync(() => {
219 process.off("SIGINT", shutdown);220 process.off("SIGINT", shutdown);
220 process.off("SIGTERM", shutdown);221 process.off("SIGTERM", shutdown);
222 process.off("SIGHUP", shutdown);
221 });223 });
222 });224 });
223 225
apps/host-selfhost/src/config.ts+25−0
@@ -45,6 +45,14 @@ export interface SelfHostConfig { · 44 unmodified lines
45 readonly organizationName: string;45 readonly organizationName: string;
46 /** URL slug for org-prefixed console paths (`/<slug>/policies`). */46 /** URL slug for org-prefixed console paths (`/<slug>/policies`). */
47 readonly orgSlug: string;47 readonly orgSlug: string;
48 /**
49 * Sandbox execution budget passed to the QuickJS runtime, or undefined for
50 * the runtime's own default (5 minutes). An operator knob in principle, but
51 * its real consumer is the e2e harness, which shrinks it to seconds so the
52 * sandbox-deadline scenario proves its race without waiting out real
53 * minutes (the same pattern as MCP_PAUSED_SESSION_IDLE_TIMEOUT_MS on cloud).
54 */
55 readonly sandboxTimeoutMs: number | undefined;
48}56}
49 57
50export const resolveDataDir = (): string =>58export const resolveDataDir = (): string =>
@@ -151,9 +159,26 @@ export const loadConfig = (): SelfHostConfig => { · 100 unmodified lines
151 bootstrapAdminName: process.env.EXECUTOR_BOOTSTRAP_ADMIN_NAME ?? "Admin",159 bootstrapAdminName: process.env.EXECUTOR_BOOTSTRAP_ADMIN_NAME ?? "Admin",
152 organizationName: process.env.EXECUTOR_ORG_NAME ?? "Default",160 organizationName: process.env.EXECUTOR_ORG_NAME ?? "Default",
153 orgSlug: resolveOrgSlug(),161 orgSlug: resolveOrgSlug(),
162 sandboxTimeoutMs: resolveSandboxTimeoutMs(),
154 };163 };
155};164};
156 165
166// A malformed value is refused rather than silently ignored: an operator who
167// sets the knob and typos it should find out at boot, not by watching a
168// runaway execution use the 5-minute default.
169const resolveSandboxTimeoutMs = (): number | undefined => {
170 const raw = process.env.EXECUTOR_SANDBOX_TIMEOUT_MS;
171 if (!raw) return undefined;
172 const parsed = Number(raw);
173 if (!Number.isFinite(parsed) || parsed <= 0) {
174 // oxlint-disable-next-line executor/no-try-catch-or-throw, executor/no-error-constructor -- boundary: refuse to boot on a malformed operator knob
175 throw new Error(
176 `EXECUTOR_SANDBOX_TIMEOUT_MS ${JSON.stringify(raw)} is not a positive number of milliseconds`,
177 );
178 }
179 return Math.floor(parsed);
180};
181
157// The org slug doubles as a URL segment (`/<slug>/policies`), so an182// The org slug doubles as a URL segment (`/<slug>/policies`), so an
158// operator-set value must fit the shared grammar and avoid reserved root183// operator-set value must fit the shared grammar and avoid reserved root
159// segments (api, mcp, login, …) — a colliding slug would shadow real routes.184// segments (api, mcp, login, …) — a colliding slug would shadow real routes.
apps/host-selfhost/src/execution.ts+6−1
@@ -65,7 +65,12 @@ export const SelfHostHostConfig: Layer.Layer<HostConfig> = Layer.sync(HostConfig · 64 unmodified lines
65 65
66export const SelfHostCodeExecutorProvider: Layer.Layer<CodeExecutorProvider> = Layer.sync(66export const SelfHostCodeExecutorProvider: Layer.Layer<CodeExecutorProvider> = Layer.sync(
67 CodeExecutorProvider,67 CodeExecutorProvider,
68 () => makeQuickJsExecutor(),68 () => {
69 const { sandboxTimeoutMs } = loadConfig();
70 return makeQuickJsExecutor(
71 sandboxTimeoutMs === undefined ? {} : { timeoutMs: sandboxTimeoutMs },
72 );
73 },
69);74);
70 75
71/**76/**
apps/local/src/executor.ts+37−18
@@ -19,7 +19,7 @@ import type { McpPluginExtension } from "@executor-js/plugin-mcp"; · 18 unmodified lines
19import executorConfig from "../executor.config";19import executorConfig from "../executor.config";
20import { localAnalytics } from "./analytics";20import { localAnalytics } from "./analytics";
21import { localDataMigrations } from "./db/data-migrations";21import { localDataMigrations } from "./db/data-migrations";
22import { openOwnedLocalDatabase } from "./db/owned-database";22import { openOwnedLocalDatabase, type OwnedLocalDatabase } from "./db/owned-database";
23 23
24interface ResolvedStorage {24interface ResolvedStorage {
25 readonly dataDir: string;25 readonly dataDir: string;
@@ -56,6 +56,16 @@ type LocalPlugins = readonly AnyPlugin[]; · 30 unmodified lines
56 56
57export interface LocalExecutorOptions {57export interface LocalExecutorOptions {
58 readonly activeToolkitSlug?: string;58 readonly activeToolkitSlug?: string;
59 /**
60 * Reuse an already-open owned database instead of opening (and locking) the
61 * data dir again. A toolkit-scoped MCP session differs from the default one
62 * only in its plugin set, so it must ride the running server's DB handle:
63 * `openOwnedLocalDatabase` takes an EXCLUSIVE lock, and a second open from
64 * inside the same process contends with the lock this process already holds.
65 * The borrowed handle is NOT closed when the derived executor disposes —
66 * whoever opened it still owns its lifetime.
67 */
68 readonly borrowedDb?: OwnedLocalDatabase;
59}69}
60 70
61const loadLocalPlugins = (options: LocalExecutorOptions = {}) =>71const loadLocalPlugins = (options: LocalExecutorOptions = {}) =>
@@ -92,6 +102,10 @@ const loadLocalPlugins = (options: LocalExecutorOptions = {}) => · 30 unmodified lines
92interface LocalExecutorBundle {102interface LocalExecutorBundle {
93 readonly executor: Executor<LocalPlugins>;103 readonly executor: Executor<LocalPlugins>;
94 readonly plugins: LocalPlugins;104 readonly plugins: LocalPlugins;
105 /** The owned DB this bundle opened (or borrowed). Surfaced so a
106 * toolkit-scoped executor can ride the SAME handle instead of contending
107 * with this process's own exclusive data-dir lock. */
108 readonly db: OwnedLocalDatabase;
95 /** Where this daemon's web UI is reachable, resolved once at boot. Surfaced109 /** Where this daemon's web UI is reachable, resolved once at boot. Surfaced
96 * so callers building user-facing links (MCP artifact deep links) use the110 * so callers building user-facing links (MCP artifact deep links) use the
97 * same origin the executor itself was configured with. */111 * same origin the executor itself was configured with. */
@@ -151,23 +165,27 @@ const createLocalExecutorLayer = (options: LocalExecutorOptions = {}) => { · 53 unmodified lines
151 const tenantId = makeTenantId(cwd);165 const tenantId = makeTenantId(cwd);
152 const tables = collectTables();166 const tables = collectTables();
153 167
154 const owned = yield* Effect.acquireRelease(168 // A borrowed handle is owned by its opener, so it is used as-is and left
155 Effect.tryPromise({169 // open on release; only a handle opened here is closed here.
156 try: () =>170 const owned = options.borrowedDb
157 openOwnedLocalDatabase({171 ? options.borrowedDb
158 dataDir: storage.dataDir,172 : yield* Effect.acquireRelease(
159 tables,173 Effect.tryPromise({
160 namespace: localNamespace,174 try: () =>
161 tenantId,175 openOwnedLocalDatabase({
176 dataDir: storage.dataDir,
177 tables,
178 namespace: localNamespace,
179 tenantId,
180 }),
181 catch: (cause) =>
182 new LocalExecutorCreateError({
183 message: CREATE_SQLITE_ERROR_MESSAGE,
184 cause,
185 }),
162 }),186 }),
163 catch: (cause) =>187 (database) => Effect.promise(() => database.close()).pipe(Effect.ignore),
164 new LocalExecutorCreateError({188 );
165 message: CREATE_SQLITE_ERROR_MESSAGE,
166 cause,
167 }),
168 }),
169 (database) => Effect.promise(() => database.close()).pipe(Effect.ignore),
170 );
171 const sqlite = owned.db;189 const sqlite = owned.db;
172 const migration = owned.migration;190 const migration = owned.migration;
173 191
@@ -243,7 +261,7 @@ const createLocalExecutorLayer = (options: LocalExecutorOptions = {}) => { · 69 unmodified lines
243 );261 );
244 }262 }
245 263
246 return { executor, plugins, webBaseUrl };264 return { executor, plugins, webBaseUrl, db: owned };
247 }),265 }),
248 );266 );
249};267};
@@ -257,6 +275,7 @@ export const createExecutorHandle = async (options: LocalExecutorOptions = {}) = · 7 unmodified lines
257 executor: bundle.executor,275 executor: bundle.executor,
258 plugins: bundle.plugins,276 plugins: bundle.plugins,
259 webBaseUrl: bundle.webBaseUrl,277 webBaseUrl: bundle.webBaseUrl,
278 db: bundle.db,
260 dispose: async () => {279 dispose: async () => {
261 await Effect.runPromise(Effect.ignore(bundle.executor.close()));280 await Effect.runPromise(Effect.ignore(bundle.executor.close()));
262 await ignorePromiseFailure("disposeRuntime", () => runtime.dispose());281 await ignorePromiseFailure("disposeRuntime", () => runtime.dispose());
apps/local/src/main.ts+5−0
@@ -137,8 +137,13 @@ export const createServerHandlers = async (token: string): Promise<ServerHandler · 136 unmodified lines
137 },137 },
138 };138 };
139 }139 }
140 // Borrow the running server's DB handle: this process already holds the
141 // data dir's exclusive ownership lock, so opening it a second time here
142 // fails against ourselves. The toolkit executor differs only in its
143 // plugin set, and the borrowed handle stays open when it disposes.
140 const handle = await createExecutorHandle({144 const handle = await createExecutorHandle({
141 activeToolkitSlug: resource.slug,145 activeToolkitSlug: resource.slug,
146 borrowedDb: (await getExecutorBundle()).db,
142 });147 });
143 const toolkitEngine = withExecutionAnalytics(148 const toolkitEngine = withExecutionAnalytics(
144 createExecutionEngine({149 createExecutionEngine({
apps/local/src/serve.ts+39−11
@@ -113,6 +113,8 @@ interface ViteChild { · 112 unmodified lines
113 readonly stop: () => Promise<void>;113 readonly stop: () => Promise<void>;
114}114}
115 115
116const viteChildSignals = ["SIGINT", "SIGTERM", "SIGHUP"] as const;
117
116async function allocatePort(): Promise<number> {118async function allocatePort(): Promise<number> {
117 const probe = Bun.serve({119 const probe = Bun.serve({
118 port: 0,120 port: 0,
@@ -127,15 +129,15 @@ async function allocatePort(): Promise<number> { · 8 unmodified lines
127async function startViteChild(): Promise<ViteChild> {129async function startViteChild(): Promise<ViteChild> {
128 const vitePort = await allocatePort();130 const vitePort = await allocatePort();
129 const cwd = resolve(import.meta.dirname, "..");131 const cwd = resolve(import.meta.dirname, "..");
132 const viteEntrypoint = resolve(cwd, "node_modules/vite/bin/vite.js");
130 const env = { ...process.env };133 const env = { ...process.env };
131 delete env.PORT;134 delete env.PORT;
132 // `bunx --bun vite` runs vite under Bun, matching the `dev:vite` script135 // Run Vite directly under Bun, matching the `dev:vite` script without a
133 // already in apps/local. --strictPort keeps the URL we hand back stable.136 // bunx wrapper that can outlive its child. --strictPort keeps the URL stable.
134 const child: Subprocess = Bun.spawn(137 const child: Subprocess = Bun.spawn(
135 [138 [
136 "bunx",139 process.execPath,
137 "--bun",140 viteEntrypoint,
138 "vite",
139 "dev",141 "dev",
140 "--port",142 "--port",
141 String(vitePort),143 String(vitePort),
@@ -158,20 +160,45 @@ async function startViteChild(): Promise<ViteChild> { · 16 unmodified lines
158 },160 },
159 );161 );
160 162
163 let stopping = false;
164 const stop = async (): Promise<void> => {
165 if (stopping) {
166 await child.exited;
167 return;
168 }
169 stopping = true;
170 for (const signal of viteChildSignals) process.off(signal, stopOnParentSignal);
171 if (child.exitCode === null) child.kill();
172 await Promise.race([child.exited, Bun.sleep(5_000)]);
173 if (child.exitCode === null) child.kill("SIGKILL");
174 await child.exited;
175 };
176 const stopOnParentSignal = (): void => {
177 // A PTY/session teardown can signal the CLI while Vite is still optimizing
178 // dependencies, before the server's normal stop handle exists. Reap the
179 // owned child immediately; the CLI's signal waiter performs full cleanup
180 // once startup has completed.
181 void stop();
182 };
183 for (const signal of viteChildSignals) process.once(signal, stopOnParentSignal);
184
161 const url = `http://127.0.0.1:${vitePort}`;185 const url = `http://127.0.0.1:${vitePort}`;
162 const deadline = Date.now() + 30_000;186 const deadline = Date.now() + 30_000;
163 while (Date.now() < deadline) {187 while (Date.now() < deadline) {
164 // oxlint-disable-next-line executor/no-try-catch-or-throw -- boundary: probing a child process that may not be listening yet188 // oxlint-disable-next-line executor/no-try-catch-or-throw -- boundary: probing a child process that may not be listening yet
165 try {189 try {
166 const r = await fetch(`${url}/`, { redirect: "manual" });190 const r = await fetch(`${url}/`, {
191 redirect: "manual",
192 // A listening socket is not proof that Vite can answer. Bound each
193 // probe so one accepted-but-stalled request cannot defeat the 30s boot
194 // deadline and wedge the entire local e2e suite.
195 signal: AbortSignal.timeout(5_000),
196 });
167 if (r.status < 500) {197 if (r.status < 500) {
168 await r.body?.cancel();198 await r.body?.cancel();
169 return {199 return {
170 url,200 url,
171 stop: async () => {201 stop,
172 child.kill();
173 await child.exited;
174 },
175 };202 };
176 }203 }
177 await r.body?.cancel();204 await r.body?.cancel();
@@ -179,12 +206,13 @@ async function startViteChild(): Promise<ViteChild> { · 1 unmodified line
179 // not up yet206 // not up yet
180 }207 }
181 if (child.exitCode !== null) {208 if (child.exitCode !== null) {
209 await stop();
182 // oxlint-disable-next-line executor/no-try-catch-or-throw, executor/no-error-constructor -- boundary: child process aborted before becoming ready210 // oxlint-disable-next-line executor/no-try-catch-or-throw, executor/no-error-constructor -- boundary: child process aborted before becoming ready
183 throw new Error(`vite dev exited with code ${child.exitCode} before becoming ready`);211 throw new Error(`vite dev exited with code ${child.exitCode} before becoming ready`);
184 }212 }
185 await Bun.sleep(150);213 await Bun.sleep(150);
186 }214 }
187 child.kill();215 await stop();
188 // oxlint-disable-next-line executor/no-try-catch-or-throw, executor/no-error-constructor -- boundary: vite never became reachable216 // oxlint-disable-next-line executor/no-try-catch-or-throw, executor/no-error-constructor -- boundary: vite never became reachable
189 throw new Error(`vite dev did not become reachable on ${url} within 30s`);217 throw new Error(`vite dev did not become reachable on ${url} within 30s`);
190}218}
e2e/local/auth.test.ts+5−3
@@ -36,8 +36,10 @@ scenario( · 35 unmodified lines
36 await page.goto(url, { waitUntil: "domcontentloaded" });36 await page.goto(url, { waitUntil: "domcontentloaded" });
37 await page.getByRole("link", { name: "Secrets" }).first().waitFor({ timeout: 30_000 });37 await page.getByRole("link", { name: "Secrets" }).first().waitFor({ timeout: 30_000 });
38 // Integrations actually LOAD (the built-in Executor integration) — proves38 // Integrations actually LOAD (the built-in Executor integration) — proves
39 // auth + data, not just the static shell.39 // auth + data, not just the static shell. Matched on the row's stable
40 await page.getByText("built-in").first().waitFor({ timeout: 30_000 });40 // testid: the list renders each integration's name + slug, never the
41 // literal "built-in" (that string is only an internal `kind`).
42 await page.getByTestId("integration-entry-executor").first().waitFor({ timeout: 30_000 });
41 // The token is moved out of the URL and persisted to localStorage.43 // The token is moved out of the URL and persisted to localStorage.
42 expect(new URL(page.url()).searchParams.has("_token")).toBe(false);44 expect(new URL(page.url()).searchParams.has("_token")).toBe(false);
43 const stored = await page.evaluate(() => localStorage.getItem("executor.authToken"));45 const stored = await page.evaluate(() => localStorage.getItem("executor.authToken"));
@@ -70,7 +72,7 @@ scenario( · 26 unmodified lines
70 await page.getByRole("button", { name: "Connect" }).click();72 await page.getByRole("button", { name: "Connect" }).click();
71 await page.getByRole("link", { name: "Secrets" }).first().waitFor({ timeout: 30_000 });73 await page.getByRole("link", { name: "Secrets" }).first().waitFor({ timeout: 30_000 });
72 // The reconnect fully restores — integrations LOAD, not a stale 401.74 // The reconnect fully restores — integrations LOAD, not a stale 401.
73 await page.getByText("built-in").first().waitFor({ timeout: 30_000 });75 await page.getByTestId("integration-entry-executor").first().waitFor({ timeout: 30_000 });
74 });76 });
75 }),77 }),
76 );78 );
e2e/local/boot-process.test.ts+101−0
@@ -0,0 +1,101 @@
1import { describe, expect, it } from "@effect/vitest";
2import { mkdtempSync, rmSync } from "node:fs";
3import { tmpdir } from "node:os";
4import { join } from "node:path";
5
6import {
7 BootProcessExitError,
8 BootReadinessTimeoutError,
9 bootProcesses,
10 isBootReadinessTimeout,
11 waitForBoot,
12} from "../setup/boot";
13import { claimAndBoot, isAddrInUse } from "../src/ports";
14
15describe("e2e boot process lifecycle", () => {
16 it("fails immediately with the boot log when a child exits", async () => {
17 const tempDir = mkdtempSync(join(tmpdir(), "executor-e2e-boot-"));
18 const logFile = join(tempDir, "boot.log");
19 let readinessProbeAborted = false;
20
21 try {
22 const processes = bootProcesses(
23 [
24 {
25 cmd: process.execPath,
26 args: [
27 "-e",
28 'console.error("Error: Port 44550 is already in use (EADDRINUSE)"); process.exit(17)',
29 ],
30 cwd: tempDir,
31 logFile,
32 },
33 ],
34 { label: "lifecycle-test" },
35 );
36
37 const startedAt = Date.now();
38 let failure: unknown;
39 try {
40 await waitForBoot(
41 processes,
42 (signal) =>
43 new Promise<never>((_resolve, reject) => {
44 signal.addEventListener(
45 "abort",
46 () => {
47 readinessProbeAborted = true;
48 // oxlint-disable-next-line executor/no-promise-reject -- boundary: the fixture models an abort-aware readiness promise
49 reject(signal.reason);
50 },
51 { once: true },
52 );
53 }),
54 );
55 } catch (error) {
56 failure = error;
57 }
58
59 expect(failure).toBeInstanceOf(BootProcessExitError);
60 expect(Date.now() - startedAt, "child exit beats the readiness timeout").toBeLessThan(5_000);
61 expect(readinessProbeAborted, "the losing readiness probe is cancelled").toBe(true);
62 expect(isAddrInUse(failure), "the port claimer can retry this boot failure").toBe(true);
63 const bootFailure = failure as BootProcessExitError;
64 expect(bootFailure.exitCode).toBe(17);
65 expect(bootFailure.logTail).toContain("Port 44550 is already in use");
66
67 await processes.teardown();
68 } finally {
69 rmSync(tempDir, { recursive: true, force: true });
70 }
71 });
72
73 it("releases a failed claim before retrying a readiness timeout", async () => {
74 const envVar = "E2E_BOOT_LIFECYCLE_TEST_PORT";
75 let attempts = 0;
76 const claimedPorts: number[] = [];
77
78 try {
79 const booted = await claimAndBoot(
80 [{ envVar, offset: 8, label: "boot lifecycle test" }],
81 async (ports) => {
82 attempts += 1;
83 claimedPorts.push(ports[envVar]!);
84 if (attempts === 1) {
85 throw new BootReadinessTimeoutError("http://127.0.0.1:1", 10, "fixture timeout");
86 }
87 return { teardown: async () => {}, value: "ready" };
88 },
89 { maxAttempts: 2, label: "lifecycle-test", retryWhen: isBootReadinessTimeout },
90 );
91
92 expect(booted.value).toBe("ready");
93 expect(attempts).toBe(2);
94 expect(claimedPorts).toHaveLength(2);
95 await booted.teardown();
96 expect(process.env[envVar], "the successful claim is cleared at teardown").toBeUndefined();
97 } finally {
98 delete process.env[envVar];
99 }
100 });
101});
e2e/local/cli-mcp-daemon-attach-stress.test.ts+52−62
@@ -21,20 +21,24 @@ import { expect } from "@effect/vitest"; · 20 unmodified lines
21import { Client } from "@modelcontextprotocol/sdk/client/index.js";21import { Client } from "@modelcontextprotocol/sdk/client/index.js";
22import { StdioClientTransport } from "@modelcontextprotocol/sdk/client/stdio.js";22import { StdioClientTransport } from "@modelcontextprotocol/sdk/client/stdio.js";
23import { Effect } from "effect";23import { Effect } from "effect";
24import { spawn, type ChildProcessWithoutNullStreams } from "node:child_process";
24import { mkdtempSync, readdirSync, rmSync } from "node:fs";25import { mkdtempSync, readdirSync, rmSync } from "node:fs";
25import { tmpdir } from "node:os";26import { tmpdir } from "node:os";
26import { join } from "node:path";27import { join } from "node:path";
27import { fileURLToPath } from "node:url";28import { fileURLToPath } from "node:url";
28import type { Subprocess } from "bun";
29 29
30import { scenario } from "../src/scenario";30import { scenario } from "../src/scenario";
31import { stopAutoSpawnedDaemon } from "./daemon-process";
31 32
32const repoRoot = fileURLToPath(new URL("../../", import.meta.url));33const repoRoot = fileURLToPath(new URL("../../", import.meta.url));
33const testScope = join(repoRoot, "apps/local");34const testScope = join(repoRoot, "apps/local");
34// Generous: a dev-mode daemon boots a Vite dev server, slow under machine load.35// Generous: a dev-mode daemon boots a Vite dev server, slow under machine load.
35const readyTimeoutMs = 150_000;36const readyTimeoutMs = 150_000;
36 37
37type DaemonProc = Subprocess<"ignore", "pipe", "pipe">;38// vitest runs this suite under NODE, not bun, so the daemon is spawned with
39// node:child_process (the rest of the e2e harness does the same). `Bun.spawn`
40// here threw `ReferenceError: Bun is not defined` on every run.
41type DaemonProc = ChildProcessWithoutNullStreams;
38 42
39const waitForDaemonReady = (43const waitForDaemonReady = (
40 proc: DaemonProc,44 proc: DaemonProc,
@@ -44,50 +48,38 @@ const waitForDaemonReady = ( · 3 unmodified lines
44 let stdoutBuffer = "";48 let stdoutBuffer = "";
45 let stderrBuffer = "";49 let stderrBuffer = "";
46 let settled = false;50 let settled = false;
47 const decoder = new TextDecoder();
48 const stdout = proc.stdout.getReader();
49 const stderr = proc.stderr.getReader();
50 const deadline = setTimeout(() => {51 const deadline = setTimeout(() => {
51 if (settled) return;52 if (settled) return;
52 settled = true;53 settled = true;
53 // oxlint-disable-next-line executor/no-promise-reject, executor/no-error-constructor -- boundary: captured daemon stderr54 // oxlint-disable-next-line executor/no-promise-reject, executor/no-error-constructor -- boundary: captured daemon stderr
54 rejectReady(new Error(`daemon did not announce ready: ${stderrBuffer}`));55 rejectReady(new Error(`daemon did not announce ready: ${stderrBuffer}`));
55 }, readyTimeoutMs);56 }, readyTimeoutMs);
56 void (async () => {57 proc.stderr.on("data", (chunk: Buffer) => {
57 while (true) {58 stderrBuffer += chunk.toString();
58 const { value, done } = await stderr.read();59 });
59 if (done) return;60 proc.stdout.on("data", (chunk: Buffer) => {
60 stderrBuffer += decoder.decode(value);61 if (settled) return;
61 }62 stdoutBuffer += chunk.toString();
62 })();63 const match = /Daemon ready on http:\/\/(?:\[[^\]]+\]|[^:\s]+):(\d+)/.exec(stdoutBuffer);
63 void (async () => {64 if (match) {
64 while (true) {65 settled = true;
65 const { value, done } = await stdout.read();66 clearTimeout(deadline);
66 if (done) {67 resolveReady({ port: Number(match[1]), stderr: () => stderrBuffer });
67 if (!settled) {
68 settled = true;
69 clearTimeout(deadline);
70 // oxlint-disable-next-line executor/no-promise-reject, executor/no-error-constructor -- boundary: captured daemon stderr
71 rejectReady(new Error(`daemon stdout closed before ready: ${stderrBuffer}`));
72 }
73 return;
74 }
75 stdoutBuffer += decoder.decode(value);
76 const match = /Daemon ready on http:\/\/(?:\[[^\]]+\]|[^:\s]+):(\d+)/.exec(stdoutBuffer);
77 if (match) {
78 settled = true;
79 clearTimeout(deadline);
80 resolveReady({ port: Number(match[1]), stderr: () => stderrBuffer });
81 return;
82 }
83 }68 }
84 })();69 });
70 proc.stdout.on("close", () => {
71 if (settled) return;
72 settled = true;
73 clearTimeout(deadline);
74 // oxlint-disable-next-line executor/no-promise-reject, executor/no-error-constructor -- boundary: captured daemon stderr
75 rejectReady(new Error(`daemon stdout closed before ready: ${stderrBuffer}`));
76 });
85 });77 });
86 78
87const spawnDaemon = (dataDir: string): DaemonProc =>79const spawnDaemon = (dataDir: string): DaemonProc =>
88 Bun.spawn(80 spawn(
81 "bun",
89 [82 [
90 "bun",
91 "run",83 "run",
92 "dev:cli",84 "dev:cli",
93 "daemon",85 "daemon",
@@ -103,17 +95,24 @@ const spawnDaemon = (dataDir: string): DaemonProc => · 9 unmodified lines
103 {95 {
104 cwd: repoRoot,96 cwd: repoRoot,
105 env: { ...process.env, EXECUTOR_DATA_DIR: dataDir },97 env: { ...process.env, EXECUTOR_DATA_DIR: dataDir },
106 stdin: "ignore",98 stdio: ["ignore", "pipe", "pipe"],
107 stdout: "pipe",99 detached: true,
108 stderr: "pipe",
109 },100 },
110 );101 ) as DaemonProc;
102
103const exited = (proc: DaemonProc): Promise<void> =>
104 proc.exitCode !== null || proc.signalCode !== null
105 ? Promise.resolve()
106 : new Promise((resolve) => proc.once("exit", () => resolve()));
111 107
112const stopProc = async (proc: DaemonProc): Promise<void> => {108const stopProc = async (proc: DaemonProc): Promise<void> => {
113 if (proc.exitCode !== null) return;109 if (proc.exitCode !== null || proc.signalCode !== null) return;
114 proc.kill("SIGTERM");110 if (proc.pid) process.kill(-proc.pid, "SIGTERM");
115 await Promise.race([proc.exited, Bun.sleep(3000)]);111 await Promise.race([exited(proc), new Promise<void>((resolve) => setTimeout(resolve, 3000))]);
116 if (proc.exitCode === null) proc.kill("SIGKILL");112 if (proc.exitCode === null && proc.signalCode === null && proc.pid) {
113 process.kill(-proc.pid, "SIGKILL");
114 await exited(proc);
115 }
117};116};
118 117
119const startForegroundDaemon = (dataDir: string) =>118const startForegroundDaemon = (dataDir: string) =>
@@ -187,28 +186,14 @@ const runOneClient = async ( · 67 unmodified lines
187 }186 }
188};187};
189 188
190/** `executor mcp` now ensures a DURABLE (detached) daemon and bridges to it, so a
191 * cold-start scenario leaves that daemon running. Stop it before removing the
192 * data dir so the test never leaks an orphan daemon. */
193const stopAutoSpawnedDaemon = (dataDir: string): void => {
194 try {
195 const manifest = JSON.parse(
196 readFileSync(join(dataDir, "server-control", "server.json"), "utf8"),
197 ) as { pid?: number };
198 if (manifest.pid) process.kill(manifest.pid, "SIGTERM");
199 } catch {
200 // no manifest (no daemon spawned) — nothing to stop.
201 }
202};
203
204const withTempData = Effect.acquireRelease(189const withTempData = Effect.acquireRelease(
205 Effect.sync(() => {190 Effect.sync(() => {
206 const root = mkdtempSync(join(tmpdir(), "executor-mcp-stress-"));191 const root = mkdtempSync(join(tmpdir(), "executor-mcp-stress-"));
207 return join(root, "data");192 return join(root, "data");
208 }),193 }),
209 (dataDir) =>194 (dataDir) =>
210 Effect.sync(() => {195 Effect.promise(async () => {
211 stopAutoSpawnedDaemon(dataDir);196 await stopAutoSpawnedDaemon(dataDir);
212 rmSync(join(dataDir, ".."), { recursive: true, force: true });197 rmSync(join(dataDir, ".."), { recursive: true, force: true });
213 }),198 }),
214);199);
@@ -321,8 +306,13 @@ scenario( · 106 unmodified lines
321 "2",306 "2",
322 );307 );
323 308
324 daemon.proc.kill("SIGKILL");309 if (daemon.proc.pid) process.kill(-daemon.proc.pid, "SIGKILL");
325 yield* Effect.promise(() => Promise.race([daemon.proc.exited, Bun.sleep(3000)]));310 yield* Effect.promise(() =>
311 Promise.race([
312 exited(daemon.proc),
313 new Promise<void>((resolve) => setTimeout(resolve, 3000)),
314 ]),
315 );
326 316
327 // The next call must settle (reject) quickly — a 10s bound well under the317 // The next call must settle (reject) quickly — a 10s bound well under the
328 // scenario timeout catches a hang.318 // scenario timeout catches a hang.
@@ -332,7 +322,7 @@ scenario( · 3 unmodified lines
332 .callTool({ name: "execute", arguments: { code: "return 3" } })322 .callTool({ name: "execute", arguments: { code: "return 3" } })
333 .then(() => "resolved" as const)323 .then(() => "resolved" as const)
334 .catch(() => "rejected" as const),324 .catch(() => "rejected" as const),
335 Bun.sleep(10_000).then(() => "timeout" as const),325 new Promise<"timeout">((resolve) => setTimeout(() => resolve("timeout"), 10_000)),
336 ]),326 ]),
337 );327 );
338 // eslint-disable-next-line no-console328 // eslint-disable-next-line no-console
e2e/local/cli-mcp-protocol.test.ts+4−17
@@ -11,12 +11,13 @@ import { StdioClientTransport as ModernStdioClientTransport } from "@modelcontex · 10 unmodified lines
11import { Client as LegacyClient } from "@modelcontextprotocol/sdk/client/index.js";11import { Client as LegacyClient } from "@modelcontextprotocol/sdk/client/index.js";
12import { StdioClientTransport as LegacyStdioClientTransport } from "@modelcontextprotocol/sdk/client/stdio.js";12import { StdioClientTransport as LegacyStdioClientTransport } from "@modelcontextprotocol/sdk/client/stdio.js";
13import { Effect } from "effect";13import { Effect } from "effect";
14import { mkdtempSync, readFileSync, rmSync } from "node:fs";14import { mkdtempSync, rmSync } from "node:fs";
15import { tmpdir } from "node:os";15import { tmpdir } from "node:os";
16import { join } from "node:path";16import { join } from "node:path";
17import { fileURLToPath } from "node:url";17import { fileURLToPath } from "node:url";
18 18
19import { scenario } from "../src/scenario";19import { scenario } from "../src/scenario";
20import { stopAutoSpawnedDaemon } from "./daemon-process";
20 21
21const repoRoot = fileURLToPath(new URL("../../", import.meta.url));22const repoRoot = fileURLToPath(new URL("../../", import.meta.url));
22const testScope = join(repoRoot, "apps/local");23const testScope = join(repoRoot, "apps/local");
@@ -33,28 +34,14 @@ const bridgeCommand = (dataDir: string) => ({ · 10 unmodified lines
33 stderr: "pipe" as const,34 stderr: "pipe" as const,
34});35});
35 36
36const stopAutoSpawnedDaemon = (dataDir: string): void => {
37 // The bridge is transient, while its auto-started daemon is detached. Reap
38 // that owner before deleting this scenario's private data directory.
39 // oxlint-disable-next-line executor/no-try-catch-or-throw -- cleanup tolerates a bridge that failed before writing its manifest
40 try {
41 const manifest = JSON.parse(
42 readFileSync(join(dataDir, "server-control", "server.json"), "utf8"),
43 ) as { readonly pid?: number };
44 if (manifest.pid) process.kill(manifest.pid, "SIGTERM");
45 } catch {
46 // No manifest means there is no auto-started daemon to stop.
47 }
48};
49
50const withTempData = Effect.acquireRelease(37const withTempData = Effect.acquireRelease(
51 Effect.sync(() => {38 Effect.sync(() => {
52 const root = mkdtempSync(join(tmpdir(), "executor-mcp-protocol-"));39 const root = mkdtempSync(join(tmpdir(), "executor-mcp-protocol-"));
53 return { root, dataDir: join(root, "data") };40 return { root, dataDir: join(root, "data") };
54 }),41 }),
55 ({ root, dataDir }) =>42 ({ root, dataDir }) =>
56 Effect.sync(() => {43 Effect.promise(async () => {
57 stopAutoSpawnedDaemon(dataDir);44 await stopAutoSpawnedDaemon(dataDir);
58 rmSync(root, { recursive: true, force: true });45 rmSync(root, { recursive: true, force: true });
59 }),46 }),
60);47);
e2e/local/daemon-process.ts+55−0
@@ -0,0 +1,55 @@
1import { readFile } from "node:fs/promises";
2import { join } from "node:path";
3
4const isAlive = (pid: number): boolean => {
5 // oxlint-disable-next-line executor/no-try-catch-or-throw -- boundary: process liveness probing reports false for an already-reaped test daemon
6 try {
7 process.kill(process.platform === "win32" ? pid : -pid, 0);
8 return true;
9 } catch {
10 return false;
11 }
12};
13
14const signal = (pid: number, name: NodeJS.Signals): void => {
15 // Auto-started daemons are detached process-group leaders. Signal the whole
16 // private group so their Vite child cannot survive a failed test.
17 // oxlint-disable-next-line executor/no-try-catch-or-throw -- boundary: Windows and pre-detach failures require a direct-pid fallback
18 try {
19 process.kill(process.platform === "win32" ? pid : -pid, name);
20 } catch {
21 // oxlint-disable-next-line executor/no-try-catch-or-throw -- cleanup tolerates a process that exited between the liveness check and signal
22 try {
23 process.kill(pid, name);
24 } catch {}
25 }
26};
27
28const waitUntilStopped = async (pid: number, timeoutMs: number): Promise<boolean> => {
29 const deadline = performance.now() + timeoutMs;
30 while (performance.now() < deadline) {
31 if (!isAlive(pid)) return true;
32 await new Promise((resolve) => setTimeout(resolve, 50));
33 }
34 return !isAlive(pid);
35};
36
37/** Stop the detached daemon elected by an `executor mcp` cold start. */
38export const stopAutoSpawnedDaemon = async (dataDir: string): Promise<void> => {
39 // oxlint-disable-next-line executor/no-try-catch-or-throw -- cleanup tolerates a bridge that failed before writing its manifest
40 try {
41 const manifest = JSON.parse(
42 await readFile(join(dataDir, "server-control", "server.json"), "utf8"),
43 ) as { readonly pid?: unknown };
44 if (!Number.isSafeInteger(manifest.pid) || (manifest.pid as number) <= 0) return;
45
46 const pid = manifest.pid as number;
47 signal(pid, "SIGTERM");
48 if (await waitUntilStopped(pid, 10_000)) return;
49
50 signal(pid, "SIGKILL");
51 await waitUntilStopped(pid, 2_000);
52 } catch {
53 // No manifest means there is no auto-started daemon to stop.
54 }
55};
e2e/local/local-server.ts+8−2
@@ -68,7 +68,7 @@ export const withLocalServer = ( · 67 unmodified lines
68 yield* Effect.all(68 yield* Effect.all(
69 [69 [
70 cli.session(70 cli.session(
71 ["bun", "run", "dev:cli", "web", "--foreground", "--port", "0"],71 ["bun", "run", "apps/cli/src/main.ts", "web", "--foreground", "--port", "0"],
72 async (term) => {72 async (term) => {
73 markRecordingStart(runDir, "terminal");73 markRecordingStart(runDir, "terminal");
74 markFocus(runDir, "terminal");74 markFocus(runDir, "terminal");
@@ -107,10 +107,16 @@ export const withLocalServer = ( · 32 unmodified lines
107 // otherwise the orphaned child wedges the terminal teardown.107 // otherwise the orphaned child wedges the terminal teardown.
108 markFocus(runDir, "terminal");108 markFocus(runDir, "terminal");
109 await term.keyboard.press("Control+C");109 await term.keyboard.press("Control+C");
110 await term.waitForExit({ timeoutMs: 15_000 });
110 },111 },
111 {112 {
112 cwd: repoRoot,113 cwd: repoRoot,
113 env: { EXECUTOR_DATA_DIR: dataDir, EXECUTOR_SCOPE_DIR: dataDir, ...options?.env },114 env: {
115 EXECUTOR_DEV: "1",
116 EXECUTOR_DATA_DIR: dataDir,
117 EXECUTOR_SCOPE_DIR: dataDir,
118 ...options?.env,
119 },
114 record: join(runDir, options?.castName ?? "terminal.cast"),120 record: join(runDir, options?.castName ?? "terminal.cast"),
115 viewport: { cols: 120, rows: 40 },121 viewport: { cols: 120, rows: 40 },
116 },122 },
e2e/local/vite-dev-routing.test.ts+3−2
@@ -80,10 +80,11 @@ const startPlainViteDev = async (): Promise<ViteDev> => { · 79 unmodified lines
80 const dataDir = mkdtempSync(join(tmpdir(), "executor-local-vite-e2e-"));80 const dataDir = mkdtempSync(join(tmpdir(), "executor-local-vite-e2e-"));
81 const port = await freePort();81 const port = await freePort();
82 const origin = `http://127.0.0.1:${port}`;82 const origin = `http://127.0.0.1:${port}`;
83 const viteEntrypoint = join(localAppDir, "node_modules", "vite", "bin", "vite.js");
83 let logs = "";84 let logs = "";
84 const child = spawn(85 const child = spawn(
85 "bunx",86 "bun",
86 ["--bun", "vite", "dev", "--host", "127.0.0.1", "--port", String(port), "--strictPort"],87 [viteEntrypoint, "dev", "--host", "127.0.0.1", "--port", String(port), "--strictPort"],
87 {88 {
88 cwd: localAppDir,89 cwd: localAppDir,
89 env: { ...process.env, EXECUTOR_DATA_DIR: dataDir, PORT: String(port) },90 env: { ...process.env, EXECUTOR_DATA_DIR: dataDir, PORT: String(port) },
e2e/scenarios/resume-after-sandbox-deadline.test.ts+35−20
@@ -8,14 +8,18 @@ · 7 unmodified lines
8// unknown execution.8// unknown execution.
9//9//
10// The journey drives exactly that shape: ONE execution with TWO approval10// The journey drives exactly that shape: ONE execution with TWO approval
11// gates. The first approval is granted late in its window (~3.5 min), so the11// gates. The first approval is granted late (70% of the sandbox budget in),
12// second pause's window reaches well past the sandbox's 5-minute mark. The12// so the second pause's window reaches well past the budget. The second
13// second approval arrives ~5.75 min after execution start — inside its OWN13// approval arrives at ~115% of the budget after execution start — inside its
14// advertised window, but past the old absolute deadline. Deliberately slow14// OWN advertised window, but past the old absolute deadline. The subject is
15// (~6 min): the elapsed time IS the subject under test. A single-pause15// that RATIO, not any absolute duration, so the delays scale off the budget
16// variant cannot express this cross-target — hosts that advertise a16// the target was booted with: selfhost boots with a seconds-long
17// 4-minute window would expire it legitimately before the sandbox clock17// EXECUTOR_SANDBOX_TIMEOUT_MS (setup/sandbox-timeout.ts) and proves the race
18// even matters.18// in ~25s; a target on the production 5-minute budget runs the original
19// ~6-minute journey (the elapsed time IS the subject — nothing is mocked). A
20// single-pause variant cannot express this cross-target — hosts that
21// advertise a 4-minute window would expire it legitimately before the
22// sandbox clock even matters.
19//23//
20// The gate is `policies.create`'s own `requiresApproval` annotation24// The gate is `policies.create`'s own `requiresApproval` annotation
21// (hermetic, same device as policy-tool-approval.test.ts); both approvals25// (hermetic, same device as policy-tool-approval.test.ts); both approvals
@@ -29,15 +33,23 @@ import { composePluginApi } from "@executor-js/api/server"; · 7 unmodified lines
29import { scenario } from "../src/scenario";33import { scenario } from "../src/scenario";
30import { Api, Mcp, Target } from "../src/services";34import { Api, Mcp, Target } from "../src/services";
31import { configuredMcpPausedSessionIdleTimeoutMs } from "../setup/mcp-session-timeouts";35import { configuredMcpPausedSessionIdleTimeoutMs } from "../setup/mcp-session-timeouts";
36import { configuredSandboxTimeoutMs } from "../setup/sandbox-timeout";
32 37
33const coreApi = composePluginApi([] as const);38const coreApi = composePluginApi([] as const);
34 39
35// Grant the first approval at 3.5 min — late but inside its 4-minute window.40const SANDBOX_BUDGET_MS = configuredSandboxTimeoutMs();
36// The second pause then opens a fresh window reaching ~7.5 min.41
37const FIRST_APPROVAL_DELAY_MS = 3.5 * 60_000;42// Grant the first approval at 70% of the budget — late but inside its window
38// Grant the second approval 2.25 min later: ~5.75 min after execution start,43// (was 3.5 of 5 min). The second pause then opens a fresh window reaching
39// past the sandbox's 5-minute budget but inside the second window.44// past the budget.
40const SECOND_APPROVAL_DELAY_MS = 2.25 * 60_000;45const FIRST_APPROVAL_DELAY_MS = 0.7 * SANDBOX_BUDGET_MS;
46// Grant the second approval 45% of the budget later: ~115% of the budget
47// after execution start, past the sandbox clock but inside the second window
48// (was 2.25 of 5 min → ~5.75 min total).
49const SECOND_APPROVAL_DELAY_MS = 0.45 * SANDBOX_BUDGET_MS;
50// The whole journey plus scheduling slack, for the idle-window guard and the
51// vitest timeout.
52const JOURNEY_MS = FIRST_APPROVAL_DELAY_MS + SECOND_APPROVAL_DELAY_MS;
41 53
42/** Sandbox code that creates two policies through the approval-gated core54/** Sandbox code that creates two policies through the approval-gated core
43 * tool. Patterns are unique-per-run and match no real tool, so the rules are55 * tool. Patterns are unique-per-run and match no real tool, so the rules are
@@ -56,19 +68,22 @@ const second = await tools.executor.coreTools.policies.create({ · 12 unmodified lines
56return JSON.stringify({ first: first.ok, second: second.ok });68return JSON.stringify({ first: first.ok, second: second.ok });
57`;69`;
58 70
59// The journey spans ~6 real minutes of paused waiting, so the host must keep71// The journey spans the whole paused waiting time, so the host must keep the
60// the paused session alive that long. The suite's default e2e override shrinks72// paused session alive that long. The suite's default e2e override shrinks
61// the paused-session idle teardown to seconds (to keep teardown tests fast),73// the paused-session idle teardown to seconds (to keep teardown tests fast),
62// which would evict the session mid-scenario for reasons unrelated to the74// which would evict the session mid-scenario for reasons unrelated to the
63// clock under test — require the production-like window instead.75// clock under test — require a window that outlasts the journey instead.
76// With a shrunken sandbox budget the journey shrinks too, so even the short
77// e2e idle window can suffice; the guard compares the two rather than
78// hardcoding either.
64const PAUSED_IDLE_WINDOW_TOO_SHORT =79const PAUSED_IDLE_WINDOW_TOO_SHORT =
65 configuredMcpPausedSessionIdleTimeoutMs() < 8 * 60_00080 configuredMcpPausedSessionIdleTimeoutMs() < JOURNEY_MS + 60_000
66 ? `the target's paused-session idle teardown (${configuredMcpPausedSessionIdleTimeoutMs()}ms) evicts the session before this ~6-minute journey completes; boot the target with MCP_PAUSED_SESSION_IDLE_TIMEOUT_MS >= 480000 to run it`81 ? `the target's paused-session idle teardown (${configuredMcpPausedSessionIdleTimeoutMs()}ms) evicts the session before this ${Math.round(JOURNEY_MS / 1000)}s journey completes; boot the target with MCP_PAUSED_SESSION_IDLE_TIMEOUT_MS >= ${JOURNEY_MS + 60_000} or a smaller E2E_SANDBOX_TIMEOUT_MS to run it`
67 : undefined;82 : undefined;
68 83
69scenario(84scenario(
70 "MCP · chained approvals granted within their windows survive the sandbox clock",85 "MCP · chained approvals granted within their windows survive the sandbox clock",
71 { timeout: 480_000, skip: PAUSED_IDLE_WINDOW_TOO_SHORT },86 { timeout: Math.max(120_000, JOURNEY_MS + 120_000), skip: PAUSED_IDLE_WINDOW_TOO_SHORT },
72 Effect.gen(function* () {87 Effect.gen(function* () {
73 const target = yield* Target;88 const target = yield* Target;
74 const apiSurface = yield* Api;89 const apiSurface = yield* Api;
e2e/selfhost/posthog-mcp-oauth.test.ts+43−20
@@ -1,8 +1,14 @@
1// Selfhost browser regression for the reported PostHog MCP OAuth dead-end. A1// Hermetic selfhost browser regression for the reported PostHog MCP OAuth
2// real Executor instance adds https://mcp.posthog.com/mcp, then starts the2// dead-end. A real Executor instance adds a wire-level OAuth-protected MCP
3// connection flow. The product guarantee: clicking Connect opens PostHog's3// server, then starts the connection flow. The product guarantee: clicking
4// OAuth authorization page through dynamic client registration, not the4// Connect reaches the authorization page through dynamic client registration,
5// bring-your-own OAuth app picker with "Automatic setup unavailable".5// not the bring-your-own OAuth app picker with "Automatic setup unavailable".
6//
7// This used to call PostHog's production MCP and OAuth sites directly. That
8// made Executor CI depend on a third party's availability, metadata, and popup
9// response time. The local fixtures implement the same RFC 9728 discovery,
10// RFC 8414 metadata, RFC 7591 registration, and authorization redirect over
11// real HTTP, while keeping the assertion deterministic.
6import { randomBytes } from "node:crypto";12import { randomBytes } from "node:crypto";
7 13
8import { expect } from "@effect/vitest";14import { expect } from "@effect/vitest";
@@ -10,22 +16,28 @@ import { Effect } from "effect"; · 1 unmodified line
10import { composePluginApi } from "@executor-js/api/server";16import { composePluginApi } from "@executor-js/api/server";
11import { deriveMcpNamespace } from "@executor-js/plugin-mcp";17import { deriveMcpNamespace } from "@executor-js/plugin-mcp";
12import { mcpHttpPlugin } from "@executor-js/plugin-mcp/api";18import { mcpHttpPlugin } from "@executor-js/plugin-mcp/api";
19import { makeGreetingMcpServer, serveMcpServerWithOAuth } from "@executor-js/plugin-mcp/testing";
13import { IntegrationSlug } from "@executor-js/sdk/shared";20import { IntegrationSlug } from "@executor-js/sdk/shared";
21import { OAuthTestServer } from "@executor-js/sdk/testing";
14 22
15import { scenario } from "../src/scenario";23import { scenario } from "../src/scenario";
16import { Api, Browser, Target } from "../src/services";24import { Api, Browser, Target } from "../src/services";
17 25
18const POSTHOG_MCP_URL = "https://mcp.posthog.com/mcp";
19const api = composePluginApi([mcpHttpPlugin()] as const);26const api = composePluginApi([mcpHttpPlugin()] as const);
20 27
21scenario(28scenario(
22 "MCP OAuth · PostHog starts OAuth from Add connection",29 "MCP OAuth · dynamic registration opens the discovered authorization server",
23 { timeout: 180_000 },30 { timeout: 180_000 },
24 Effect.scoped(31 Effect.scoped(
25 Effect.gen(function* () {32 Effect.gen(function* () {
26 const target = yield* Target;33 const target = yield* Target;
27 const browser = yield* Browser;34 const browser = yield* Browser;
28 const { client: makeApiClient } = yield* Api;35 const { client: makeApiClient } = yield* Api;
36 const oauth = yield* OAuthTestServer;
37 const server = yield* serveMcpServerWithOAuth(
38 () => makeGreetingMcpServer({ name: "dcr-regression-mcp" }),
39 { path: "/mcp" },
40 );
29 const identity = yield* target.newIdentity();41 const identity = yield* target.newIdentity();
30 const client = yield* makeApiClient(api, identity);42 const client = yield* makeApiClient(api, identity);
31 const displayName = `PostHog MCP ${randomBytes(3).toString("hex")}`;43 const displayName = `PostHog MCP ${randomBytes(3).toString("hex")}`;
@@ -33,9 +45,9 @@ scenario( · 1 unmodified line
33 45
34 yield* Effect.gen(function* () {46 yield* Effect.gen(function* () {
35 yield* browser.session(identity, async ({ page, step }) => {47 yield* browser.session(identity, async ({ page, step }) => {
36 await step("Open the add-MCP flow pointed at PostHog", async () => {48 await step("Open the add-MCP flow pointed at the OAuth server", async () => {
37 const addUrl = new URL("/integrations/add/mcp", target.baseUrl);49 const addUrl = new URL("/integrations/add/mcp", target.baseUrl);
38 addUrl.searchParams.set("url", POSTHOG_MCP_URL);50 addUrl.searchParams.set("url", server.endpoint);
39 await page.goto(addUrl.toString(), { waitUntil: "networkidle" });51 await page.goto(addUrl.toString(), { waitUntil: "networkidle" });
40 await page.getByText("How does this server authenticate?").waitFor({ timeout: 30_000 });52 await page.getByText("How does this server authenticate?").waitFor({ timeout: 30_000 });
41 await page.getByText("Method 1 · Detected").waitFor();53 await page.getByText("Method 1 · Detected").waitFor();
@@ -59,26 +71,37 @@ scenario( · 17 unmodified lines
59 const popupPromise = page.waitForEvent("popup", { timeout: 30_000 });71 const popupPromise = page.waitForEvent("popup", { timeout: 30_000 });
60 await page.getByRole("button", { name: "Connect", exact: true }).click();72 await page.getByRole("button", { name: "Connect", exact: true }).click();
61 const popup = await popupPromise;73 const popup = await popupPromise;
62 await popup.waitForURL(/^https:\/\/oauth\.posthog\.com\/oauth\/authorize\//, {74 await popup.waitForURL((url) => url.origin === new URL(oauth.issuerUrl).origin, {
63 timeout: 30_000,75 timeout: 30_000,
64 });76 });
65 await popup.waitForLoadState("domcontentloaded", { timeout: 30_000 });77 await popup.waitForLoadState("domcontentloaded", { timeout: 30_000 });
66 78
67 const authorizeUrl = new URL(popup.url());79 const authorizeUrl = new URL(popup.url());
68 expect(authorizeUrl.origin, "OAuth opened PostHog's authorization host").toBe(80 expect(authorizeUrl.origin, "OAuth opened the discovered authorization host").toBe(
69 "https://oauth.posthog.com",81 new URL(oauth.authorizationEndpoint).origin,
70 );82 );
71 expect(authorizeUrl.pathname, "OAuth opened the authorize endpoint").toBe(
72 "/oauth/authorize/",
73 );
74 expect(
75 authorizeUrl.searchParams.get("resource"),
76 "resource targets the MCP endpoint",
77 ).toBe(POSTHOG_MCP_URL);
78 await popup.close();83 await popup.close();
79 });84 });
80 });85 });
86
87 const oauthRequests = yield* oauth.requests;
88 expect(
89 oauthRequests.some(
90 (request) => request.method === "POST" && request.path === "/register",
91 ),
92 "the connection flow dynamically registered its OAuth client",
93 ).toBe(true);
94 const authorizeRequest = oauthRequests.find(
95 (request) => request.method === "GET" && request.path === "/authorize",
96 );
97 expect(
98 authorizeRequest,
99 "the popup reached the discovered authorize endpoint",
100 ).toBeDefined();
101 expect(authorizeRequest?.query.resource, "resource targets the MCP endpoint").toBe(
102 server.endpoint,
103 );
81 }).pipe(Effect.ensuring(client.mcp.removeServer({ params: { slug } }).pipe(Effect.ignore)));104 }).pipe(Effect.ensuring(client.mcp.removeServer({ params: { slug } }).pipe(Effect.ignore)));
82 }),105 }),
83 ),106 ).pipe(Effect.provide(OAuthTestServer.layer())),
84);107);
e2e/setup/boot.ts+122−13
@@ -3,7 +3,46 @@ · 2 unmodified lines
3// what runs (their dev stack, their stub flags); this file only owns process3// what runs (their dev stack, their stub flags); this file only owns process
4// lifecycle, so it stays target-agnostic.4// lifecycle, so it stays target-agnostic.
5import { spawn, type ChildProcess } from "node:child_process";5import { spawn, type ChildProcess } from "node:child_process";
6import { openSync } from "node:fs";6import { closeSync, openSync, readFileSync } from "node:fs";
7import { setTimeout as sleep } from "node:timers/promises";
8
9const BOOT_LOG_TAIL_BYTES = 8_000;
10const HTTP_PROBE_TIMEOUT_MS = 5_000;
11
12/** A long-lived boot process exited before its service became ready. */
13export class BootProcessExitError extends Error {
14 readonly _tag = "BootProcessExitError";
15
16 constructor(
17 readonly command: string,
18 readonly exitCode: number | null,
19 readonly signal: NodeJS.Signals | null,
20 readonly logTail: string,
21 ) {
22 const outcome = exitCode === null ? `signal ${signal ?? "unknown"}` : `code ${exitCode}`;
23 const diagnostic = logTail ? `\nLast boot log output:\n${logTail}` : "";
24 super(`Boot process ${JSON.stringify(command)} exited with ${outcome}${diagnostic}`);
25 this.name = "BootProcessExitError";
26 }
27}
28
29/** A service process stayed alive but did not become HTTP-ready in time. */
30export class BootReadinessTimeoutError extends Error {
31 readonly _tag = "BootReadinessTimeoutError";
32
33 constructor(
34 readonly url: string,
35 readonly timeoutMs: number,
36 readonly lastError: unknown,
37 ) {
38 super(`Timed out after ${timeoutMs}ms waiting for ${url}: ${String(lastError)}`);
39 this.name = "BootReadinessTimeoutError";
40 }
41}
42
43/** Whether an acquisition failed because a live process never became ready. */
44export const isBootReadinessTimeout = (error: unknown): boolean =>
45 error instanceof BootReadinessTimeoutError;
7 46
8export interface BootedProcesses {47export interface BootedProcesses {
9 readonly teardown: () => Promise<void>;48 readonly teardown: () => Promise<void>;
@@ -11,6 +50,21 @@ export interface BootedProcesses { · 1 unmodified line
11 readonly pids: ReadonlyArray<number>;50 readonly pids: ReadonlyArray<number>;
12}51}
13 52
53/** A spawned process tree that can report an exit during startup. */
54export interface MonitoredBootedProcesses extends BootedProcesses {
55 /** Rejects as soon as any child exits before readiness/teardown. */
56 readonly unexpectedExit: Promise<never>;
57}
58
59const readLogTail = (logFile: string | undefined): string => {
60 if (!logFile) return "";
61 try {
62 return readFileSync(logFile, "utf8").slice(-BOOT_LOG_TAIL_BYTES).trim();
63 } catch {
64 return "";
65 }
66};
67
14export const bootProcesses = (68export const bootProcesses = (
15 procs: ReadonlyArray<{69 procs: ReadonlyArray<{
16 readonly cmd: string;70 readonly cmd: string;
@@ -21,9 +75,10 @@ export const bootProcesses = ( · 4 unmodified lines
21 readonly logFile?: string;75 readonly logFile?: string;
22 }>,76 }>,
23 options: { readonly label: string },77 options: { readonly label: string },
24): BootedProcesses => {78): MonitoredBootedProcesses => {
25 const children: ChildProcess[] = [];79 const children: ChildProcess[] = [];
26 let tearingDown = false;80 let tearingDown = false;
81 const unexpectedExits: Array<Promise<never>> = [];
27 for (const proc of procs) {82 for (const proc of procs) {
28 const log = proc.logFile ? openSync(proc.logFile, "a") : undefined;83 const log = proc.logFile ? openSync(proc.logFile, "a") : undefined;
29 const child = spawn(proc.cmd, [...proc.args], {84 const child = spawn(proc.cmd, [...proc.args], {
@@ -36,11 +91,35 @@ export const bootProcesses = ( · 6 unmodified lines
36 // kill and squat the port into the NEXT invocation's waitForHttp.91 // kill and squat the port into the NEXT invocation's waitForHttp.
37 detached: true,92 detached: true,
38 });93 });
39 child.on("exit", (code) => {94 if (log !== undefined) closeSync(log);
40 if (code !== 0 && code !== null && !tearingDown) {95 unexpectedExits.push(
41 console.error(`[e2e:${options.label}] ${proc.cmd} exited with ${code}`);96 new Promise<never>((_resolve, reject) => {
42 }97 child.once("error", (cause) => {
43 });98 if (tearingDown) return;
99 // oxlint-disable-next-line executor/no-promise-reject -- boundary: adapt the child-process error event to the startup race
100 reject(
101 new BootProcessExitError(
102 [proc.cmd, ...proc.args].join(" "),
103 child.exitCode,
104 child.signalCode,
105 `${readLogTail(proc.logFile)}\n${String(cause)}`.trim(),
106 ),
107 );
108 });
109 child.once("exit", (code, signal) => {
110 if (tearingDown) return;
111 const error = new BootProcessExitError(
112 [proc.cmd, ...proc.args].join(" "),
113 code,
114 signal,
115 readLogTail(proc.logFile),
116 );
117 console.error(`[e2e:${options.label}] ${error.message}`);
118 // oxlint-disable-next-line executor/no-promise-reject -- boundary: adapt the child-process exit event to the startup race
119 reject(error);
120 });
121 }),
122 );
44 children.push(child);123 children.push(child);
45 }124 }
46 125
@@ -77,28 +156,58 @@ export const bootProcesses = ( · 30 unmodified lines
77 }156 }
78 },157 },
79 pids: children.flatMap((child) => (child.pid === undefined ? [] : [child.pid])),158 pids: children.flatMap((child) => (child.pid === undefined ? [] : [child.pid])),
159 unexpectedExit: Promise.race(unexpectedExits),
80 };160 };
81};161};
82 162
163/**
164 * Wait for a boot probe while also observing the spawned process tree. The
165 * losing readiness probe is aborted, so an early process exit neither hides
166 * behind the full HTTP timeout nor leaves a polling timer alive.
167 */
168export const waitForBoot = async <A>(
169 processes: MonitoredBootedProcesses,
170 ready: (signal: AbortSignal) => Promise<A>,
171): Promise<A> => {
172 const controller = new AbortController();
173 try {
174 return await Promise.race([ready(controller.signal), processes.unexpectedExit]);
175 } finally {
176 controller.abort();
177 }
178};
179
83export const waitForHttp = async (180export const waitForHttp = async (
84 url: string,181 url: string,
85 options: { readonly timeoutMs?: number; readonly expectRedirect?: boolean } = {},182 options: {
183 readonly timeoutMs?: number;
184 readonly expectRedirect?: boolean;
185 readonly signal?: AbortSignal;
186 } = {},
86): Promise<void> => {187): Promise<void> => {
87 const deadline = Date.now() + (options.timeoutMs ?? 90_000);188 const timeoutMs = options.timeoutMs ?? 90_000;
189 const deadline = performance.now() + timeoutMs;
88 let lastError: unknown;190 let lastError: unknown;
89 while (Date.now() < deadline) {191 while (performance.now() < deadline) {
192 options.signal?.throwIfAborted();
90 try {193 try {
91 const response = await fetch(url, { redirect: "manual" });194 const remainingMs = Math.max(1, deadline - performance.now());
195 const probeTimeout = AbortSignal.timeout(Math.min(HTTP_PROBE_TIMEOUT_MS, remainingMs));
196 const signal = options.signal
197 ? AbortSignal.any([options.signal, probeTimeout])
198 : probeTimeout;
199 const response = await fetch(url, { redirect: "manual", signal });
92 // During a cold vite compile /api/* falls back to the SPA's 200 HTML —200 // During a cold vite compile /api/* falls back to the SPA's 200 HTML —
93 // expectRedirect waits for the real handler (302) instead.201 // expectRedirect waits for the real handler (302) instead.
94 if (options.expectRedirect ? response.status === 302 : response.status < 500) return;202 if (options.expectRedirect ? response.status === 302 : response.status < 500) return;
95 lastError = new Error(`status ${response.status}`);203 lastError = new Error(`status ${response.status}`);
96 } catch (error) {204 } catch (error) {
205 options.signal?.throwIfAborted();
97 lastError = error;206 lastError = error;
98 }207 }
99 await new Promise((resolve) => setTimeout(resolve, 400));208 await sleep(400, undefined, { signal: options.signal });
100 }209 }
101 throw new Error(`timed out waiting for ${url}: ${String(lastError)}`);210 throw new BootReadinessTimeoutError(url, timeoutMs, lastError);
102};211};
103 212
104/**213/**
e2e/setup/cloud.boot.ts+5−3
@@ -10,7 +10,7 @@ import { fileURLToPath } from "node:url"; · 9 unmodified lines
10// Vendored fork import (same pattern as mcporter).10// Vendored fork import (same pattern as mcporter).
11import { createEmulator } from "@executor-js/emulate";11import { createEmulator } from "@executor-js/emulate";
12 12
13import { bootProcesses, waitForHttp } from "./boot";13import { bootProcesses, waitForBoot, waitForHttp } from "./boot";
14import { AUTUMN_PLAN_SEED } from "./autumn-plans";14import { AUTUMN_PLAN_SEED } from "./autumn-plans";
15import { E2E_EXECUTION_RATE_LIMIT } from "./execution-limits";15import { E2E_EXECUTION_RATE_LIMIT } from "./execution-limits";
16 16
@@ -168,9 +168,11 @@ export const bootCloud = async (options: CloudBootOptions): Promise<CloudBooted> · 151 unmodified lines
168 168
169 try {169 try {
170 const local = `http://127.0.0.1:${options.cloudPort}`;170 const local = `http://127.0.0.1:${options.cloudPort}`;
171 await waitForHttp(local);171 await waitForBoot(procs, (signal) => waitForHttp(local, { signal }));
172 // The API plane is ready when login actually redirects to AuthKit.172 // The API plane is ready when login actually redirects to AuthKit.
173 await waitForHttp(`${local}/api/auth/login`, { expectRedirect: true });173 await waitForBoot(procs, (signal) =>
174 waitForHttp(`${local}/api/auth/login`, { expectRedirect: true, signal }),
175 );
174 } catch (error) {176 } catch (error) {
175 await teardown();177 await teardown();
176 throw error;178 throw error;
e2e/setup/cloud.globalsetup.ts+2−2
@@ -8,7 +8,7 @@ import { resolve } from "node:path"; · 7 unmodified lines
8 8
9import { claimAndBoot } from "../src/ports";9import { claimAndBoot } from "../src/ports";
10import { E2E_COOKIE_PASSWORD, E2E_WORKOS_CLIENT_ID } from "../targets/cloud";10import { E2E_COOKIE_PASSWORD, E2E_WORKOS_CLIENT_ID } from "../targets/cloud";
11import { waitForHttp } from "./boot";11import { isBootReadinessTimeout, waitForHttp } from "./boot";
12import { bootCloud } from "./cloud.boot";12import { bootCloud } from "./cloud.boot";
13import { ensureE2eMcpSessionTimeoutEnv } from "./mcp-session-timeouts";13import { ensureE2eMcpSessionTimeoutEnv } from "./mcp-session-timeouts";
14import { bootMotel, motelExporterEnv } from "./motel";14import { bootMotel, motelExporterEnv } from "./motel";
@@ -93,7 +93,7 @@ export default async function setup(): Promise<(() => Promise<void>) | void> { · 78 unmodified lines
93 });93 });
94 return { teardown: cloud.teardown, value: cloud };94 return { teardown: cloud.teardown, value: cloud };
95 },95 },
96 { label: "cloud" },96 { label: "cloud", retryWhen: isBootReadinessTimeout },
97 );97 );
98 // Publish the Autumn emulator URL to the test workers (they inherit this98 // Publish the Autumn emulator URL to the test workers (they inherit this
99 // process's env): scenarios that assert on tracked usage yield the Autumn99 // process's env): scenarios that assert on tracked usage yield the Autumn
e2e/setup/cloudflare.boot.ts+7−2
@@ -9,7 +9,7 @@ import { execFile } from "node:child_process"; · 8 unmodified lines
9import { fileURLToPath } from "node:url";9import { fileURLToPath } from "node:url";
10import { promisify } from "node:util";10import { promisify } from "node:util";
11 11
12import { bootProcesses, waitForHttp, type BootedProcesses } from "./boot";12import { bootProcesses, waitForBoot, waitForHttp, type BootedProcesses } from "./boot";
13 13
14export const cloudflareDir = fileURLToPath(new URL("../../apps/host-cloudflare/", import.meta.url));14export const cloudflareDir = fileURLToPath(new URL("../../apps/host-cloudflare/", import.meta.url));
15const wranglerBin = fileURLToPath(15const wranglerBin = fileURLToPath(
@@ -63,7 +63,12 @@ export const bootCloudflare = async (options: CloudflareBootOptions): Promise<Bo · 47 unmodified lines
63 try {63 try {
64 // dev-auth: /api/account/me answers 200 as the dev admin once the worker is64 // dev-auth: /api/account/me answers 200 as the dev admin once the worker is
65 // up (workerd boot + esbuild + D1 schema bring-up take a beat on first run).65 // up (workerd boot + esbuild + D1 schema bring-up take a beat on first run).
66 await waitForHttp(`http://127.0.0.1:${options.port}/api/account/me`, { timeoutMs: 120_000 });66 await waitForBoot(procs, (signal) =>
67 waitForHttp(`http://127.0.0.1:${options.port}/api/account/me`, {
68 timeoutMs: 120_000,
69 signal,
70 }),
71 );
67 } catch (error) {72 } catch (error) {
68 await procs.teardown();73 await procs.teardown();
69 throw error;74 throw error;
e2e/setup/motel.ts+3−3
@@ -8,7 +8,7 @@ import { mkdirSync, rmSync } from "node:fs"; · 7 unmodified lines
8import { join } from "node:path";8import { join } from "node:path";
9import { fileURLToPath } from "node:url";9import { fileURLToPath } from "node:url";
10 10
11import { bootProcesses, waitForHttp, type BootedProcesses } from "./boot";11import { bootProcesses, waitForBoot, waitForHttp, type MonitoredBootedProcesses } from "./boot";
12 12
13export const MOTEL_PORT = 4796;13export const MOTEL_PORT = 4796;
14export const MOTEL_URL = `http://127.0.0.1:${MOTEL_PORT}`;14export const MOTEL_URL = `http://127.0.0.1:${MOTEL_PORT}`;
@@ -28,7 +28,7 @@ export const bootMotel = async (): Promise<SuiteMotel | null> => { · 13 unmodified lines
28 rmSync(dataDir, { recursive: true, force: true });28 rmSync(dataDir, { recursive: true, force: true });
29 mkdirSync(dataDir, { recursive: true });29 mkdirSync(dataDir, { recursive: true });
30 30
31 let procs: BootedProcesses | null = null;31 let procs: MonitoredBootedProcesses | null = null;
32 // oxlint-disable-next-line executor/no-try-catch-or-throw -- boundary: optional infrastructure; a motel-less host still runs the suite32 // oxlint-disable-next-line executor/no-try-catch-or-throw -- boundary: optional infrastructure; a motel-less host still runs the suite
33 try {33 try {
34 procs = bootProcesses(34 procs = bootProcesses(
@@ -45,7 +45,7 @@ export const bootMotel = async (): Promise<SuiteMotel | null> => { · 10 unmodified lines
45 ],45 ],
46 { label: "motel" },46 { label: "motel" },
47 );47 );
48 await waitForHttp(`${MOTEL_URL}/api/health`);48 await waitForBoot(procs, (signal) => waitForHttp(`${MOTEL_URL}/api/health`, { signal }));
49 console.log(`[e2e] traces → suite motel at ${MOTEL_URL}`);49 console.log(`[e2e] traces → suite motel at ${MOTEL_URL}`);
50 return { url: MOTEL_URL, teardown: procs.teardown };50 return { url: MOTEL_URL, teardown: procs.teardown };
51 } catch (error) {51 } catch (error) {
e2e/setup/sandbox-timeout.ts+28−0
@@ -0,0 +1,28 @@
1// The sandbox execution budget shared between a target's boot env and the
2// sandbox-deadline scenario, so they cannot drift apart (same pattern as
3// execution-limits.ts). The scenario proves a RATIO — approvals granted
4// inside their own windows survive an execution that outlives the sandbox's
5// absolute budget — so the budget's magnitude is free to shrink: on selfhost
6// the boot recipe passes E2E_SANDBOX_TIMEOUT_MS through to the server as
7// EXECUTOR_SANDBOX_TIMEOUT_MS and the scenario scales its approval delays to
8// match, turning a ~6-minute real-time wait into seconds. Targets that cannot
9// shrink the budget (cloud's dynamic-worker deadline is not env-tunable) run
10// against the production default and skip via their paused-session window
11// guard instead.
12export const E2E_SANDBOX_TIMEOUT_MS = 20_000;
13
14export const SANDBOX_TIMEOUT_ENV = "E2E_SANDBOX_TIMEOUT_MS";
15
16const PRODUCTION_SANDBOX_TIMEOUT_MS = 5 * 60_000;
17
18const positiveMilliseconds = (raw: string | undefined): number | undefined => {
19 if (!raw) return undefined;
20 const parsed = Number(raw);
21 if (!Number.isFinite(parsed) || parsed <= 0) return undefined;
22 return Math.floor(parsed);
23};
24
25/** The sandbox budget the current target enforces: the harness override when
26 * the target was booted with one, else the production default. */
27export const configuredSandboxTimeoutMs = (): number =>
28 positiveMilliseconds(process.env[SANDBOX_TIMEOUT_ENV]) ?? PRODUCTION_SANDBOX_TIMEOUT_MS;
e2e/setup/selfhost.boot.ts+10−2
@@ -5,7 +5,7 @@ import { rmSync } from "node:fs"; · 4 unmodified lines
5import { resolve } from "node:path";5import { resolve } from "node:path";
6import { fileURLToPath } from "node:url";6import { fileURLToPath } from "node:url";
7 7
8import { bootProcesses, waitForHttp, type BootedProcesses } from "./boot";8import { bootProcesses, waitForBoot, waitForHttp, type BootedProcesses } from "./boot";
9 9
10export const selfhostDir = fileURLToPath(new URL("../../apps/host-selfhost/", import.meta.url));10export const selfhostDir = fileURLToPath(new URL("../../apps/host-selfhost/", import.meta.url));
11 11
@@ -21,6 +21,9 @@ export interface SelfhostBootOptions { · 9 unmodified lines
21 /** vite --host (e.g. "0.0.0.0" to be tailnet-reachable). */21 /** vite --host (e.g. "0.0.0.0" to be tailnet-reachable). */
22 readonly host?: string;22 readonly host?: string;
23 readonly logFile?: string;23 readonly logFile?: string;
24 /** Shrink the sandbox execution budget (EXECUTOR_SANDBOX_TIMEOUT_MS) so
25 * deadline scenarios prove their race in seconds. Omit for production. */
26 readonly sandboxTimeoutMs?: number;
24}27}
25 28
26export const bootSelfhost = async (options: SelfhostBootOptions): Promise<BootedProcesses> => {29export const bootSelfhost = async (options: SelfhostBootOptions): Promise<BootedProcesses> => {
@@ -51,6 +54,9 @@ export const bootSelfhost = async (options: SelfhostBootOptions): Promise<Booted · 24 unmodified lines
51 // instance at them; the hosted SSRF guard would otherwise block54 // instance at them; the hosted SSRF guard would otherwise block
52 // outbound probes/dials to localhost. Hermetic test instance only.55 // outbound probes/dials to localhost. Hermetic test instance only.
53 EXECUTOR_ALLOW_LOCAL_NETWORK: "true",56 EXECUTOR_ALLOW_LOCAL_NETWORK: "true",
57 ...(options.sandboxTimeoutMs !== undefined
58 ? { EXECUTOR_SANDBOX_TIMEOUT_MS: String(options.sandboxTimeoutMs) }
59 : {}),
54 },60 },
55 logFile: options.logFile,61 logFile: options.logFile,
56 },62 },
@@ -62,7 +68,9 @@ export const bootSelfhost = async (options: SelfhostBootOptions): Promise<Booted · 5 unmodified lines
62 // Probe via `localhost`, not 127.0.0.1 — without --host, vite binds the68 // Probe via `localhost`, not 127.0.0.1 — without --host, vite binds the
63 // resolver's first answer for localhost (::1 on macOS), so the IPv469 // resolver's first answer for localhost (::1 on macOS), so the IPv4
64 // loopback literal never answers.70 // loopback literal never answers.
65 await waitForHttp(`http://localhost:${options.port}`);71 await waitForBoot(procs, (signal) =>
72 waitForHttp(`http://localhost:${options.port}`, { signal }),
73 );
66 } catch (error) {74 } catch (error) {
67 await procs.teardown();75 await procs.teardown();
68 throw error;76 throw error;
e2e/setup/selfhost.globalsetup.ts+8−2
@@ -7,7 +7,8 @@ import { resolve } from "node:path"; · 6 unmodified lines
7 7
8import { claimAndBoot } from "../src/ports";8import { claimAndBoot } from "../src/ports";
9import { SELFHOST_ADMIN } from "../targets/selfhost";9import { SELFHOST_ADMIN } from "../targets/selfhost";
10import { waitForHttp } from "./boot";10import { isBootReadinessTimeout, waitForHttp } from "./boot";
11import { E2E_SANDBOX_TIMEOUT_MS, SANDBOX_TIMEOUT_ENV } from "./sandbox-timeout";
11import { bootSelfhost } from "./selfhost.boot";12import { bootSelfhost } from "./selfhost.boot";
12import { RUNS_DIR } from "../src/scenario";13import { RUNS_DIR } from "../src/scenario";
13 14
@@ -37,6 +38,10 @@ export default async function setup(): Promise<(() => Promise<void>) | void> { · 23 unmodified lines
37 [{ envVar: "E2E_SELFHOST_PORT", offset: 4, label: "selfhost vite dev" }],38 [{ envVar: "E2E_SELFHOST_PORT", offset: 4, label: "selfhost vite dev" }],
38 async (ports) => {39 async (ports) => {
39 const port = ports.E2E_SELFHOST_PORT!;40 const port = ports.E2E_SELFHOST_PORT!;
41 // Shrink the sandbox execution budget and publish the value to the test
42 // workers (spawned after this globalsetup, so they inherit the env): the
43 // sandbox-deadline scenario reads it to scale its approval delays.
44 process.env[SANDBOX_TIMEOUT_ENV] = String(E2E_SANDBOX_TIMEOUT_MS);
40 // Fresh data dir per suite run — hermetic; in-suite isolation comes from45 // Fresh data dir per suite run — hermetic; in-suite isolation comes from
41 // fresh identities, not resets (bootSelfhost wipes it).46 // fresh identities, not resets (bootSelfhost wipes it).
42 const procs = await bootSelfhost({47 const procs = await bootSelfhost({
@@ -44,10 +49,11 @@ export default async function setup(): Promise<(() => Promise<void>) | void> { · 1 unmodified line
44 webBaseUrl: `http://localhost:${port}`,49 webBaseUrl: `http://localhost:${port}`,
45 admin: SELFHOST_ADMIN,50 admin: SELFHOST_ADMIN,
46 logFile: bootLogFile,51 logFile: bootLogFile,
52 sandboxTimeoutMs: E2E_SANDBOX_TIMEOUT_MS,
47 });53 });
48 return { teardown: procs.teardown, value: procs };54 return { teardown: procs.teardown, value: procs };
49 },55 },
50 { label: "selfhost" },56 { label: "selfhost", retryWhen: isBootReadinessTimeout },
51 );57 );
52 return teardown;58 return teardown;
53}59}
e2e/src/ports.ts+23−5
@@ -207,6 +207,16 @@ export const claimPorts = async (claims: ReadonlyArray<PortClaim>): Promise<Clai · 206 unmodified lines
207 return {207 return {
208 ports,208 ports,
209 release: async () => {209 release: async () => {
210 // Values published by this claim are not operator pins. Clear them
211 // when the acquisition is released so a failed `claimAndBoot` attempt
212 // can genuinely probe and claim again instead of treating its own
213 // stale E2E_* value as an explicit override on every retry.
214 for (const claim of unpinned) {
215 const published = ports[claim.envVar];
216 if (published !== undefined && process.env[claim.envVar] === String(published)) {
217 delete process.env[claim.envVar];
218 }
219 }
210 const held = heldLocks.get(block);220 const held = heldLocks.get(block);
211 if (!held) return;221 if (!held) return;
212 heldLocks.delete(block);222 heldLocks.delete(block);
@@ -238,8 +248,9 @@ export const isAddrInUse = (error: unknown): boolean => { · 25 unmodified lines
238 * ephemeral) an outbound socket can still grab a just-released probe port before248 * ephemeral) an outbound socket can still grab a just-released probe port before
239 * the service binds it. When that happens the boot throws EADDRINUSE; we release249 * the service binds it. When that happens the boot throws EADDRINUSE; we release
240 * the block (freeing its lock so `claimPorts` walks past it) and re-claim + retry250 * the block (freeing its lock so `claimPorts` walks past it) and re-claim + retry
241 * up to `maxAttempts` times. Any non-EADDRINUSE boot failure — or exhausting the251 * up to `maxAttempts` times. Callers may also classify another idempotent
242 * retries — releases and rethrows, so a genuinely broken boot still surfaces.252 * acquisition failure with `retryWhen` (for example, a bounded Vite readiness
253 * timeout). Unclassified failures and exhausted retries surface unchanged.
243 *254 *
244 * `boot` receives the freshly claimed ports and must return its teardown; the255 * `boot` receives the freshly claimed ports and must return its teardown; the
245 * returned `teardown` chains the caller's teardown then releases the block.256 * returned `teardown` chains the caller's teardown then releases the block.
@@ -247,7 +258,12 @@ export const isAddrInUse = (error: unknown): boolean => { · 1 unmodified line
247export const claimAndBoot = async <T>(258export const claimAndBoot = async <T>(
248 claims: ReadonlyArray<PortClaim>,259 claims: ReadonlyArray<PortClaim>,
249 boot: (ports: Record<string, number>) => Promise<{ teardown: () => Promise<void>; value: T }>,260 boot: (ports: Record<string, number>) => Promise<{ teardown: () => Promise<void>; value: T }>,
250 options: { readonly maxAttempts?: number; readonly label?: string } = {},261 options: {
262 readonly maxAttempts?: number;
263 readonly label?: string;
264 /** Additional acquisition failures that are safe to retry from scratch. */
265 readonly retryWhen?: (error: unknown) => boolean;
266 } = {},
251): Promise<{ ports: Record<string, number>; teardown: () => Promise<void>; value: T }> => {267): Promise<{ ports: Record<string, number>; teardown: () => Promise<void>; value: T }> => {
252 const maxAttempts = options.maxAttempts ?? 3;268 const maxAttempts = options.maxAttempts ?? 3;
253 const label = options.label ?? "boot";269 const label = options.label ?? "boot";
@@ -267,13 +283,15 @@ export const claimAndBoot = async <T>( · 13 unmodified lines
267 } catch (error) {283 } catch (error) {
268 await release();284 await release();
269 lastError = error;285 lastError = error;
270 if (!isAddrInUse(error) || attempt === maxAttempts) throw error;286 const retryable = isAddrInUse(error) || options.retryWhen?.(error) === true;
287 if (!retryable || attempt === maxAttempts) throw error;
271 const collided = claims288 const collided = claims
272 .map((claim) => ports[claim.envVar])289 .map((claim) => ports[claim.envVar])
273 .filter((port): port is number => port !== undefined)290 .filter((port): port is number => port !== undefined)
274 .join(", ");291 .join(", ");
292 const reason = isAddrInUse(error) ? `hit EADDRINUSE on port(s) ${collided}` : String(error);
275 console.warn(293 console.warn(
276 `[e2e] ${label} hit EADDRINUSE on port(s) ${collided} (attempt ${attempt}/${maxAttempts}); re-claiming a fresh block and retrying`,294 `[e2e] ${label} acquisition failed (${reason}, attempt ${attempt}/${maxAttempts}); re-claiming a fresh block and retrying`,
277 );295 );
278 }296 }
279 }297 }
packages/hosts/mcp/src/stdio-integration.test.ts+57−3
@@ -4,8 +4,8 @@ import { StdioClientTransport as ModernStdioClientTransport } from "@modelcontex · 3 unmodified lines
4import { Client } from "@modelcontextprotocol/sdk/client/index.js";4import { Client } from "@modelcontextprotocol/sdk/client/index.js";
5import { StdioClientTransport } from "@modelcontextprotocol/sdk/client/stdio.js";5import { StdioClientTransport } from "@modelcontextprotocol/sdk/client/stdio.js";
6import { ElicitRequestSchema } from "@modelcontextprotocol/sdk/types.js";6import { ElicitRequestSchema } from "@modelcontextprotocol/sdk/types.js";
7import { Effect } from "effect";7import { Effect, Schema } from "effect";
8import { mkdtempSync } from "node:fs";8import { mkdtempSync, readFileSync, rmSync } from "node:fs";
9import { tmpdir } from "node:os";9import { tmpdir } from "node:os";
10import { join, resolve } from "node:path";10import { join, resolve } from "node:path";
11 11
@@ -17,6 +17,60 @@ const stdioServer = { · 5 unmodified lines
17 command: "bun",17 command: "bun",
18 args: ["run", stdioServerEntry],18 args: ["run", stdioServerEntry],
19};19};
20const decodeDaemonManifest = Schema.decodeUnknownSync(
21 Schema.fromJsonString(Schema.Struct({ pid: Schema.Number })),
22);
23
24const isProcessGroupAlive = (pid: number): boolean => {
25 // oxlint-disable-next-line executor/no-try-catch-or-throw -- boundary: process liveness probing reports false after the test daemon is reaped
26 try {
27 process.kill(process.platform === "win32" ? pid : -pid, 0);
28 return true;
29 } catch {
30 return false;
31 }
32};
33
34const signalProcessGroup = (pid: number, signal: NodeJS.Signals): void => {
35 // oxlint-disable-next-line executor/no-try-catch-or-throw -- boundary: Windows and pre-detach failures require a direct-pid fallback
36 try {
37 process.kill(process.platform === "win32" ? pid : -pid, signal);
38 } catch {
39 // oxlint-disable-next-line executor/no-try-catch-or-throw -- cleanup tolerates a daemon that exited between liveness check and signal
40 try {
41 process.kill(pid, signal);
42 } catch {}
43 }
44};
45
46const stopAutoSpawnedDaemon = async (dataDir: string): Promise<void> => {
47 // oxlint-disable-next-line executor/no-try-catch-or-throw -- cleanup tolerates startup that failed before publishing a manifest
48 try {
49 const manifest = decodeDaemonManifest(
50 readFileSync(join(dataDir, "server-control", "server.json"), "utf8"),
51 );
52 if (!Number.isSafeInteger(manifest.pid) || manifest.pid <= 0) return;
53
54 const pid = manifest.pid;
55 signalProcessGroup(pid, "SIGTERM");
56 const deadline = performance.now() + 10_000;
57 while (isProcessGroupAlive(pid) && performance.now() < deadline) {
58 await new Promise((resolve) => setTimeout(resolve, 50));
59 }
60 if (isProcessGroupAlive(pid)) signalProcessGroup(pid, "SIGKILL");
61 } catch {
62 // No manifest means there is no auto-started daemon to stop.
63 }
64};
65
66const withTempData = Effect.acquireRelease(
67 Effect.sync(() => mkdtempSync(join(tmpdir(), "executor-mcp-test-"))),
68 (dataDir) =>
69 Effect.promise(async () => {
70 await stopAutoSpawnedDaemon(dataDir);
71 rmSync(dataDir, { recursive: true, force: true });
72 }),
73);
20 74
21describe("MCP stdio integration", () => {75describe("MCP stdio integration", () => {
22 it.effect(76 it.effect(
@@ -25,7 +79,7 @@ describe("MCP stdio integration", () => { · 2 unmodified lines
25 Effect.gen(function* () {79 Effect.gen(function* () {
26 // Fresh temp dir so the test doesn't migrate against the developer's80 // Fresh temp dir so the test doesn't migrate against the developer's
27 // real ~/.executor/data.db.81 // real ~/.executor/data.db.
28 const dataDir = mkdtempSync(join(tmpdir(), "executor-mcp-test-"));82 const dataDir = yield* withTempData;
29 const transport = new StdioClientTransport({83 const transport = new StdioClientTransport({
30 command: "bun",84 command: "bun",
31 args: ["run", cliEntry, "mcp", "--scope", testScope],85 args: ["run", cliEntry, "mcp", "--scope", testScope],

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t