History

Make the test suite parallel-safe and speed up CI (#1273)

* Make self-host tests CI-safe * Await cloud db teardown * Give local e2e boots more headroom * Keep main CI runs alive * Cache CI dependencies and image layers * Wait for recorded introspection requests in graphql plugin tests * Size self-host test budget for loaded runners * Probe test http servers for readiness before use * Die on recorder wait timeout instead of typing a string error * Type the recorder wait timeout as a tagged error * Update e2e locators for the integration copy renames * Keep stale data rendered while a refresh is waiting * Guard create-org input against hydration clobbering * Match the new sign-in heading in the selfhost callback test

Rhys Sullivan committed 4 months ago

Commit 8652c991 parent 9c7e4a627 files · +252 −102Browse files

Unified · Split

27 files
  1. .github/workflows/ci.yml +74 −2
  2. apps/cloud/src/db/db.ts +6 −10
  3. apps/host-selfhost/src/boot.test.ts +14 −8
  4. apps/host-selfhost/src/multi-user.test.ts +9 −3
  5. apps/host-selfhost/src/scope-isolation.test.ts +11 −5
  6. apps/host-selfhost/src/secrets-integration.test.ts +14 −7
  7. apps/host-selfhost/src/sources-mcp.test.ts +10 −3
  8. apps/host-selfhost/src/sources.test.ts +14 −7
  9. apps/host-selfhost/vitest.config.ts +13 −25
  10. e2e/cloud/auth-routing-flow.test.ts +11 −1
  11. e2e/local/local-server.ts +4 −4
  12. e2e/local/stdio-mcp.test.ts +2 −2
  13. e2e/scenarios/auth-methods-ui.test.ts +2 −2
  14. e2e/scenarios/google-photos-preset-ui.test.ts +1 −1
  15. e2e/scenarios/mcp-catalog-sync-ui.test.ts +2 −2
  16. e2e/scenarios/provider-plugins-ui.test.ts +2 −2
  17. e2e/selfhost/auth-methods-ui.test.ts +2 −2
  18. e2e/selfhost/mcp-auth-required-add.test.ts +2 −2
  19. e2e/selfhost/oauth-callback-unauthenticated.test.ts +1 −1
  20. e2e/selfhost/posthog-mcp-oauth.test.ts +2 −2
  21. e2e/vitest.config.ts +1 −1
  22. package.json +1 −1
  23. packages/core/sdk/src/testing.ts +12 −1
  24. packages/plugins/graphql/src/sdk/plugin.test.ts +6 −5
  25. packages/plugins/graphql/src/testing/index.ts +23 −0
  26. packages/react/src/lib/async-result.test.ts +8 −2
  27. packages/react/src/lib/async-result.ts +5 −1
.github/workflows/ci.yml+74−2
@@ -7,8 +7,8 @@ on: · 6 unmodified lines
7 - main7 - main
8 8
9concurrency:9concurrency:
10 group: ci-${{ github.ref }}10 group: ci-${{ github.event_name == 'push' && format('{0}-{1}', github.ref, github.sha) || github.ref }}
11 cancel-in-progress: true11 cancel-in-progress: ${{ github.event_name == 'pull_request' }}
12 12
13jobs:13jobs:
14 format:14 format:
@@ -21,6 +21,14 @@ jobs: · 6 unmodified lines
21 with:21 with:
22 bun-version: 1.3.1122 bun-version: 1.3.11
23 23
24 - name: Cache Bun package cache
25 uses: actions/cache@v4
26 with:
27 path: ~/.bun/install/cache
28 key: ${{ runner.os }}-bun-1.3.11-${{ hashFiles('bun.lock') }}
29 restore-keys: |
30 ${{ runner.os }}-bun-1.3.11-
31
24 - run: bun install --frozen-lockfile32 - run: bun install --frozen-lockfile
25 33
26 - run: bun run format:check34 - run: bun run format:check
@@ -35,6 +43,14 @@ jobs: · 8 unmodified lines
35 with:43 with:
36 bun-version: 1.3.1144 bun-version: 1.3.11
37 45
46 - name: Cache Bun package cache
47 uses: actions/cache@v4
48 with:
49 path: ~/.bun/install/cache
50 key: ${{ runner.os }}-bun-1.3.11-${{ hashFiles('bun.lock') }}
51 restore-keys: |
52 ${{ runner.os }}-bun-1.3.11-
53
38 - run: bun install --frozen-lockfile54 - run: bun install --frozen-lockfile
39 55
40 - run: bun run lint56 - run: bun run lint
@@ -49,6 +65,14 @@ jobs: · 8 unmodified lines
49 with:65 with:
50 bun-version: 1.3.1166 bun-version: 1.3.11
51 67
68 - name: Cache Bun package cache
69 uses: actions/cache@v4
70 with:
71 path: ~/.bun/install/cache
72 key: ${{ runner.os }}-bun-1.3.11-${{ hashFiles('bun.lock') }}
73 restore-keys: |
74 ${{ runner.os }}-bun-1.3.11-
75
52 - run: bun install --frozen-lockfile76 - run: bun install --frozen-lockfile
53 77
54 - run: bun run typecheck78 - run: bun run typecheck
@@ -56,6 +80,8 @@ jobs: · 1 unmodified line
56 test:80 test:
57 name: Test81 name: Test
58 runs-on: ubuntu-latest82 runs-on: ubuntu-latest
83 env:
84 TURBO_TEST_CONCURRENCY: 3
59 steps:85 steps:
60 - uses: actions/checkout@v486 - uses: actions/checkout@v4
61 87
@@ -63,6 +89,14 @@ jobs: · 1 unmodified line
63 with:89 with:
64 bun-version: 1.3.1190 bun-version: 1.3.11
65 91
92 - name: Cache Bun package cache
93 uses: actions/cache@v4
94 with:
95 path: ~/.bun/install/cache
96 key: ${{ runner.os }}-bun-1.3.11-${{ hashFiles('bun.lock') }}
97 restore-keys: |
98 ${{ runner.os }}-bun-1.3.11-
99
66 # apps/cloud's test script invokes `node` directly; undici 8.x (pulled100 # apps/cloud's test script invokes `node` directly; undici 8.x (pulled
67 # in by @cloudflare/vitest-pool-workers) calls webidl.markAsUncloneable101 # in by @cloudflare/vitest-pool-workers) calls webidl.markAsUncloneable
68 # which only exists in Node 22.10+. Pin a known-good runtime.102 # which only exists in Node 22.10+. Pin a known-good runtime.
@@ -105,6 +139,14 @@ jobs: · 36 unmodified lines
105 with:139 with:
106 bun-version: 1.3.11140 bun-version: 1.3.11
107 141
142 - name: Cache Bun package cache
143 uses: actions/cache@v4
144 with:
145 path: ~/.bun/install/cache
146 key: ${{ runner.os }}-bun-1.3.11-${{ hashFiles('bun.lock') }}
147 restore-keys: |
148 ${{ runner.os }}-bun-1.3.11-
149
108 # The dev stacks spawn Node sidecars (vite/workerd tooling); pin the150 # The dev stacks spawn Node sidecars (vite/workerd tooling); pin the
109 # same known-good runtime the unit-test job uses.151 # same known-good runtime the unit-test job uses.
110 - uses: actions/setup-node@v4152 - uses: actions/setup-node@v4
@@ -113,6 +155,12 @@ jobs: · 2 unmodified lines
113 155
114 - run: bun install --frozen-lockfile156 - run: bun install --frozen-lockfile
115 157
158 - name: Cache Playwright browsers
159 uses: actions/cache@v4
160 with:
161 path: ~/.cache/ms-playwright
162 key: ${{ runner.os }}-playwright-1.60.0
163
116 # Install from e2e so bunx resolves ITS pinned playwright (the version164 # Install from e2e so bunx resolves ITS pinned playwright (the version
117 # the tests run against) rather than floating to the latest.165 # the tests run against) rather than floating to the latest.
118 - name: Install Playwright Chromium166 - name: Install Playwright Chromium
@@ -152,6 +200,14 @@ jobs: · 33 unmodified lines
152 with:200 with:
153 bun-version: 1.3.11201 bun-version: 1.3.11
154 202
203 - name: Cache Bun package cache
204 uses: actions/cache@v4
205 with:
206 path: ~/.bun/install/cache
207 key: ${{ runner.os }}-bun-1.3.11-${{ hashFiles('bun.lock') }}
208 restore-keys: |
209 ${{ runner.os }}-bun-1.3.11-
210
155 # The local scenarios boot a real `executor web` (which spawns a Node211 # The local scenarios boot a real `executor web` (which spawns a Node
156 # sidecar) and some drive a browser, so pin Node 22 and install Chromium.212 # sidecar) and some drive a browser, so pin Node 22 and install Chromium.
157 - uses: actions/setup-node@v4213 - uses: actions/setup-node@v4
@@ -160,6 +216,12 @@ jobs: · 2 unmodified lines
160 216
161 - run: bun install --frozen-lockfile217 - run: bun install --frozen-lockfile
162 218
219 - name: Cache Playwright browsers
220 uses: actions/cache@v4
221 with:
222 path: ~/.cache/ms-playwright
223 key: ${{ runner.os }}-playwright-1.60.0
224
163 # `chromium` and the new `chromium-headless-shell` ship as separate225 # `chromium` and the new `chromium-headless-shell` ship as separate
164 # downloads; the browser-driven scenarios launch the headless shell.226 # downloads; the browser-driven scenarios launch the headless shell.
165 # Install from e2e so bunx resolves ITS pinned playwright (the version the227 # Install from e2e so bunx resolves ITS pinned playwright (the version the
@@ -190,6 +252,14 @@ jobs: · 24 unmodified lines
190 with:252 with:
191 bun-version: 1.3.11253 bun-version: 1.3.11
192 254
255 - name: Cache Bun package cache
256 uses: actions/cache@v4
257 with:
258 path: ~/.bun/install/cache
259 key: ${{ runner.os }}-bun-1.3.11-${{ hashFiles('bun.lock') }}
260 restore-keys: |
261 ${{ runner.os }}-bun-1.3.11-
262
193 - run: bun install --frozen-lockfile263 - run: bun install --frozen-lockfile
194 264
195 - name: Build web app265 - name: Build web app
@@ -221,3 +291,5 @@ jobs: · 25 unmodified lines
221 file: apps/host-selfhost/Dockerfile291 file: apps/host-selfhost/Dockerfile
222 push: false292 push: false
223 tags: executor-selfhost:ci293 tags: executor-selfhost:ci
294 cache-from: type=gha
295 cache-to: type=gha,mode=max
apps/cloud/src/db/db.ts+6−10
@@ -73,16 +73,12 @@ const makePostgresResource = (): DbResource => { · 72 unmodified lines
73 sql,73 sql,
74 db: drizzle(sql, { schema: combinedSchema }) as DrizzleDb,74 db: drizzle(sql, { schema: combinedSchema }) as DrizzleDb,
75 close: () =>75 close: () =>
76 Effect.sync(() => {76 Effect.ignore(
77 void Effect.runFork(77 Effect.tryPromise({
78 Effect.ignore(78 try: () => sql.end({ timeout: 0 }),
79 Effect.tryPromise({79 catch: (cause) => cause,
80 try: () => sql.end({ timeout: 0 }),80 }),
81 catch: (cause) => cause,81 ),
82 }),
83 ),
84 );
85 }),
86 };82 };
87};83};
88 84
apps/host-selfhost/src/boot.test.ts+14−8
@@ -2,20 +2,26 @@ import { mkdtempSync } from "node:fs"; · 1 unmodified line
2import { tmpdir } from "node:os";2import { tmpdir } from "node:os";
3import { join } from "node:path";3import { join } from "node:path";
4 4
5import { afterAll, expect, test } from "@effect/vitest";5import { afterAll, beforeAll, expect, test } from "@effect/vitest";
6 6
7// Config reads the environment, so point it at a throwaway data dir before7// Config reads the environment, so point it at a throwaway data dir before
8// importing the app graph.8// importing the app graph.
9process.env.EXECUTOR_DATA_DIR = mkdtempSync(join(tmpdir(), "eh-boot-"));9process.env.EXECUTOR_DATA_DIR = mkdtempSync(join(tmpdir(), "eh-boot-"));
10 10
11const { makeSelfHostTestApp, singleAdminIdentityLayer } = await import("./testing/test-app");11let handler!: (request: Request) => Promise<Response>;
12let dispose: () => Promise<void> = async () => {};
12 13
13const { handler, dispose } = await makeSelfHostTestApp({14beforeAll(async () => {
14 identity: singleAdminIdentityLayer({15 const { makeSelfHostTestApp, singleAdminIdentityLayer } = await import("./testing/test-app");
15 userId: "admin",16 const app = await makeSelfHostTestApp({
16 organizationId: "default-org",17 identity: singleAdminIdentityLayer({
17 organizationName: "Default",18 userId: "admin",
18 }),19 organizationId: "default-org",
20 organizationName: "Default",
21 }),
22 });
23 handler = app.handler;
24 dispose = app.dispose;
19});25});
20afterAll(() => dispose());26afterAll(() => dispose());
21 27
apps/host-selfhost/src/multi-user.test.ts+9−3
@@ -2,7 +2,7 @@ import { mkdtempSync } from "node:fs"; · 1 unmodified line
2import { tmpdir } from "node:os";2import { tmpdir } from "node:os";
3import { join } from "node:path";3import { join } from "node:path";
4 4
5import { afterAll, expect, test } from "@effect/vitest";5import { afterAll, beforeAll, expect, test } from "@effect/vitest";
6import { connectionIdentifier } from "@executor-js/sdk/shared";6import { connectionIdentifier } from "@executor-js/sdk/shared";
7 7
8import { mintInviteCode } from "./testing/mint-invite";8import { mintInviteCode } from "./testing/mint-invite";
@@ -13,9 +13,15 @@ process.env.BETTER_AUTH_SECRET = "multi-user-secret-0123456789-abcdefghij-klmn"; · 4 unmodified lines
13process.env.EXECUTOR_BOOTSTRAP_ADMIN_EMAIL = "admin@multi.test";13process.env.EXECUTOR_BOOTSTRAP_ADMIN_EMAIL = "admin@multi.test";
14process.env.EXECUTOR_BOOTSTRAP_ADMIN_PASSWORD = "admin-pass-123456";14process.env.EXECUTOR_BOOTSTRAP_ADMIN_PASSWORD = "admin-pass-123456";
15 15
16const { makeSelfHostApiHandler } = await import("./app");16let handler!: (request: Request) => Promise<Response>;
17let dispose: () => Promise<void> = async () => {};
17 18
18const { handler, dispose } = await makeSelfHostApiHandler();19beforeAll(async () => {
20 const { makeSelfHostApiHandler } = await import("./app");
21 const app = await makeSelfHostApiHandler();
22 handler = app.handler;
23 dispose = app.dispose;
24});
19afterAll(() => dispose());25afterAll(() => dispose());
20 26
21const BASE = "http://localhost:4788";27const BASE = "http://localhost:4788";
apps/host-selfhost/src/scope-isolation.test.ts+11−5
@@ -2,7 +2,7 @@ import { mkdtempSync } from "node:fs"; · 1 unmodified line
2import { tmpdir } from "node:os";2import { tmpdir } from "node:os";
3import { join } from "node:path";3import { join } from "node:path";
4 4
5import { afterAll, expect, test } from "@effect/vitest";5import { afterAll, beforeAll, expect, test } from "@effect/vitest";
6import { connectionIdentifier } from "@executor-js/sdk/shared";6import { connectionIdentifier } from "@executor-js/sdk/shared";
7 7
8process.env.EXECUTOR_DATA_DIR = mkdtempSync(join(tmpdir(), "eh-iso-"));8process.env.EXECUTOR_DATA_DIR = mkdtempSync(join(tmpdir(), "eh-iso-"));
@@ -13,10 +13,16 @@ process.env.EXECUTOR_DATA_DIR = mkdtempSync(join(tmpdir(), "eh-iso-")); · 4 unmodified lines
13// request-scoped. Each identity is its own (org, user): in v2 the org is the13// request-scoped. Each identity is its own (org, user): in v2 the org is the
14// tenant (catalog partition) and the user is the acting subject (drives14// tenant (catalog partition) and the user is the acting subject (drives
15// `owner: "user"` rows).15// `owner: "user"` rows).
16const { makeSelfHostTestApp, headerIdentityLayer } = await import("./testing/test-app");16let handler!: (request: Request) => Promise<Response>;
17let dispose: () => Promise<void> = async () => {};
17 18
18const { handler, dispose } = await makeSelfHostTestApp({19beforeAll(async () => {
19 identity: headerIdentityLayer,20 const { makeSelfHostTestApp, headerIdentityLayer } = await import("./testing/test-app");
21 const app = await makeSelfHostTestApp({
22 identity: headerIdentityLayer,
23 });
24 handler = app.handler;
25 dispose = app.dispose;
20});26});
21afterAll(() => dispose());27afterAll(() => dispose());
22 28
@@ -135,7 +141,7 @@ test("concurrent requests with distinct identities get disjoint, correct executo · 112 unmodified lines
135 expect(addresses.some((a) => a.includes(connectionNameForUser(other)))).toBe(false);141 expect(addresses.some((a) => a.includes(connectionNameForUser(other)))).toBe(false);
136 }142 }
137 });143 });
138}, 30_000);144});
139 145
140test("a request with no identity is rejected", async () => {146test("a request with no identity is rejected", async () => {
141 const res = await handler(new Request("http://localhost/api/connections"));147 const res = await handler(new Request("http://localhost/api/connections"));
apps/host-selfhost/src/secrets-integration.test.ts+14−7
@@ -4,7 +4,7 @@ import { tmpdir } from "node:os"; · 3 unmodified lines
4import { join } from "node:path";4import { join } from "node:path";
5 5
6import { Effect, Layer } from "effect";6import { Effect, Layer } from "effect";
7import { afterAll, expect, test } from "@effect/vitest";7import { afterAll, beforeAll, expect, test } from "@effect/vitest";
8 8
9import { AuthTemplateSlug, ConnectionName, IntegrationSlug } from "@executor-js/sdk";9import { AuthTemplateSlug, ConnectionName, IntegrationSlug } from "@executor-js/sdk";
10import { makeScopedExecutor } from "@executor-js/api/server";10import { makeScopedExecutor } from "@executor-js/api/server";
@@ -32,13 +32,20 @@ const createScopedExecutor = ( · 21 unmodified lines
32 Effect.provide(SelfHostScopedExecutorSeams),32 Effect.provide(SelfHostScopedExecutorSeams),
33 );33 );
34 34
35const dbHandle = await createSelfHostDb({35let dbLayer!: Layer.Layer<SelfHostDb>;
36 path: dbPath,36let dbHandle: Awaited<ReturnType<typeof createSelfHostDb>> | undefined;
37 namespace: "executor_selfhost",37
38 version: "1.0.0",38beforeAll(async () => {
39 dbHandle = await createSelfHostDb({
40 path: dbPath,
41 namespace: "executor_selfhost",
42 version: "1.0.0",
43 });
44 dbLayer = Layer.succeed(SelfHostDb)(dbHandle);
45});
46afterAll(async () => {
47 await dbHandle?.close();
39});48});
40const dbLayer = Layer.succeed(SelfHostDb)(dbHandle);
41afterAll(() => dbHandle.close());
42 49
43const TINY_SPEC = JSON.stringify({50const TINY_SPEC = JSON.stringify({
44 openapi: "3.0.0",51 openapi: "3.0.0",
apps/host-selfhost/src/sources-mcp.test.ts+10−3
@@ -3,7 +3,7 @@ import { tmpdir } from "node:os"; · 2 unmodified lines
3import { join } from "node:path";3import { join } from "node:path";
4 4
5import { Effect, Layer } from "effect";5import { Effect, Layer } from "effect";
6import { afterAll, expect, test } from "@effect/vitest";6import { afterAll, beforeAll, expect, test } from "@effect/vitest";
7 7
8import { AuthTemplateSlug, ConnectionName, IntegrationSlug } from "@executor-js/sdk";8import { AuthTemplateSlug, ConnectionName, IntegrationSlug } from "@executor-js/sdk";
9import { makeScopedExecutor } from "@executor-js/api/server";9import { makeScopedExecutor } from "@executor-js/api/server";
@@ -50,8 +50,15 @@ const TINY_SPEC = JSON.stringify({ · 40 unmodified lines
50 },50 },
51});51});
52 52
53const { makeSelfHostApiHandler } = await import("./app");53let handler!: (request: Request) => Promise<Response>;
54const { handler, dispose } = await makeSelfHostApiHandler({ dbPath });54let dispose: () => Promise<void> = async () => {};
55
56beforeAll(async () => {
57 const { makeSelfHostApiHandler } = await import("./app");
58 const app = await makeSelfHostApiHandler({ dbPath });
59 handler = app.handler;
60 dispose = app.dispose;
61});
55afterAll(() => dispose());62afterAll(() => dispose());
56 63
57const BASE = "http://localhost:4788";64const BASE = "http://localhost:4788";
apps/host-selfhost/src/sources.test.ts+14−7
@@ -3,7 +3,7 @@ import { tmpdir } from "node:os"; · 2 unmodified lines
3import { join } from "node:path";3import { join } from "node:path";
4 4
5import { Effect, Layer } from "effect";5import { Effect, Layer } from "effect";
6import { afterAll, expect, test } from "@effect/vitest";6import { afterAll, beforeAll, expect, test } from "@effect/vitest";
7 7
8import { AuthTemplateSlug, ConnectionName, IntegrationSlug } from "@executor-js/sdk";8import { AuthTemplateSlug, ConnectionName, IntegrationSlug } from "@executor-js/sdk";
9import { makeScopedExecutor } from "@executor-js/api/server";9import { makeScopedExecutor } from "@executor-js/api/server";
@@ -28,13 +28,20 @@ const createScopedExecutor = ( · 18 unmodified lines
28const dataDir = mkdtempSync(join(tmpdir(), "eh-src-"));28const dataDir = mkdtempSync(join(tmpdir(), "eh-src-"));
29process.env.EXECUTOR_DATA_DIR = dataDir;29process.env.EXECUTOR_DATA_DIR = dataDir;
30 30
31const dbHandle = await createSelfHostDb({31let dbLayer!: Layer.Layer<SelfHostDb>;
32 path: join(dataDir, "data.db"),32let dbHandle: Awaited<ReturnType<typeof createSelfHostDb>> | undefined;
33 namespace: "executor_selfhost",33
34 version: "1.0.0",34beforeAll(async () => {
35 dbHandle = await createSelfHostDb({
36 path: join(dataDir, "data.db"),
37 namespace: "executor_selfhost",
38 version: "1.0.0",
39 });
40 dbLayer = Layer.succeed(SelfHostDb)(dbHandle);
41});
42afterAll(async () => {
43 await dbHandle?.close();
35});44});
36const dbLayer = Layer.succeed(SelfHostDb)(dbHandle);
37afterAll(() => dbHandle.close());
38 45
39// Inline OpenAPI spec so the test doesn't depend on the network to register.46// Inline OpenAPI spec so the test doesn't depend on the network to register.
40const TINY_SPEC = JSON.stringify({47const TINY_SPEC = JSON.stringify({
apps/host-selfhost/vitest.config.ts+13−25
@@ -4,30 +4,18 @@ export default defineConfig({ · 3 unmodified lines
4 test: {4 test: {
5 include: ["src/**/*.test.ts"],5 include: ["src/**/*.test.ts"],
6 passWithNoTests: true,6 passWithNoTests: true,
7 // These are integration suites: most boot a full self-host app graph7 // These are integration suites: several files boot a full self-host app
8 // (Better Auth + libSQL + MCP + plugins) at module load, then drive it8 // graph (Better Auth + libSQL + MCP + plugins) in beforeAll, then drive it
9 // over the in-memory handler, and some (scope-isolation) fire dozens of9 // over the in-memory handler. Each boot is CPU-heavy and every query
10 // concurrent requests through it. Each boot is CPU-heavy and every query10 // serializes through the one shared libSQL connection, so parallel files can
11 // serializes through the one shared libSQL connection, so when many files11 // oversubscribe CI and starve in-flight requests. Run files serially and
12 // boot at once they oversubscribe the CPU and starve the event loop: an12 // give lifecycle hooks room for the cold graph boot. Tests that fan out
13 // in-flight request stalls indefinitely and the test times out. Which13 // many concurrent requests (scope-isolation) stretch well past 30s when
14 // files lose the race is nondeterministic, which is the CI flakiness.14 // sibling turbo tasks contend for the same cores, so the budget is sized
15 //15 // for a loaded runner, not an idle one.
16 // Vitest sizes its fork pool to the core count, so a many-core machine16 fileParallelism: false,
17 // oversubscribes HARDER than a small CI runner. Bound the concurrency to17 maxWorkers: 1,
18 // an absolute number instead, so behavior is the same everywhere: at most18 testTimeout: 120_000,
19 // two heavy boots run together, which fits a 4-vCPU runner without19 hookTimeout: 60_000,
20 // starvation while still running the suite in parallel. Pair it with
21 // integration-grade timeouts as headroom for the slowest boot. This keeps
22 // every assertion intact (no logic or coverage changed) and makes the
23 // suite deterministic rather than load-dependent.
24 testTimeout: 30_000,
25 hookTimeout: 30_000,
26 poolOptions: {
27 forks: {
28 maxForks: 2,
29 minForks: 1,
30 },
31 },
32 },20 },
33});21});
e2e/cloud/auth-routing-flow.test.ts+11−1
@@ -56,7 +56,17 @@ scenario( · 55 unmodified lines
56 });56 });
57 57
58 await step("Create the first org → canonical dashboard at /<slug>", async () => {58 await step("Create the first org → canonical dashboard at /<slug>", async () => {
59 await page.getByPlaceholder("Northwind Labs").fill("Flow Test Org");59 const orgName = "Flow Test Org";
60 const orgNameInput = page.getByPlaceholder("Northwind Labs");
61 await page.waitForLoadState("networkidle");
62 await orgNameInput.fill(orgName);
63 await page.waitForTimeout(250);
64 if ((await orgNameInput.inputValue()) !== orgName) {
65 await orgNameInput.fill(orgName);
66 }
67 expect(await orgNameInput.inputValue(), "org name survives create-org hydration").toBe(
68 orgName,
69 );
60 await page.getByRole("button", { name: "Create organization" }).click();70 await page.getByRole("button", { name: "Create organization" }).click();
61 await page.getByText("Connect your MCP client").waitFor({ timeout: 30_000 });71 await page.getByText("Connect your MCP client").waitFor({ timeout: 30_000 });
62 await page.getByRole("button", { name: "Continue to app" }).click();72 await page.getByRole("button", { name: "Continue to app" }).click();
e2e/local/local-server.ts+4−4
@@ -76,15 +76,15 @@ export const withLocalServer = ( · 75 unmodified lines
76 return TOKEN_URL.test(current.text);76 return TOKEN_URL.test(current.text);
77 },77 },
78 // A cold `executor web` runs vite's optimizeDeps before it prints78 // A cold `executor web` runs vite's optimizeDeps before it prints
79 // the URL; 180s is generous headroom over the warm ~3s boot. Kept79 // the URL; 240s leaves headroom for isolated CI cold boots. Kept
80 // BELOW each scenario's test timeout (240s) so that on a slow or80 // BELOW each scenario's test timeout (300s) so that on a slow or
81 // wedged boot THIS error surfaces, with the terminal tail, rather81 // wedged boot THIS error surfaces, with the terminal tail, rather
82 // than racing vitest's generic timeout at the same deadline.82 // than racing vitest's generic timeout at the same deadline.
83 { timeoutMs: 180_000 },83 { timeoutMs: 240_000 },
84 )84 )
85 .catch((cause: unknown) => {85 .catch((cause: unknown) => {
86 throw new Error(86 throw new Error(
87 `executor web --foreground printed no ?_token URL within 180s (${String(cause)}):\n${lastText.slice(-600)}`,87 `executor web --foreground printed no ?_token URL within 240s (${String(cause)}):\n${lastText.slice(-600)}`,
88 );88 );
89 });89 });
90 const url = TOKEN_URL.exec(snapshot.text)?.[0];90 const url = TOKEN_URL.exec(snapshot.text)?.[0];
e2e/local/stdio-mcp.test.ts+2−2
@@ -40,13 +40,13 @@ const SECRET = "s3cr3t-from-the-vault"; · 39 unmodified lines
40scenario(40scenario(
41 "Local · a stdio MCP server's tools are detected on a fresh install, with env stored as a secret",41 "Local · a stdio MCP server's tools are detected on a fresh install, with env stored as a secret",
42 // Must stay STRICTLY greater than the boot-URL wait in `withLocalServer`42 // Must stay STRICTLY greater than the boot-URL wait in `withLocalServer`
43 // (currently 180s). When this CI job runs `stdio-mcp.test.ts` alone it always43 // (currently 240s). When this CI job runs `stdio-mcp.test.ts` alone it always
44 // pays a cold `vite optimizeDeps` boot (no prior file to warm the cache), the44 // pays a cold `vite optimizeDeps` boot (no prior file to warm the cache), the
45 // one variable step. If this test timeout equals the boot wait, both deadlines45 // one variable step. If this test timeout equals the boot wait, both deadlines
46 // fire together and vitest's generic "Test timed out" wins, swallowing the46 // fire together and vitest's generic "Test timed out" wins, swallowing the
47 // harness's "printed no ?_token URL\n<terminal tail>" error that tells us what47 // harness's "printed no ?_token URL\n<terminal tail>" error that tells us what
48 // boot actually got stuck on. Keep the gap so the boot diagnostic surfaces.48 // boot actually got stuck on. Keep the gap so the boot diagnostic surfaces.
49 { timeout: 240_000 },49 { timeout: 300_000 },
50 Effect.gen(function* () {50 Effect.gen(function* () {
51 const cli = yield* Cli;51 const cli = yield* Cli;
52 const runDir = yield* RunDir;52 const runDir = yield* RunDir;
e2e/scenarios/auth-methods-ui.test.ts+2−2
@@ -55,8 +55,8 @@ scenario( · 54 unmodified lines
55 await headerName.waitFor();55 await headerName.waitFor();
56 });56 });
57 57
58 await step("Add the source with both methods", async () => {58 await step("Add the integration with both methods", async () => {
59 await page.getByRole("button", { name: "Add source" }).click();59 await page.getByRole("button", { name: "Add integration" }).click();
60 // onComplete routes to the new integration's detail hub.60 // onComplete routes to the new integration's detail hub.
61 await page.waitForURL(/\/integrations\/(?!add\b)[^/?]+$/, { timeout: 30_000 });61 await page.waitForURL(/\/integrations\/(?!add\b)[^/?]+$/, { timeout: 30_000 });
62 await page.getByText("Connections").first().waitFor();62 await page.getByText("Connections").first().waitFor();
e2e/scenarios/google-photos-preset-ui.test.ts+1−1
@@ -26,7 +26,7 @@ scenario( · 25 unmodified lines
26 const dialog = page.getByRole("dialog", { name: "Connect an integration" });26 const dialog = page.getByRole("dialog", { name: "Connect an integration" });
27 await dialog.getByRole("link", { name: /^Google Photos\b/ }).click();27 await dialog.getByRole("link", { name: /^Google Photos\b/ }).click();
28 await page.waitForURL(/\/integrations\/add\/google/);28 await page.waitForURL(/\/integrations\/add\/google/);
29 await page.getByRole("heading", { name: "Add Google" }).waitFor();29 await page.getByRole("heading", { name: "Add Google integration" }).waitFor();
30 });30 });
31 31
32 await step("The Photos preset defaults to the focused namespace and products", async () => {32 await step("The Photos preset defaults to the focused namespace and products", async () => {
e2e/scenarios/mcp-catalog-sync-ui.test.ts+2−2
@@ -44,8 +44,8 @@ scenario( · 43 unmodified lines
44 await page.getByText("Method 1 · Detected").waitFor();44 await page.getByText("Method 1 · Detected").waitFor();
45 });45 });
46 46
47 await step("Add the source", async () => {47 await step("Add the integration", async () => {
48 await page.getByRole("button", { name: "Add source" }).click();48 await page.getByRole("button", { name: "Add integration" }).click();
49 await page.waitForURL(/\/integrations\/(?!add\b)[^/?]+$/, { timeout: 30_000 });49 await page.waitForURL(/\/integrations\/(?!add\b)[^/?]+$/, { timeout: 30_000 });
50 await page.getByText("Connections").first().waitFor();50 await page.getByText("Connections").first().waitFor();
51 });51 });
e2e/scenarios/provider-plugins-ui.test.ts+2−2
@@ -45,7 +45,7 @@ scenario( · 44 unmodified lines
45 await page.goto("/integrations/add/google", {45 await page.goto("/integrations/add/google", {
46 waitUntil: "domcontentloaded",46 waitUntil: "domcontentloaded",
47 });47 });
48 await page.getByRole("heading", { name: "Add Google" }).waitFor();48 await page.getByRole("heading", { name: "Add Google integration" }).waitFor();
49 await page.getByText("Customize your Google connection").waitFor();49 await page.getByText("Customize your Google connection").waitFor();
50 await page.getByText("Gmail").first().waitFor();50 await page.getByText("Gmail").first().waitFor();
51 await page.getByText("Google Calendar").first().waitFor();51 await page.getByText("Google Calendar").first().waitFor();
@@ -55,7 +55,7 @@ scenario( · 3 unmodified lines
55 await page.goto("/integrations/add/microsoft", {55 await page.goto("/integrations/add/microsoft", {
56 waitUntil: "domcontentloaded",56 waitUntil: "domcontentloaded",
57 });57 });
58 await page.getByRole("heading", { name: "Add Microsoft Graph" }).waitFor();58 await page.getByRole("heading", { name: "Add Microsoft integration" }).waitFor();
59 await page.getByText("Customize Microsoft Graph").waitFor();59 await page.getByText("Customize Microsoft Graph").waitFor();
60 expect(await page.getByText("All Microsoft Graph", { exact: true }).count()).toBe(0);60 expect(await page.getByText("All Microsoft Graph", { exact: true }).count()).toBe(0);
61 await page.getByText("Productivity").waitFor();61 await page.getByText("Productivity").waitFor();
e2e/selfhost/auth-methods-ui.test.ts+2−2
@@ -64,8 +64,8 @@ scenario( · 63 unmodified lines
64 await page.getByPlaceholder("Authorization").last().waitFor();64 await page.getByPlaceholder("Authorization").last().waitFor();
65 });65 });
66 66
67 await step("Add the source with both methods", async () => {67 await step("Add the integration with both methods", async () => {
68 await page.getByRole("button", { name: "Add source" }).click();68 await page.getByRole("button", { name: "Add integration" }).click();
69 await page.waitForURL(/\/integrations\/(?!add\b)[^/?]+$/, {69 await page.waitForURL(/\/integrations\/(?!add\b)[^/?]+$/, {
70 timeout: 30_000,70 timeout: 30_000,
71 });71 });
e2e/selfhost/mcp-auth-required-add.test.ts+2−2
@@ -67,9 +67,9 @@ scenario( · 66 unmodified lines
67 await page.getByText("Auth required").first().waitFor();67 await page.getByText("Auth required").first().waitFor();
68 });68 });
69 69
70 await step("Add the source with the declared method", async () => {70 await step("Add the integration with the declared method", async () => {
71 await page.getByPlaceholder("e.g. Linear").fill(name);71 await page.getByPlaceholder("e.g. Linear").fill(name);
72 await page.getByRole("button", { name: "Add source" }).click();72 await page.getByRole("button", { name: "Add integration" }).click();
73 // onComplete routes to the new integration's detail hub.73 // onComplete routes to the new integration's detail hub.
74 await page.waitForURL(/\/integrations\/(?!add\b)[^/?]+$/, { timeout: 30_000 });74 await page.waitForURL(/\/integrations\/(?!add\b)[^/?]+$/, { timeout: 30_000 });
75 const landedSlug = new URL(page.url()).pathname.split("/").filter(Boolean).at(-1);75 const landedSlug = new URL(page.url()).pathname.split("/").filter(Boolean).at(-1);
e2e/selfhost/oauth-callback-unauthenticated.test.ts+1−1
@@ -113,7 +113,7 @@ scenario( · 112 unmodified lines
113 await step("Provider sends a signed-out browser to the OAuth callback", async () => {113 await step("Provider sends a signed-out browser to the OAuth callback", async () => {
114 const response = await page.goto(callbackPath, { waitUntil: "networkidle" });114 const response = await page.goto(callbackPath, { waitUntil: "networkidle" });
115 expect(response?.status(), "the callback redirects into the login flow").toBe(200);115 expect(response?.status(), "the callback redirects into the login flow").toBe(200);
116 await page.getByText("Sign in to your instance").waitFor();116 await page.getByRole("heading", { name: "Sign in" }).waitFor();
117 });117 });
118 118
119 const loginUrl = new URL(page.url());119 const loginUrl = new URL(page.url());
e2e/selfhost/posthog-mcp-oauth.test.ts+2−2
@@ -42,9 +42,9 @@ scenario( · 41 unmodified lines
42 await page.getByText("OAuth metadata is discovered from this server").waitFor();42 await page.getByText("OAuth metadata is discovered from this server").waitFor();
43 });43 });
44 44
45 await step("Add the PostHog MCP source", async () => {45 await step("Add the PostHog MCP integration", async () => {
46 await page.getByPlaceholder("e.g. Linear").fill(displayName);46 await page.getByPlaceholder("e.g. Linear").fill(displayName);
47 await page.getByRole("button", { name: "Add source" }).click();47 await page.getByRole("button", { name: "Add integration" }).click();
48 await page.waitForURL(/\/integrations\/(?!add\b)[^/?]+$/, { timeout: 30_000 });48 await page.waitForURL(/\/integrations\/(?!add\b)[^/?]+$/, { timeout: 30_000 });
49 const landedSlug = new URL(page.url()).pathname.split("/").filter(Boolean).at(-1);49 const landedSlug = new URL(page.url()).pathname.split("/").filter(Boolean).at(-1);
50 expect(landedSlug, "the add flow lands on the created integration").toBe(String(slug));50 expect(landedSlug, "the add flow lands on the created integration").toBe(String(slug));
e2e/vitest.config.ts+1−1
@@ -110,7 +110,7 @@ export default defineConfig({ · 109 unmodified lines
110 include: ["local/**/*.test.ts"],110 include: ["local/**/*.test.ts"],
111 globalSetup: [],111 globalSetup: [],
112 fileParallelism: false,112 fileParallelism: false,
113 testTimeout: 240_000,113 testTimeout: 300_000,
114 }),114 }),
115 // The supervised CLI daemon inside a guest VM, one project per OS. The115 // The supervised CLI daemon inside a guest VM, one project per OS. The
116 // globalsetup provisions a VM, `executor service install`s the daemon, and116 // globalsetup provisions a VM, `executor service install`s the daemon, and
package.json+1−1
@@ -35,7 +35,7 @@ · 34 unmodified lines
35 "dev": "turbo run dev --filter='!@executor-js/desktop' --filter='!@executor-js/cloud'",35 "dev": "turbo run dev --filter='!@executor-js/desktop' --filter='!@executor-js/cloud'",
36 "dev:desktop": "turbo run dev",36 "dev:desktop": "turbo run dev",
37 "dev:cli": "EXECUTOR_DEV=1 EXECUTOR_DATA_DIR=${EXECUTOR_DATA_DIR:-apps/local/.executor-dev} bun run apps/cli/src/main.ts",37 "dev:cli": "EXECUTOR_DEV=1 EXECUTOR_DATA_DIR=${EXECUTOR_DATA_DIR:-apps/local/.executor-dev} bun run apps/cli/src/main.ts",
38 "test": "turbo run test --filter=!@executor-js/e2e",38 "test": "turbo run test --filter=!@executor-js/e2e ${TURBO_TEST_CONCURRENCY:+--concurrency=$TURBO_TEST_CONCURRENCY}",
39 "test:e2e": "bun run --cwd e2e test",39 "test:e2e": "bun run --cwd e2e test",
40 "test:release:bootstrap": "vitest run tests/release-bootstrap-smoke.test.ts",40 "test:release:bootstrap": "vitest run tests/release-bootstrap-smoke.test.ts",
41 "build:packages": "bun run --filter='@executor-js/fumadb' build && bun run --filter='@executor-js/codemode-core' build && bun run --filter='@executor-js/runtime-quickjs' build && bun run --filter='@executor-js/sdk' build && bun run --filter='@executor-js/config' build && bun run --filter='@executor-js/execution' build && bun run --filter='@executor-js/cli' build && bun run --filter='@executor-js/plugin-*' build",41 "build:packages": "bun run --filter='@executor-js/fumadb' build && bun run --filter='@executor-js/codemode-core' build && bun run --filter='@executor-js/runtime-quickjs' build && bun run --filter='@executor-js/sdk' build && bun run --filter='@executor-js/config' build && bun run --filter='@executor-js/execution' build && bun run --filter='@executor-js/cli' build && bun run --filter='@executor-js/plugin-*' build",
packages/core/sdk/src/testing.ts+12−1
@@ -1,5 +1,6 @@
1import { createConnection } from "node:net";
1import * as NodeHttpServer from "@effect/platform-node/NodeHttpServer";2import * as NodeHttpServer from "@effect/platform-node/NodeHttpServer";
2import { Context, Data, Effect, Layer, Predicate, Scope as EffectScope } from "effect";3import { Context, Data, Effect, Layer, Predicate, Schedule, Scope as EffectScope } from "effect";
3import {4import {
4 HttpClient,5 HttpClient,
5 HttpRouter,6 HttpRouter,
@@ -109,6 +110,16 @@ const makeTestHttpServer = ( · 103 unmodified lines
109 return yield* new TestHttpServerAddressError({ address });110 return yield* new TestHttpServerAddressError({ address });
110 }111 }
111 const client = Context.get(context, HttpClient.HttpClient);112 const client = Context.get(context, HttpClient.HttpClient);
113 // Under a loaded machine the listen callback can fire before the socket
114 // reliably accepts; probe at the TCP level (no HTTP request, so handlers
115 // that record requests never see it) until a connect succeeds.
116 yield* Effect.callback<void, TestHttpServerServeError>((resume) => {
117 const socket = createConnection({ host: "127.0.0.1", port: address.port }, () => {
118 socket.end();
119 resume(Effect.void);
120 });
121 socket.on("error", (cause) => resume(Effect.fail(new TestHttpServerServeError({ cause }))));
122 }).pipe(Effect.retry(Schedule.both(Schedule.spaced("10 millis"), Schedule.recurs(100))));
112 const baseUrl = `http://127.0.0.1:${address.port}`;123 const baseUrl = `http://127.0.0.1:${address.port}`;
113 return {124 return {
114 baseUrl,125 baseUrl,
packages/plugins/graphql/src/sdk/plugin.test.ts+6−5
@@ -32,6 +32,7 @@ import { · 31 unmodified lines
32 makeGreetingGraphqlSchema,32 makeGreetingGraphqlSchema,
33 serveGraphqlFailureTestServer,33 serveGraphqlFailureTestServer,
34 serveGraphqlTestServer,34 serveGraphqlTestServer,
35 waitForRecordedRequests,
35} from "../testing";36} from "../testing";
36 37
37// removed: v1 secret browser-handoff, credential-binding scopes, usagesForSecret/38// removed: v1 secret browser-handoff, credential-binding scopes, usagesForSecret/
@@ -313,9 +314,8 @@ describe("graphqlPlugin real protocol server", () => { · 275 unmodified lines
313 value: "unused",314 value: "unused",
314 });315 });
315 316
316 const afterConnect = yield* server.requests;317 yield* waitForRecordedRequests(server.requests, (requests) =>
317 expect(afterConnect.some((request) => request.payload.query?.includes("__schema"))).toBe(318 requests.some((request) => request.payload.query?.includes("__schema")),
318 true,
319 );319 );
320 320
321 const tools = yield* executor.tools.list();321 const tools = yield* executor.tools.list();
@@ -550,11 +550,12 @@ describe("graphqlPlugin real protocol server", () => { · 228 unmodified lines
550 value: "connect-token",550 value: "connect-token",
551 });551 });
552 552
553 const afterConnect = yield* server.requests;553 const afterConnect = yield* waitForRecordedRequests(server.requests, (requests) =>
554 requests.some((request) => request.payload.query?.includes("__schema")),
555 );
554 const introspectionRequests = afterConnect.filter((request) =>556 const introspectionRequests = afterConnect.filter((request) =>
555 request.payload.query?.includes("__schema"),557 request.payload.query?.includes("__schema"),
556 );558 );
557 expect(introspectionRequests.length).toBeGreaterThan(0);
558 expect(introspectionRequests[0]?.headers.authorization).toBe("Bearer connect-token");559 expect(introspectionRequests[0]?.headers.authorization).toBe("Bearer connect-token");
559 560
560 // The introspected operations become per-connection tools.561 // The introspected operations become per-connection tools.
packages/plugins/graphql/src/testing/index.ts+23−0
@@ -5,6 +5,7 @@ import { · 4 unmodified lines
5 Layer,5 Layer,
6 Predicate,6 Predicate,
7 Ref,7 Ref,
8 Schedule,
8 Schema as EffectSchema,9 Schema as EffectSchema,
9 Scope,10 Scope,
10} from "effect";11} from "effect";
@@ -54,6 +55,28 @@ export interface GraphqlTestServerShape { · 43 unmodified lines
54 readonly clearRequests: Effect.Effect<void>;55 readonly clearRequests: Effect.Effect<void>;
55}56}
56 57
58export class GraphqlTestRecorderTimeoutError extends Data.TaggedError(
59 "GraphqlTestRecorderTimeoutError",
60)<{
61 readonly message: string;
62}> {}
63
64/** Poll the request recorder until `predicate` matches. Yoga captures requests
65 * asynchronously (`captureRequest` runs via `Effect.runPromise` in the handler
66 * context), so connect/invoke can resolve before the ref updates under load. */
67export const waitForRecordedRequests = (
68 requests: Effect.Effect<readonly GraphqlTestRequest[]>,
69 predicate: (requests: readonly GraphqlTestRequest[]) => boolean,
70 message = "timed out waiting for matching recorded GraphQL requests",
71): Effect.Effect<readonly GraphqlTestRequest[], GraphqlTestRecorderTimeoutError> =>
72 requests.pipe(
73 Effect.filterOrFail(
74 (all) => predicate(all),
75 () => new GraphqlTestRecorderTimeoutError({ message }),
76 ),
77 Effect.retry(Schedule.both(Schedule.spaced("50 millis"), Schedule.recurs(100))),
78 );
79
57class GraphqlTestServerAddressError extends Data.TaggedError("GraphqlTestServerAddressError")<{80class GraphqlTestServerAddressError extends Data.TaggedError("GraphqlTestServerAddressError")<{
58 readonly address: unknown;81 readonly address: unknown;
59}> {}82}> {}
packages/react/src/lib/async-result.test.ts+8−2
@@ -5,14 +5,20 @@ import * as AsyncResult from "effect/unstable/reactivity/AsyncResult"; · 4 unmodified lines
5import { isAsyncResultLoading } from "./async-result";5import { isAsyncResultLoading } from "./async-result";
6 6
7describe("isAsyncResultLoading", () => {7describe("isAsyncResultLoading", () => {
8 it("treats initial and waiting async results as loading", () => {8 it("treats initial and waiting-without-value async results as loading", () => {
9 expect(isAsyncResultLoading(AsyncResult.initial())).toBe(true);9 expect(isAsyncResultLoading(AsyncResult.initial())).toBe(true);
10 expect(isAsyncResultLoading(AsyncResult.initial(true))).toBe(true);10 expect(isAsyncResultLoading(AsyncResult.initial(true))).toBe(true);
11 expect(isAsyncResultLoading(AsyncResult.success(["cached"], { waiting: true }))).toBe(true);11 expect(isAsyncResultLoading(AsyncResult.failure(Cause.fail("boom"), { waiting: true }))).toBe(
12 true,
13 );
12 });14 });
13 15
14 it("does not treat settled success or failure as loading", () => {16 it("does not treat settled success or failure as loading", () => {
15 expect(isAsyncResultLoading(AsyncResult.success(["ready"]))).toBe(false);17 expect(isAsyncResultLoading(AsyncResult.success(["ready"]))).toBe(false);
16 expect(isAsyncResultLoading(AsyncResult.failure(Cause.fail("boom")))).toBe(false);18 expect(isAsyncResultLoading(AsyncResult.failure(Cause.fail("boom")))).toBe(false);
17 });19 });
20
21 it("keeps stale values rendered while a refresh is waiting", () => {
22 expect(isAsyncResultLoading(AsyncResult.success(["cached"], { waiting: true }))).toBe(false);
23 });
18});24});
packages/react/src/lib/async-result.ts+5−1
@@ -1,5 +1,9 @@
1import * as AsyncResult from "effect/unstable/reactivity/AsyncResult";1import * as AsyncResult from "effect/unstable/reactivity/AsyncResult";
2import * as Option from "effect/Option";
2 3
3export function isAsyncResultLoading<A, E>(result: AsyncResult.AsyncResult<A, E>): boolean {4export function isAsyncResultLoading<A, E>(result: AsyncResult.AsyncResult<A, E>): boolean {
4 return AsyncResult.isInitial(result) || AsyncResult.isWaiting(result);5 return (
6 AsyncResult.isInitial(result) ||
7 (AsyncResult.isWaiting(result) && Option.isNone(AsyncResult.value(result)))
8 );
5}9}