History

Make the test suite parallel-safe and speed up CI (#1273)

* Make self-host tests CI-safe * Await cloud db teardown * Give local e2e boots more headroom * Keep main CI runs alive * Cache CI dependencies and image layers * Wait for recorded introspection requests in graphql plugin tests * Size self-host test budget for loaded runners * Probe test http servers for readiness before use * Die on recorder wait timeout instead of typing a string error * Type the recorder wait timeout as a tagged error * Update e2e locators for the integration copy renames * Keep stale data rendered while a refresh is waiting * Guard create-org input against hydration clobbering * Match the new sign-in heading in the selfhost callback test

Rhys Sullivan committed 4 months ago

Commit 8652c991 parent 9c7e4a627 files · +252 −102Browse files

Showing the changes to apps (8 of 27 files). Show all

Unified · Split

8 files
  1. apps/cloud/src/db/db.ts +6 −10
  2. apps/host-selfhost/src/boot.test.ts +14 −8
  3. apps/host-selfhost/src/multi-user.test.ts +9 −3
  4. apps/host-selfhost/src/scope-isolation.test.ts +11 −5
  5. apps/host-selfhost/src/secrets-integration.test.ts +14 −7
  6. apps/host-selfhost/src/sources-mcp.test.ts +10 −3
  7. apps/host-selfhost/src/sources.test.ts +14 −7
  8. apps/host-selfhost/vitest.config.ts +13 −25
apps/cloud/src/db/db.ts+6−10
@@ -73,16 +73,12 @@ const makePostgresResource = (): DbResource => { · 72 unmodified lines
7373 sql,
7474 db: drizzle(sql, { schema: combinedSchema }) as DrizzleDb,
7575 close: () =>
76− Effect.sync(() => {
77− void Effect.runFork(
78− Effect.ignore(
79− Effect.tryPromise({
80− try: () => sql.end({ timeout: 0 }),
81− catch: (cause) => cause,
82− }),
83− ),
84− );
85− }),
76+ Effect.ignore(
77+ Effect.tryPromise({
78+ try: () => sql.end({ timeout: 0 }),
79+ catch: (cause) => cause,
80+ }),
81+ ),
8682 };
8783 };
8884
apps/host-selfhost/src/boot.test.ts+14−8
@@ -2,20 +2,26 @@ import { mkdtempSync } from "node:fs"; · 1 unmodified line
22 import { tmpdir } from "node:os";
33 import { join } from "node:path";
44
5−import { afterAll, expect, test } from "@effect/vitest";
5+import { afterAll, beforeAll, expect, test } from "@effect/vitest";
66
77 // Config reads the environment, so point it at a throwaway data dir before
88 // importing the app graph.
99 process.env.EXECUTOR_DATA_DIR = mkdtempSync(join(tmpdir(), "eh-boot-"));
1010
11−const { makeSelfHostTestApp, singleAdminIdentityLayer } = await import("./testing/test-app");
11+let handler!: (request: Request) => Promise<Response>;
12+let dispose: () => Promise<void> = async () => {};
1213
13−const { handler, dispose } = await makeSelfHostTestApp({
14− identity: singleAdminIdentityLayer({
15− userId: "admin",
16− organizationId: "default-org",
17− organizationName: "Default",
18− }),
14+beforeAll(async () => {
15+ const { makeSelfHostTestApp, singleAdminIdentityLayer } = await import("./testing/test-app");
16+ const app = await makeSelfHostTestApp({
17+ identity: singleAdminIdentityLayer({
18+ userId: "admin",
19+ organizationId: "default-org",
20+ organizationName: "Default",
21+ }),
22+ });
23+ handler = app.handler;
24+ dispose = app.dispose;
1925 });
2026 afterAll(() => dispose());
2127
apps/host-selfhost/src/multi-user.test.ts+9−3
@@ -2,7 +2,7 @@ import { mkdtempSync } from "node:fs"; · 1 unmodified line
22 import { tmpdir } from "node:os";
33 import { join } from "node:path";
44
5−import { afterAll, expect, test } from "@effect/vitest";
5+import { afterAll, beforeAll, expect, test } from "@effect/vitest";
66 import { connectionIdentifier } from "@executor-js/sdk/shared";
77
88 import { mintInviteCode } from "./testing/mint-invite";
@@ -13,9 +13,15 @@ process.env.BETTER_AUTH_SECRET = "multi-user-secret-0123456789-abcdefghij-klmn"; · 4 unmodified lines
1313 process.env.EXECUTOR_BOOTSTRAP_ADMIN_EMAIL = "admin@multi.test";
1414 process.env.EXECUTOR_BOOTSTRAP_ADMIN_PASSWORD = "admin-pass-123456";
1515
16−const { makeSelfHostApiHandler } = await import("./app");
16+let handler!: (request: Request) => Promise<Response>;
17+let dispose: () => Promise<void> = async () => {};
1718
18−const { handler, dispose } = await makeSelfHostApiHandler();
19+beforeAll(async () => {
20+ const { makeSelfHostApiHandler } = await import("./app");
21+ const app = await makeSelfHostApiHandler();
22+ handler = app.handler;
23+ dispose = app.dispose;
24+});
1925 afterAll(() => dispose());
2026
2127 const BASE = "http://localhost:4788";
apps/host-selfhost/src/scope-isolation.test.ts+11−5
@@ -2,7 +2,7 @@ import { mkdtempSync } from "node:fs"; · 1 unmodified line
22 import { tmpdir } from "node:os";
33 import { join } from "node:path";
44
5−import { afterAll, expect, test } from "@effect/vitest";
5+import { afterAll, beforeAll, expect, test } from "@effect/vitest";
66 import { connectionIdentifier } from "@executor-js/sdk/shared";
77
88 process.env.EXECUTOR_DATA_DIR = mkdtempSync(join(tmpdir(), "eh-iso-"));
@@ -13,10 +13,16 @@ process.env.EXECUTOR_DATA_DIR = mkdtempSync(join(tmpdir(), "eh-iso-")); · 4 unmodified lines
1313 // request-scoped. Each identity is its own (org, user): in v2 the org is the
1414 // tenant (catalog partition) and the user is the acting subject (drives
1515 // `owner: "user"` rows).
16−const { makeSelfHostTestApp, headerIdentityLayer } = await import("./testing/test-app");
16+let handler!: (request: Request) => Promise<Response>;
17+let dispose: () => Promise<void> = async () => {};
1718
18−const { handler, dispose } = await makeSelfHostTestApp({
19− identity: headerIdentityLayer,
19+beforeAll(async () => {
20+ const { makeSelfHostTestApp, headerIdentityLayer } = await import("./testing/test-app");
21+ const app = await makeSelfHostTestApp({
22+ identity: headerIdentityLayer,
23+ });
24+ handler = app.handler;
25+ dispose = app.dispose;
2026 });
2127 afterAll(() => dispose());
2228
@@ -135,7 +141,7 @@ test("concurrent requests with distinct identities get disjoint, correct executo · 112 unmodified lines
135141 expect(addresses.some((a) => a.includes(connectionNameForUser(other)))).toBe(false);
136142 }
137143 });
138−}, 30_000);
144+});
139145
140146 test("a request with no identity is rejected", async () => {
141147 const res = await handler(new Request("http://localhost/api/connections"));
apps/host-selfhost/src/secrets-integration.test.ts+14−7
@@ -4,7 +4,7 @@ import { tmpdir } from "node:os"; · 3 unmodified lines
44 import { join } from "node:path";
55
66 import { Effect, Layer } from "effect";
7−import { afterAll, expect, test } from "@effect/vitest";
7+import { afterAll, beforeAll, expect, test } from "@effect/vitest";
88
99 import { AuthTemplateSlug, ConnectionName, IntegrationSlug } from "@executor-js/sdk";
1010 import { makeScopedExecutor } from "@executor-js/api/server";
@@ -32,13 +32,20 @@ const createScopedExecutor = ( · 21 unmodified lines
3232 Effect.provide(SelfHostScopedExecutorSeams),
3333 );
3434
35−const dbHandle = await createSelfHostDb({
36− path: dbPath,
37− namespace: "executor_selfhost",
38− version: "1.0.0",
35+let dbLayer!: Layer.Layer<SelfHostDb>;
36+let dbHandle: Awaited<ReturnType<typeof createSelfHostDb>> | undefined;
37+
38+beforeAll(async () => {
39+ dbHandle = await createSelfHostDb({
40+ path: dbPath,
41+ namespace: "executor_selfhost",
42+ version: "1.0.0",
43+ });
44+ dbLayer = Layer.succeed(SelfHostDb)(dbHandle);
45+});
46+afterAll(async () => {
47+ await dbHandle?.close();
3948 });
40−const dbLayer = Layer.succeed(SelfHostDb)(dbHandle);
41−afterAll(() => dbHandle.close());
4249
4350 const TINY_SPEC = JSON.stringify({
4451 openapi: "3.0.0",
apps/host-selfhost/src/sources-mcp.test.ts+10−3
@@ -3,7 +3,7 @@ import { tmpdir } from "node:os"; · 2 unmodified lines
33 import { join } from "node:path";
44
55 import { Effect, Layer } from "effect";
6−import { afterAll, expect, test } from "@effect/vitest";
6+import { afterAll, beforeAll, expect, test } from "@effect/vitest";
77
88 import { AuthTemplateSlug, ConnectionName, IntegrationSlug } from "@executor-js/sdk";
99 import { makeScopedExecutor } from "@executor-js/api/server";
@@ -50,8 +50,15 @@ const TINY_SPEC = JSON.stringify({ · 40 unmodified lines
5050 },
5151 });
5252
53−const { makeSelfHostApiHandler } = await import("./app");
54−const { handler, dispose } = await makeSelfHostApiHandler({ dbPath });
53+let handler!: (request: Request) => Promise<Response>;
54+let dispose: () => Promise<void> = async () => {};
55+
56+beforeAll(async () => {
57+ const { makeSelfHostApiHandler } = await import("./app");
58+ const app = await makeSelfHostApiHandler({ dbPath });
59+ handler = app.handler;
60+ dispose = app.dispose;
61+});
5562 afterAll(() => dispose());
5663
5764 const BASE = "http://localhost:4788";
apps/host-selfhost/src/sources.test.ts+14−7
@@ -3,7 +3,7 @@ import { tmpdir } from "node:os"; · 2 unmodified lines
33 import { join } from "node:path";
44
55 import { Effect, Layer } from "effect";
6−import { afterAll, expect, test } from "@effect/vitest";
6+import { afterAll, beforeAll, expect, test } from "@effect/vitest";
77
88 import { AuthTemplateSlug, ConnectionName, IntegrationSlug } from "@executor-js/sdk";
99 import { makeScopedExecutor } from "@executor-js/api/server";
@@ -28,13 +28,20 @@ const createScopedExecutor = ( · 18 unmodified lines
2828 const dataDir = mkdtempSync(join(tmpdir(), "eh-src-"));
2929 process.env.EXECUTOR_DATA_DIR = dataDir;
3030
31−const dbHandle = await createSelfHostDb({
32− path: join(dataDir, "data.db"),
33− namespace: "executor_selfhost",
34− version: "1.0.0",
31+let dbLayer!: Layer.Layer<SelfHostDb>;
32+let dbHandle: Awaited<ReturnType<typeof createSelfHostDb>> | undefined;
33+
34+beforeAll(async () => {
35+ dbHandle = await createSelfHostDb({
36+ path: join(dataDir, "data.db"),
37+ namespace: "executor_selfhost",
38+ version: "1.0.0",
39+ });
40+ dbLayer = Layer.succeed(SelfHostDb)(dbHandle);
41+});
42+afterAll(async () => {
43+ await dbHandle?.close();
3544 });
36−const dbLayer = Layer.succeed(SelfHostDb)(dbHandle);
37−afterAll(() => dbHandle.close());
3845
3946 // Inline OpenAPI spec so the test doesn't depend on the network to register.
4047 const TINY_SPEC = JSON.stringify({
apps/host-selfhost/vitest.config.ts+13−25
@@ -4,30 +4,18 @@ export default defineConfig({ · 3 unmodified lines
44 test: {
55 include: ["src/**/*.test.ts"],
66 passWithNoTests: true,
7− // These are integration suites: most boot a full self-host app graph
8− // (Better Auth + libSQL + MCP + plugins) at module load, then drive it
9− // over the in-memory handler, and some (scope-isolation) fire dozens of
10− // concurrent requests through it. Each boot is CPU-heavy and every query
11− // serializes through the one shared libSQL connection, so when many files
12− // boot at once they oversubscribe the CPU and starve the event loop: an
13− // in-flight request stalls indefinitely and the test times out. Which
14− // files lose the race is nondeterministic, which is the CI flakiness.
15− //
16− // Vitest sizes its fork pool to the core count, so a many-core machine
17− // oversubscribes HARDER than a small CI runner. Bound the concurrency to
18− // an absolute number instead, so behavior is the same everywhere: at most
19− // two heavy boots run together, which fits a 4-vCPU runner without
20− // starvation while still running the suite in parallel. Pair it with
21− // integration-grade timeouts as headroom for the slowest boot. This keeps
22− // every assertion intact (no logic or coverage changed) and makes the
23− // suite deterministic rather than load-dependent.
24− testTimeout: 30_000,
25− hookTimeout: 30_000,
26− poolOptions: {
27− forks: {
28− maxForks: 2,
29− minForks: 1,
30− },
31− },
7+ // These are integration suites: several files boot a full self-host app
8+ // graph (Better Auth + libSQL + MCP + plugins) in beforeAll, then drive it
9+ // over the in-memory handler. Each boot is CPU-heavy and every query
10+ // serializes through the one shared libSQL connection, so parallel files can
11+ // oversubscribe CI and starve in-flight requests. Run files serially and
12+ // give lifecycle hooks room for the cold graph boot. Tests that fan out
13+ // many concurrent requests (scope-isolation) stretch well past 30s when
14+ // sibling turbo tasks contend for the same cores, so the budget is sized
15+ // for a loaded runner, not an idle one.
16+ fileParallelism: false,
17+ maxWorkers: 1,
18+ testTimeout: 120_000,
19+ hookTimeout: 60_000,
3220 },
3321 });