History

Make the test suite parallel-safe and speed up CI (#1273)

* Make self-host tests CI-safe * Await cloud db teardown * Give local e2e boots more headroom * Keep main CI runs alive * Cache CI dependencies and image layers * Wait for recorded introspection requests in graphql plugin tests * Size self-host test budget for loaded runners * Probe test http servers for readiness before use * Die on recorder wait timeout instead of typing a string error * Type the recorder wait timeout as a tagged error * Update e2e locators for the integration copy renames * Keep stale data rendered while a refresh is waiting * Guard create-org input against hydration clobbering * Match the new sign-in heading in the selfhost callback test

Rhys Sullivan committed 4 months ago

Commit 8652c991 parent 9c7e4a627 files · +252 −102Browse files

Showing the changes to apps (8 of 27 files). Show all

Unified · Split

8 files
  1. apps/cloud/src/db/db.ts +6 −10
  2. apps/host-selfhost/src/boot.test.ts +14 −8
  3. apps/host-selfhost/src/multi-user.test.ts +9 −3
  4. apps/host-selfhost/src/scope-isolation.test.ts +11 −5
  5. apps/host-selfhost/src/secrets-integration.test.ts +14 −7
  6. apps/host-selfhost/src/sources-mcp.test.ts +10 −3
  7. apps/host-selfhost/src/sources.test.ts +14 −7
  8. apps/host-selfhost/vitest.config.ts +13 −25
apps/cloud/src/db/db.ts+6−10
@@ -73,16 +73,12 @@ const makePostgresResource = (): DbResource => { · 72 unmodified lines
73 sql,73 sql,
74 db: drizzle(sql, { schema: combinedSchema }) as DrizzleDb,74 db: drizzle(sql, { schema: combinedSchema }) as DrizzleDb,
75 close: () =>75 close: () =>
76 Effect.sync(() => {76 Effect.ignore(
77 void Effect.runFork(77 Effect.tryPromise({
78 Effect.ignore(78 try: () => sql.end({ timeout: 0 }),
79 Effect.tryPromise({79 catch: (cause) => cause,
80 try: () => sql.end({ timeout: 0 }),80 }),
81 catch: (cause) => cause,81 ),
82 }),
83 ),
84 );
85 }),
86 };82 };
87};83};
88 84
apps/host-selfhost/src/boot.test.ts+14−8
@@ -2,20 +2,26 @@ import { mkdtempSync } from "node:fs"; · 1 unmodified line
2import { tmpdir } from "node:os";2import { tmpdir } from "node:os";
3import { join } from "node:path";3import { join } from "node:path";
4 4
5import { afterAll, expect, test } from "@effect/vitest";5import { afterAll, beforeAll, expect, test } from "@effect/vitest";
6 6
7// Config reads the environment, so point it at a throwaway data dir before7// Config reads the environment, so point it at a throwaway data dir before
8// importing the app graph.8// importing the app graph.
9process.env.EXECUTOR_DATA_DIR = mkdtempSync(join(tmpdir(), "eh-boot-"));9process.env.EXECUTOR_DATA_DIR = mkdtempSync(join(tmpdir(), "eh-boot-"));
10 10
11const { makeSelfHostTestApp, singleAdminIdentityLayer } = await import("./testing/test-app");11let handler!: (request: Request) => Promise<Response>;
12let dispose: () => Promise<void> = async () => {};
12 13
13const { handler, dispose } = await makeSelfHostTestApp({14beforeAll(async () => {
14 identity: singleAdminIdentityLayer({15 const { makeSelfHostTestApp, singleAdminIdentityLayer } = await import("./testing/test-app");
15 userId: "admin",16 const app = await makeSelfHostTestApp({
16 organizationId: "default-org",17 identity: singleAdminIdentityLayer({
17 organizationName: "Default",18 userId: "admin",
18 }),19 organizationId: "default-org",
20 organizationName: "Default",
21 }),
22 });
23 handler = app.handler;
24 dispose = app.dispose;
19});25});
20afterAll(() => dispose());26afterAll(() => dispose());
21 27
apps/host-selfhost/src/multi-user.test.ts+9−3
@@ -2,7 +2,7 @@ import { mkdtempSync } from "node:fs"; · 1 unmodified line
2import { tmpdir } from "node:os";2import { tmpdir } from "node:os";
3import { join } from "node:path";3import { join } from "node:path";
4 4
5import { afterAll, expect, test } from "@effect/vitest";5import { afterAll, beforeAll, expect, test } from "@effect/vitest";
6import { connectionIdentifier } from "@executor-js/sdk/shared";6import { connectionIdentifier } from "@executor-js/sdk/shared";
7 7
8import { mintInviteCode } from "./testing/mint-invite";8import { mintInviteCode } from "./testing/mint-invite";
@@ -13,9 +13,15 @@ process.env.BETTER_AUTH_SECRET = "multi-user-secret-0123456789-abcdefghij-klmn"; · 4 unmodified lines
13process.env.EXECUTOR_BOOTSTRAP_ADMIN_EMAIL = "admin@multi.test";13process.env.EXECUTOR_BOOTSTRAP_ADMIN_EMAIL = "admin@multi.test";
14process.env.EXECUTOR_BOOTSTRAP_ADMIN_PASSWORD = "admin-pass-123456";14process.env.EXECUTOR_BOOTSTRAP_ADMIN_PASSWORD = "admin-pass-123456";
15 15
16const { makeSelfHostApiHandler } = await import("./app");16let handler!: (request: Request) => Promise<Response>;
17let dispose: () => Promise<void> = async () => {};
17 18
18const { handler, dispose } = await makeSelfHostApiHandler();19beforeAll(async () => {
20 const { makeSelfHostApiHandler } = await import("./app");
21 const app = await makeSelfHostApiHandler();
22 handler = app.handler;
23 dispose = app.dispose;
24});
19afterAll(() => dispose());25afterAll(() => dispose());
20 26
21const BASE = "http://localhost:4788";27const BASE = "http://localhost:4788";
apps/host-selfhost/src/scope-isolation.test.ts+11−5
@@ -2,7 +2,7 @@ import { mkdtempSync } from "node:fs"; · 1 unmodified line
2import { tmpdir } from "node:os";2import { tmpdir } from "node:os";
3import { join } from "node:path";3import { join } from "node:path";
4 4
5import { afterAll, expect, test } from "@effect/vitest";5import { afterAll, beforeAll, expect, test } from "@effect/vitest";
6import { connectionIdentifier } from "@executor-js/sdk/shared";6import { connectionIdentifier } from "@executor-js/sdk/shared";
7 7
8process.env.EXECUTOR_DATA_DIR = mkdtempSync(join(tmpdir(), "eh-iso-"));8process.env.EXECUTOR_DATA_DIR = mkdtempSync(join(tmpdir(), "eh-iso-"));
@@ -13,10 +13,16 @@ process.env.EXECUTOR_DATA_DIR = mkdtempSync(join(tmpdir(), "eh-iso-")); · 4 unmodified lines
13// request-scoped. Each identity is its own (org, user): in v2 the org is the13// request-scoped. Each identity is its own (org, user): in v2 the org is the
14// tenant (catalog partition) and the user is the acting subject (drives14// tenant (catalog partition) and the user is the acting subject (drives
15// `owner: "user"` rows).15// `owner: "user"` rows).
16const { makeSelfHostTestApp, headerIdentityLayer } = await import("./testing/test-app");16let handler!: (request: Request) => Promise<Response>;
17let dispose: () => Promise<void> = async () => {};
17 18
18const { handler, dispose } = await makeSelfHostTestApp({19beforeAll(async () => {
19 identity: headerIdentityLayer,20 const { makeSelfHostTestApp, headerIdentityLayer } = await import("./testing/test-app");
21 const app = await makeSelfHostTestApp({
22 identity: headerIdentityLayer,
23 });
24 handler = app.handler;
25 dispose = app.dispose;
20});26});
21afterAll(() => dispose());27afterAll(() => dispose());
22 28
@@ -135,7 +141,7 @@ test("concurrent requests with distinct identities get disjoint, correct executo · 112 unmodified lines
135 expect(addresses.some((a) => a.includes(connectionNameForUser(other)))).toBe(false);141 expect(addresses.some((a) => a.includes(connectionNameForUser(other)))).toBe(false);
136 }142 }
137 });143 });
138}, 30_000);144});
139 145
140test("a request with no identity is rejected", async () => {146test("a request with no identity is rejected", async () => {
141 const res = await handler(new Request("http://localhost/api/connections"));147 const res = await handler(new Request("http://localhost/api/connections"));
apps/host-selfhost/src/secrets-integration.test.ts+14−7
@@ -4,7 +4,7 @@ import { tmpdir } from "node:os"; · 3 unmodified lines
4import { join } from "node:path";4import { join } from "node:path";
5 5
6import { Effect, Layer } from "effect";6import { Effect, Layer } from "effect";
7import { afterAll, expect, test } from "@effect/vitest";7import { afterAll, beforeAll, expect, test } from "@effect/vitest";
8 8
9import { AuthTemplateSlug, ConnectionName, IntegrationSlug } from "@executor-js/sdk";9import { AuthTemplateSlug, ConnectionName, IntegrationSlug } from "@executor-js/sdk";
10import { makeScopedExecutor } from "@executor-js/api/server";10import { makeScopedExecutor } from "@executor-js/api/server";
@@ -32,13 +32,20 @@ const createScopedExecutor = ( · 21 unmodified lines
32 Effect.provide(SelfHostScopedExecutorSeams),32 Effect.provide(SelfHostScopedExecutorSeams),
33 );33 );
34 34
35const dbHandle = await createSelfHostDb({35let dbLayer!: Layer.Layer<SelfHostDb>;
36 path: dbPath,36let dbHandle: Awaited<ReturnType<typeof createSelfHostDb>> | undefined;
37 namespace: "executor_selfhost",37
38 version: "1.0.0",38beforeAll(async () => {
39 dbHandle = await createSelfHostDb({
40 path: dbPath,
41 namespace: "executor_selfhost",
42 version: "1.0.0",
43 });
44 dbLayer = Layer.succeed(SelfHostDb)(dbHandle);
45});
46afterAll(async () => {
47 await dbHandle?.close();
39});48});
40const dbLayer = Layer.succeed(SelfHostDb)(dbHandle);
41afterAll(() => dbHandle.close());
42 49
43const TINY_SPEC = JSON.stringify({50const TINY_SPEC = JSON.stringify({
44 openapi: "3.0.0",51 openapi: "3.0.0",
apps/host-selfhost/src/sources-mcp.test.ts+10−3
@@ -3,7 +3,7 @@ import { tmpdir } from "node:os"; · 2 unmodified lines
3import { join } from "node:path";3import { join } from "node:path";
4 4
5import { Effect, Layer } from "effect";5import { Effect, Layer } from "effect";
6import { afterAll, expect, test } from "@effect/vitest";6import { afterAll, beforeAll, expect, test } from "@effect/vitest";
7 7
8import { AuthTemplateSlug, ConnectionName, IntegrationSlug } from "@executor-js/sdk";8import { AuthTemplateSlug, ConnectionName, IntegrationSlug } from "@executor-js/sdk";
9import { makeScopedExecutor } from "@executor-js/api/server";9import { makeScopedExecutor } from "@executor-js/api/server";
@@ -50,8 +50,15 @@ const TINY_SPEC = JSON.stringify({ · 40 unmodified lines
50 },50 },
51});51});
52 52
53const { makeSelfHostApiHandler } = await import("./app");53let handler!: (request: Request) => Promise<Response>;
54const { handler, dispose } = await makeSelfHostApiHandler({ dbPath });54let dispose: () => Promise<void> = async () => {};
55
56beforeAll(async () => {
57 const { makeSelfHostApiHandler } = await import("./app");
58 const app = await makeSelfHostApiHandler({ dbPath });
59 handler = app.handler;
60 dispose = app.dispose;
61});
55afterAll(() => dispose());62afterAll(() => dispose());
56 63
57const BASE = "http://localhost:4788";64const BASE = "http://localhost:4788";
apps/host-selfhost/src/sources.test.ts+14−7
@@ -3,7 +3,7 @@ import { tmpdir } from "node:os"; · 2 unmodified lines
3import { join } from "node:path";3import { join } from "node:path";
4 4
5import { Effect, Layer } from "effect";5import { Effect, Layer } from "effect";
6import { afterAll, expect, test } from "@effect/vitest";6import { afterAll, beforeAll, expect, test } from "@effect/vitest";
7 7
8import { AuthTemplateSlug, ConnectionName, IntegrationSlug } from "@executor-js/sdk";8import { AuthTemplateSlug, ConnectionName, IntegrationSlug } from "@executor-js/sdk";
9import { makeScopedExecutor } from "@executor-js/api/server";9import { makeScopedExecutor } from "@executor-js/api/server";
@@ -28,13 +28,20 @@ const createScopedExecutor = ( · 18 unmodified lines
28const dataDir = mkdtempSync(join(tmpdir(), "eh-src-"));28const dataDir = mkdtempSync(join(tmpdir(), "eh-src-"));
29process.env.EXECUTOR_DATA_DIR = dataDir;29process.env.EXECUTOR_DATA_DIR = dataDir;
30 30
31const dbHandle = await createSelfHostDb({31let dbLayer!: Layer.Layer<SelfHostDb>;
32 path: join(dataDir, "data.db"),32let dbHandle: Awaited<ReturnType<typeof createSelfHostDb>> | undefined;
33 namespace: "executor_selfhost",33
34 version: "1.0.0",34beforeAll(async () => {
35 dbHandle = await createSelfHostDb({
36 path: join(dataDir, "data.db"),
37 namespace: "executor_selfhost",
38 version: "1.0.0",
39 });
40 dbLayer = Layer.succeed(SelfHostDb)(dbHandle);
41});
42afterAll(async () => {
43 await dbHandle?.close();
35});44});
36const dbLayer = Layer.succeed(SelfHostDb)(dbHandle);
37afterAll(() => dbHandle.close());
38 45
39// Inline OpenAPI spec so the test doesn't depend on the network to register.46// Inline OpenAPI spec so the test doesn't depend on the network to register.
40const TINY_SPEC = JSON.stringify({47const TINY_SPEC = JSON.stringify({
apps/host-selfhost/vitest.config.ts+13−25
@@ -4,30 +4,18 @@ export default defineConfig({ · 3 unmodified lines
4 test: {4 test: {
5 include: ["src/**/*.test.ts"],5 include: ["src/**/*.test.ts"],
6 passWithNoTests: true,6 passWithNoTests: true,
7 // These are integration suites: most boot a full self-host app graph7 // These are integration suites: several files boot a full self-host app
8 // (Better Auth + libSQL + MCP + plugins) at module load, then drive it8 // graph (Better Auth + libSQL + MCP + plugins) in beforeAll, then drive it
9 // over the in-memory handler, and some (scope-isolation) fire dozens of9 // over the in-memory handler. Each boot is CPU-heavy and every query
10 // concurrent requests through it. Each boot is CPU-heavy and every query10 // serializes through the one shared libSQL connection, so parallel files can
11 // serializes through the one shared libSQL connection, so when many files11 // oversubscribe CI and starve in-flight requests. Run files serially and
12 // boot at once they oversubscribe the CPU and starve the event loop: an12 // give lifecycle hooks room for the cold graph boot. Tests that fan out
13 // in-flight request stalls indefinitely and the test times out. Which13 // many concurrent requests (scope-isolation) stretch well past 30s when
14 // files lose the race is nondeterministic, which is the CI flakiness.14 // sibling turbo tasks contend for the same cores, so the budget is sized
15 //15 // for a loaded runner, not an idle one.
16 // Vitest sizes its fork pool to the core count, so a many-core machine16 fileParallelism: false,
17 // oversubscribes HARDER than a small CI runner. Bound the concurrency to17 maxWorkers: 1,
18 // an absolute number instead, so behavior is the same everywhere: at most18 testTimeout: 120_000,
19 // two heavy boots run together, which fits a 4-vCPU runner without19 hookTimeout: 60_000,
20 // starvation while still running the suite in parallel. Pair it with
21 // integration-grade timeouts as headroom for the slowest boot. This keeps
22 // every assertion intact (no logic or coverage changed) and makes the
23 // suite deterministic rather than load-dependent.
24 testTimeout: 30_000,
25 hookTimeout: 30_000,
26 poolOptions: {
27 forks: {
28 maxForks: 2,
29 minForks: 1,
30 },
31 },
32 },20 },
33});21});