Make self-host tests CI-safe

Rhys Sullivan committed 4 months ago

Commit f88bf971 parent 9a1856e9 files · +84 −58Browse files

Unified · Split

9 files

.github/workflows/ci.yml +2 −0
apps/host-selfhost/src/boot.test.ts +14 −8
apps/host-selfhost/src/multi-user.test.ts +9 −3
apps/host-selfhost/src/scope-isolation.test.ts +11 −5
apps/host-selfhost/src/secrets-integration.test.ts +14 −7
apps/host-selfhost/src/sources-mcp.test.ts +10 −3
apps/host-selfhost/src/sources.test.ts +14 −7
apps/host-selfhost/vitest.config.ts +9 −24
package.json +1 −1

@@ -56,6 +56,8 @@ jobs: · 55 unmodified lines
56	56	test:
57	57	name: Test
58	58	runs-on: ubuntu-latest
	59	+ env:
	60	+ TURBO_TEST_CONCURRENCY: 3
59	61	steps:
60	62	- uses: actions/checkout@v4
61	63

apps/host-selfhost/src/boot.test.ts+14−8

@@ -2,20 +2,26 @@ import { mkdtempSync } from "node:fs"; · 1 unmodified line
2	2	import { tmpdir } from "node:os";
3	3	import { join } from "node:path";
4	4
5		−import { afterAll, expect, test } from "@effect/vitest";
	5	+import { afterAll, beforeAll, expect, test } from "@effect/vitest";
6	6
7	7	// Config reads the environment, so point it at a throwaway data dir before
8	8	// importing the app graph.
9	9	process.env.EXECUTOR_DATA_DIR = mkdtempSync(join(tmpdir(), "eh-boot-"));
10	10
11		−const { makeSelfHostTestApp, singleAdminIdentityLayer } = await import("./testing/test-app");
	11	+let handler!: (request: Request) => Promise<Response>;
	12	+let dispose: () => Promise<void> = async () => {};
12	13
13		−const { handler, dispose } = await makeSelfHostTestApp({
14		− identity: singleAdminIdentityLayer({
15		− userId: "admin",
16		− organizationId: "default-org",
17		− organizationName: "Default",
18		− }),
	14	+beforeAll(async () => {
	15	+ const { makeSelfHostTestApp, singleAdminIdentityLayer } = await import("./testing/test-app");
	16	+ const app = await makeSelfHostTestApp({
	17	+ identity: singleAdminIdentityLayer({
	18	+ userId: "admin",
	19	+ organizationId: "default-org",
	20	+ organizationName: "Default",
	21	+ }),
	22	+ });
	23	+ handler = app.handler;
	24	+ dispose = app.dispose;
19	25	});
20	26	afterAll(() => dispose());
21	27

apps/host-selfhost/src/multi-user.test.ts+9−3

@@ -2,7 +2,7 @@ import { mkdtempSync } from "node:fs"; · 1 unmodified line
2	2	import { tmpdir } from "node:os";
3	3	import { join } from "node:path";
4	4
5		−import { afterAll, expect, test } from "@effect/vitest";
	5	+import { afterAll, beforeAll, expect, test } from "@effect/vitest";
6	6	import { connectionIdentifier } from "@executor-js/sdk/shared";
7	7
8	8	import { mintInviteCode } from "./testing/mint-invite";
@@ -13,9 +13,15 @@ process.env.BETTER_AUTH_SECRET = "multi-user-secret-0123456789-abcdefghij-klmn"; · 4 unmodified lines
13	13	process.env.EXECUTOR_BOOTSTRAP_ADMIN_EMAIL = "admin@multi.test";
14	14	process.env.EXECUTOR_BOOTSTRAP_ADMIN_PASSWORD = "admin-pass-123456";
15	15
16		−const { makeSelfHostApiHandler } = await import("./app");
	16	+let handler!: (request: Request) => Promise<Response>;
	17	+let dispose: () => Promise<void> = async () => {};
17	18
18		−const { handler, dispose } = await makeSelfHostApiHandler();
	19	+beforeAll(async () => {
	20	+ const { makeSelfHostApiHandler } = await import("./app");
	21	+ const app = await makeSelfHostApiHandler();
	22	+ handler = app.handler;
	23	+ dispose = app.dispose;
	24	+});
19	25	afterAll(() => dispose());
20	26
21	27	const BASE = "http://localhost:4788";

apps/host-selfhost/src/scope-isolation.test.ts+11−5

@@ -2,7 +2,7 @@ import { mkdtempSync } from "node:fs"; · 1 unmodified line
2	2	import { tmpdir } from "node:os";
3	3	import { join } from "node:path";
4	4
5		−import { afterAll, expect, test } from "@effect/vitest";
	5	+import { afterAll, beforeAll, expect, test } from "@effect/vitest";
6	6	import { connectionIdentifier } from "@executor-js/sdk/shared";
7	7
8	8	process.env.EXECUTOR_DATA_DIR = mkdtempSync(join(tmpdir(), "eh-iso-"));
@@ -13,10 +13,16 @@ process.env.EXECUTOR_DATA_DIR = mkdtempSync(join(tmpdir(), "eh-iso-")); · 4 unmodified lines
13	13	// request-scoped. Each identity is its own (org, user): in v2 the org is the
14	14	// tenant (catalog partition) and the user is the acting subject (drives
15	15	// `owner: "user"` rows).
16		−const { makeSelfHostTestApp, headerIdentityLayer } = await import("./testing/test-app");
	16	+let handler!: (request: Request) => Promise<Response>;
	17	+let dispose: () => Promise<void> = async () => {};
17	18
18		−const { handler, dispose } = await makeSelfHostTestApp({
19		− identity: headerIdentityLayer,
	19	+beforeAll(async () => {
	20	+ const { makeSelfHostTestApp, headerIdentityLayer } = await import("./testing/test-app");
	21	+ const app = await makeSelfHostTestApp({
	22	+ identity: headerIdentityLayer,
	23	+ });
	24	+ handler = app.handler;
	25	+ dispose = app.dispose;
20	26	});
21	27	afterAll(() => dispose());
22	28
@@ -135,7 +141,7 @@ test("concurrent requests with distinct identities get disjoint, correct executo · 112 unmodified lines
135	141	expect(addresses.some((a) => a.includes(connectionNameForUser(other)))).toBe(false);
136	142	}
137	143	});
138		−}, 30_000);
	144	+});
139	145
140	146	test("a request with no identity is rejected", async () => {
141	147	const res = await handler(new Request("http://localhost/api/connections"));

apps/host-selfhost/src/secrets-integration.test.ts+14−7

@@ -4,7 +4,7 @@ import { tmpdir } from "node:os"; · 3 unmodified lines
4	4	import { join } from "node:path";
5	5
6	6	import { Effect, Layer } from "effect";
7		−import { afterAll, expect, test } from "@effect/vitest";
	7	+import { afterAll, beforeAll, expect, test } from "@effect/vitest";
8	8
9	9	import { AuthTemplateSlug, ConnectionName, IntegrationSlug } from "@executor-js/sdk";
10	10	import { makeScopedExecutor } from "@executor-js/api/server";
@@ -32,13 +32,20 @@ const createScopedExecutor = ( · 21 unmodified lines
32	32	Effect.provide(SelfHostScopedExecutorSeams),
33	33	);
34	34
35		−const dbHandle = await createSelfHostDb({
36		− path: dbPath,
37		− namespace: "executor_selfhost",
38		− version: "1.0.0",
	35	+let dbLayer!: Layer.Layer<SelfHostDb>;
	36	+let dbHandle: Awaited<ReturnType<typeof createSelfHostDb>> \| undefined;
	37	+
	38	+beforeAll(async () => {
	39	+ dbHandle = await createSelfHostDb({
	40	+ path: dbPath,
	41	+ namespace: "executor_selfhost",
	42	+ version: "1.0.0",
	43	+ });
	44	+ dbLayer = Layer.succeed(SelfHostDb)(dbHandle);
	45	+});
	46	+afterAll(async () => {
	47	+ await dbHandle?.close();
39	48	});
40		−const dbLayer = Layer.succeed(SelfHostDb)(dbHandle);
41		−afterAll(() => dbHandle.close());
42	49
43	50	const TINY_SPEC = JSON.stringify({
44	51	openapi: "3.0.0",

apps/host-selfhost/src/sources-mcp.test.ts+10−3

@@ -3,7 +3,7 @@ import { tmpdir } from "node:os"; · 2 unmodified lines
3	3	import { join } from "node:path";
4	4
5	5	import { Effect, Layer } from "effect";
6		−import { afterAll, expect, test } from "@effect/vitest";
	6	+import { afterAll, beforeAll, expect, test } from "@effect/vitest";
7	7
8	8	import { AuthTemplateSlug, ConnectionName, IntegrationSlug } from "@executor-js/sdk";
9	9	import { makeScopedExecutor } from "@executor-js/api/server";
@@ -50,8 +50,15 @@ const TINY_SPEC = JSON.stringify({ · 40 unmodified lines
50	50	},
51	51	});
52	52
53		−const { makeSelfHostApiHandler } = await import("./app");
54		−const { handler, dispose } = await makeSelfHostApiHandler({ dbPath });
	53	+let handler!: (request: Request) => Promise<Response>;
	54	+let dispose: () => Promise<void> = async () => {};
	55	+
	56	+beforeAll(async () => {
	57	+ const { makeSelfHostApiHandler } = await import("./app");
	58	+ const app = await makeSelfHostApiHandler({ dbPath });
	59	+ handler = app.handler;
	60	+ dispose = app.dispose;
	61	+});
55	62	afterAll(() => dispose());
56	63
57	64	const BASE = "http://localhost:4788";

apps/host-selfhost/src/sources.test.ts+14−7

@@ -3,7 +3,7 @@ import { tmpdir } from "node:os"; · 2 unmodified lines
3	3	import { join } from "node:path";
4	4
5	5	import { Effect, Layer } from "effect";
6		−import { afterAll, expect, test } from "@effect/vitest";
	6	+import { afterAll, beforeAll, expect, test } from "@effect/vitest";
7	7
8	8	import { AuthTemplateSlug, ConnectionName, IntegrationSlug } from "@executor-js/sdk";
9	9	import { makeScopedExecutor } from "@executor-js/api/server";
@@ -28,13 +28,20 @@ const createScopedExecutor = ( · 18 unmodified lines
28	28	const dataDir = mkdtempSync(join(tmpdir(), "eh-src-"));
29	29	process.env.EXECUTOR_DATA_DIR = dataDir;
30	30
31		−const dbHandle = await createSelfHostDb({
32		− path: join(dataDir, "data.db"),
33		− namespace: "executor_selfhost",
34		− version: "1.0.0",
	31	+let dbLayer!: Layer.Layer<SelfHostDb>;
	32	+let dbHandle: Awaited<ReturnType<typeof createSelfHostDb>> \| undefined;
	33	+
	34	+beforeAll(async () => {
	35	+ dbHandle = await createSelfHostDb({
	36	+ path: join(dataDir, "data.db"),
	37	+ namespace: "executor_selfhost",
	38	+ version: "1.0.0",
	39	+ });
	40	+ dbLayer = Layer.succeed(SelfHostDb)(dbHandle);
	41	+});
	42	+afterAll(async () => {
	43	+ await dbHandle?.close();
35	44	});
36		−const dbLayer = Layer.succeed(SelfHostDb)(dbHandle);
37		−afterAll(() => dbHandle.close());
38	45
39	46	// Inline OpenAPI spec so the test doesn't depend on the network to register.
40	47	const TINY_SPEC = JSON.stringify({

apps/host-selfhost/vitest.config.ts+9−24

@@ -4,30 +4,15 @@ export default defineConfig({ · 3 unmodified lines
4	4	test: {
5	5	include: ["src/*/.test.ts"],
6	6	passWithNoTests: true,
7		− // These are integration suites: most boot a full self-host app graph
8		− // (Better Auth + libSQL + MCP + plugins) at module load, then drive it
9		− // over the in-memory handler, and some (scope-isolation) fire dozens of
10		− // concurrent requests through it. Each boot is CPU-heavy and every query
11		− // serializes through the one shared libSQL connection, so when many files
12		− // boot at once they oversubscribe the CPU and starve the event loop: an
13		− // in-flight request stalls indefinitely and the test times out. Which
14		− // files lose the race is nondeterministic, which is the CI flakiness.
15		− //
16		− // Vitest sizes its fork pool to the core count, so a many-core machine
17		− // oversubscribes HARDER than a small CI runner. Bound the concurrency to
18		− // an absolute number instead, so behavior is the same everywhere: at most
19		− // two heavy boots run together, which fits a 4-vCPU runner without
20		− // starvation while still running the suite in parallel. Pair it with
21		− // integration-grade timeouts as headroom for the slowest boot. This keeps
22		− // every assertion intact (no logic or coverage changed) and makes the
23		− // suite deterministic rather than load-dependent.
	7	+ // These are integration suites: several files boot a full self-host app
	8	+ // graph (Better Auth + libSQL + MCP + plugins) in beforeAll, then drive it
	9	+ // over the in-memory handler. Each boot is CPU-heavy and every query
	10	+ // serializes through the one shared libSQL connection, so parallel files can
	11	+ // oversubscribe CI and starve in-flight requests. Run files serially and
	12	+ // give lifecycle hooks room for the cold graph boot.
	13	+ fileParallelism: false,
	14	+ maxWorkers: 1,
24	15	testTimeout: 30_000,
25		− hookTimeout: 30_000,
26		− poolOptions: {
27		− forks: {
28		− maxForks: 2,
29		− minForks: 1,
30		− },
31		− },
	16	+ hookTimeout: 60_000,
32	17	},
33	18	});

package.json+1−1

@@ -35,7 +35,7 @@ · 34 unmodified lines
35	35	"dev": "turbo run dev --filter='!@executor-js/desktop' --filter='!@executor-js/cloud'",
36	36	"dev:desktop": "turbo run dev",
37	37	"dev:cli": "EXECUTOR_DEV=1 EXECUTOR_DATA_DIR=${EXECUTOR_DATA_DIR:-apps/local/.executor-dev} bun run apps/cli/src/main.ts",
38		− "test": "turbo run test --filter=!@executor-js/e2e",
	38	+ "test": "turbo run test --filter=!@executor-js/e2e ${TURBO_TEST_CONCURRENCY:+--concurrency=$TURBO_TEST_CONCURRENCY}",
39	39	"test:e2e": "bun run --cwd e2e test",
40	40	"test:release:bootstrap": "vitest run tests/release-bootstrap-smoke.test.ts",
41	41	"build:packages": "bun run --filter='@executor-js/fumadb' build && bun run --filter='@executor-js/codemode-core' build && bun run --filter='@executor-js/runtime-quickjs' build && bun run --filter='@executor-js/sdk' build && bun run --filter='@executor-js/config' build && bun run --filter='@executor-js/execution' build && bun run --filter='@executor-js/cli' build && bun run --filter='@executor-js/plugin-*' build",