Make the test suite parallel-safe and speed up CI (#1273)

* Make self-host tests CI-safe * Await cloud db teardown * Give local e2e boots more headroom * Keep main CI runs alive * Cache CI dependencies and image layers * Wait for recorded introspection requests in graphql plugin tests * Size self-host test budget for loaded runners * Probe test http servers for readiness before use * Die on recorder wait timeout instead of typing a string error * Type the recorder wait timeout as a tagged error * Update e2e locators for the integration copy renames * Keep stale data rendered while a refresh is waiting * Guard create-org input against hydration clobbering * Match the new sign-in heading in the selfhost callback test

Rhys Sullivan committed 4 months ago

Commit 8652c991 parent 9c7e4a627 files · +252 −102Browse files

Showing the changes to e2e (12 of 27 files). Show all

Unified · Split

12 files

e2e/cloud/auth-routing-flow.test.ts +11 −1
e2e/local/local-server.ts +4 −4
e2e/local/stdio-mcp.test.ts +2 −2
e2e/scenarios/auth-methods-ui.test.ts +2 −2
e2e/scenarios/google-photos-preset-ui.test.ts +1 −1
e2e/scenarios/mcp-catalog-sync-ui.test.ts +2 −2
e2e/scenarios/provider-plugins-ui.test.ts +2 −2
e2e/selfhost/auth-methods-ui.test.ts +2 −2
e2e/selfhost/mcp-auth-required-add.test.ts +2 −2
e2e/selfhost/oauth-callback-unauthenticated.test.ts +1 −1
e2e/selfhost/posthog-mcp-oauth.test.ts +2 −2
e2e/vitest.config.ts +1 −1

e2e/cloud/auth-routing-flow.test.ts+11−1

@@ -56,7 +56,17 @@ scenario( · 55 unmodified lines
56	56	});
57	57
58	58	await step("Create the first org → canonical dashboard at /<slug>", async () => {
59		− await page.getByPlaceholder("Northwind Labs").fill("Flow Test Org");
	59	+ const orgName = "Flow Test Org";
	60	+ const orgNameInput = page.getByPlaceholder("Northwind Labs");
	61	+ await page.waitForLoadState("networkidle");
	62	+ await orgNameInput.fill(orgName);
	63	+ await page.waitForTimeout(250);
	64	+ if ((await orgNameInput.inputValue()) !== orgName) {
	65	+ await orgNameInput.fill(orgName);
	66	+ }
	67	+ expect(await orgNameInput.inputValue(), "org name survives create-org hydration").toBe(
	68	+ orgName,
	69	+ );
60	70	await page.getByRole("button", { name: "Create organization" }).click();
61	71	await page.getByText("Connect your MCP client").waitFor({ timeout: 30_000 });
62	72	await page.getByRole("button", { name: "Continue to app" }).click();

e2e/local/local-server.ts+4−4

@@ -76,15 +76,15 @@ export const withLocalServer = ( · 75 unmodified lines
76	76	return TOKEN_URL.test(current.text);
77	77	},
78	78	// A cold `executor web` runs vite's optimizeDeps before it prints
79		− // the URL; 180s is generous headroom over the warm ~3s boot. Kept
80		− // BELOW each scenario's test timeout (240s) so that on a slow or
	79	+ // the URL; 240s leaves headroom for isolated CI cold boots. Kept
	80	+ // BELOW each scenario's test timeout (300s) so that on a slow or
81	81	// wedged boot THIS error surfaces, with the terminal tail, rather
82	82	// than racing vitest's generic timeout at the same deadline.
83		− { timeoutMs: 180_000 },
	83	+ { timeoutMs: 240_000 },
84	84	)
85	85	.catch((cause: unknown) => {
86	86	throw new Error(
87		− `executor web --foreground printed no ?_token URL within 180s (${String(cause)}):\n${lastText.slice(-600)}`,
	87	+ `executor web --foreground printed no ?_token URL within 240s (${String(cause)}):\n${lastText.slice(-600)}`,
88	88	);
89	89	});
90	90	const url = TOKEN_URL.exec(snapshot.text)?.[0];

e2e/local/stdio-mcp.test.ts+2−2

@@ -40,13 +40,13 @@ const SECRET = "s3cr3t-from-the-vault"; · 39 unmodified lines
40	40	scenario(
41	41	"Local · a stdio MCP server's tools are detected on a fresh install, with env stored as a secret",
42	42	// Must stay STRICTLY greater than the boot-URL wait in `withLocalServer`
43		− // (currently 180s). When this CI job runs `stdio-mcp.test.ts` alone it always
	43	+ // (currently 240s). When this CI job runs `stdio-mcp.test.ts` alone it always
44	44	// pays a cold `vite optimizeDeps` boot (no prior file to warm the cache), the
45	45	// one variable step. If this test timeout equals the boot wait, both deadlines
46	46	// fire together and vitest's generic "Test timed out" wins, swallowing the
47	47	// harness's "printed no ?_token URL\n<terminal tail>" error that tells us what
48	48	// boot actually got stuck on. Keep the gap so the boot diagnostic surfaces.
49		− { timeout: 240_000 },
	49	+ { timeout: 300_000 },
50	50	Effect.gen(function* () {
51	51	const cli = yield* Cli;
52	52	const runDir = yield* RunDir;

e2e/scenarios/auth-methods-ui.test.ts+2−2

@@ -55,8 +55,8 @@ scenario( · 54 unmodified lines
55	55	await headerName.waitFor();
56	56	});
57	57
58		− await step("Add the source with both methods", async () => {
59		− await page.getByRole("button", { name: "Add source" }).click();
	58	+ await step("Add the integration with both methods", async () => {
	59	+ await page.getByRole("button", { name: "Add integration" }).click();
60	60	// onComplete routes to the new integration's detail hub.
61	61	await page.waitForURL(/\/integrations\/(?!add\b)[^/?]+$/, { timeout: 30_000 });
62	62	await page.getByText("Connections").first().waitFor();

e2e/scenarios/google-photos-preset-ui.test.ts+1−1

@@ -26,7 +26,7 @@ scenario( · 25 unmodified lines
26	26	const dialog = page.getByRole("dialog", { name: "Connect an integration" });
27	27	await dialog.getByRole("link", { name: /^Google Photos\b/ }).click();
28	28	await page.waitForURL(/\/integrations\/add\/google/);
29		− await page.getByRole("heading", { name: "Add Google" }).waitFor();
	29	+ await page.getByRole("heading", { name: "Add Google integration" }).waitFor();
30	30	});
31	31
32	32	await step("The Photos preset defaults to the focused namespace and products", async () => {

e2e/scenarios/mcp-catalog-sync-ui.test.ts+2−2

@@ -44,8 +44,8 @@ scenario( · 43 unmodified lines
44	44	await page.getByText("Method 1 · Detected").waitFor();
45	45	});
46	46
47		− await step("Add the source", async () => {
48		− await page.getByRole("button", { name: "Add source" }).click();
	47	+ await step("Add the integration", async () => {
	48	+ await page.getByRole("button", { name: "Add integration" }).click();
49	49	await page.waitForURL(/\/integrations\/(?!add\b)[^/?]+$/, { timeout: 30_000 });
50	50	await page.getByText("Connections").first().waitFor();
51	51	});

e2e/scenarios/provider-plugins-ui.test.ts+2−2

@@ -45,7 +45,7 @@ scenario( · 44 unmodified lines
45	45	await page.goto("/integrations/add/google", {
46	46	waitUntil: "domcontentloaded",
47	47	});
48		− await page.getByRole("heading", { name: "Add Google" }).waitFor();
	48	+ await page.getByRole("heading", { name: "Add Google integration" }).waitFor();
49	49	await page.getByText("Customize your Google connection").waitFor();
50	50	await page.getByText("Gmail").first().waitFor();
51	51	await page.getByText("Google Calendar").first().waitFor();
@@ -55,7 +55,7 @@ scenario( · 3 unmodified lines
55	55	await page.goto("/integrations/add/microsoft", {
56	56	waitUntil: "domcontentloaded",
57	57	});
58		− await page.getByRole("heading", { name: "Add Microsoft Graph" }).waitFor();
	58	+ await page.getByRole("heading", { name: "Add Microsoft integration" }).waitFor();
59	59	await page.getByText("Customize Microsoft Graph").waitFor();
60	60	expect(await page.getByText("All Microsoft Graph", { exact: true }).count()).toBe(0);
61	61	await page.getByText("Productivity").waitFor();

e2e/selfhost/auth-methods-ui.test.ts+2−2

@@ -64,8 +64,8 @@ scenario( · 63 unmodified lines
64	64	await page.getByPlaceholder("Authorization").last().waitFor();
65	65	});
66	66
67		− await step("Add the source with both methods", async () => {
68		− await page.getByRole("button", { name: "Add source" }).click();
	67	+ await step("Add the integration with both methods", async () => {
	68	+ await page.getByRole("button", { name: "Add integration" }).click();
69	69	await page.waitForURL(/\/integrations\/(?!add\b)[^/?]+$/, {
70	70	timeout: 30_000,
71	71	});

e2e/selfhost/mcp-auth-required-add.test.ts+2−2

@@ -67,9 +67,9 @@ scenario( · 66 unmodified lines
67	67	await page.getByText("Auth required").first().waitFor();
68	68	});
69	69
70		− await step("Add the source with the declared method", async () => {
	70	+ await step("Add the integration with the declared method", async () => {
71	71	await page.getByPlaceholder("e.g. Linear").fill(name);
72		− await page.getByRole("button", { name: "Add source" }).click();
	72	+ await page.getByRole("button", { name: "Add integration" }).click();
73	73	// onComplete routes to the new integration's detail hub.
74	74	await page.waitForURL(/\/integrations\/(?!add\b)[^/?]+$/, { timeout: 30_000 });
75	75	const landedSlug = new URL(page.url()).pathname.split("/").filter(Boolean).at(-1);

e2e/selfhost/oauth-callback-unauthenticated.test.ts+1−1

@@ -113,7 +113,7 @@ scenario( · 112 unmodified lines
113	113	await step("Provider sends a signed-out browser to the OAuth callback", async () => {
114	114	const response = await page.goto(callbackPath, { waitUntil: "networkidle" });
115	115	expect(response?.status(), "the callback redirects into the login flow").toBe(200);
116		− await page.getByText("Sign in to your instance").waitFor();
	116	+ await page.getByRole("heading", { name: "Sign in" }).waitFor();
117	117	});
118	118
119	119	const loginUrl = new URL(page.url());

e2e/selfhost/posthog-mcp-oauth.test.ts+2−2

@@ -42,9 +42,9 @@ scenario( · 41 unmodified lines
42	42	await page.getByText("OAuth metadata is discovered from this server").waitFor();
43	43	});
44	44
45		− await step("Add the PostHog MCP source", async () => {
	45	+ await step("Add the PostHog MCP integration", async () => {
46	46	await page.getByPlaceholder("e.g. Linear").fill(displayName);
47		− await page.getByRole("button", { name: "Add source" }).click();
	47	+ await page.getByRole("button", { name: "Add integration" }).click();
48	48	await page.waitForURL(/\/integrations\/(?!add\b)[^/?]+$/, { timeout: 30_000 });
49	49	const landedSlug = new URL(page.url()).pathname.split("/").filter(Boolean).at(-1);
50	50	expect(landedSlug, "the add flow lands on the created integration").toBe(String(slug));

e2e/vitest.config.ts+1−1

@@ -110,7 +110,7 @@ export default defineConfig({ · 109 unmodified lines
110	110	include: ["local/*/.test.ts"],
111	111	globalSetup: [],
112	112	fileParallelism: false,
113		− testTimeout: 240_000,
	113	+ testTimeout: 300_000,
114	114	}),
115	115	// The supervised CLI daemon inside a guest VM, one project per OS. The
116	116	// globalsetup provisions a VM, `executor service install`s the daemon, and