History

Make the test suite parallel-safe and speed up CI (#1273)

* Make self-host tests CI-safe * Await cloud db teardown * Give local e2e boots more headroom * Keep main CI runs alive * Cache CI dependencies and image layers * Wait for recorded introspection requests in graphql plugin tests * Size self-host test budget for loaded runners * Probe test http servers for readiness before use * Die on recorder wait timeout instead of typing a string error * Type the recorder wait timeout as a tagged error * Update e2e locators for the integration copy renames * Keep stale data rendered while a refresh is waiting * Guard create-org input against hydration clobbering * Match the new sign-in heading in the selfhost callback test

Rhys Sullivan committed 4 months ago

Commit 8652c991 parent 9c7e4a627 files · +252 −102Browse files

Showing the changes to e2e (12 of 27 files). Show all

Unified · Split

12 files
  1. e2e/cloud/auth-routing-flow.test.ts +11 −1
  2. e2e/local/local-server.ts +4 −4
  3. e2e/local/stdio-mcp.test.ts +2 −2
  4. e2e/scenarios/auth-methods-ui.test.ts +2 −2
  5. e2e/scenarios/google-photos-preset-ui.test.ts +1 −1
  6. e2e/scenarios/mcp-catalog-sync-ui.test.ts +2 −2
  7. e2e/scenarios/provider-plugins-ui.test.ts +2 −2
  8. e2e/selfhost/auth-methods-ui.test.ts +2 −2
  9. e2e/selfhost/mcp-auth-required-add.test.ts +2 −2
  10. e2e/selfhost/oauth-callback-unauthenticated.test.ts +1 −1
  11. e2e/selfhost/posthog-mcp-oauth.test.ts +2 −2
  12. e2e/vitest.config.ts +1 −1
e2e/cloud/auth-routing-flow.test.ts+11−1
@@ -56,7 +56,17 @@ scenario( · 55 unmodified lines
56 });56 });
57 57
58 await step("Create the first org → canonical dashboard at /<slug>", async () => {58 await step("Create the first org → canonical dashboard at /<slug>", async () => {
59 await page.getByPlaceholder("Northwind Labs").fill("Flow Test Org");59 const orgName = "Flow Test Org";
60 const orgNameInput = page.getByPlaceholder("Northwind Labs");
61 await page.waitForLoadState("networkidle");
62 await orgNameInput.fill(orgName);
63 await page.waitForTimeout(250);
64 if ((await orgNameInput.inputValue()) !== orgName) {
65 await orgNameInput.fill(orgName);
66 }
67 expect(await orgNameInput.inputValue(), "org name survives create-org hydration").toBe(
68 orgName,
69 );
60 await page.getByRole("button", { name: "Create organization" }).click();70 await page.getByRole("button", { name: "Create organization" }).click();
61 await page.getByText("Connect your MCP client").waitFor({ timeout: 30_000 });71 await page.getByText("Connect your MCP client").waitFor({ timeout: 30_000 });
62 await page.getByRole("button", { name: "Continue to app" }).click();72 await page.getByRole("button", { name: "Continue to app" }).click();
e2e/local/local-server.ts+4−4
@@ -76,15 +76,15 @@ export const withLocalServer = ( · 75 unmodified lines
76 return TOKEN_URL.test(current.text);76 return TOKEN_URL.test(current.text);
77 },77 },
78 // A cold `executor web` runs vite's optimizeDeps before it prints78 // A cold `executor web` runs vite's optimizeDeps before it prints
79 // the URL; 180s is generous headroom over the warm ~3s boot. Kept79 // the URL; 240s leaves headroom for isolated CI cold boots. Kept
80 // BELOW each scenario's test timeout (240s) so that on a slow or80 // BELOW each scenario's test timeout (300s) so that on a slow or
81 // wedged boot THIS error surfaces, with the terminal tail, rather81 // wedged boot THIS error surfaces, with the terminal tail, rather
82 // than racing vitest's generic timeout at the same deadline.82 // than racing vitest's generic timeout at the same deadline.
83 { timeoutMs: 180_000 },83 { timeoutMs: 240_000 },
84 )84 )
85 .catch((cause: unknown) => {85 .catch((cause: unknown) => {
86 throw new Error(86 throw new Error(
87 `executor web --foreground printed no ?_token URL within 180s (${String(cause)}):\n${lastText.slice(-600)}`,87 `executor web --foreground printed no ?_token URL within 240s (${String(cause)}):\n${lastText.slice(-600)}`,
88 );88 );
89 });89 });
90 const url = TOKEN_URL.exec(snapshot.text)?.[0];90 const url = TOKEN_URL.exec(snapshot.text)?.[0];
e2e/local/stdio-mcp.test.ts+2−2
@@ -40,13 +40,13 @@ const SECRET = "s3cr3t-from-the-vault"; · 39 unmodified lines
40scenario(40scenario(
41 "Local · a stdio MCP server's tools are detected on a fresh install, with env stored as a secret",41 "Local · a stdio MCP server's tools are detected on a fresh install, with env stored as a secret",
42 // Must stay STRICTLY greater than the boot-URL wait in `withLocalServer`42 // Must stay STRICTLY greater than the boot-URL wait in `withLocalServer`
43 // (currently 180s). When this CI job runs `stdio-mcp.test.ts` alone it always43 // (currently 240s). When this CI job runs `stdio-mcp.test.ts` alone it always
44 // pays a cold `vite optimizeDeps` boot (no prior file to warm the cache), the44 // pays a cold `vite optimizeDeps` boot (no prior file to warm the cache), the
45 // one variable step. If this test timeout equals the boot wait, both deadlines45 // one variable step. If this test timeout equals the boot wait, both deadlines
46 // fire together and vitest's generic "Test timed out" wins, swallowing the46 // fire together and vitest's generic "Test timed out" wins, swallowing the
47 // harness's "printed no ?_token URL\n<terminal tail>" error that tells us what47 // harness's "printed no ?_token URL\n<terminal tail>" error that tells us what
48 // boot actually got stuck on. Keep the gap so the boot diagnostic surfaces.48 // boot actually got stuck on. Keep the gap so the boot diagnostic surfaces.
49 { timeout: 240_000 },49 { timeout: 300_000 },
50 Effect.gen(function* () {50 Effect.gen(function* () {
51 const cli = yield* Cli;51 const cli = yield* Cli;
52 const runDir = yield* RunDir;52 const runDir = yield* RunDir;
e2e/scenarios/auth-methods-ui.test.ts+2−2
@@ -55,8 +55,8 @@ scenario( · 54 unmodified lines
55 await headerName.waitFor();55 await headerName.waitFor();
56 });56 });
57 57
58 await step("Add the source with both methods", async () => {58 await step("Add the integration with both methods", async () => {
59 await page.getByRole("button", { name: "Add source" }).click();59 await page.getByRole("button", { name: "Add integration" }).click();
60 // onComplete routes to the new integration's detail hub.60 // onComplete routes to the new integration's detail hub.
61 await page.waitForURL(/\/integrations\/(?!add\b)[^/?]+$/, { timeout: 30_000 });61 await page.waitForURL(/\/integrations\/(?!add\b)[^/?]+$/, { timeout: 30_000 });
62 await page.getByText("Connections").first().waitFor();62 await page.getByText("Connections").first().waitFor();
e2e/scenarios/google-photos-preset-ui.test.ts+1−1
@@ -26,7 +26,7 @@ scenario( · 25 unmodified lines
26 const dialog = page.getByRole("dialog", { name: "Connect an integration" });26 const dialog = page.getByRole("dialog", { name: "Connect an integration" });
27 await dialog.getByRole("link", { name: /^Google Photos\b/ }).click();27 await dialog.getByRole("link", { name: /^Google Photos\b/ }).click();
28 await page.waitForURL(/\/integrations\/add\/google/);28 await page.waitForURL(/\/integrations\/add\/google/);
29 await page.getByRole("heading", { name: "Add Google" }).waitFor();29 await page.getByRole("heading", { name: "Add Google integration" }).waitFor();
30 });30 });
31 31
32 await step("The Photos preset defaults to the focused namespace and products", async () => {32 await step("The Photos preset defaults to the focused namespace and products", async () => {
e2e/scenarios/mcp-catalog-sync-ui.test.ts+2−2
@@ -44,8 +44,8 @@ scenario( · 43 unmodified lines
44 await page.getByText("Method 1 · Detected").waitFor();44 await page.getByText("Method 1 · Detected").waitFor();
45 });45 });
46 46
47 await step("Add the source", async () => {47 await step("Add the integration", async () => {
48 await page.getByRole("button", { name: "Add source" }).click();48 await page.getByRole("button", { name: "Add integration" }).click();
49 await page.waitForURL(/\/integrations\/(?!add\b)[^/?]+$/, { timeout: 30_000 });49 await page.waitForURL(/\/integrations\/(?!add\b)[^/?]+$/, { timeout: 30_000 });
50 await page.getByText("Connections").first().waitFor();50 await page.getByText("Connections").first().waitFor();
51 });51 });
e2e/scenarios/provider-plugins-ui.test.ts+2−2
@@ -45,7 +45,7 @@ scenario( · 44 unmodified lines
45 await page.goto("/integrations/add/google", {45 await page.goto("/integrations/add/google", {
46 waitUntil: "domcontentloaded",46 waitUntil: "domcontentloaded",
47 });47 });
48 await page.getByRole("heading", { name: "Add Google" }).waitFor();48 await page.getByRole("heading", { name: "Add Google integration" }).waitFor();
49 await page.getByText("Customize your Google connection").waitFor();49 await page.getByText("Customize your Google connection").waitFor();
50 await page.getByText("Gmail").first().waitFor();50 await page.getByText("Gmail").first().waitFor();
51 await page.getByText("Google Calendar").first().waitFor();51 await page.getByText("Google Calendar").first().waitFor();
@@ -55,7 +55,7 @@ scenario( · 3 unmodified lines
55 await page.goto("/integrations/add/microsoft", {55 await page.goto("/integrations/add/microsoft", {
56 waitUntil: "domcontentloaded",56 waitUntil: "domcontentloaded",
57 });57 });
58 await page.getByRole("heading", { name: "Add Microsoft Graph" }).waitFor();58 await page.getByRole("heading", { name: "Add Microsoft integration" }).waitFor();
59 await page.getByText("Customize Microsoft Graph").waitFor();59 await page.getByText("Customize Microsoft Graph").waitFor();
60 expect(await page.getByText("All Microsoft Graph", { exact: true }).count()).toBe(0);60 expect(await page.getByText("All Microsoft Graph", { exact: true }).count()).toBe(0);
61 await page.getByText("Productivity").waitFor();61 await page.getByText("Productivity").waitFor();
e2e/selfhost/auth-methods-ui.test.ts+2−2
@@ -64,8 +64,8 @@ scenario( · 63 unmodified lines
64 await page.getByPlaceholder("Authorization").last().waitFor();64 await page.getByPlaceholder("Authorization").last().waitFor();
65 });65 });
66 66
67 await step("Add the source with both methods", async () => {67 await step("Add the integration with both methods", async () => {
68 await page.getByRole("button", { name: "Add source" }).click();68 await page.getByRole("button", { name: "Add integration" }).click();
69 await page.waitForURL(/\/integrations\/(?!add\b)[^/?]+$/, {69 await page.waitForURL(/\/integrations\/(?!add\b)[^/?]+$/, {
70 timeout: 30_000,70 timeout: 30_000,
71 });71 });
e2e/selfhost/mcp-auth-required-add.test.ts+2−2
@@ -67,9 +67,9 @@ scenario( · 66 unmodified lines
67 await page.getByText("Auth required").first().waitFor();67 await page.getByText("Auth required").first().waitFor();
68 });68 });
69 69
70 await step("Add the source with the declared method", async () => {70 await step("Add the integration with the declared method", async () => {
71 await page.getByPlaceholder("e.g. Linear").fill(name);71 await page.getByPlaceholder("e.g. Linear").fill(name);
72 await page.getByRole("button", { name: "Add source" }).click();72 await page.getByRole("button", { name: "Add integration" }).click();
73 // onComplete routes to the new integration's detail hub.73 // onComplete routes to the new integration's detail hub.
74 await page.waitForURL(/\/integrations\/(?!add\b)[^/?]+$/, { timeout: 30_000 });74 await page.waitForURL(/\/integrations\/(?!add\b)[^/?]+$/, { timeout: 30_000 });
75 const landedSlug = new URL(page.url()).pathname.split("/").filter(Boolean).at(-1);75 const landedSlug = new URL(page.url()).pathname.split("/").filter(Boolean).at(-1);
e2e/selfhost/oauth-callback-unauthenticated.test.ts+1−1
@@ -113,7 +113,7 @@ scenario( · 112 unmodified lines
113 await step("Provider sends a signed-out browser to the OAuth callback", async () => {113 await step("Provider sends a signed-out browser to the OAuth callback", async () => {
114 const response = await page.goto(callbackPath, { waitUntil: "networkidle" });114 const response = await page.goto(callbackPath, { waitUntil: "networkidle" });
115 expect(response?.status(), "the callback redirects into the login flow").toBe(200);115 expect(response?.status(), "the callback redirects into the login flow").toBe(200);
116 await page.getByText("Sign in to your instance").waitFor();116 await page.getByRole("heading", { name: "Sign in" }).waitFor();
117 });117 });
118 118
119 const loginUrl = new URL(page.url());119 const loginUrl = new URL(page.url());
e2e/selfhost/posthog-mcp-oauth.test.ts+2−2
@@ -42,9 +42,9 @@ scenario( · 41 unmodified lines
42 await page.getByText("OAuth metadata is discovered from this server").waitFor();42 await page.getByText("OAuth metadata is discovered from this server").waitFor();
43 });43 });
44 44
45 await step("Add the PostHog MCP source", async () => {45 await step("Add the PostHog MCP integration", async () => {
46 await page.getByPlaceholder("e.g. Linear").fill(displayName);46 await page.getByPlaceholder("e.g. Linear").fill(displayName);
47 await page.getByRole("button", { name: "Add source" }).click();47 await page.getByRole("button", { name: "Add integration" }).click();
48 await page.waitForURL(/\/integrations\/(?!add\b)[^/?]+$/, { timeout: 30_000 });48 await page.waitForURL(/\/integrations\/(?!add\b)[^/?]+$/, { timeout: 30_000 });
49 const landedSlug = new URL(page.url()).pathname.split("/").filter(Boolean).at(-1);49 const landedSlug = new URL(page.url()).pathname.split("/").filter(Boolean).at(-1);
50 expect(landedSlug, "the add flow lands on the created integration").toBe(String(slug));50 expect(landedSlug, "the add flow lands on the created integration").toBe(String(slug));
e2e/vitest.config.ts+1−1
@@ -110,7 +110,7 @@ export default defineConfig({ · 109 unmodified lines
110 include: ["local/**/*.test.ts"],110 include: ["local/**/*.test.ts"],
111 globalSetup: [],111 globalSetup: [],
112 fileParallelism: false,112 fileParallelism: false,
113 testTimeout: 240_000,113 testTimeout: 300_000,
114 }),114 }),
115 // The supervised CLI daemon inside a guest VM, one project per OS. The115 // The supervised CLI daemon inside a guest VM, one project per OS. The
116 // globalsetup provisions a VM, `executor service install`s the daemon, and116 // globalsetup provisions a VM, `executor service install`s the daemon, and