The self-host e2e project never ran in CI, so it drifted red while the app moved on. Repair the failing scenarios (stale connect-modal selectors, a racy action-bar position read, a shared-admin connection-count assertion, a multi-tenant-only org-slug 404 step, and a cloud-shaped toolkit MCP URL), add a documented skip affordance to the scenario helper, and quarantine the two Microsoft emulator scenarios that need a canonical block-YAML Graph spec (tracked separately). Cherry-picked from origin/fix-selfhost-e2e-and-ci (PR #1239); its CI job is superseded by the cloud+selfhost matrix job already on this branch.
Unified · Split
| @@ -411,15 +411,11 @@ export const routeTree = rootRouteImport · 410 unmodified lines | ||
| 411 | 411 | ._addFileTypes<FileRouteTypes>() |
| 412 | 412 | |
| 413 | 413 | import type { getRouter } from './router.tsx' |
| 414 | − | |
| 415 | 414 | import type { startInstance } from './start.ts' |
| 416 | − | |
| 417 | 415 | declare module '@tanstack/react-start' { |
| 418 | 416 | interface Register { |
| 419 | 417 | ssr: true |
| 420 | − | |
| 421 | 418 | router: Awaited<ReturnType<typeof getRouter>> |
| 422 | − | |
| 423 | 419 | config: Awaited<ReturnType<typeof startInstance.getOptions>> |
| 424 | 420 | } |
| 425 | 421 | } |
| @@ -40,7 +40,7 @@ const coreApi = composePluginApi([] as const); // tools/integrations/connections · 39 unmodified lines | ||
| 40 | 40 | |
| 41 | 41 | scenario( |
| 42 | 42 | "Tools · a fresh workspace advertises the built-in tools", |
| 43 | − {}, // options: { timeout?: number } | |
| 43 | + {}, // options: { timeout?: number; skip?: string (reason — registers as skipped) } | |
| 44 | 44 | Effect.gen(function* () { |
| 45 | 45 | const target = yield* Target; |
| 46 | 46 | const { client } = yield* Api; |
| @@ -31,7 +31,13 @@ scenario( · 30 unmodified lines | ||
| 31 | 31 | const { client } = yield* Api; |
| 32 | 32 | const identity = yield* target.newIdentity(); |
| 33 | 33 | const api = yield* client(coreApi, identity); |
| 34 | + // The list call itself exercises the endpoint on every target (a failure | |
| 35 | + // fails the test). Only isolated-identity targets (a fresh org per identity) | |
| 36 | + // can additionally guarantee the list is empty. Selfhost shares one | |
| 37 | + // bootstrap admin, so other scenarios' connections legitimately appear here; | |
| 38 | + // asserting a global count there is exactly what e2e/AGENTS.md forbids. | |
| 34 | 39 | const connections = yield* api.connections.list({ query: {} }); |
| 35 | − expect(connections.length, "no connections leak across identities").toBe(0); | |
| 40 | + if (target.name === "selfhost") return; | |
| 41 | + expect(connections.length, "a fresh org starts with no connections").toBe(0); | |
| 36 | 42 | }), |
| 37 | 43 | ); |
| @@ -152,7 +152,9 @@ scenario( · 151 unmodified lines | ||
| 152 | 152 | .waitFor({ timeout: 15_000 }); |
| 153 | 153 | }); |
| 154 | 154 | await step("Paste the Resend API key and connect", async () => { |
| 155 | − const credential = page.getByPlaceholder(/paste the value \/ token/i); | |
| 155 | + // Affixed single-input bearer field: value input placeholder is | |
| 156 | + // "token" (scoped to the dialog to stay unique). | |
| 157 | + const credential = page.getByRole("dialog").getByPlaceholder("token"); | |
| 156 | 158 | await credential.waitFor({ timeout: 15_000 }); |
| 157 | 159 | await credential.fill(apiKey); |
| 158 | 160 | await page.getByRole("button", { name: "Add connection", exact: true }).click(); |
| @@ -209,7 +209,10 @@ const runScenario = (input: { · 208 unmodified lines | ||
| 209 | 209 | }); |
| 210 | 210 | |
| 211 | 211 | await step("Paste the emulator API key", async () => { |
| 212 | − const credential = page.getByPlaceholder(/paste the value \/ token/i); | |
| 212 | + // The single-input bearer method renders an affixed field ("Authorization: | |
| 213 | + // Bearer " prefix) whose value input placeholder is "token". Scope to the | |
| 214 | + // dialog so the match stays unique. | |
| 215 | + const credential = page.getByRole("dialog").getByPlaceholder("token"); | |
| 213 | 216 | await credential.waitFor({ timeout: 15_000 }); |
| 214 | 217 | await credential.fill(apiKey); |
| 215 | 218 | }); |
| @@ -80,7 +80,18 @@ return { ok: result.ok, path: item.path, result: result.ok ? result.data : resul · 79 unmodified lines | ||
| 80 | 80 | |
| 81 | 81 | scenario( |
| 82 | 82 | "Microsoft · client credentials against the emulator mint a Graph connection and call /users", |
| 83 | − { timeout: 180_000 }, | |
| 83 | + { | |
| 84 | + // Blocked (pre-existing, not this PR): `microsoft.addGraph` only accepts the | |
| 85 | + // canonical Graph spec in the streamable block-YAML profile — it structurally | |
| 86 | + // splits the doc to avoid OOMing the 128MB Workers isolate on the real 37MB | |
| 87 | + // spec (packages/plugins/microsoft/src/sdk/graph.ts), and hard-errors on | |
| 88 | + // anything else. The @executor-js/emulate Microsoft emulator serves a small | |
| 89 | + // custom Graph spec that isn't in that profile, so addGraph rejects it. Fix | |
| 90 | + // needs the emulator to serve a block-YAML-profile Graph spec (or a | |
| 91 | + // non-Workers compile path); tracked separately. | |
| 92 | + skip: "microsoft.addGraph requires the canonical block-YAML Graph spec; the emulator spec is not in that profile", | |
| 93 | + timeout: 180_000, | |
| 94 | + }, | |
| 84 | 95 | Effect.scoped( |
| 85 | 96 | Effect.gen(function* () { |
| 86 | 97 | const target = yield* Target; |
| @@ -310,7 +310,18 @@ const requireOAuthClientCredential = (credential: IssuedCredential) => · 309 unmodified lines | ||
| 310 | 310 | |
| 311 | 311 | scenario( |
| 312 | 312 | "OAuth client · agent hands off, the human enters the secret in the browser, and the app connects", |
| 313 | − { timeout: 240_000 }, | |
| 313 | + { | |
| 314 | + // Blocked (pre-existing, not this PR): this scenario drives the handoff | |
| 315 | + // through `microsoft.addGraph`, which only accepts the canonical Graph spec | |
| 316 | + // in the streamable block-YAML profile (structural split to avoid OOMing the | |
| 317 | + // 128MB Workers isolate on the 37MB doc — packages/plugins/microsoft/src/sdk/ | |
| 318 | + // graph.ts). The @executor-js/emulate Microsoft emulator serves a small spec | |
| 319 | + // outside that profile, so addGraph hard-errors. The other two OAuth-client | |
| 320 | + // scenarios in this file (createHandoff, approval-gating) do not touch Graph | |
| 321 | + // and pass. Fix needs a block-YAML-profile emulator spec; tracked separately. | |
| 322 | + skip: "drives microsoft.addGraph, which requires the canonical block-YAML Graph spec the emulator does not serve", | |
| 323 | + timeout: 240_000, | |
| 324 | + }, | |
| 314 | 325 | Effect.gen(function* () { |
| 315 | 326 | const target = yield* Target; |
| 316 | 327 | const { client: makeApiClient } = yield* Api; |
| @@ -52,23 +52,15 @@ scenario( · 51 unmodified lines | ||
| 52 | 52 | }); |
| 53 | 53 | |
| 54 | 54 | await step( |
| 55 | − "Submitting does not reflow the bar, then lands on the integration", | |
| 55 | + "Submitting commits the source and lands on the created integration", | |
| 56 | 56 | async () => { |
| 57 | 57 | // The reported ghost was the bar painting doubled when the submit |
| 58 | − // button changed width on click. With a stable-width loading button the | |
| 59 | − // row must not move: Cancel stays put while the add is in flight. | |
| 60 | − const cancel = page.getByRole("button", { name: "Cancel" }); | |
| 61 | − const before = await cancel.boundingBox(); | |
| 58 | + // button changed width on click. The single-node counts (above and | |
| 59 | + // below) are the hard regression cover for that; the floating action | |
| 60 | + // bar unmounts the instant the router navigates, so there is no | |
| 61 | + // reliable in-flight frame to measure its position without racing the | |
| 62 | + // teardown. Assert the submit completes and lands on the integration. | |
| 62 | 63 | await page.getByRole("button", { name: "Add integration" }).click(); |
| 63 | − // The submit button marks itself data-loading synchronously on click. | |
| 64 | − await page | |
| 65 | − .locator('[data-slot="button"][data-loading]') | |
| 66 | − .first() | |
| 67 | − .waitFor({ timeout: 5_000 }); | |
| 68 | − const during = await cancel.boundingBox(); | |
| 69 | − expect(Math.round(during?.x ?? -1), "Cancel does not move when submitting").toBe( | |
| 70 | − Math.round(before?.x ?? -2), | |
| 71 | − ); | |
| 72 | 64 | await page.waitForURL(/\/integrations\/(?!add\b)[^/?]+$/, { timeout: 30_000 }); |
| 73 | 65 | await page.getByText("Connections").first().waitFor(); |
| 74 | 66 | }, |
| @@ -45,10 +45,17 @@ scenario( · 44 unmodified lines | ||
| 45 | 45 | await page.getByText("Policies").first().waitFor(); |
| 46 | 46 | }); |
| 47 | 47 | |
| 48 | − await step("An unknown org slug is a wrong address, not a redirect", async () => { | |
| 49 | − await page.goto("/zz-no-such-org/policies", { waitUntil: "networkidle" }); | |
| 50 | − await page.getByText("Page not found").waitFor({ timeout: 30_000 }); | |
| 51 | − }); | |
| 48 | + // The "unknown slug is a 404" contract is multi-tenant only. Selfhost is | |
| 49 | + // single-tenant: /account/me always returns the instance org regardless of | |
| 50 | + // the URL segment, so the slug is cosmetic and an unknown one canonicalizes | |
| 51 | + // onto the shell rather than 404ing. Cloud enforces the not-found; selfhost | |
| 52 | + // legitimately does not. | |
| 53 | + if (target.name !== "selfhost") { | |
| 54 | + await step("An unknown org slug is a wrong address, not a redirect", async () => { | |
| 55 | + await page.goto("/zz-no-such-org/policies", { waitUntil: "networkidle" }); | |
| 56 | + await page.getByText("Page not found").waitFor({ timeout: 30_000 }); | |
| 57 | + }); | |
| 58 | + } | |
| 52 | 59 | |
| 53 | 60 | await step("In-shell navigation keeps the slug prefix", async () => { |
| 54 | 61 | await page.goto(`/${slug}`, { waitUntil: "networkidle" }); |
| @@ -142,7 +142,9 @@ scenario( · 141 unmodified lines | ||
| 142 | 142 | }); |
| 143 | 143 | |
| 144 | 144 | await step("Connect through the new method", async () => { |
| 145 | − await page.getByPlaceholder("paste the value / token").fill(token); | |
| 145 | + // Custom "Authorization: Bearer " method renders the affixed field, | |
| 146 | + // whose value input placeholder is "token". | |
| 147 | + await page.getByRole("dialog").getByPlaceholder("token").fill(token); | |
| 146 | 148 | await page.getByRole("button", { name: "Add connection" }).click(); |
| 147 | 149 | await page.getByText("Connection added").waitFor(); |
| 148 | 150 | }); |
| @@ -0,0 +1,164 @@ | ||
| 1 | +import { randomBytes } from "node:crypto"; | |
| 2 | + | |
| 3 | +import { expect } from "@effect/vitest"; | |
| 4 | +import { Effect } from "effect"; | |
| 5 | +import { composePluginApi } from "@executor-js/api/server"; | |
| 6 | +import { openApiHttpPlugin } from "@executor-js/plugin-openapi/api"; | |
| 7 | +import { | |
| 8 | + AuthTemplateSlug, | |
| 9 | + ConnectionName, | |
| 10 | + decodeOAuthCallbackState, | |
| 11 | + IntegrationSlug, | |
| 12 | + OAuthClientSlug, | |
| 13 | +} from "@executor-js/sdk/shared"; | |
| 14 | +import { serveOAuthTestServer } from "@executor-js/sdk/testing"; | |
| 15 | + | |
| 16 | +import { scenario } from "../src/scenario"; | |
| 17 | +import { Api, Target } from "../src/services"; | |
| 18 | + | |
| 19 | +const api = composePluginApi([openApiHttpPlugin()] as const); | |
| 20 | + | |
| 21 | +const unique = (prefix: string) => `${prefix}_${randomBytes(4).toString("hex")}`; | |
| 22 | + | |
| 23 | +const oauthIntegrationSpec = (oauth: { | |
| 24 | + readonly authorizationEndpoint: string; | |
| 25 | + readonly tokenEndpoint: string; | |
| 26 | +}) => | |
| 27 | + ({ | |
| 28 | + spec: { | |
| 29 | + kind: "blob" as const, | |
| 30 | + value: JSON.stringify({ | |
| 31 | + openapi: "3.0.3", | |
| 32 | + info: { title: "OAuth-protected API", version: "1.0.0" }, | |
| 33 | + paths: { | |
| 34 | + "/me": { | |
| 35 | + get: { | |
| 36 | + operationId: "getMe", | |
| 37 | + tags: ["default"], | |
| 38 | + responses: { "200": { description: "the caller" } }, | |
| 39 | + }, | |
| 40 | + }, | |
| 41 | + }, | |
| 42 | + }), | |
| 43 | + }, | |
| 44 | + baseUrl: "http://127.0.0.1:59999", | |
| 45 | + authenticationTemplate: [ | |
| 46 | + { | |
| 47 | + slug: "oauth", | |
| 48 | + kind: "oauth2" as const, | |
| 49 | + authorizationUrl: oauth.authorizationEndpoint, | |
| 50 | + tokenUrl: oauth.tokenEndpoint, | |
| 51 | + scopes: ["read"], | |
| 52 | + }, | |
| 53 | + ], | |
| 54 | + }) as const; | |
| 55 | + | |
| 56 | +// Better Auth email sign-in → session cookie, so the callback (a browser GET | |
| 57 | +// behind the session) can be driven with a plain authenticated fetch. Mirrors | |
| 58 | +// what the API surface does internally; kept local to keep this a black-box HTTP | |
| 59 | +// journey with no browser dependency. | |
| 60 | +const sessionCookie = (baseUrl: string, credentials: { email: string; password: string }) => | |
| 61 | + Effect.promise(async () => { | |
| 62 | + const response = await fetch(new URL("/api/auth/sign-in/email", baseUrl), { | |
| 63 | + method: "POST", | |
| 64 | + headers: { "content-type": "application/json", origin: new URL(baseUrl).origin }, | |
| 65 | + body: JSON.stringify(credentials), | |
| 66 | + }); | |
| 67 | + const cookie = (response.headers.getSetCookie?.() ?? []).map((c) => c.split(";")[0]).join("; "); | |
| 68 | + if (!cookie) throw new Error(`sign-in set no cookie (${response.status})`); | |
| 69 | + return cookie; | |
| 70 | + }); | |
| 71 | + | |
| 72 | +// Regression guard for the org-wrapped callback state. Self-host binds every | |
| 73 | +// request to an org slug ("default"), so `oauth.start` wraps the raw session | |
| 74 | +// token in the state it sends the provider. The provider echoes that wrapped | |
| 75 | +// value back on the callback; the shared popup callback must unwrap it to the | |
| 76 | +// raw token before looking up the session. Before the fix it passed the wrapped | |
| 77 | +// value straight to `oauth.complete`, which looks up by the raw token and failed | |
| 78 | +// with "OAuth session expired or not found". | |
| 79 | +scenario( | |
| 80 | + "OAuth callback · a self-host org-context popup callback completes with the wrapped state", | |
| 81 | + {}, | |
| 82 | + Effect.gen(function* () { | |
| 83 | + const target = yield* Target; | |
| 84 | + const { client: makeApiClient } = yield* Api; | |
| 85 | + const oauth = yield* serveOAuthTestServer(); | |
| 86 | + const identity = yield* target.newIdentity(); | |
| 87 | + const client = yield* makeApiClient(api, identity); | |
| 88 | + | |
| 89 | + const integration = IntegrationSlug.make(unique("selfhostorgstate")); | |
| 90 | + yield* client.openapi.addSpec({ | |
| 91 | + payload: { ...oauthIntegrationSpec(oauth), slug: integration }, | |
| 92 | + }); | |
| 93 | + | |
| 94 | + const clientSlug = OAuthClientSlug.make(unique("selfhostorgstate")); | |
| 95 | + yield* client.oauth.createClient({ | |
| 96 | + payload: { | |
| 97 | + owner: "org", | |
| 98 | + slug: clientSlug, | |
| 99 | + authorizationUrl: oauth.authorizationEndpoint, | |
| 100 | + tokenUrl: oauth.tokenEndpoint, | |
| 101 | + grant: "authorization_code", | |
| 102 | + clientId: "test-client", | |
| 103 | + clientSecret: "test-secret", | |
| 104 | + }, | |
| 105 | + }); | |
| 106 | + | |
| 107 | + const started = yield* client.oauth.start({ | |
| 108 | + payload: { | |
| 109 | + client: clientSlug, | |
| 110 | + clientOwner: "org", | |
| 111 | + owner: "org", | |
| 112 | + name: ConnectionName.make("main"), | |
| 113 | + integration, | |
| 114 | + template: AuthTemplateSlug.make("oauth"), | |
| 115 | + }, | |
| 116 | + }); | |
| 117 | + expect(started.status, "oauth.start begins at the provider").toBe("redirect"); | |
| 118 | + const authorizationUrl = started.status === "redirect" ? started.authorizationUrl : ""; | |
| 119 | + | |
| 120 | + // The bug's precondition: the state sent to the provider is NOT the raw | |
| 121 | + // session token, it is the org-slug-wrapped envelope. If this stops being | |
| 122 | + // true the callback path below no longer exercises the regression. | |
| 123 | + const providerState = new URL(authorizationUrl).searchParams.get("state") ?? ""; | |
| 124 | + expect( | |
| 125 | + decodeOAuthCallbackState(providerState), | |
| 126 | + "self-host org context wraps the OAuth state with the org slug before redirecting", | |
| 127 | + ).not.toBeNull(); | |
| 128 | + | |
| 129 | + const authorize = yield* Effect.promise(() => fetch(authorizationUrl, { redirect: "manual" })); | |
| 130 | + expect(authorize.status, "the provider asks the user to log in").toBe(302); | |
| 131 | + const consent = yield* Effect.promise(() => | |
| 132 | + fetch(authorize.headers.get("location") ?? "", { | |
| 133 | + method: "POST", | |
| 134 | + redirect: "manual", | |
| 135 | + headers: { | |
| 136 | + authorization: `Basic ${Buffer.from("alice:password").toString("base64")}`, | |
| 137 | + }, | |
| 138 | + }), | |
| 139 | + ); | |
| 140 | + expect(consent.status, "provider consent redirects back to Executor").toBe(302); | |
| 141 | + const callback = new URL(consent.headers.get("location") ?? ""); | |
| 142 | + const callbackPath = `${callback.pathname}${callback.search}`; | |
| 143 | + expect( | |
| 144 | + callback.searchParams.get("state"), | |
| 145 | + "the provider echoes the wrapped state back on the callback", | |
| 146 | + ).toBe(providerState); | |
| 147 | + | |
| 148 | + const cookie = yield* sessionCookie(target.baseUrl, identity.credentials!); | |
| 149 | + const response = yield* Effect.promise(() => | |
| 150 | + fetch(new URL(callbackPath, target.baseUrl), { headers: { cookie } }), | |
| 151 | + ); | |
| 152 | + expect(response.status, "the callback renders its popup HTML").toBe(200); | |
| 153 | + const html = yield* Effect.promise(() => response.text()); | |
| 154 | + | |
| 155 | + expect( | |
| 156 | + html, | |
| 157 | + "the wrapped state is unwrapped to the raw token, so the session is found and completes", | |
| 158 | + ).toContain("Connected"); | |
| 159 | + expect( | |
| 160 | + html, | |
| 161 | + "the raw session token is recovered from the wrapped state (no expired-session error)", | |
| 162 | + ).not.toContain("OAuth session expired or not found"); | |
| 163 | + }).pipe(Effect.scoped), | |
| 164 | +); | |
| @@ -107,10 +107,14 @@ scenario( · 106 unmodified lines | ||
| 107 | 107 | }, |
| 108 | 108 | }); |
| 109 | 109 | |
| 110 | − const toolkitUrl = new URL( | |
| 111 | − `/e2e-org/mcp/toolkits/${toolkit.slug}`, | |
| 112 | − target.baseUrl, | |
| 113 | − ).toString(); | |
| 110 | + // Self-host advertises the BARE MCP path (no org prefix — see the | |
| 111 | + // host-selfhost __root shell and `toolkitUrlFor`, which only prefixes a | |
| 112 | + // slug when one is present, i.e. on cloud). A made-up `/e2e-org` prefix is | |
| 113 | + // a cloud-shaped URL self-host never serves as canonical: the server's | |
| 114 | + // RFC 9728 protected-resource doc reports the bare resource, and MCP SDK | |
| 115 | + // 1.29's `selectResourceURL` rejects the prefix/bare mismatch. Connect to | |
| 116 | + // the URL self-host actually publishes. | |
| 117 | + const toolkitUrl = new URL(`/mcp/toolkits/${toolkit.slug}`, target.baseUrl).toString(); | |
| 114 | 118 | const toolkitSession = mcp.session(identity, { url: toolkitUrl }); |
| 115 | 119 | const toolkitTools = yield* toolkitSession.listTools(); |
| 116 | 120 | expect(toolkitTools, "the toolkit endpoint still advertises execute").toContain("execute"); |
| @@ -54,6 +54,10 @@ export const slugify = (text: string): string => · 53 unmodified lines | ||
| 54 | 54 | |
| 55 | 55 | export interface ScenarioOptions { |
| 56 | 56 | readonly timeout?: number; |
| 57 | + /** When set, the scenario is registered as skipped (vitest `it.skip`) and its | |
| 58 | + * body never runs. Use ONLY for a scenario blocked on a tracked, out-of-scope | |
| 59 | + * issue; state the reason here so the skip is self-documenting in the source. */ | |
| 60 | + readonly skip?: string; | |
| 57 | 61 | } |
| 58 | 62 | |
| 59 | 63 | type AllServices = |
| @@ -114,6 +118,13 @@ export const scenario = ( · 54 unmodified lines | ||
| 114 | 118 | options: ScenarioOptions, |
| 115 | 119 | body: Effect.Effect<void, unknown, AllServices | HttpClient.HttpClient>, |
| 116 | 120 | ): void => { |
| 121 | + if (options.skip) { | |
| 122 | + // Blocked on a tracked, out-of-scope issue (see the scenario's `skip` | |
| 123 | + // reason). Registered as skipped so the suite stays green and the gap stays | |
| 124 | + // visible in the test report rather than silently deleted. | |
| 125 | + it.skip(name, () => Effect.void); | |
| 126 | + return; | |
| 127 | + } | |
| 117 | 128 | const target = resolveTarget(); |
| 118 | 129 | const dir = join(RUNS_DIR, target.name, slugify(name)); |
| 119 | 130 | const context = contextFor(target, dir); |