History

Add regression e2e for surfaced OAuth setup failure causes

Rhys Sullivan committed 2 months ago

Commit 12ac1f41 parent 10e16a51 file · +202 −0Browse files

Unified · Split

e2e/scenarios/oauth-setup-failure-cause.test.ts+202−0
@@ -0,0 +1,202 @@
1+// When OAuth integration setup fails — malformed discovery metadata, a broken
2+// authorization server — the sandbox must see the typed failure's own message
3+// (`oauth_probe_error` / `oauth_start_error` with the discovery cause), never
4+// the scrubbed "Internal tool error [hex]" defect.
5+//
6+// Regression guard for the swallowed-cause report in #1330: `oauth.probe` and
7+// `oauth.start` against a broken server used to reach agents only as
8+// "Internal tool error [id]", with the real OAuthProbeError/OAuthStartError
9+// cause visible solely in the daemon log. The user-actionable error contract
10+// now carries the authored message across the tool-dispatch boundary.
11+import { randomBytes } from "node:crypto";
12+import { createServer } from "node:http";
13+
14+import { expect } from "@effect/vitest";
15+import { Effect } from "effect";
16+import { composePluginApi } from "@executor-js/api/server";
17+import { mcpHttpPlugin } from "@executor-js/plugin-mcp/api";
18+import { IntegrationSlug, OAuthClientSlug } from "@executor-js/sdk/shared";
19+
20+import { scenario } from "../src/scenario";
21+import { Api, Mcp, Target } from "../src/services";
22+import type { McpSession } from "../src/surfaces/mcp";
23+
24+const api = composePluginApi([mcpHttpPlugin()] as const);
25+
26+const unique = (prefix: string) => `${prefix}-${randomBytes(4).toString("hex")}`;
27+
28+/** A server whose every response is 200 with a non-JSON body: RFC 9728/8414
29+ * discovery reaches it fine and then fails on malformed metadata — the "the
30+ * provider is broken, tell me HOW" case. */
31+const serveBrokenMetadata = () =>
32+ Effect.acquireRelease(
33+ Effect.callback<{ readonly url: string; readonly close: () => void }>((resume) => {
34+ const server = createServer((_request, response) => {
35+ response.writeHead(200, { "content-type": "application/json" });
36+ response.end("this is not metadata{");
37+ });
38+ server.listen(0, "127.0.0.1", () => {
39+ const address = server.address();
40+ const port = typeof address === "object" && address ? address.port : 0;
41+ resume(
42+ Effect.succeed({
43+ url: `http://127.0.0.1:${port}`,
44+ close: () => {
45+ server.close();
46+ server.closeAllConnections();
47+ },
48+ }),
49+ );
50+ });
51+ }),
52+ (server) => Effect.sync(server.close),
53+ );
54+
55+/** Run `execute`, auto-approving any paused (approval-gated) calls, and parse
56+ * the sandbox's JSON return value. */
57+const executeJson = (session: McpSession, code: string) =>
58+ Effect.gen(function* () {
59+ let result = yield* session.call("execute", { code });
60+ let guard = 0;
61+ while (result.text.includes("executionId:") && guard < 10) {
62+ result = yield* session.approvePaused(result.text);
63+ guard += 1;
64+ }
65+ expect(result.ok, `execute completed (got: ${result.text.slice(0, 400)})`).toBe(true);
66+ return JSON.parse(result.text) as {
67+ readonly ok: boolean;
68+ readonly code?: string;
69+ readonly message?: string;
70+ };
71+ });
72+
73+const probeCode = (url: string) => `
74+const result = await tools.executor.coreTools.oauth.probe({ url: ${JSON.stringify(url)} });
75+return result.ok
76+ ? { ok: true }
77+ : { ok: false, code: result.error.code, message: result.error.message };
78+`;
79+
80+const startCode = (input: {
81+ readonly client: string;
82+ readonly integration: string;
83+ readonly connection: string;
84+}) => `
85+const started = await tools.executor.coreTools.oauth.start({
86+ client: ${JSON.stringify(input.client)},
87+ clientOwner: "org",
88+ owner: "org",
89+ name: ${JSON.stringify(input.connection)},
90+ integration: ${JSON.stringify(input.integration)},
91+ template: "oauth2",
92+});
93+return started.ok
94+ ? { ok: true, status: started.data.status }
95+ : { ok: false, code: started.error.code, message: started.error.message };
96+`;
97+
98+// ---------------------------------------------------------------------------
99+// oauth.probe — a broken discovery endpoint reports WHAT failed.
100+// ---------------------------------------------------------------------------
101+
102+scenario(
103+ "OAuth setup · a failed oauth.probe surfaces the discovery cause, not an opaque internal error",
104+ {},
105+ Effect.scoped(
106+ Effect.gen(function* () {
107+ const target = yield* Target;
108+ const mcp = yield* Mcp;
109+ const identity = yield* target.newIdentity();
110+ const session = mcp.session(identity);
111+ const broken = yield* serveBrokenMetadata();
112+
113+ yield* session.listTools();
114+ const result = yield* executeJson(session, probeCode(`${broken.url}/mcp`));
115+
116+ expect(result.ok, "the probe against broken metadata fails").toBe(false);
117+ expect(result.code, "the failure carries the typed probe code").toBe("oauth_probe_error");
118+ expect(
119+ result.message,
120+ "the failure names the discovery problem, so the caller can act on it",
121+ ).toContain("metadata is malformed");
122+ expect(result.message, "the opaque defect mask is not used").not.toContain(
123+ "Internal tool error",
124+ );
125+ }),
126+ ),
127+);
128+
129+// ---------------------------------------------------------------------------
130+// oauth.start — a scope-discovery failure reports its cause chain.
131+// ---------------------------------------------------------------------------
132+
133+scenario(
134+ "OAuth setup · a failed oauth.start scope discovery surfaces its cause, not an opaque internal error",
135+ {},
136+ Effect.scoped(
137+ Effect.gen(function* () {
138+ const target = yield* Target;
139+ const { client: makeApiClient } = yield* Api;
140+ const mcp = yield* Mcp;
141+ const identity = yield* target.newIdentity();
142+ const session = mcp.session(identity);
143+ const client = yield* makeApiClient(api, identity);
144+ const broken = yield* serveBrokenMetadata();
145+
146+ // An OAuth2 MCP integration with NO declared scopes: `oauth.start` must
147+ // discover them from the server's metadata — which is broken.
148+ const slug = unique("oauth-cause-mcp");
149+ yield* client.mcp.addServer({
150+ payload: {
151+ transport: "remote",
152+ name: "Broken-metadata MCP",
153+ endpoint: `${broken.url}/mcp`,
154+ slug,
155+ authenticationTemplate: [{ kind: "oauth2" }],
156+ },
157+ });
158+ yield* Effect.addFinalizer(() =>
159+ client.mcp
160+ .removeServer({ params: { slug: IntegrationSlug.make(slug) } })
161+ .pipe(Effect.ignore),
162+ );
163+
164+ const clientSlug = OAuthClientSlug.make(unique("oauth-cause-app"));
165+ yield* client.oauth.createClient({
166+ payload: {
167+ owner: "org",
168+ slug: clientSlug,
169+ authorizationUrl: `${broken.url}/authorize`,
170+ tokenUrl: `${broken.url}/token`,
171+ grant: "authorization_code",
172+ clientId: "test-client",
173+ clientSecret: "test-secret",
174+ resource: `${broken.url}/mcp`,
175+ },
176+ });
177+ yield* Effect.addFinalizer(() =>
178+ client.oauth
179+ .removeClient({ params: { slug: clientSlug }, payload: { owner: "org" } })
180+ .pipe(Effect.ignore),
181+ );
182+
183+ yield* session.listTools();
184+ const result = yield* executeJson(
185+ session,
186+ startCode({ client: String(clientSlug), integration: slug, connection: "main" }),
187+ );
188+
189+ expect(result.ok, "the start against broken metadata fails").toBe(false);
190+ expect(result.code, "the failure carries the typed start code").toBe("oauth_start_error");
191+ expect(result.message, "the failure names the scope-discovery step that broke").toContain(
192+ "Failed to discover OAuth scopes",
193+ );
194+ expect(result.message, "the failure carries the discovery cause beneath it").toContain(
195+ "metadata is malformed",
196+ );
197+ expect(result.message, "the opaque defect mask is not used").not.toContain(
198+ "Internal tool error",
199+ );
200+ }),
201+ ),
202+);