History

Add regression e2e for surfaced OAuth setup failure causes

Rhys Sullivan committed 2 months ago

Commit 12ac1f41 parent 10e16a51 file · +202 −0Browse files

Unified · Split

e2e/scenarios/oauth-setup-failure-cause.test.ts+202−0
@@ -0,0 +1,202 @@
1// When OAuth integration setup fails — malformed discovery metadata, a broken
2// authorization server — the sandbox must see the typed failure's own message
3// (`oauth_probe_error` / `oauth_start_error` with the discovery cause), never
4// the scrubbed "Internal tool error [hex]" defect.
5//
6// Regression guard for the swallowed-cause report in #1330: `oauth.probe` and
7// `oauth.start` against a broken server used to reach agents only as
8// "Internal tool error [id]", with the real OAuthProbeError/OAuthStartError
9// cause visible solely in the daemon log. The user-actionable error contract
10// now carries the authored message across the tool-dispatch boundary.
11import { randomBytes } from "node:crypto";
12import { createServer } from "node:http";
13
14import { expect } from "@effect/vitest";
15import { Effect } from "effect";
16import { composePluginApi } from "@executor-js/api/server";
17import { mcpHttpPlugin } from "@executor-js/plugin-mcp/api";
18import { IntegrationSlug, OAuthClientSlug } from "@executor-js/sdk/shared";
19
20import { scenario } from "../src/scenario";
21import { Api, Mcp, Target } from "../src/services";
22import type { McpSession } from "../src/surfaces/mcp";
23
24const api = composePluginApi([mcpHttpPlugin()] as const);
25
26const unique = (prefix: string) => `${prefix}-${randomBytes(4).toString("hex")}`;
27
28/** A server whose every response is 200 with a non-JSON body: RFC 9728/8414
29 * discovery reaches it fine and then fails on malformed metadata — the "the
30 * provider is broken, tell me HOW" case. */
31const serveBrokenMetadata = () =>
32 Effect.acquireRelease(
33 Effect.callback<{ readonly url: string; readonly close: () => void }>((resume) => {
34 const server = createServer((_request, response) => {
35 response.writeHead(200, { "content-type": "application/json" });
36 response.end("this is not metadata{");
37 });
38 server.listen(0, "127.0.0.1", () => {
39 const address = server.address();
40 const port = typeof address === "object" && address ? address.port : 0;
41 resume(
42 Effect.succeed({
43 url: `http://127.0.0.1:${port}`,
44 close: () => {
45 server.close();
46 server.closeAllConnections();
47 },
48 }),
49 );
50 });
51 }),
52 (server) => Effect.sync(server.close),
53 );
54
55/** Run `execute`, auto-approving any paused (approval-gated) calls, and parse
56 * the sandbox's JSON return value. */
57const executeJson = (session: McpSession, code: string) =>
58 Effect.gen(function* () {
59 let result = yield* session.call("execute", { code });
60 let guard = 0;
61 while (result.text.includes("executionId:") && guard < 10) {
62 result = yield* session.approvePaused(result.text);
63 guard += 1;
64 }
65 expect(result.ok, `execute completed (got: ${result.text.slice(0, 400)})`).toBe(true);
66 return JSON.parse(result.text) as {
67 readonly ok: boolean;
68 readonly code?: string;
69 readonly message?: string;
70 };
71 });
72
73const probeCode = (url: string) => `
74const result = await tools.executor.coreTools.oauth.probe({ url: ${JSON.stringify(url)} });
75return result.ok
76 ? { ok: true }
77 : { ok: false, code: result.error.code, message: result.error.message };
78`;
79
80const startCode = (input: {
81 readonly client: string;
82 readonly integration: string;
83 readonly connection: string;
84}) => `
85const started = await tools.executor.coreTools.oauth.start({
86 client: ${JSON.stringify(input.client)},
87 clientOwner: "org",
88 owner: "org",
89 name: ${JSON.stringify(input.connection)},
90 integration: ${JSON.stringify(input.integration)},
91 template: "oauth2",
92});
93return started.ok
94 ? { ok: true, status: started.data.status }
95 : { ok: false, code: started.error.code, message: started.error.message };
96`;
97
98// ---------------------------------------------------------------------------
99// oauth.probe — a broken discovery endpoint reports WHAT failed.
100// ---------------------------------------------------------------------------
101
102scenario(
103 "OAuth setup · a failed oauth.probe surfaces the discovery cause, not an opaque internal error",
104 {},
105 Effect.scoped(
106 Effect.gen(function* () {
107 const target = yield* Target;
108 const mcp = yield* Mcp;
109 const identity = yield* target.newIdentity();
110 const session = mcp.session(identity);
111 const broken = yield* serveBrokenMetadata();
112
113 yield* session.listTools();
114 const result = yield* executeJson(session, probeCode(`${broken.url}/mcp`));
115
116 expect(result.ok, "the probe against broken metadata fails").toBe(false);
117 expect(result.code, "the failure carries the typed probe code").toBe("oauth_probe_error");
118 expect(
119 result.message,
120 "the failure names the discovery problem, so the caller can act on it",
121 ).toContain("metadata is malformed");
122 expect(result.message, "the opaque defect mask is not used").not.toContain(
123 "Internal tool error",
124 );
125 }),
126 ),
127);
128
129// ---------------------------------------------------------------------------
130// oauth.start — a scope-discovery failure reports its cause chain.
131// ---------------------------------------------------------------------------
132
133scenario(
134 "OAuth setup · a failed oauth.start scope discovery surfaces its cause, not an opaque internal error",
135 {},
136 Effect.scoped(
137 Effect.gen(function* () {
138 const target = yield* Target;
139 const { client: makeApiClient } = yield* Api;
140 const mcp = yield* Mcp;
141 const identity = yield* target.newIdentity();
142 const session = mcp.session(identity);
143 const client = yield* makeApiClient(api, identity);
144 const broken = yield* serveBrokenMetadata();
145
146 // An OAuth2 MCP integration with NO declared scopes: `oauth.start` must
147 // discover them from the server's metadata — which is broken.
148 const slug = unique("oauth-cause-mcp");
149 yield* client.mcp.addServer({
150 payload: {
151 transport: "remote",
152 name: "Broken-metadata MCP",
153 endpoint: `${broken.url}/mcp`,
154 slug,
155 authenticationTemplate: [{ kind: "oauth2" }],
156 },
157 });
158 yield* Effect.addFinalizer(() =>
159 client.mcp
160 .removeServer({ params: { slug: IntegrationSlug.make(slug) } })
161 .pipe(Effect.ignore),
162 );
163
164 const clientSlug = OAuthClientSlug.make(unique("oauth-cause-app"));
165 yield* client.oauth.createClient({
166 payload: {
167 owner: "org",
168 slug: clientSlug,
169 authorizationUrl: `${broken.url}/authorize`,
170 tokenUrl: `${broken.url}/token`,
171 grant: "authorization_code",
172 clientId: "test-client",
173 clientSecret: "test-secret",
174 resource: `${broken.url}/mcp`,
175 },
176 });
177 yield* Effect.addFinalizer(() =>
178 client.oauth
179 .removeClient({ params: { slug: clientSlug }, payload: { owner: "org" } })
180 .pipe(Effect.ignore),
181 );
182
183 yield* session.listTools();
184 const result = yield* executeJson(
185 session,
186 startCode({ client: String(clientSlug), integration: slug, connection: "main" }),
187 );
188
189 expect(result.ok, "the start against broken metadata fails").toBe(false);
190 expect(result.code, "the failure carries the typed start code").toBe("oauth_start_error");
191 expect(result.message, "the failure names the scope-discovery step that broke").toContain(
192 "Failed to discover OAuth scopes",
193 );
194 expect(result.message, "the failure carries the discovery cause beneath it").toContain(
195 "metadata is malformed",
196 );
197 expect(result.message, "the opaque defect mask is not used").not.toContain(
198 "Internal tool error",
199 );
200 }),
201 ),
202);