| @@ -0,0 +1,252 @@ |
| | 1 | // Repro for the user report: "OAuth custom-MCP tokens are stored in a |
| | 2 | // non-durable keychain on headless/sandbox daemons → connections silently die |
| | 3 | // on every restart/recreate." |
| | 4 | // |
| | 5 | // The local app's plugin order (apps/local/executor.config.ts) registers |
| | 6 | // keychainPlugin() BEFORE fileSecretsPlugin(), and defaultWritableProvider() |
| | 7 | // picks the first writable provider — so wherever the keyring write-probe |
| | 8 | // passes, every minted OAuth access/refresh token lands in the keychain |
| | 9 | // (connection.provider = "keychain"). On sandbox hosts that keychain is an |
| | 10 | // in-memory store: only EXECUTOR_DATA_DIR (data.db + auth.json) survives a |
| | 11 | // sandbox stop/recreate, and data.db holds just the item REFERENCE |
| | 12 | // (oauth:org:<slug>:default:refresh), not the secret. After recreate the |
| | 13 | // connection is a zombie: health says expired — "Stored refresh token could |
| | 14 | // not be resolved." — and the user must re-auth. Repeatedly. |
| | 15 | // |
| | 16 | // The scenario models the sandbox lifecycle with two real `executor web` boots |
| | 17 | // on ONE kept data dir (what the sandbox persists), wiping this run's keychain |
| | 18 | // service between them (what the sandbox loses): |
| | 19 | // |
| | 20 | // boot 1: OAuth-connect a custom MCP server (real authorization-code flow |
| | 21 | // against a live test AS) → connection is healthy. |
| | 22 | // stop/recreate: keychain wiped, EXECUTOR_DATA_DIR kept. |
| | 23 | // boot 2: the SAME connection must still be healthy — durable-by-default is |
| | 24 | // the product guarantee. With the bug, this is where the zombie |
| | 25 | // appears (expired: "Stored refresh token could not be resolved."). |
| | 26 | // |
| | 27 | // Note: the repro only bites where the keychain probe passes (macOS dev |
| | 28 | // machines, sandbox hosts with a kernel keyring). On headless CI without a |
| | 29 | // reachable keyring the keychain provider never registers and the file |
| | 30 | // provider is already the default — the scenario then passes vacuously, which |
| | 31 | // mirrors production reality. |
| | 32 | import { execFileSync } from "node:child_process"; |
| | 33 | import { randomBytes } from "node:crypto"; |
| | 34 | import { mkdtempSync, rmSync } from "node:fs"; |
| | 35 | import { tmpdir } from "node:os"; |
| | 36 | import { join } from "node:path"; |
| | 37 | |
| | 38 | import { expect } from "@effect/vitest"; |
| | 39 | import { Effect } from "effect"; |
| | 40 | import { HttpApiClient } from "effect/unstable/httpapi"; |
| | 41 | import { FetchHttpClient, HttpClient, HttpClientRequest } from "effect/unstable/http"; |
| | 42 | import { composePluginApi } from "@executor-js/api/server"; |
| | 43 | import { mcpHttpPlugin } from "@executor-js/plugin-mcp/api"; |
| | 44 | import { makeEchoMcpServer, serveMcpServer } from "@executor-js/plugin-mcp/testing"; |
| | 45 | import { |
| | 46 | AuthTemplateSlug, |
| | 47 | ConnectionName, |
| | 48 | IntegrationSlug, |
| | 49 | OAuthClientSlug, |
| | 50 | } from "@executor-js/sdk/shared"; |
| | 51 | import { serveOAuthTestServer } from "@executor-js/sdk/testing"; |
| | 52 | |
| | 53 | import { scenario } from "../src/scenario"; |
| | 54 | import { Cli, RunDir } from "../src/services"; |
| | 55 | import { withLocalServer, type ServerHandle } from "./local-server"; |
| | 56 | |
| | 57 | const api = composePluginApi([mcpHttpPlugin()] as const); |
| | 58 | |
| | 59 | const name = ConnectionName.make("default"); |
| | 60 | const template = AuthTemplateSlug.make("oauth2"); |
| | 61 | |
| | 62 | /** Drive the test AS's consent by hand (authorize → login → code), without |
| | 63 | * visiting the app's callback: `oauth.complete` takes the code directly. */ |
| | 64 | const completeAuthorization = (authorizationUrl: string) => |
| | 65 | Effect.promise(async () => { |
| | 66 | const authorize = await fetch(authorizationUrl, { redirect: "manual" }); |
| | 67 | const loginUrl = authorize.headers.get("location"); |
| | 68 | if (!loginUrl) throw new Error(`authorize did not redirect: ${authorize.status}`); |
| | 69 | const login = await fetch(loginUrl, { |
| | 70 | method: "POST", |
| | 71 | headers: { authorization: `Basic ${Buffer.from("alice:password").toString("base64")}` }, |
| | 72 | redirect: "manual", |
| | 73 | }); |
| | 74 | const callbackUrl = login.headers.get("location"); |
| | 75 | if (!callbackUrl) throw new Error(`login did not redirect: ${login.status}`); |
| | 76 | const code = new URL(callbackUrl).searchParams.get("code"); |
| | 77 | if (!code) throw new Error("callback carried no authorization code"); |
| | 78 | return code; |
| | 79 | }); |
| | 80 | |
| | 81 | /** Simulate the non-durable secret store being lost on sandbox recreate: drop |
| | 82 | * every entry under this run's keychain service. On macOS `security` deletes |
| | 83 | * one matching item per call, so loop until none remain. When the fix routes |
| | 84 | * tokens to the file provider this is a no-op — nothing lives there. */ |
| | 85 | const wipeKeychainService = (serviceName: string): void => { |
| | 86 | if (process.platform !== "darwin") return; |
| | 87 | for (let i = 0; i < 32; i++) { |
| | 88 | try { |
| | 89 | execFileSync("security", ["delete-generic-password", "-s", serviceName], { |
| | 90 | stdio: "ignore", |
| | 91 | }); |
| | 92 | } catch { |
| | 93 | return; // no more entries under this service |
| | 94 | } |
| | 95 | } |
| | 96 | }; |
| | 97 | |
| | 98 | const apiClient = (server: ServerHandle) => |
| | 99 | HttpApiClient.make(api, { |
| | 100 | baseUrl: new URL("/api", server.origin).toString(), |
| | 101 | transformClient: HttpClient.mapRequest((request) => |
| | 102 | HttpClientRequest.setHeader(request, "authorization", `Bearer ${server.token}`), |
| | 103 | ), |
| | 104 | }).pipe(Effect.provide(FetchHttpClient.layer)); |
| | 105 | |
| | 106 | scenario( |
| | 107 | "Local · an OAuth MCP connection survives a sandbox stop/recreate (data dir kept, keychain lost)", |
| | 108 | // Two full `executor web` boots; each allows up to 240s for a cold vite |
| | 109 | // start, so the test deadline must clear both plus the OAuth journey. |
| | 110 | { timeout: 600_000 }, |
| | 111 | Effect.scoped( |
| | 112 | Effect.gen(function* () { |
| | 113 | const cli = yield* Cli; |
| | 114 | const runDir = yield* RunDir; |
| | 115 | |
| | 116 | // The AS mints instantly-expiring access tokens so EVERY health check |
| | 117 | // exercises the refresh path — the exact path that dies when the stored |
| | 118 | // refresh token evaporates with the keychain. |
| | 119 | const oauth = yield* serveOAuthTestServer({ |
| | 120 | scopes: ["mcp.read"], |
| | 121 | tokenExpiresInSeconds: 0, |
| | 122 | }); |
| | 123 | // A real MCP upstream that accepts exactly the AS's issued bearers, so |
| | 124 | // "healthy" is a genuine dial-and-list verdict, not a stub's opinion. |
| | 125 | const mcp = yield* serveMcpServer(() => makeEchoMcpServer({ name: "durability-mcp" }), { |
| | 126 | auth: { |
| | 127 | validateAuthorization: (authorization) => oauth.acceptsAuthorizationHeader(authorization), |
| | 128 | }, |
| | 129 | }); |
| | 130 | |
| | 131 | const suffix = randomBytes(4).toString("hex"); |
| | 132 | const slug = IntegrationSlug.make(`oauth-durability-${suffix}`); |
| | 133 | const clientSlug = OAuthClientSlug.make(`oauth-durability-client-${suffix}`); |
| | 134 | // A unique keychain service per run: the wipe below can only ever touch |
| | 135 | // this scenario's entries, and leftovers never collide across runs. |
| | 136 | const keychainService = `executor-e2e-durability-${suffix}`; |
| | 137 | // The data dir is the durable surface (what a sandbox backs up); it is |
| | 138 | // deliberately KEPT across the two boots and cleaned only at the end. |
| | 139 | const dataDir = mkdtempSync(join(tmpdir(), "executor-durability-e2e-")); |
| | 140 | const serverEnv = { EXECUTOR_KEYCHAIN_SERVICE_NAME: keychainService }; |
| | 141 | |
| | 142 | yield* Effect.ensuring( |
| | 143 | Effect.gen(function* () { |
| | 144 | // ---- Boot 1: connect the OAuth MCP integration. ------------------- |
| | 145 | yield* withLocalServer( |
| | 146 | cli, |
| | 147 | runDir, |
| | 148 | (server) => |
| | 149 | Effect.gen(function* () { |
| | 150 | const client = yield* apiClient(server); |
| | 151 | |
| | 152 | yield* client.mcp.addServer({ |
| | 153 | payload: { |
| | 154 | transport: "remote", |
| | 155 | name: "Durability MCP", |
| | 156 | endpoint: mcp.url, |
| | 157 | slug: String(slug), |
| | 158 | authenticationTemplate: [{ kind: "oauth2" }], |
| | 159 | }, |
| | 160 | }); |
| | 161 | yield* client.oauth.createClient({ |
| | 162 | payload: { |
| | 163 | owner: "org", |
| | 164 | slug: clientSlug, |
| | 165 | grant: "authorization_code", |
| | 166 | authorizationUrl: oauth.authorizationEndpoint, |
| | 167 | tokenUrl: oauth.tokenEndpoint, |
| | 168 | clientId: "test-client", |
| | 169 | clientSecret: "test-secret", |
| | 170 | // MCP oauth2 templates discover scopes via the protected |
| | 171 | // resource's metadata; the client must name that resource. |
| | 172 | resource: oauth.mcpResourceUrl, |
| | 173 | originIntegration: slug, |
| | 174 | }, |
| | 175 | }); |
| | 176 | |
| | 177 | const started = yield* client.oauth.start({ |
| | 178 | payload: { |
| | 179 | client: clientSlug, |
| | 180 | clientOwner: "org", |
| | 181 | owner: "org", |
| | 182 | name, |
| | 183 | integration: slug, |
| | 184 | template, |
| | 185 | }, |
| | 186 | }); |
| | 187 | expect(started.status, "oauth.start redirects to the authorization server").toBe( |
| | 188 | "redirect", |
| | 189 | ); |
| | 190 | if (started.status !== "redirect") return yield* Effect.die("no redirect"); |
| | 191 | const code = yield* completeAuthorization(started.authorizationUrl); |
| | 192 | yield* client.oauth.complete({ payload: { state: started.state, code } }); |
| | 193 | |
| | 194 | const connection = yield* client.connections.get({ |
| | 195 | params: { owner: "org", integration: slug, name }, |
| | 196 | }); |
| | 197 | // Where did the minted tokens actually land? This is the root |
| | 198 | // cause made visible: "keychain" = non-durable on sandbox hosts. |
| | 199 | console.info( |
| | 200 | `[durability repro] minted connection provider: ${String(connection.provider)}`, |
| | 201 | ); |
| | 202 | |
| | 203 | const healthy = yield* client.connections.checkHealth({ |
| | 204 | params: { owner: "org", integration: slug, name }, |
| | 205 | query: {}, |
| | 206 | }); |
| | 207 | expect( |
| | 208 | healthy.status, |
| | 209 | `the fresh OAuth connection is healthy (detail: ${healthy.detail ?? "-"})`, |
| | 210 | ).toBe("healthy"); |
| | 211 | }), |
| | 212 | { dataDir, env: serverEnv, castName: "terminal-boot1.cast" }, |
| | 213 | ); |
| | 214 | |
| | 215 | // ---- Sandbox stop/recreate: keychain gone, data dir kept. --------- |
| | 216 | wipeKeychainService(keychainService); |
| | 217 | |
| | 218 | // ---- Boot 2: the connection must still work. ---------------------- |
| | 219 | yield* withLocalServer( |
| | 220 | cli, |
| | 221 | runDir, |
| | 222 | (server) => |
| | 223 | Effect.gen(function* () { |
| | 224 | const client = yield* apiClient(server); |
| | 225 | |
| | 226 | const health = yield* client.connections.checkHealth({ |
| | 227 | params: { owner: "org", integration: slug, name }, |
| | 228 | query: {}, |
| | 229 | }); |
| | 230 | console.info( |
| | 231 | `[durability repro] post-recreate health: ${health.status} — ${health.detail ?? "-"}`, |
| | 232 | ); |
| | 233 | // THE guarantee: OAuth credentials live in the durable store |
| | 234 | // (EXECUTOR_DATA_DIR), so losing the host's ephemeral keychain |
| | 235 | // must not kill the connection. With the bug this reads |
| | 236 | // expired: "Stored refresh token could not be resolved." |
| | 237 | expect( |
| | 238 | health.status, |
| | 239 | `the OAuth connection survives a sandbox recreate (detail: ${health.detail ?? "-"})`, |
| | 240 | ).toBe("healthy"); |
| | 241 | }), |
| | 242 | { dataDir, env: serverEnv, castName: "terminal-boot2.cast" }, |
| | 243 | ); |
| | 244 | }), |
| | 245 | Effect.sync(() => { |
| | 246 | wipeKeychainService(keychainService); |
| | 247 | rmSync(dataDir, { recursive: true, force: true }); |
| | 248 | }), |
| | 249 | ); |
| | 250 | }), |
| | 251 | ), |
| | 252 | ); |