History

Default OAuth tokens to the durable file secret store on local

Rhys Sullivan committed 3 months ago

Commit 1e60a3e1 parent fc1e5893 files · +275 −4Browse files

Unified · Split

3 files
  1. apps/local/executor.config.ts +6 −1
  2. e2e/local/local-server.ts +17 −3
  3. e2e/local/oauth-token-durability.test.ts +252 −0
apps/local/executor.config.ts+6−1
@@ -50,8 +50,13 @@ export default defineExecutorConfig({ · 49 unmodified lines
5050 allowPrivateGitHosts: true,
5151 }),
5252 toolkitsPlugin({ activeToolkitSlug }),
53− keychainPlugin(),
53+ // The durable file store must register before keychain: the first
54+ // writable provider becomes the default for minted OAuth tokens, and on
55+ // sandbox/headless hosts the keychain is an in-memory keyring that a
56+ // stop/recreate wipes while only EXECUTOR_DATA_DIR is persisted.
57+ // Keychain stays registered for explicit external refs.
5458 fileSecretsPlugin(),
59+ keychainPlugin(),
5560 onepasswordHttpPlugin(),
5661 desktopSettingsPlugin({
5762 webBaseUrl:
e2e/local/local-server.ts+17−3
@@ -43,10 +43,18 @@ export const withLocalServer = ( · 42 unmodified lines
4343 /** Extra env merged into the `executor web` process, e.g. to force a
4444 * published-version signal for the update check. */
4545 readonly env?: Record<string, string>;
46+ /** Reuse an existing data dir instead of a throwaway one (and keep it on
47+ * exit) — for durability scenarios that boot the server twice against the
48+ * same persisted state. The caller owns cleanup. */
49+ readonly dataDir?: string;
50+ /** Terminal cast filename (default "terminal.cast") so multi-boot
51+ * scenarios keep one recording per boot. */
52+ readonly castName?: string;
4653 },
4754 ): Effect.Effect<void> =>
4855 Effect.gen(function* () {
49− const dataDir = mkdtempSync(join(tmpdir(), "executor-local-e2e-"));
56+ const ownsDataDir = options?.dataDir === undefined;
57+ const dataDir = options?.dataDir ?? mkdtempSync(join(tmpdir(), "executor-local-e2e-"));
5058
5159 let publishUrl!: (url: string) => void;
5260 const urlReady = new Promise<string>((res) => {
@@ -103,7 +111,7 @@ export const withLocalServer = ( · 50 unmodified lines
103111 {
104112 cwd: repoRoot,
105113 env: { EXECUTOR_DATA_DIR: dataDir, EXECUTOR_SCOPE_DIR: dataDir, ...options?.env },
106− record: join(runDir, "terminal.cast"),
114+ record: join(runDir, options?.castName ?? "terminal.cast"),
107115 viewport: { cols: 120, rows: 40 },
108116 },
109117 ),
@@ -119,5 +127,11 @@ export const withLocalServer = ( · 9 unmodified lines
119127 }),
120128 ],
121129 { concurrency: "unbounded" },
122− ).pipe(Effect.ensuring(Effect.sync(() => rmSync(dataDir, { recursive: true, force: true }))));
130+ ).pipe(
131+ Effect.ensuring(
132+ ownsDataDir
133+ ? Effect.sync(() => rmSync(dataDir, { recursive: true, force: true }))
134+ : Effect.void,
135+ ),
136+ );
123137 });
e2e/local/oauth-token-durability.test.ts+252−0
@@ -0,0 +1,252 @@
1+// Repro for the user report: "OAuth custom-MCP tokens are stored in a
2+// non-durable keychain on headless/sandbox daemons → connections silently die
3+// on every restart/recreate."
4+//
5+// The local app's plugin order (apps/local/executor.config.ts) registers
6+// keychainPlugin() BEFORE fileSecretsPlugin(), and defaultWritableProvider()
7+// picks the first writable provider — so wherever the keyring write-probe
8+// passes, every minted OAuth access/refresh token lands in the keychain
9+// (connection.provider = "keychain"). On sandbox hosts that keychain is an
10+// in-memory store: only EXECUTOR_DATA_DIR (data.db + auth.json) survives a
11+// sandbox stop/recreate, and data.db holds just the item REFERENCE
12+// (oauth:org:<slug>:default:refresh), not the secret. After recreate the
13+// connection is a zombie: health says expired — "Stored refresh token could
14+// not be resolved." — and the user must re-auth. Repeatedly.
15+//
16+// The scenario models the sandbox lifecycle with two real `executor web` boots
17+// on ONE kept data dir (what the sandbox persists), wiping this run's keychain
18+// service between them (what the sandbox loses):
19+//
20+// boot 1: OAuth-connect a custom MCP server (real authorization-code flow
21+// against a live test AS) → connection is healthy.
22+// stop/recreate: keychain wiped, EXECUTOR_DATA_DIR kept.
23+// boot 2: the SAME connection must still be healthy — durable-by-default is
24+// the product guarantee. With the bug, this is where the zombie
25+// appears (expired: "Stored refresh token could not be resolved.").
26+//
27+// Note: the repro only bites where the keychain probe passes (macOS dev
28+// machines, sandbox hosts with a kernel keyring). On headless CI without a
29+// reachable keyring the keychain provider never registers and the file
30+// provider is already the default — the scenario then passes vacuously, which
31+// mirrors production reality.
32+import { execFileSync } from "node:child_process";
33+import { randomBytes } from "node:crypto";
34+import { mkdtempSync, rmSync } from "node:fs";
35+import { tmpdir } from "node:os";
36+import { join } from "node:path";
37+
38+import { expect } from "@effect/vitest";
39+import { Effect } from "effect";
40+import { HttpApiClient } from "effect/unstable/httpapi";
41+import { FetchHttpClient, HttpClient, HttpClientRequest } from "effect/unstable/http";
42+import { composePluginApi } from "@executor-js/api/server";
43+import { mcpHttpPlugin } from "@executor-js/plugin-mcp/api";
44+import { makeEchoMcpServer, serveMcpServer } from "@executor-js/plugin-mcp/testing";
45+import {
46+ AuthTemplateSlug,
47+ ConnectionName,
48+ IntegrationSlug,
49+ OAuthClientSlug,
50+} from "@executor-js/sdk/shared";
51+import { serveOAuthTestServer } from "@executor-js/sdk/testing";
52+
53+import { scenario } from "../src/scenario";
54+import { Cli, RunDir } from "../src/services";
55+import { withLocalServer, type ServerHandle } from "./local-server";
56+
57+const api = composePluginApi([mcpHttpPlugin()] as const);
58+
59+const name = ConnectionName.make("default");
60+const template = AuthTemplateSlug.make("oauth2");
61+
62+/** Drive the test AS's consent by hand (authorize → login → code), without
63+ * visiting the app's callback: `oauth.complete` takes the code directly. */
64+const completeAuthorization = (authorizationUrl: string) =>
65+ Effect.promise(async () => {
66+ const authorize = await fetch(authorizationUrl, { redirect: "manual" });
67+ const loginUrl = authorize.headers.get("location");
68+ if (!loginUrl) throw new Error(`authorize did not redirect: ${authorize.status}`);
69+ const login = await fetch(loginUrl, {
70+ method: "POST",
71+ headers: { authorization: `Basic ${Buffer.from("alice:password").toString("base64")}` },
72+ redirect: "manual",
73+ });
74+ const callbackUrl = login.headers.get("location");
75+ if (!callbackUrl) throw new Error(`login did not redirect: ${login.status}`);
76+ const code = new URL(callbackUrl).searchParams.get("code");
77+ if (!code) throw new Error("callback carried no authorization code");
78+ return code;
79+ });
80+
81+/** Simulate the non-durable secret store being lost on sandbox recreate: drop
82+ * every entry under this run's keychain service. On macOS `security` deletes
83+ * one matching item per call, so loop until none remain. When the fix routes
84+ * tokens to the file provider this is a no-op — nothing lives there. */
85+const wipeKeychainService = (serviceName: string): void => {
86+ if (process.platform !== "darwin") return;
87+ for (let i = 0; i < 32; i++) {
88+ try {
89+ execFileSync("security", ["delete-generic-password", "-s", serviceName], {
90+ stdio: "ignore",
91+ });
92+ } catch {
93+ return; // no more entries under this service
94+ }
95+ }
96+};
97+
98+const apiClient = (server: ServerHandle) =>
99+ HttpApiClient.make(api, {
100+ baseUrl: new URL("/api", server.origin).toString(),
101+ transformClient: HttpClient.mapRequest((request) =>
102+ HttpClientRequest.setHeader(request, "authorization", `Bearer ${server.token}`),
103+ ),
104+ }).pipe(Effect.provide(FetchHttpClient.layer));
105+
106+scenario(
107+ "Local · an OAuth MCP connection survives a sandbox stop/recreate (data dir kept, keychain lost)",
108+ // Two full `executor web` boots; each allows up to 240s for a cold vite
109+ // start, so the test deadline must clear both plus the OAuth journey.
110+ { timeout: 600_000 },
111+ Effect.scoped(
112+ Effect.gen(function* () {
113+ const cli = yield* Cli;
114+ const runDir = yield* RunDir;
115+
116+ // The AS mints instantly-expiring access tokens so EVERY health check
117+ // exercises the refresh path — the exact path that dies when the stored
118+ // refresh token evaporates with the keychain.
119+ const oauth = yield* serveOAuthTestServer({
120+ scopes: ["mcp.read"],
121+ tokenExpiresInSeconds: 0,
122+ });
123+ // A real MCP upstream that accepts exactly the AS's issued bearers, so
124+ // "healthy" is a genuine dial-and-list verdict, not a stub's opinion.
125+ const mcp = yield* serveMcpServer(() => makeEchoMcpServer({ name: "durability-mcp" }), {
126+ auth: {
127+ validateAuthorization: (authorization) => oauth.acceptsAuthorizationHeader(authorization),
128+ },
129+ });
130+
131+ const suffix = randomBytes(4).toString("hex");
132+ const slug = IntegrationSlug.make(`oauth-durability-${suffix}`);
133+ const clientSlug = OAuthClientSlug.make(`oauth-durability-client-${suffix}`);
134+ // A unique keychain service per run: the wipe below can only ever touch
135+ // this scenario's entries, and leftovers never collide across runs.
136+ const keychainService = `executor-e2e-durability-${suffix}`;
137+ // The data dir is the durable surface (what a sandbox backs up); it is
138+ // deliberately KEPT across the two boots and cleaned only at the end.
139+ const dataDir = mkdtempSync(join(tmpdir(), "executor-durability-e2e-"));
140+ const serverEnv = { EXECUTOR_KEYCHAIN_SERVICE_NAME: keychainService };
141+
142+ yield* Effect.ensuring(
143+ Effect.gen(function* () {
144+ // ---- Boot 1: connect the OAuth MCP integration. -------------------
145+ yield* withLocalServer(
146+ cli,
147+ runDir,
148+ (server) =>
149+ Effect.gen(function* () {
150+ const client = yield* apiClient(server);
151+
152+ yield* client.mcp.addServer({
153+ payload: {
154+ transport: "remote",
155+ name: "Durability MCP",
156+ endpoint: mcp.url,
157+ slug: String(slug),
158+ authenticationTemplate: [{ kind: "oauth2" }],
159+ },
160+ });
161+ yield* client.oauth.createClient({
162+ payload: {
163+ owner: "org",
164+ slug: clientSlug,
165+ grant: "authorization_code",
166+ authorizationUrl: oauth.authorizationEndpoint,
167+ tokenUrl: oauth.tokenEndpoint,
168+ clientId: "test-client",
169+ clientSecret: "test-secret",
170+ // MCP oauth2 templates discover scopes via the protected
171+ // resource's metadata; the client must name that resource.
172+ resource: oauth.mcpResourceUrl,
173+ originIntegration: slug,
174+ },
175+ });
176+
177+ const started = yield* client.oauth.start({
178+ payload: {
179+ client: clientSlug,
180+ clientOwner: "org",
181+ owner: "org",
182+ name,
183+ integration: slug,
184+ template,
185+ },
186+ });
187+ expect(started.status, "oauth.start redirects to the authorization server").toBe(
188+ "redirect",
189+ );
190+ if (started.status !== "redirect") return yield* Effect.die("no redirect");
191+ const code = yield* completeAuthorization(started.authorizationUrl);
192+ yield* client.oauth.complete({ payload: { state: started.state, code } });
193+
194+ const connection = yield* client.connections.get({
195+ params: { owner: "org", integration: slug, name },
196+ });
197+ // Where did the minted tokens actually land? This is the root
198+ // cause made visible: "keychain" = non-durable on sandbox hosts.
199+ console.info(
200+ `[durability repro] minted connection provider: ${String(connection.provider)}`,
201+ );
202+
203+ const healthy = yield* client.connections.checkHealth({
204+ params: { owner: "org", integration: slug, name },
205+ query: {},
206+ });
207+ expect(
208+ healthy.status,
209+ `the fresh OAuth connection is healthy (detail: ${healthy.detail ?? "-"})`,
210+ ).toBe("healthy");
211+ }),
212+ { dataDir, env: serverEnv, castName: "terminal-boot1.cast" },
213+ );
214+
215+ // ---- Sandbox stop/recreate: keychain gone, data dir kept. ---------
216+ wipeKeychainService(keychainService);
217+
218+ // ---- Boot 2: the connection must still work. ----------------------
219+ yield* withLocalServer(
220+ cli,
221+ runDir,
222+ (server) =>
223+ Effect.gen(function* () {
224+ const client = yield* apiClient(server);
225+
226+ const health = yield* client.connections.checkHealth({
227+ params: { owner: "org", integration: slug, name },
228+ query: {},
229+ });
230+ console.info(
231+ `[durability repro] post-recreate health: ${health.status} — ${health.detail ?? "-"}`,
232+ );
233+ // THE guarantee: OAuth credentials live in the durable store
234+ // (EXECUTOR_DATA_DIR), so losing the host's ephemeral keychain
235+ // must not kill the connection. With the bug this reads
236+ // expired: "Stored refresh token could not be resolved."
237+ expect(
238+ health.status,
239+ `the OAuth connection survives a sandbox recreate (detail: ${health.detail ?? "-"})`,
240+ ).toBe("healthy");
241+ }),
242+ { dataDir, env: serverEnv, castName: "terminal-boot2.cast" },
243+ );
244+ }),
245+ Effect.sync(() => {
246+ wipeKeychainService(keychainService);
247+ rmSync(dataDir, { recursive: true, force: true });
248+ }),
249+ );
250+ }),
251+ ),
252+);