// Repro for the user report: "OAuth custom-MCP tokens are stored in a// non-durable keychain on headless/sandbox daemons → connections silently die// on every restart/recreate."//// The local app's plugin order (apps/local/executor.config.ts) registers// keychainPlugin() BEFORE fileSecretsPlugin(), and defaultWritableProvider()// picks the first writable provider — so wherever the keyring write-probe// passes, every minted OAuth access/refresh token lands in the keychain// (connection.provider = "keychain"). On sandbox hosts that keychain is an// in-memory store: only EXECUTOR_DATA_DIR (data.db + auth.json) survives a// sandbox stop/recreate, and data.db holds just the item REFERENCE// (oauth:org:<slug>:default:refresh), not the secret. After recreate the// connection is a zombie: health says expired — "Stored refresh token could// not be resolved." — and the user must re-auth. Repeatedly.//// The scenario models the sandbox lifecycle with two real `executor web` boots// on ONE kept data dir (what the sandbox persists), wiping this run's keychain// service between them (what the sandbox loses)://// boot 1: OAuth-connect a custom MCP server (real authorization-code flow// against a live test AS) → connection is healthy.// stop/recreate: keychain wiped, EXECUTOR_DATA_DIR kept.// boot 2: the SAME connection must still be healthy — durable-by-default is// the product guarantee. With the bug, this is where the zombie// appears (expired: "Stored refresh token could not be resolved.").//// Note: the repro only bites where the keychain probe passes (macOS dev// machines, sandbox hosts with a kernel keyring). On headless CI without a// reachable keyring the keychain provider never registers and the file// provider is already the default — the scenario then passes vacuously, which// mirrors production reality.import { execFileSync } from "node:child_process";import { randomBytes } from "node:crypto";import { mkdtempSync, rmSync } from "node:fs";import { tmpdir } from "node:os";import { join } from "node:path"; import { expect } from "@effect/vitest";import { Effect } from "effect";import { HttpApiClient } from "effect/unstable/httpapi";import { FetchHttpClient, HttpClient, HttpClientRequest } from "effect/unstable/http";import { composePluginApi } from "@executor-js/api/server";import { mcpHttpPlugin } from "@executor-js/plugin-mcp/api";import { makeEchoMcpServer, serveMcpServer } from "@executor-js/plugin-mcp/testing";import { AuthTemplateSlug, ConnectionName, IntegrationSlug, OAuthClientSlug,} from "@executor-js/sdk/shared";import { serveOAuthTestServer } from "@executor-js/sdk/testing"; import { scenario } from "../src/scenario";import { Cli, RunDir } from "../src/services";import { withLocalServer, type ServerHandle } from "./local-server"; const api = composePluginApi([mcpHttpPlugin()] as const); const name = ConnectionName.make("default");const template = AuthTemplateSlug.make("oauth2"); /** Drive the test AS's consent by hand (authorize → login → code), without * visiting the app's callback: `oauth.complete` takes the code directly. */const completeAuthorization = (authorizationUrl: string) => Effect.promise(async () => { const authorize = await fetch(authorizationUrl, { redirect: "manual" }); const loginUrl = authorize.headers.get("location"); if (!loginUrl) throw new Error(`authorize did not redirect: ${authorize.status}`); const login = await fetch(loginUrl, { method: "POST", headers: { authorization: `Basic ${Buffer.from("alice:password").toString("base64")}` }, redirect: "manual", }); const callbackUrl = login.headers.get("location"); if (!callbackUrl) throw new Error(`login did not redirect: ${login.status}`); const code = new URL(callbackUrl).searchParams.get("code"); if (!code) throw new Error("callback carried no authorization code"); return code; }); /** Simulate the non-durable secret store being lost on sandbox recreate: drop * every entry under this run's keychain service. On macOS `security` deletes * one matching item per call, so loop until none remain. When the fix routes * tokens to the file provider this is a no-op — nothing lives there. */const wipeKeychainService = (serviceName: string): void => { if (process.platform !== "darwin") return; for (let i = 0; i < 32; i++) { try { execFileSync("security", ["delete-generic-password", "-s", serviceName], { stdio: "ignore", }); } catch { return; // no more entries under this service } }}; const apiClient = (server: ServerHandle) => HttpApiClient.make(api, { baseUrl: new URL("/api", server.origin).toString(), transformClient: HttpClient.mapRequest((request) => HttpClientRequest.setHeader(request, "authorization", `Bearer ${server.token}`), ), }).pipe(Effect.provide(FetchHttpClient.layer)); scenario( "Local · an OAuth MCP connection survives a sandbox stop/recreate (data dir kept, keychain lost)", // Two full `executor web` boots; each allows up to 240s for a cold vite // start, so the test deadline must clear both plus the OAuth journey. { timeout: 600_000 }, Effect.scoped( Effect.gen(function* () { const cli = yield* Cli; const runDir = yield* RunDir; // The AS mints instantly-expiring access tokens so EVERY health check // exercises the refresh path — the exact path that dies when the stored // refresh token evaporates with the keychain. const oauth = yield* serveOAuthTestServer({ scopes: ["mcp.read"], tokenExpiresInSeconds: 0, }); // A real MCP upstream that accepts exactly the AS's issued bearers, so // "healthy" is a genuine dial-and-list verdict, not a stub's opinion. const mcp = yield* serveMcpServer(() => makeEchoMcpServer({ name: "durability-mcp" }), { auth: { validateAuthorization: (authorization) => oauth.acceptsAuthorizationHeader(authorization), }, }); const suffix = randomBytes(4).toString("hex"); const slug = IntegrationSlug.make(`oauth-durability-${suffix}`); const clientSlug = OAuthClientSlug.make(`oauth-durability-client-${suffix}`); // A unique keychain service per run: the wipe below can only ever touch // this scenario's entries, and leftovers never collide across runs. const keychainService = `executor-e2e-durability-${suffix}`; // The data dir is the durable surface (what a sandbox backs up); it is // deliberately KEPT across the two boots and cleaned only at the end. const dataDir = mkdtempSync(join(tmpdir(), "executor-durability-e2e-")); const serverEnv = { EXECUTOR_KEYCHAIN_SERVICE_NAME: keychainService }; yield* Effect.ensuring( Effect.gen(function* () { // ---- Boot 1: connect the OAuth MCP integration. ------------------- yield* withLocalServer( cli, runDir, (server) => Effect.gen(function* () { const client = yield* apiClient(server); yield* client.mcp.addServer({ payload: { transport: "remote", name: "Durability MCP", endpoint: mcp.url, slug: String(slug), authenticationTemplate: [{ kind: "oauth2" }], }, }); yield* client.oauth.createClient({ payload: { owner: "org", slug: clientSlug, grant: "authorization_code", authorizationUrl: oauth.authorizationEndpoint, tokenUrl: oauth.tokenEndpoint, clientId: "test-client", clientSecret: "test-secret", // MCP oauth2 templates discover scopes via the protected // resource's metadata; the client must name that resource. resource: oauth.mcpResourceUrl, originIntegration: slug, }, }); const started = yield* client.oauth.start({ payload: { client: clientSlug, clientOwner: "org", owner: "org", name, integration: slug, template, }, }); expect(started.status, "oauth.start redirects to the authorization server").toBe( "redirect", ); if (started.status !== "redirect") return yield* Effect.die("no redirect"); const code = yield* completeAuthorization(started.authorizationUrl); yield* client.oauth.complete({ payload: { state: started.state, code } }); const connection = yield* client.connections.get({ params: { owner: "org", integration: slug, name }, }); // Where did the minted tokens actually land? This is the root // cause made visible: "keychain" = non-durable on sandbox hosts. console.info( `[durability repro] minted connection provider: ${String(connection.provider)}`, ); const healthy = yield* client.connections.checkHealth({ params: { owner: "org", integration: slug, name }, query: {}, }); expect( healthy.status, `the fresh OAuth connection is healthy (detail: ${healthy.detail ?? "-"})`, ).toBe("healthy"); }), { dataDir, env: serverEnv, castName: "terminal-boot1.cast" }, ); // ---- Sandbox stop/recreate: keychain gone, data dir kept. --------- wipeKeychainService(keychainService); // ---- Boot 2: the connection must still work. ---------------------- yield* withLocalServer( cli, runDir, (server) => Effect.gen(function* () { const client = yield* apiClient(server); const health = yield* client.connections.checkHealth({ params: { owner: "org", integration: slug, name }, query: {}, }); console.info( `[durability repro] post-recreate health: ${health.status} — ${health.detail ?? "-"}`, ); // THE guarantee: OAuth credentials live in the durable store // (EXECUTOR_DATA_DIR), so losing the host's ephemeral keychain // must not kill the connection. With the bug this reads // expired: "Stored refresh token could not be resolved." expect( health.status, `the OAuth connection survives a sandbox recreate (detail: ${health.detail ?? "-"})`, ).toBe("healthy"); }), { dataDir, env: serverEnv, castName: "terminal-boot2.cast" }, ); }), Effect.sync(() => { wipeKeychainService(keychainService); rmSync(dataDir, { recursive: true, force: true }); }), ); }), ),);
// Repro for the user report: "OAuth custom-MCP tokens are stored in a
// non-durable keychain on headless/sandbox daemons → connections silently die
// on every restart/recreate."
//
// The local app's plugin order (apps/local/executor.config.ts) registers
// keychainPlugin() BEFORE fileSecretsPlugin(), and defaultWritableProvider()
// picks the first writable provider — so wherever the keyring write-probe
// passes, every minted OAuth access/refresh token lands in the keychain
// (connection.provider = "keychain"). On sandbox hosts that keychain is an
// in-memory store: only EXECUTOR_DATA_DIR (data.db + auth.json) survives a
// sandbox stop/recreate, and data.db holds just the item REFERENCE
// (oauth:org:<slug>:default:refresh), not the secret. After recreate the
// connection is a zombie: health says expired — "Stored refresh token could
// not be resolved." — and the user must re-auth. Repeatedly.
//
// The scenario models the sandbox lifecycle with two real `executor web` boots
// on ONE kept data dir (what the sandbox persists), wiping this run's keychain
// service between them (what the sandbox loses):
//
// boot 1: OAuth-connect a custom MCP server (real authorization-code flow
// against a live test AS) → connection is healthy.
// stop/recreate: keychain wiped, EXECUTOR_DATA_DIR kept.
// boot 2: the SAME connection must still be healthy — durable-by-default is
// the product guarantee. With the bug, this is where the zombie
// appears (expired: "Stored refresh token could not be resolved.").
//
// Note: the repro only bites where the keychain probe passes (macOS dev
// machines, sandbox hosts with a kernel keyring). On headless CI without a
// reachable keyring the keychain provider never registers and the file
// provider is already the default — the scenario then passes vacuously, which
// mirrors production reality.
import { execFileSync } from "node:child_process";
import { randomBytes } from "node:crypto";
import { mkdtempSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { expect } from "@effect/vitest";
import { Effect } from "effect";
import { HttpApiClient } from "effect/unstable/httpapi";
import { FetchHttpClient, HttpClient, HttpClientRequest } from "effect/unstable/http";
import { composePluginApi } from "@executor-js/api/server";
import { mcpHttpPlugin } from "@executor-js/plugin-mcp/api";
import { makeEchoMcpServer, serveMcpServer } from "@executor-js/plugin-mcp/testing";
import {
AuthTemplateSlug,
ConnectionName,
IntegrationSlug,
OAuthClientSlug,
} from "@executor-js/sdk/shared";
import { serveOAuthTestServer } from "@executor-js/sdk/testing";
import { scenario } from "../src/scenario";
import { Cli, RunDir } from "../src/services";
import { withLocalServer, type ServerHandle } from "./local-server";
const api = composePluginApi([mcpHttpPlugin()] as const);
const name = ConnectionName.make("default");
const template = AuthTemplateSlug.make("oauth2");
/** Drive the test AS's consent by hand (authorize → login → code), without
* visiting the app's callback: `oauth.complete` takes the code directly. */
const completeAuthorization = (authorizationUrl: string) =>
Effect.promise(async () => {
const authorize = await fetch(authorizationUrl, { redirect: "manual" });
const loginUrl = authorize.headers.get("location");
if (!loginUrl) throw new Error(`authorize did not redirect: ${authorize.status}`);
const login = await fetch(loginUrl, {
method: "POST",
headers: { authorization: `Basic ${Buffer.from("alice:password").toString("base64")}` },
redirect: "manual",
});
const callbackUrl = login.headers.get("location");
if (!callbackUrl) throw new Error(`login did not redirect: ${login.status}`);
const code = new URL(callbackUrl).searchParams.get("code");
if (!code) throw new Error("callback carried no authorization code");
return code;
});
/** Simulate the non-durable secret store being lost on sandbox recreate: drop
* every entry under this run's keychain service. On macOS `security` deletes
* one matching item per call, so loop until none remain. When the fix routes
* tokens to the file provider this is a no-op — nothing lives there. */
const wipeKeychainService = (serviceName: string): void => {
if (process.platform !== "darwin") return;
for (let i = 0; i < 32; i++) {
try {
execFileSync("security", ["delete-generic-password", "-s", serviceName], {
stdio: "ignore",
});
} catch {
return; // no more entries under this service
}
}
};
const apiClient = (server: ServerHandle) =>
HttpApiClient.make(api, {
baseUrl: new URL("/api", server.origin).toString(),
transformClient: HttpClient.mapRequest((request) =>
HttpClientRequest.setHeader(request, "authorization", `Bearer ${server.token}`),
),
}).pipe(Effect.provide(FetchHttpClient.layer));
scenario(
"Local · an OAuth MCP connection survives a sandbox stop/recreate (data dir kept, keychain lost)",
// Two full `executor web` boots; each allows up to 240s for a cold vite
// start, so the test deadline must clear both plus the OAuth journey.
{ timeout: 600_000 },
Effect.scoped(
Effect.gen(function* () {
const cli = yield* Cli;
const runDir = yield* RunDir;
// The AS mints instantly-expiring access tokens so EVERY health check
// exercises the refresh path — the exact path that dies when the stored
// refresh token evaporates with the keychain.
const oauth = yield* serveOAuthTestServer({
scopes: ["mcp.read"],
tokenExpiresInSeconds: 0,
});
// A real MCP upstream that accepts exactly the AS's issued bearers, so
// "healthy" is a genuine dial-and-list verdict, not a stub's opinion.
const mcp = yield* serveMcpServer(() => makeEchoMcpServer({ name: "durability-mcp" }), {
auth: {
validateAuthorization: (authorization) => oauth.acceptsAuthorizationHeader(authorization),
},
});
const suffix = randomBytes(4).toString("hex");
const slug = IntegrationSlug.make(`oauth-durability-${suffix}`);
const clientSlug = OAuthClientSlug.make(`oauth-durability-client-${suffix}`);
// A unique keychain service per run: the wipe below can only ever touch
// this scenario's entries, and leftovers never collide across runs.
const keychainService = `executor-e2e-durability-${suffix}`;
// The data dir is the durable surface (what a sandbox backs up); it is
// deliberately KEPT across the two boots and cleaned only at the end.
const dataDir = mkdtempSync(join(tmpdir(), "executor-durability-e2e-"));
const serverEnv = { EXECUTOR_KEYCHAIN_SERVICE_NAME: keychainService };
yield* Effect.ensuring(
Effect.gen(function* () {
// ---- Boot 1: connect the OAuth MCP integration. -------------------
yield* withLocalServer(
cli,
runDir,
(server) =>
Effect.gen(function* () {
const client = yield* apiClient(server);
yield* client.mcp.addServer({
payload: {
transport: "remote",
name: "Durability MCP",
endpoint: mcp.url,
slug: String(slug),
authenticationTemplate: [{ kind: "oauth2" }],
},
});
yield* client.oauth.createClient({
payload: {
owner: "org",
slug: clientSlug,
grant: "authorization_code",
authorizationUrl: oauth.authorizationEndpoint,
tokenUrl: oauth.tokenEndpoint,
clientId: "test-client",
clientSecret: "test-secret",
// MCP oauth2 templates discover scopes via the protected
// resource's metadata; the client must name that resource.
resource: oauth.mcpResourceUrl,
originIntegration: slug,
},
});
const started = yield* client.oauth.start({
payload: {
client: clientSlug,
clientOwner: "org",
owner: "org",
name,
integration: slug,
template,
},
});
expect(started.status, "oauth.start redirects to the authorization server").toBe(
"redirect",
);
if (started.status !== "redirect") return yield* Effect.die("no redirect");
const code = yield* completeAuthorization(started.authorizationUrl);
yield* client.oauth.complete({ payload: { state: started.state, code } });
const connection = yield* client.connections.get({
params: { owner: "org", integration: slug, name },
});
// Where did the minted tokens actually land? This is the root
// cause made visible: "keychain" = non-durable on sandbox hosts.
console.info(
`[durability repro] minted connection provider: ${String(connection.provider)}`,
);
const healthy = yield* client.connections.checkHealth({
params: { owner: "org", integration: slug, name },
query: {},
});
expect(
healthy.status,
`the fresh OAuth connection is healthy (detail: ${healthy.detail ?? "-"})`,
).toBe("healthy");
}),
{ dataDir, env: serverEnv, castName: "terminal-boot1.cast" },
);
// ---- Sandbox stop/recreate: keychain gone, data dir kept. ---------
wipeKeychainService(keychainService);
// ---- Boot 2: the connection must still work. ----------------------
yield* withLocalServer(
cli,
runDir,
(server) =>
Effect.gen(function* () {
const client = yield* apiClient(server);
const health = yield* client.connections.checkHealth({
params: { owner: "org", integration: slug, name },
query: {},
});
console.info(
`[durability repro] post-recreate health: ${health.status} — ${health.detail ?? "-"}`,
);
// THE guarantee: OAuth credentials live in the durable store
// (EXECUTOR_DATA_DIR), so losing the host's ephemeral keychain
// must not kill the connection. With the bug this reads
// expired: "Stored refresh token could not be resolved."
expect(
health.status,
`the OAuth connection survives a sandbox recreate (detail: ${health.detail ?? "-"})`,
).toBe("healthy");
}),
{ dataDir, env: serverEnv, castName: "terminal-boot2.cast" },
);
}),
Effect.sync(() => {
wipeKeychainService(keychainService);
rmSync(dataDir, { recursive: true, force: true });
}),
);
}),
),
);