executor mcp no longer starts a server in-process. It ensures a durable detached daemon and bridges stdio JSON-RPC to that owner over HTTP. Concurrent cold starts run a race-safe election: one process becomes the owner and the rest wait for its manifest and attach instead of failing. The owner's lifetime is independent of any MCP client, so many clients, the web UI, and the desktop app share one local server. This builds on the merged start-lock primitive so no client ever owns the database, replacing the earlier approach where the first mcp process started a server in-process and bridged to itself. Adds a cold-start election probe across the cli VM targets plus a local attach stress test; the one-winner, rest-attach behavior is verified on macOS, Linux, and Windows.
Unified · Split
| @@ -30,6 +30,7 @@ · 29 unmodified lines | |||
| 30 | "@executor-js/runtime-quickjs": "workspace:*", | 30 | "@executor-js/runtime-quickjs": "workspace:*", |
| 31 | "@executor-js/sdk": "workspace:*", | 31 | "@executor-js/sdk": "workspace:*", |
| 32 | "@jitl/quickjs-wasmfile-release-sync": "catalog:", | 32 | "@jitl/quickjs-wasmfile-release-sync": "catalog:", |
| 33 | "@modelcontextprotocol/sdk": "^1.29.0", | ||
| 33 | "@sentry/bun": "^10.57.0", | 34 | "@sentry/bun": "^10.57.0", |
| 34 | "effect": "catalog:", | 35 | "effect": "catalog:", |
| 35 | "quickjs-emscripten": "catalog:" | 36 | "quickjs-emscripten": "catalog:" |
| @@ -63,6 +63,9 @@ import type { PlatformError } from "effect/PlatformError"; · 62 unmodified lines | |||
| 63 | import * as Effect from "effect/Effect"; | 63 | import * as Effect from "effect/Effect"; |
| 64 | import * as Option from "effect/Option"; | 64 | import * as Option from "effect/Option"; |
| 65 | import * as Cause from "effect/Cause"; | 65 | import * as Cause from "effect/Cause"; |
| 66 | import { StreamableHTTPClientTransport } from "@modelcontextprotocol/sdk/client/streamableHttp.js"; | ||
| 67 | import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js"; | ||
| 68 | import type { JSONRPCMessage } from "@modelcontextprotocol/sdk/types.js"; | ||
| 66 | 69 | ||
| 67 | import { ExecutorApi, checkForUpdate } from "@executor-js/api"; | 70 | import { ExecutorApi, checkForUpdate } from "@executor-js/api"; |
| 68 | import { | 71 | import { |
| @@ -84,15 +87,12 @@ import { · 15 unmodified lines | |||
| 84 | } from "./device-login"; | 87 | } from "./device-login"; |
| 85 | import { | 88 | import { |
| 86 | startServer, | 89 | startServer, |
| 87 | runMcpStdioServer, | ||
| 88 | getExecutor, | ||
| 89 | rotateLocalAuthToken, | 90 | rotateLocalAuthToken, |
| 90 | localAuthTokenPath, | 91 | localAuthTokenPath, |
| 91 | findDataDirOwnershipHeld, | 92 | findDataDirOwnershipHeld, |
| 92 | type ServerInstance, | 93 | type ServerInstance, |
| 93 | type StartServerOptions, | 94 | type StartServerOptions, |
| 94 | } from "@executor-js/local"; | 95 | } from "@executor-js/local"; |
| 95 | import { makeQuickJsExecutor } from "@executor-js/runtime-quickjs"; | ||
| 96 | import { fetchIntegrations } from "./integrations"; | 96 | import { fetchIntegrations } from "./integrations"; |
| 97 | import { | 97 | import { |
| 98 | buildDaemonSpawnSpec, | 98 | buildDaemonSpawnSpec, |
| @@ -501,77 +501,134 @@ const waitForDaemonStartupTarget = (input: { · 402 unmodified lines | |||
| 501 | // Serialize daemon startup behind a filesystem lock so concurrent CLI invocations don't | 501 | // Serialize daemon startup behind a filesystem lock so concurrent CLI invocations don't |
| 502 | // each spawn their own daemon. The post-lock pointer recheck catches the case where | 502 | // each spawn their own daemon. The post-lock pointer recheck catches the case where |
| 503 | // another invocation finished bootstrapping while we were waiting for the lock. | 503 | // another invocation finished bootstrapping while we were waiting for the lock. |
| 504 | const spawnAndWaitForDaemon = (input: { | 504 | // A storm of concurrent cold starts should elect ONE owner with the rest |
| 505 | // attaching, never N-1 hard failures. Each attempt either wins the per-scope | ||
| 506 | // start lock and spawns the daemon, or waits for the current holder's manifest. | ||
| 507 | // Re-acquiring after a wait timeout is what recovers the one window the wait | ||
| 508 | // alone cannot: acquireDaemonStartLock reclaims a STALE lock at acquisition, so | ||
| 509 | // a holder that took the lock then died BEFORE spawning (no daemon, no manifest) | ||
| 510 | // is recovered when a loser loops back and re-acquires. | ||
| 511 | const MAX_DAEMON_ELECTION_ATTEMPTS = 3; | ||
| 512 | |||
| 513 | /** The stable message acquireDaemonStartLock fails with on genuine lock | ||
| 514 | * contention. Only this should be treated as "another process is electing"; | ||
| 515 | * any other failure (e.g. an unwritable data dir) must propagate, not masquerade | ||
| 516 | * as a race the caller should wait out. */ | ||
| 517 | const isStartLockContention = (error: Error): boolean => | ||
| 518 | error.message.includes("Another daemon startup is already in progress"); | ||
| 519 | |||
| 520 | const spawnDaemonAsLockHolder = (input: { | ||
| 505 | host: string; | 521 | host: string; |
| 506 | scopeId: string; | 522 | scopeId: string; |
| 507 | preferredPort: number; | 523 | preferredPort: number; |
| 508 | allowedHosts: ReadonlyArray<string>; | 524 | allowedHosts: ReadonlyArray<string>; |
| 509 | }): Effect.Effect<string, Error, FileSystem.FileSystem | PlatformPath.Path> => | 525 | }): Effect.Effect<string, Error, FileSystem.FileSystem | PlatformPath.Path> => |
| 510 | Effect.gen(function* () { | 526 | Effect.gen(function* () { |
| 511 | const lock = yield* acquireDaemonStartLock({ hostname: input.host, scopeId: input.scopeId }); | 527 | const existing = yield* readDaemonPointer({ hostname: input.host, scopeId: input.scopeId }); |
| 512 | 528 | if (existing && isPidAlive(existing.pid)) { | |
| 513 | try { | 529 | const existingUrl = daemonBaseUrl(existing.hostname, existing.port); |
| 514 | const existing = yield* readDaemonPointer({ hostname: input.host, scopeId: input.scopeId }); | 530 | if (yield* isServerReachable(existingUrl)) { |
| 515 | if (existing && isPidAlive(existing.pid)) { | 531 | return existingUrl; |
| 516 | const existingUrl = daemonBaseUrl(existing.hostname, existing.port); | ||
| 517 | if (yield* isServerReachable(existingUrl)) { | ||
| 518 | return existingUrl; | ||
| 519 | } | ||
| 520 | } | 532 | } |
| 533 | } | ||
| 521 | 534 | ||
| 522 | const selectedPort = yield* chooseDaemonPort({ | 535 | const selectedPort = yield* chooseDaemonPort({ |
| 523 | preferredPort: input.preferredPort, | 536 | preferredPort: input.preferredPort, |
| 524 | hostname: input.host, | 537 | hostname: input.host, |
| 525 | }); | 538 | }); |
| 526 | 539 | ||
| 527 | if (selectedPort !== input.preferredPort) { | 540 | if (selectedPort !== input.preferredPort) { |
| 528 | console.error( | 541 | console.error( |
| 529 | `Port ${input.preferredPort} is in use. Starting daemon on available port ${selectedPort} instead.`, | 542 | `Port ${input.preferredPort} is in use. Starting daemon on available port ${selectedPort} instead.`, |
| 530 | ); | 543 | ); |
| 531 | } | 544 | } |
| 532 | 545 | ||
| 533 | const spec = yield* Effect.try({ | 546 | const spec = yield* Effect.try({ |
| 534 | try: () => | 547 | try: () => |
| 535 | buildDaemonSpawnSpec({ | 548 | buildDaemonSpawnSpec({ |
| 536 | port: selectedPort, | 549 | port: selectedPort, |
| 537 | hostname: input.host, | 550 | hostname: input.host, |
| 538 | isDevMode, | 551 | isDevMode, |
| 539 | scriptPath: script, | 552 | scriptPath: script, |
| 540 | executablePath: process.execPath, | 553 | executablePath: process.execPath, |
| 541 | allowedHosts: input.allowedHosts, | 554 | allowedHosts: input.allowedHosts, |
| 542 | }), | 555 | }), |
| 543 | catch: (cause) => | 556 | catch: (cause) => |
| 544 | cause instanceof Error | 557 | cause instanceof Error |
| 545 | ? cause | 558 | ? cause |
| 546 | : new Error(`Failed to build daemon command: ${String(cause)}`), | 559 | : new Error(`Failed to build daemon command: ${String(cause)}`), |
| 547 | }); | 560 | }); |
| 548 | 561 | ||
| 549 | const startBaseUrl = daemonBaseUrl(input.host, selectedPort); | 562 | const startBaseUrl = daemonBaseUrl(input.host, selectedPort); |
| 550 | console.error(`Starting daemon on ${input.host}:${selectedPort}...`); | 563 | console.error(`Starting daemon on ${input.host}:${selectedPort}...`); |
| 551 | const child = yield* spawnDetached({ | 564 | const child = yield* spawnDetached({ |
| 552 | command: spec.command, | 565 | command: spec.command, |
| 553 | args: spec.args, | 566 | args: spec.args, |
| 554 | env: process.env, | 567 | env: process.env, |
| 555 | }); | 568 | }); |
| 556 | 569 | ||
| 557 | const readyBaseUrl = yield* waitForDaemonStartupTarget({ requestedBaseUrl: startBaseUrl }); | 570 | const readyBaseUrl = yield* waitForDaemonStartupTarget({ requestedBaseUrl: startBaseUrl }); |
| 558 | 571 | ||
| 559 | if (!readyBaseUrl) { | 572 | if (!readyBaseUrl) { |
| 560 | yield* terminateSpawnedDetachedProcess(child).pipe(Effect.ignore); | 573 | yield* terminateSpawnedDetachedProcess(child).pipe(Effect.ignore); |
| 561 | return yield* Effect.fail( | 574 | return yield* Effect.fail( |
| 562 | new Error( | 575 | new Error( |
| 563 | [ | 576 | [ |
| 564 | `Daemon did not become reachable at ${startBaseUrl} and no reachable local server manifest appeared within ${DAEMON_BOOT_TIMEOUT_MS}ms.`, | 577 | `Daemon did not become reachable at ${startBaseUrl} and no reachable local server manifest appeared within ${DAEMON_BOOT_TIMEOUT_MS}ms.`, |
| 565 | `Run in foreground to inspect logs: ${cliPrefix} daemon run --foreground --port ${selectedPort} --hostname ${input.host}`, | 578 | `Run in foreground to inspect logs: ${cliPrefix} daemon run --foreground --port ${selectedPort} --hostname ${input.host}`, |
| 566 | ].join("\n"), | 579 | ].join("\n"), |
| 567 | ), | 580 | ), |
| 581 | ); | ||
| 582 | } | ||
| 583 | |||
| 584 | return readyBaseUrl; | ||
| 585 | }); | ||
| 586 | |||
| 587 | const spawnAndWaitForDaemon = (input: { | ||
| 588 | host: string; | ||
| 589 | scopeId: string; | ||
| 590 | preferredPort: number; | ||
| 591 | allowedHosts: ReadonlyArray<string>; | ||
| 592 | }): Effect.Effect<string, Error, FileSystem.FileSystem | PlatformPath.Path> => | ||
| 593 | Effect.gen(function* () { | ||
| 594 | const requestedBaseUrl = daemonBaseUrl(input.host, input.preferredPort); | ||
| 595 | |||
| 596 | for (let attempt = 1; attempt <= MAX_DAEMON_ELECTION_ATTEMPTS; attempt++) { | ||
| 597 | const acquired = yield* acquireDaemonStartLock({ | ||
| 598 | hostname: input.host, | ||
| 599 | scopeId: input.scopeId, | ||
| 600 | }).pipe( | ||
| 601 | Effect.map((lock) => ({ held: true as const, lock })), | ||
| 602 | Effect.catch((error) => | ||
| 603 | isStartLockContention(error) | ||
| 604 | ? Effect.succeed({ held: false as const, lock: null }) | ||
| 605 | : Effect.fail(error), | ||
| 606 | ), | ||
| 607 | ); | ||
| 608 | |||
| 609 | if (acquired.held) { | ||
| 610 | const lock = acquired.lock; | ||
| 611 | return yield* spawnDaemonAsLockHolder(input).pipe( | ||
| 612 | Effect.ensuring(releaseDaemonStartLock(lock).pipe(Effect.ignore)), | ||
| 568 | ); | 613 | ); |
| 569 | } | 614 | } |
| 570 | 615 | ||
| 571 | return readyBaseUrl; | 616 | // Lost the lock: wait for the current holder to advertise a manifest. |
| 572 | } finally { | 617 | const ready = yield* waitForDaemonStartupTarget({ requestedBaseUrl }); |
| 573 | yield* releaseDaemonStartLock(lock).pipe(Effect.ignore); | 618 | if (ready) return ready; |
| 619 | // Timed out with no manifest. The holder may have died mid-startup; loop to | ||
| 620 | // re-acquire, which reclaims its now-stale lock. | ||
| 574 | } | 621 | } |
| 622 | |||
| 623 | return yield* Effect.fail( | ||
| 624 | new Error( | ||
| 625 | [ | ||
| 626 | `Could not elect or attach to a local Executor daemon after ${MAX_DAEMON_ELECTION_ATTEMPTS} attempts.`, | ||
| 627 | "A daemon startup may be stuck. Stop any partial daemon and retry, or run it in the foreground:", | ||
| 628 | `${cliPrefix} daemon run --foreground --port ${input.preferredPort} --hostname ${input.host}`, | ||
| 629 | ].join("\n"), | ||
| 630 | ), | ||
| 631 | ); | ||
| 575 | }); | 632 | }); |
| 576 | 633 | ||
| 577 | // Auto-start a local daemon on demand so commands like `executor call` work without the | 634 | // Auto-start a local daemon on demand so commands like `executor call` work without the |
| @@ -1254,125 +1311,146 @@ const runBackgroundDaemonStart = (input: { · 676 unmodified lines | |||
| 1254 | }).pipe(Effect.mapError(toError)); | 1311 | }).pipe(Effect.mapError(toError)); |
| 1255 | 1312 | ||
| 1256 | // --------------------------------------------------------------------------- | 1313 | // --------------------------------------------------------------------------- |
| 1257 | // Stdio MCP session | 1314 | // Stdio MCP session: a pure stdio <-> HTTP bridge to the owning local daemon. |
| 1258 | // --------------------------------------------------------------------------- | 1315 | // --------------------------------------------------------------------------- |
| 1259 | 1316 | ||
| 1260 | const withStdoutReroutedToStderr = async <A>(body: () => Promise<A>): Promise<A> => { | 1317 | const mcpUrlForActiveLocalServer = ( |
| 1261 | const originalWrite = process.stdout.write; | 1318 | connection: ExecutorServerConnection, |
| 1262 | const originalLog = console.log; | 1319 | elicitationMode: "browser" | "model", |
| 1263 | const originalInfo = console.info; | 1320 | ): URL => { |
| 1264 | const originalDebug = console.debug; | 1321 | const url = new URL("/mcp", connection.origin); |
| 1265 | const stderrWrite = process.stderr.write.bind(process.stderr); | 1322 | if (elicitationMode === "browser") { |
| 1323 | url.searchParams.set("elicitation_mode", "browser"); | ||
| 1324 | } | ||
| 1325 | return url; | ||
| 1326 | }; | ||
| 1327 | |||
| 1328 | /** | ||
| 1329 | * Bridge a stdio MCP client to a local server's HTTP `/mcp` endpoint. `executor | ||
| 1330 | * mcp` owns NO database: it forwards JSON-RPC between the client's stdin/stdout | ||
| 1331 | * and the daemon over Streamable HTTP. That keeps any number of MCP clients (plus | ||
| 1332 | * the web UI and the desktop app) attached to the single owning daemon at once, | ||
| 1333 | * and means a transient MCP client exiting never takes the server down. Resolves | ||
| 1334 | * when stdin closes or the daemon connection drops; close is best-effort. | ||
| 1335 | */ | ||
| 1336 | const runMcpHttpBridge = async (input: { | ||
| 1337 | readonly manifest: ExecutorLocalServerManifest; | ||
| 1338 | readonly elicitationMode: "browser" | "model"; | ||
| 1339 | }): Promise<void> => { | ||
| 1340 | const stdio = new StdioServerTransport(); | ||
| 1341 | const authorization = getExecutorServerAuthorizationHeader(input.manifest.connection); | ||
| 1342 | const http = new StreamableHTTPClientTransport( | ||
| 1343 | mcpUrlForActiveLocalServer(input.manifest.connection, input.elicitationMode), | ||
| 1344 | authorization ? { requestInit: { headers: { Authorization: authorization } } } : undefined, | ||
| 1345 | ); | ||
| 1346 | |||
| 1347 | let finished = false; | ||
| 1348 | let closing = false; | ||
| 1349 | let closePromise: Promise<void> | null = null; | ||
| 1350 | let resolveExit: () => void = () => {}; | ||
| 1351 | const waitForExit = new Promise<void>((resolve) => { | ||
| 1352 | resolveExit = resolve; | ||
| 1353 | }); | ||
| 1354 | |||
| 1355 | const finish = () => { | ||
| 1356 | if (finished) return; | ||
| 1357 | finished = true; | ||
| 1358 | process.off("SIGINT", shutdown); | ||
| 1359 | process.off("SIGTERM", shutdown); | ||
| 1360 | process.stdin.off("end", shutdown); | ||
| 1361 | process.stdin.off("close", shutdown); | ||
| 1362 | resolveExit(); | ||
| 1363 | }; | ||
| 1364 | |||
| 1365 | const closeBoth = (): Promise<void> => { | ||
| 1366 | if (!closePromise) { | ||
| 1367 | closing = true; | ||
| 1368 | closePromise = Promise.allSettled([stdio.close(), http.close()]).then(() => undefined); | ||
| 1369 | } | ||
| 1370 | return closePromise; | ||
| 1371 | }; | ||
| 1266 | 1372 | ||
| 1267 | process.stdout.write = ((...args: Parameters<typeof process.stdout.write>) => | 1373 | function shutdown() { |
| 1268 | stderrWrite(...args)) as typeof process.stdout.write; | 1374 | finish(); |
| 1269 | console.log = console.error.bind(console); | 1375 | void closeBoth(); |
| 1270 | console.info = console.error.bind(console); | 1376 | } |
| 1271 | console.debug = console.error.bind(console); | 1377 | |
| 1378 | const isAbortDuringClose = (error: Error): boolean => | ||
| 1379 | error.name === "AbortError" || error.message.toLowerCase().includes("aborted"); | ||
| 1380 | |||
| 1381 | const reportError = (context: string, cause: unknown) => { | ||
| 1382 | const error = toError(cause); | ||
| 1383 | if (closing && isAbortDuringClose(error)) return; | ||
| 1384 | console.error(`Executor MCP bridge ${context}: ${error.message}`); | ||
| 1385 | }; | ||
| 1386 | |||
| 1387 | const forwardMessage = | ||
| 1388 | (send: (message: JSONRPCMessage) => Promise<void>, context: string) => | ||
| 1389 | (message: JSONRPCMessage) => { | ||
| 1390 | void send(message).then(undefined, (cause: unknown) => { | ||
| 1391 | reportError(context, cause); | ||
| 1392 | shutdown(); | ||
| 1393 | }); | ||
| 1394 | }; | ||
| 1395 | |||
| 1396 | process.once("SIGINT", shutdown); | ||
| 1397 | process.once("SIGTERM", shutdown); | ||
| 1398 | process.stdin.once("end", shutdown); | ||
| 1399 | process.stdin.once("close", shutdown); | ||
| 1400 | |||
| 1401 | stdio.onclose = shutdown; | ||
| 1402 | http.onclose = shutdown; | ||
| 1403 | stdio.onerror = (error) => reportError("stdio transport error", error); | ||
| 1404 | http.onerror = (error) => reportError("daemon transport error", error); | ||
| 1405 | stdio.onmessage = forwardMessage((message) => http.send(message), "failed to send to daemon"); | ||
| 1406 | http.onmessage = forwardMessage((message) => stdio.send(message), "failed to send to stdio"); | ||
| 1272 | 1407 | ||
| 1273 | try { | 1408 | try { |
| 1274 | return await body(); | 1409 | await http.start(); |
| 1410 | await stdio.start(); | ||
| 1411 | await waitForExit; | ||
| 1275 | } finally { | 1412 | } finally { |
| 1276 | process.stdout.write = originalWrite; | 1413 | finish(); |
| 1277 | console.log = originalLog; | 1414 | await closeBoth(); |
| 1278 | console.info = originalInfo; | ||
| 1279 | console.debug = originalDebug; | ||
| 1280 | } | 1415 | } |
| 1281 | }; | 1416 | }; |
| 1282 | 1417 | ||
| 1283 | const runStdioMcpSession = (input: { readonly elicitationMode: "browser" | "model" }) => | 1418 | const runStdioMcpSession = (input: { readonly elicitationMode: "browser" | "model" }) => |
| 1284 | Effect.gen(function* () { | 1419 | Effect.gen(function* () { |
| 1285 | // No process-level startup lock: the DB ownership lock inside startServer | 1420 | // `executor mcp` never owns the local database. If a local server is already |
| 1286 | // (openOwnedLocalDatabase) is the real gate. assertNoOtherActiveLocalServer | 1421 | // running, bridge this stdio client to it; otherwise ensure a durable |
| 1287 | // is a friendly fast-path that may race without being unsafe. | 1422 | // background daemon is up and bridge to that. ensureDaemon is the race-safe |
| 1288 | yield* assertNoOtherActiveLocalServer(); | 1423 | // election: concurrent cold starts elect one owner and the losers wait for |
| 1289 | const web = yield* Effect.tryPromise({ | 1424 | // its manifest (waitForDaemonStartupTarget) rather than failing. Bridging |
| 1290 | try: () => | 1425 | // means many MCP clients, the web UI, and the desktop app share one owner, |
| 1291 | withStdoutReroutedToStderr(async () => { | 1426 | // and that owner's lifetime is never tied to a transient MCP client. |
| 1292 | const host = "127.0.0.1"; | 1427 | const active = yield* readActiveLocalServerManifest(); |
| 1293 | const port = await Effect.runPromise( | 1428 | if (active) { |
| 1294 | chooseDaemonPort({ preferredPort: DEFAULT_PORT, hostname: host }), | ||
| 1295 | ); | ||
| 1296 | const baseUrl = `http://localhost:${port}`; | ||
| 1297 | const restoreWebBaseUrl = installDefaultExecutorWebBaseUrl(baseUrl); | ||
| 1298 | |||
| 1299 | try { | ||
| 1300 | const executor = await getExecutor(); | ||
| 1301 | const server = await startServer({ | ||
| 1302 | port, | ||
| 1303 | hostname: host, | ||
| 1304 | embeddedWebUI, | ||
| 1305 | }); | ||
| 1306 | const serverBaseUrl = `http://localhost:${server.port}`; | ||
| 1307 | return { executor, server, baseUrl: serverBaseUrl, restoreWebBaseUrl }; | ||
| 1308 | } catch (cause) { | ||
| 1309 | restoreWebBaseUrl(); | ||
| 1310 | throw cause; | ||
| 1311 | } | ||
| 1312 | }), | ||
| 1313 | catch: (cause) => cause, | ||
| 1314 | }).pipe( | ||
| 1315 | Effect.catch((cause) => { | ||
| 1316 | const ownership = findDataDirOwnershipHeld(cause); | ||
| 1317 | if (!ownership) return Effect.fail(toError(cause)); | ||
| 1318 | return Effect.gen(function* () { | ||
| 1319 | const manifest = yield* readReachableLocalServerHint(); | ||
| 1320 | const path = yield* PlatformPath.Path; | ||
| 1321 | const dataDir = resolveExecutorDataDir(path); | ||
| 1322 | if (manifest) { | ||
| 1323 | return yield* Effect.fail( | ||
| 1324 | new Error( | ||
| 1325 | [ | ||
| 1326 | `A local Executor server already owns ${dataDir} at ${manifest.connection.origin}.`, | ||
| 1327 | "The stdio MCP server needs exclusive access to the local database.", | ||
| 1328 | `Use the running HTTP MCP endpoint instead: ${manifest.connection.origin}/mcp`, | ||
| 1329 | ].join("\n"), | ||
| 1330 | ), | ||
| 1331 | ); | ||
| 1332 | } | ||
| 1333 | return yield* Effect.fail( | ||
| 1334 | new Error( | ||
| 1335 | [ | ||
| 1336 | "Executor data directory is owned by another live process, but no reachable local server was advertised.", | ||
| 1337 | `Data directory: ${dataDir}`, | ||
| 1338 | `Ownership lock: ${ownership.lockPath}`, | ||
| 1339 | "Wait for the existing process to finish starting, or stop it and retry.", | ||
| 1340 | ].join("\n"), | ||
| 1341 | ), | ||
| 1342 | ); | ||
| 1343 | }); | ||
| 1344 | }), | ||
| 1345 | ); | ||
| 1346 | yield* publishLocalServerManifest({ | ||
| 1347 | kind: "foreground", | ||
| 1348 | connection: normalizeExecutorServerConnection({ | ||
| 1349 | kind: "http", | ||
| 1350 | origin: web.baseUrl, | ||
| 1351 | displayName: "CLI MCP", | ||
| 1352 | auth: { kind: "bearer", token: web.server.authToken }, | ||
| 1353 | }), | ||
| 1354 | }); | ||
| 1355 | |||
| 1356 | try { | ||
| 1357 | yield* Effect.promise(() => | 1429 | yield* Effect.promise(() => |
| 1358 | runMcpStdioServer({ | 1430 | runMcpHttpBridge({ manifest: active, elicitationMode: input.elicitationMode }), |
| 1359 | executor: web.executor, | 1431 | ); |
| 1360 | codeExecutor: makeQuickJsExecutor(), | 1432 | return; |
| 1361 | elicitationMode: | 1433 | } |
| 1362 | input.elicitationMode === "browser" | 1434 | |
| 1363 | ? { | 1435 | // No reachable owner yet: ensure one. If we lose the election (another |
| 1364 | mode: "browser" as const, | 1436 | // process became owner first), ensureDaemon may fail, but the winner's |
| 1365 | approvalUrl: (executionId) => | 1437 | // manifest is then reachable, so re-read it and bridge to that instead. |
| 1366 | `${web.baseUrl}/resume/${encodeURIComponent(executionId)}`, | 1438 | const elected = yield* ensureDaemon(DEFAULT_BASE_URL).pipe( |
| 1367 | } | 1439 | Effect.flatMap(() => readActiveLocalServerManifest()), |
| 1368 | : { mode: input.elicitationMode }, | 1440 | Effect.catch((error) => |
| 1369 | }), | 1441 | readActiveLocalServerManifest().pipe( |
| 1442 | Effect.flatMap((manifest) => (manifest ? Effect.succeed(manifest) : Effect.fail(error))), | ||
| 1443 | ), | ||
| 1444 | ), | ||
| 1445 | ); | ||
| 1446 | if (!elected) { | ||
| 1447 | return yield* Effect.fail( | ||
| 1448 | new Error("The local Executor daemon started but did not advertise a reachable manifest."), | ||
| 1370 | ); | 1449 | ); |
| 1371 | } finally { | ||
| 1372 | web.restoreWebBaseUrl(); | ||
| 1373 | yield* Effect.promise(() => web.server.stop()); | ||
| 1374 | yield* removeLocalServerManifestIfOwnedBy({ pid: process.pid }).pipe(Effect.ignore); | ||
| 1375 | } | 1450 | } |
| 1451 | yield* Effect.promise(() => | ||
| 1452 | runMcpHttpBridge({ manifest: elected, elicitationMode: input.elicitationMode }), | ||
| 1453 | ); | ||
| 1376 | }); | 1454 | }); |
| 1377 | 1455 | ||
| 1378 | const scope = Options.string("scope").pipe( | 1456 | const scope = Options.string("scope").pipe( |
| @@ -42,6 +42,7 @@ · 41 unmodified lines | |||
| 42 | "@executor-js/runtime-quickjs": "workspace:*", | 42 | "@executor-js/runtime-quickjs": "workspace:*", |
| 43 | "@executor-js/sdk": "workspace:*", | 43 | "@executor-js/sdk": "workspace:*", |
| 44 | "@jitl/quickjs-wasmfile-release-sync": "catalog:", | 44 | "@jitl/quickjs-wasmfile-release-sync": "catalog:", |
| 45 | "@modelcontextprotocol/sdk": "^1.29.0", | ||
| 45 | "@sentry/bun": "^10.57.0", | 46 | "@sentry/bun": "^10.57.0", |
| 46 | "effect": "catalog:", | 47 | "effect": "catalog:", |
| 47 | "quickjs-emscripten": "catalog:", | 48 | "quickjs-emscripten": "catalog:", |
| @@ -0,0 +1,160 @@ | |||
| 1 | // Cross-OS proof of the Phase 1 daemon election (apps/cli/src/main.ts). On a real | ||
| 2 | // guest OS, fire N `executor` clients at a COLD data dir simultaneously: each | ||
| 3 | // auto-starts a daemon via resolveExecutorServerConnection -> ensureDaemon -> | ||
| 4 | // spawnAndWaitForDaemon (the election). Before Phase 1 exactly one client won and | ||
| 5 | // the other N-1 hard-failed on the start-lock; the fix makes the losers wait for | ||
| 6 | // the winner's manifest and attach. So the assertion is simply: ALL N succeed, | ||
| 7 | // exactly one owner is elected, and it is reachable. A second (warm) wave proves | ||
| 8 | // the steady-state attach path. Uses a separate data dir from the service daemon | ||
| 9 | // that globalsetup installed on ~/.executor, so the two never collide. | ||
| 10 | // | ||
| 11 | // Runs on the cli-* VM targets (cli-linux / cli-macos on local tart; cli-windows | ||
| 12 | // on EC2). Drive everything over the same SSH the other cli tests use. | ||
| 13 | import { execFile } from "node:child_process"; | ||
| 14 | import { promisify } from "node:util"; | ||
| 15 | |||
| 16 | import { expect, it } from "@effect/vitest"; | ||
| 17 | |||
| 18 | const execFileAsync = promisify(execFile); | ||
| 19 | |||
| 20 | const SSH_OPTS = [ | ||
| 21 | "-o", | ||
| 22 | "StrictHostKeyChecking=no", | ||
| 23 | "-o", | ||
| 24 | "UserKnownHostsFile=/dev/null", | ||
| 25 | "-o", | ||
| 26 | "ConnectTimeout=15", | ||
| 27 | ] as const; | ||
| 28 | |||
| 29 | type GuestOs = "macos" | "linux" | "windows"; | ||
| 30 | |||
| 31 | const guestOs = (): GuestOs => { | ||
| 32 | const os = process.env.E2E_VM_OS; | ||
| 33 | if (os === "macos" || os === "linux" || os === "windows") return os; | ||
| 34 | throw new Error(`Unsupported E2E_VM_OS: ${os ?? "<unset>"}`); | ||
| 35 | }; | ||
| 36 | |||
| 37 | const sshInvocation = (command: string): { command: string; args: ReadonlyArray<string> } => { | ||
| 38 | const host = process.env.E2E_CLI_VM_HOST; | ||
| 39 | if (!host) throw new Error("E2E_CLI_VM_HOST is not set"); | ||
| 40 | const os = guestOs(); | ||
| 41 | const wrapped = | ||
| 42 | os === "linux" ? `export XDG_RUNTIME_DIR=/run/user/$(id -u); ${command}` : command; | ||
| 43 | const keyPath = process.env.E2E_CLI_SSH_KEY; | ||
| 44 | const user = os === "windows" ? "Administrator" : "admin"; | ||
| 45 | return keyPath | ||
| 46 | ? { command: "ssh", args: ["-i", keyPath, ...SSH_OPTS, `${user}@${host}`, wrapped] } | ||
| 47 | : { | ||
| 48 | command: process.env.E2E_SSHPASS_BIN ?? "/opt/homebrew/bin/sshpass", | ||
| 49 | args: ["-p", "admin", "ssh", ...SSH_OPTS, `${user}@${host}`, wrapped], | ||
| 50 | }; | ||
| 51 | }; | ||
| 52 | |||
| 53 | const ssh = async (command: string): Promise<{ stdout: string; stderr: string; code: number }> => { | ||
| 54 | const invocation = sshInvocation(command); | ||
| 55 | try { | ||
| 56 | const { stdout, stderr } = await execFileAsync(invocation.command, [...invocation.args], { | ||
| 57 | maxBuffer: 64 * 1024 * 1024, | ||
| 58 | }); | ||
| 59 | return { stdout, stderr, code: 0 }; | ||
| 60 | } catch (error) { | ||
| 61 | const err = error as { stdout?: string; stderr?: string; code?: number }; | ||
| 62 | return { | ||
| 63 | stdout: err.stdout ?? "", | ||
| 64 | stderr: err.stderr ?? "", | ||
| 65 | code: typeof err.code === "number" ? err.code : 1, | ||
| 66 | }; | ||
| 67 | } | ||
| 68 | }; | ||
| 69 | |||
| 70 | const exePath = (): string => { | ||
| 71 | const dir = process.env.E2E_CLI_BIN_DIR ?? (guestOs() === "windows" ? "C:/ed" : "~/ed"); | ||
| 72 | return guestOs() === "windows" ? `${dir}/executor.exe` : `${dir}/executor`; | ||
| 73 | }; | ||
| 74 | |||
| 75 | const CLIENTS = 6; | ||
| 76 | |||
| 77 | // One concurrent wave of N cold/warm clients against data dir D, emitting a | ||
| 78 | // single parseable summary line. Unix (linux+macos) flavor; the election logic | ||
| 79 | // under test is OS-agnostic, the shell around it is not. | ||
| 80 | const unixWaveScript = (exe: string, dir: string, n: number): string => | ||
| 81 | [ | ||
| 82 | "set -u", | ||
| 83 | `EXE=${exe}`, | ||
| 84 | `D=${dir}`, | ||
| 85 | // `timeout` is GNU coreutils: present on Linux, absent on the macOS guest | ||
| 86 | // (where it would be `gtimeout`). Fall back to no wrapper; the test's own | ||
| 87 | // 300s budget and `tools search` self-completing are the backstops. | ||
| 88 | "TO=$(command -v timeout 2>/dev/null || command -v gtimeout 2>/dev/null || true)", | ||
| 89 | "run_client() {", | ||
| 90 | ' if [ -n "$TO" ]; then', | ||
| 91 | ` "$TO" 120 env EXECUTOR_DATA_DIR="$D" EXECUTOR_SCOPE_DIR="$D" "$EXE" tools search "probe-$1" >"$D/out-$1" 2>&1`, | ||
| 92 | " else", | ||
| 93 | ` env EXECUTOR_DATA_DIR="$D" EXECUTOR_SCOPE_DIR="$D" "$EXE" tools search "probe-$1" >"$D/out-$1" 2>&1`, | ||
| 94 | " fi", | ||
| 95 | ' echo $? >"$D/rc-$1"', | ||
| 96 | "}", | ||
| 97 | `i=1; while [ $i -le ${n} ]; do run_client $i & i=$((i+1)); done; wait`, | ||
| 98 | "ok=0; spawned=0; i=1", | ||
| 99 | `while [ $i -le ${n} ]; do`, | ||
| 100 | ' rc=$(cat "$D/rc-$i" 2>/dev/null || echo X); [ "$rc" = "0" ] && ok=$((ok+1))', | ||
| 101 | ' grep -q "Starting daemon" "$D/out-$i" 2>/dev/null && spawned=$((spawned+1))', | ||
| 102 | " i=$((i+1)); done", | ||
| 103 | `manifests=$(ls "$D"/daemon-active-* 2>/dev/null | wc -l | tr -d ' ')`, | ||
| 104 | `port=$(cat "$D"/daemon-localhost-*.json 2>/dev/null | sed -n 's/.*"port":[ ]*\\([0-9]*\\).*/\\1/p' | head -1)`, | ||
| 105 | `health=$(curl -s -o /dev/null -w '%{http_code}' --max-time 5 "http://localhost:$port/api/health" 2>/dev/null || echo 000)`, | ||
| 106 | `echo "PROBE_SUMMARY ok=$ok n=${n} spawned=$spawned manifests=$manifests port=$port health=$health"`, | ||
| 107 | ].join("\n"); | ||
| 108 | |||
| 109 | const parseSummary = (stdout: string): Record<string, string> => { | ||
| 110 | const line = stdout.split("\n").find((l) => l.includes("PROBE_SUMMARY")); | ||
| 111 | if (!line) throw new Error(`no PROBE_SUMMARY in:\n${stdout}`); | ||
| 112 | const out: Record<string, string> = {}; | ||
| 113 | for (const tok of line.replace("PROBE_SUMMARY", "").trim().split(/\s+/)) { | ||
| 114 | const [k, v] = tok.split("="); | ||
| 115 | if (k) out[k] = v ?? ""; | ||
| 116 | } | ||
| 117 | return out; | ||
| 118 | }; | ||
| 119 | |||
| 120 | it("Cold-start election: N simultaneous clients all attach to one elected daemon", async () => { | ||
| 121 | if (guestOs() === "windows") { | ||
| 122 | // This SSH/unix-shell flavor does not cover the Windows guest. The same | ||
| 123 | // election is proven on real Windows by election-cold-start.win.ps1, run by | ||
| 124 | // hand against the long-lived dockur (QEMU) Windows guest (no EC2 spend): | ||
| 125 | // cold ok=6 n=6 spawned=1 ... / warm ok=6 n=6 spawned=0 ... (one winner, rest attach). | ||
| 126 | console.warn( | ||
| 127 | "election-cold-start: see election-cold-start.win.ps1 for the Windows proof; skipping", | ||
| 128 | ); | ||
| 129 | return; | ||
| 130 | } | ||
| 131 | const exe = exePath(); | ||
| 132 | const dir = "/tmp/election-probe"; | ||
| 133 | await ssh(`rm -rf ${dir} && mkdir -p ${dir}`); | ||
| 134 | |||
| 135 | // WAVE 1: cold. No daemon owns `dir`. All N race the start-lock. | ||
| 136 | const cold = await ssh(unixWaveScript(exe, dir, CLIENTS)); | ||
| 137 | const c = parseSummary(cold.stdout); | ||
| 138 | console.log(`[cold] ${cold.stdout.split("\n").find((l) => l.includes("PROBE_SUMMARY"))}`); | ||
| 139 | expect(Number(c.ok), `all ${CLIENTS} cold clients succeeded (stdout:\n${cold.stdout})`).toBe( | ||
| 140 | CLIENTS, | ||
| 141 | ); | ||
| 142 | expect(Number(c.manifests), "exactly one daemon was elected").toBe(1); | ||
| 143 | expect(c.health, "the elected daemon answers /api/health").toBe("200"); | ||
| 144 | |||
| 145 | // WAVE 2: warm. A daemon now owns `dir`; every client should attach. | ||
| 146 | const warm = await ssh(unixWaveScript(exe, dir, CLIENTS)); | ||
| 147 | const w = parseSummary(warm.stdout); | ||
| 148 | console.log(`[warm] ${warm.stdout.split("\n").find((l) => l.includes("PROBE_SUMMARY"))}`); | ||
| 149 | expect(Number(w.ok), `all ${CLIENTS} warm clients attached (stdout:\n${warm.stdout})`).toBe( | ||
| 150 | CLIENTS, | ||
| 151 | ); | ||
| 152 | expect(Number(w.manifests), "still exactly one daemon").toBe(1); | ||
| 153 | expect(w.health, "daemon still reachable").toBe("200"); | ||
| 154 | |||
| 155 | // Leave the guest clean: stop only the daemon we elected (not the service | ||
| 156 | // daemon globalsetup installed on ~/.executor). | ||
| 157 | await ssh( | ||
| 158 | `pid=$(cat ${dir}/daemon-active-* 2>/dev/null | sed -n 's/.*"pid":[ ]*\\([0-9]*\\).*/\\1/p' | head -1); [ -n "$pid" ] && kill "$pid" 2>/dev/null; rm -rf ${dir}; true`, | ||
| 159 | ); | ||
| 160 | }, 300_000); | ||
| @@ -0,0 +1,56 @@ | |||
| 1 | # Windows companion to election-cold-start.test.ts: the Phase 1 daemon election, | ||
| 2 | # proven on real Windows. The vitest cli-windows target provisions via EC2; this | ||
| 3 | # script is the same probe, run by hand against any reachable Windows guest (e.g. | ||
| 4 | # a long-lived dockur/QEMU box) when avoiding cloud spend. | ||
| 5 | # | ||
| 6 | # How to run: | ||
| 7 | # 1. cross-build the binary: cd apps/cli; bun run src/build.ts binary --target executor-windows-x64 | ||
| 8 | # 2. push the bin dir to the guest C:\ed (scp the dist bin dir to Administrator@<guest>:C:/ed/) | ||
| 9 | # 3. push + run this script: ssh Administrator@<guest> 'powershell -ExecutionPolicy Bypass -File C:\ed\probe.ps1' | ||
| 10 | # | ||
| 11 | # Expected (matches Linux/macOS): one winner spawns, the rest attach, all succeed. | ||
| 12 | # PROBE_SUMMARY-cold ok=6 n=6 spawned=1 manifests=1 port=4788 health=200 | ||
| 13 | # PROBE_SUMMARY-warm ok=6 n=6 spawned=0 manifests=1 port=4788 health=200 | ||
| 14 | param([int]$N = 6) | ||
| 15 | $Exe = "C:\ed\executor.exe" | ||
| 16 | $D = "C:\Users\administrator\eprobe" | ||
| 17 | $ErrorActionPreference = "SilentlyContinue" | ||
| 18 | |||
| 19 | function Run-Wave($label) { | ||
| 20 | $procs = @() | ||
| 21 | for ($i = 1; $i -le $N; $i++) { | ||
| 22 | $o = "$D\out-$label-$i.txt"; $e = "$D\err-$label-$i.txt" | ||
| 23 | $p = Start-Process -FilePath $Exe -ArgumentList "tools", "search", "p$i" ` | ||
| 24 | -PassThru -NoNewWindow -RedirectStandardOutput $o -RedirectStandardError $e | ||
| 25 | $procs += $p | ||
| 26 | } | ||
| 27 | foreach ($p in $procs) { [void]$p.WaitForExit(120000) } | ||
| 28 | # Success signal is the round-trip result in the client's stdout, not the | ||
| 29 | # Start-Process ExitCode (which comes back null under -NoNewWindow on PS 5.1). | ||
| 30 | # The election winner prints "Starting daemon" to STDERR; losers attach silently. | ||
| 31 | $ok = 0; $spawned = 0 | ||
| 32 | for ($i = 1; $i -le $N; $i++) { | ||
| 33 | if (Select-String -Path "$D\out-$label-$i.txt" -Pattern '"total"' -Quiet) { $ok++ } | ||
| 34 | $sp = (Select-String -Path "$D\out-$label-$i.txt" -Pattern "Starting daemon" -Quiet) ` | ||
| 35 | -or (Select-String -Path "$D\err-$label-$i.txt" -Pattern "Starting daemon" -Quiet) | ||
| 36 | if ($sp) { $spawned++ } | ||
| 37 | } | ||
| 38 | $manifests = (Get-ChildItem "$D\daemon-active-*" -EA SilentlyContinue | Measure-Object).Count | ||
| 39 | $port = "" | ||
| 40 | $pf = Get-ChildItem "$D\daemon-localhost-*.json" -EA SilentlyContinue | Select-Object -First 1 | ||
| 41 | if ($pf) { $port = (Get-Content $pf.FullName -Raw | ConvertFrom-Json).port } | ||
| 42 | $health = "000" | ||
| 43 | if ($port) { | ||
| 44 | try { $health = [string](Invoke-WebRequest -UseBasicParsing -TimeoutSec 5 "http://localhost:$port/api/health").StatusCode } catch { $health = "ERR" } | ||
| 45 | } | ||
| 46 | Write-Output "PROBE_SUMMARY-$label ok=$ok n=$N spawned=$spawned manifests=$manifests port=$port health=$health" | ||
| 47 | } | ||
| 48 | |||
| 49 | Remove-Item -Recurse -Force $D -EA SilentlyContinue | ||
| 50 | New-Item -ItemType Directory -Force -Path $D | Out-Null | ||
| 51 | $env:EXECUTOR_DATA_DIR = $D | ||
| 52 | $env:EXECUTOR_SCOPE_DIR = $D | ||
| 53 | Run-Wave "cold" | ||
| 54 | Run-Wave "warm" | ||
| 55 | $am = Get-ChildItem "$D\daemon-active-*" -EA SilentlyContinue | Select-Object -First 1 | ||
| 56 | if ($am) { $dp = (Get-Content $am.FullName -Raw | ConvertFrom-Json).pid; if ($dp) { Stop-Process -Id $dp -Force -EA SilentlyContinue } } | ||
| @@ -0,0 +1,342 @@ | |||
| 1 | // Stress for PR 1033 (`executor mcp` attaches to the active local daemon). The | ||
| 2 | // happy path is covered by cli-mcp-daemon-attach.test.ts; this hammers the | ||
| 3 | // concurrency the bridge introduced: | ||
| 4 | // | ||
| 5 | // A. ATTACH STORM — one daemon, many stdio bridges at once, each firing a | ||
| 6 | // burst of execute() calls. Stresses the StdioServerTransport ↔ | ||
| 7 | // StreamableHTTPClientTransport forwarding under load and the daemon's MCP | ||
| 8 | // session handling. A dropped/misrouted JSON-RPC reply shows up as a wrong | ||
| 9 | // or missing result. | ||
| 10 | // B. COLD-START RACE — NO daemon, many `executor mcp` started simultaneously. | ||
| 11 | // They race acquireLocalServerStartLock + the double-check | ||
| 12 | // readActiveLocalServerManifest the PR added: exactly one must start a | ||
| 13 | // server, the rest must bridge to it. If the guard is wrong, the losers | ||
| 14 | // trip the data-dir singleton ("another active local server") or collide on | ||
| 15 | // a port — surfacing as client failures. | ||
| 16 | // C. KILL UNDER LOAD — kill the daemon mid-bridge; the http transport's | ||
| 17 | // onclose must tear the bridge down so the `executor mcp` process exits | ||
| 18 | // instead of hanging (a leaked stdio child per crashed daemon would be the | ||
| 19 | // bug). | ||
| 20 | import { expect } from "@effect/vitest"; | ||
| 21 | import { Client } from "@modelcontextprotocol/sdk/client/index.js"; | ||
| 22 | import { StdioClientTransport } from "@modelcontextprotocol/sdk/client/stdio.js"; | ||
| 23 | import { Effect } from "effect"; | ||
| 24 | import { mkdtempSync, readdirSync, rmSync } from "node:fs"; | ||
| 25 | import { tmpdir } from "node:os"; | ||
| 26 | import { join } from "node:path"; | ||
| 27 | import { fileURLToPath } from "node:url"; | ||
| 28 | import type { Subprocess } from "bun"; | ||
| 29 | |||
| 30 | import { scenario } from "../src/scenario"; | ||
| 31 | |||
| 32 | const repoRoot = fileURLToPath(new URL("../../", import.meta.url)); | ||
| 33 | const testScope = join(repoRoot, "apps/local"); | ||
| 34 | // Generous: a dev-mode daemon boots a Vite dev server, slow under machine load. | ||
| 35 | const readyTimeoutMs = 150_000; | ||
| 36 | |||
| 37 | type DaemonProc = Subprocess<"ignore", "pipe", "pipe">; | ||
| 38 | |||
| 39 | const waitForDaemonReady = ( | ||
| 40 | proc: DaemonProc, | ||
| 41 | ): Promise<{ readonly port: number; readonly stderr: () => string }> => | ||
| 42 | // oxlint-disable-next-line executor/no-promise-reject -- boundary: local e2e watches a real daemon process | ||
| 43 | new Promise((resolveReady, rejectReady) => { | ||
| 44 | let stdoutBuffer = ""; | ||
| 45 | let stderrBuffer = ""; | ||
| 46 | let settled = false; | ||
| 47 | const decoder = new TextDecoder(); | ||
| 48 | const stdout = proc.stdout.getReader(); | ||
| 49 | const stderr = proc.stderr.getReader(); | ||
| 50 | const deadline = setTimeout(() => { | ||
| 51 | if (settled) return; | ||
| 52 | settled = true; | ||
| 53 | // oxlint-disable-next-line executor/no-promise-reject, executor/no-error-constructor -- boundary: captured daemon stderr | ||
| 54 | rejectReady(new Error(`daemon did not announce ready: ${stderrBuffer}`)); | ||
| 55 | }, readyTimeoutMs); | ||
| 56 | void (async () => { | ||
| 57 | while (true) { | ||
| 58 | const { value, done } = await stderr.read(); | ||
| 59 | if (done) return; | ||
| 60 | stderrBuffer += decoder.decode(value); | ||
| 61 | } | ||
| 62 | })(); | ||
| 63 | void (async () => { | ||
| 64 | while (true) { | ||
| 65 | const { value, done } = await stdout.read(); | ||
| 66 | if (done) { | ||
| 67 | if (!settled) { | ||
| 68 | settled = true; | ||
| 69 | clearTimeout(deadline); | ||
| 70 | // oxlint-disable-next-line executor/no-promise-reject, executor/no-error-constructor -- boundary: captured daemon stderr | ||
| 71 | rejectReady(new Error(`daemon stdout closed before ready: ${stderrBuffer}`)); | ||
| 72 | } | ||
| 73 | return; | ||
| 74 | } | ||
| 75 | stdoutBuffer += decoder.decode(value); | ||
| 76 | const match = /Daemon ready on http:\/\/(?:\[[^\]]+\]|[^:\s]+):(\d+)/.exec(stdoutBuffer); | ||
| 77 | if (match) { | ||
| 78 | settled = true; | ||
| 79 | clearTimeout(deadline); | ||
| 80 | resolveReady({ port: Number(match[1]), stderr: () => stderrBuffer }); | ||
| 81 | return; | ||
| 82 | } | ||
| 83 | } | ||
| 84 | })(); | ||
| 85 | }); | ||
| 86 | |||
| 87 | const spawnDaemon = (dataDir: string): DaemonProc => | ||
| 88 | Bun.spawn( | ||
| 89 | [ | ||
| 90 | "bun", | ||
| 91 | "run", | ||
| 92 | "dev:cli", | ||
| 93 | "daemon", | ||
| 94 | "run", | ||
| 95 | "--foreground", | ||
| 96 | "--port", | ||
| 97 | "0", | ||
| 98 | "--hostname", | ||
| 99 | "127.0.0.1", | ||
| 100 | "--scope", | ||
| 101 | testScope, | ||
| 102 | ], | ||
| 103 | { | ||
| 104 | cwd: repoRoot, | ||
| 105 | env: { ...process.env, EXECUTOR_DATA_DIR: dataDir }, | ||
| 106 | stdin: "ignore", | ||
| 107 | stdout: "pipe", | ||
| 108 | stderr: "pipe", | ||
| 109 | }, | ||
| 110 | ); | ||
| 111 | |||
| 112 | const stopProc = async (proc: DaemonProc): Promise<void> => { | ||
| 113 | if (proc.exitCode !== null) return; | ||
| 114 | proc.kill("SIGTERM"); | ||
| 115 | await Promise.race([proc.exited, Bun.sleep(3000)]); | ||
| 116 | if (proc.exitCode === null) proc.kill("SIGKILL"); | ||
| 117 | }; | ||
| 118 | |||
| 119 | const startForegroundDaemon = (dataDir: string) => | ||
| 120 | Effect.acquireRelease( | ||
| 121 | Effect.gen(function* () { | ||
| 122 | const proc = spawnDaemon(dataDir); | ||
| 123 | const ready = yield* Effect.promise(() => waitForDaemonReady(proc)).pipe( | ||
| 124 | Effect.tapError(() => Effect.promise(() => stopProc(proc))), | ||
| 125 | ); | ||
| 126 | return { proc, port: ready.port, stderr: ready.stderr }; | ||
| 127 | }), | ||
| 128 | ({ proc }) => Effect.promise(() => stopProc(proc)), | ||
| 129 | ); | ||
| 130 | |||
| 131 | interface ClientReport { | ||
| 132 | readonly id: number; | ||
| 133 | readonly ok: boolean; | ||
| 134 | readonly tools: number; | ||
| 135 | readonly results: ReadonlyArray<string>; | ||
| 136 | readonly error?: string; | ||
| 137 | } | ||
| 138 | |||
| 139 | /** One `executor mcp` stdio bridge: connect, list tools, fire `callCount` | ||
| 140 | * execute() calls (each computing a unique value), collect the text results, | ||
| 141 | * always close. Never throws — failures are captured in the report so the | ||
| 142 | * scenario can assert across the whole fleet. */ | ||
| 143 | const runOneClient = async ( | ||
| 144 | id: number, | ||
| 145 | dataDir: string, | ||
| 146 | callCount: number, | ||
| 147 | ): Promise<ClientReport> => { | ||
| 148 | const transport = new StdioClientTransport({ | ||
| 149 | command: "bun", | ||
| 150 | args: ["run", "dev:cli", "mcp", "--scope", testScope], | ||
| 151 | cwd: repoRoot, | ||
| 152 | env: { ...process.env, EXECUTOR_DATA_DIR: dataDir }, | ||
| 153 | stderr: "pipe", | ||
| 154 | }); | ||
| 155 | const client = new Client({ name: `stress-client-${id}`, version: "1.0.0" }); | ||
| 156 | const results: string[] = []; | ||
| 157 | let errBuf = ""; | ||
| 158 | transport.stderr?.on("data", (d: Buffer) => { | ||
| 159 | errBuf += d.toString(); | ||
| 160 | }); | ||
| 161 | // oxlint-disable-next-line executor/no-try-catch-or-throw -- boundary: capture per-client failure for fleet-wide assertions | ||
| 162 | try { | ||
| 163 | await client.connect(transport); | ||
| 164 | const { tools } = await client.listTools(); | ||
| 165 | for (let i = 0; i < callCount; i++) { | ||
| 166 | // A value unique to (client, call) so a misrouted reply is detectable. | ||
| 167 | const expr = `return ${id} * 1000 + ${i}`; | ||
| 168 | const r = await client.callTool({ name: "execute", arguments: { code: expr } }); | ||
| 169 | const text = (r.content as Array<{ type: string; text: string }>)[0]?.text ?? ""; | ||
| 170 | results.push(text); | ||
| 171 | } | ||
| 172 | return { id, ok: true, tools: tools.length, results }; | ||
| 173 | } catch (error) { | ||
| 174 | return { | ||
| 175 | id, | ||
| 176 | ok: false, | ||
| 177 | tools: 0, | ||
| 178 | results, | ||
| 179 | error: | ||
| 180 | (error instanceof Error ? error.message : String(error)) + | ||
| 181 | (errBuf | ||
| 182 | ? `\n stderr: ${errBuf.trim().split("\n").slice(-6).join("\n ")}` | ||
| 183 | : ""), | ||
| 184 | }; | ||
| 185 | } finally { | ||
| 186 | await transport.close().catch(() => undefined); | ||
| 187 | } | ||
| 188 | }; | ||
| 189 | |||
| 190 | /** `executor mcp` now ensures a DURABLE (detached) daemon and bridges to it, so a | ||
| 191 | * cold-start scenario leaves that daemon running. Stop it before removing the | ||
| 192 | * data dir so the test never leaks an orphan daemon. */ | ||
| 193 | const stopAutoSpawnedDaemon = (dataDir: string): void => { | ||
| 194 | try { | ||
| 195 | const manifest = JSON.parse( | ||
| 196 | readFileSync(join(dataDir, "server-control", "server.json"), "utf8"), | ||
| 197 | ) as { pid?: number }; | ||
| 198 | if (manifest.pid) process.kill(manifest.pid, "SIGTERM"); | ||
| 199 | } catch { | ||
| 200 | // no manifest (no daemon spawned) — nothing to stop. | ||
| 201 | } | ||
| 202 | }; | ||
| 203 | |||
| 204 | const withTempData = Effect.acquireRelease( | ||
| 205 | Effect.sync(() => { | ||
| 206 | const root = mkdtempSync(join(tmpdir(), "executor-mcp-stress-")); | ||
| 207 | return join(root, "data"); | ||
| 208 | }), | ||
| 209 | (dataDir) => | ||
| 210 | Effect.sync(() => { | ||
| 211 | stopAutoSpawnedDaemon(dataDir); | ||
| 212 | rmSync(join(dataDir, ".."), { recursive: true, force: true }); | ||
| 213 | }), | ||
| 214 | ); | ||
| 215 | |||
| 216 | const summarize = (label: string, reports: ReadonlyArray<ClientReport>): void => { | ||
| 217 | const ok = reports.filter((r) => r.ok).length; | ||
| 218 | const calls = reports.reduce((n, r) => n + r.results.length, 0); | ||
| 219 | const failures = reports.filter((r) => !r.ok).map((r) => `#${r.id}: ${r.error}`); | ||
| 220 | // eslint-disable-next-line no-console | ||
| 221 | console.log( | ||
| 222 | `[stress:${label}] clients ${ok}/${reports.length} ok, ${calls} execute calls` + | ||
| 223 | (failures.length ? `\n failures:\n ${failures.join("\n ")}` : ""), | ||
| 224 | ); | ||
| 225 | }; | ||
| 226 | |||
| 227 | const CONCURRENCY = Number(process.env.E2E_MCP_STRESS_CLIENTS ?? "10"); | ||
| 228 | const CALLS = Number(process.env.E2E_MCP_STRESS_CALLS ?? "5"); | ||
| 229 | |||
| 230 | scenario( | ||
| 231 | "Local CLI MCP · an attach storm of concurrent stdio bridges all execute correctly", | ||
| 232 | { timeout: 240_000 }, | ||
| 233 | Effect.gen(function* () { | ||
| 234 | const dataDir = yield* withTempData; | ||
| 235 | const daemon = yield* startForegroundDaemon(dataDir); | ||
| 236 | |||
| 237 | // Many bridges attach to the one daemon at once and each fires a burst. | ||
| 238 | const reports = yield* Effect.promise(() => | ||
| 239 | Promise.all(Array.from({ length: CONCURRENCY }, (_, id) => runOneClient(id, dataDir, CALLS))), | ||
| 240 | ); | ||
| 241 | summarize("attach-storm", reports); | ||
| 242 | |||
| 243 | for (const report of reports) { | ||
| 244 | expect(report.ok, `client #${report.id} failed: ${report.error ?? ""}`).toBe(true); | ||
| 245 | expect(report.tools, `client #${report.id} listed tools`).toBeGreaterThan(0); | ||
| 246 | // Every reply is the value THIS client asked for — no cross-talk between | ||
| 247 | // concurrent bridges sharing the daemon. | ||
| 248 | report.results.forEach((text, i) => { | ||
| 249 | expect(text, `client #${report.id} call ${i} got its own result`).toContain( | ||
| 250 | String(report.id * 1000 + i), | ||
| 251 | ); | ||
| 252 | }); | ||
| 253 | } | ||
| 254 | expect(daemon.proc.exitCode, `daemon survived the storm:\n${daemon.stderr()}`).toBeNull(); | ||
| 255 | }).pipe(Effect.scoped), | ||
| 256 | ); | ||
| 257 | |||
| 258 | scenario( | ||
| 259 | "Local CLI MCP · a cold-start race elects exactly one server and every client attaches", | ||
| 260 | { timeout: 240_000 }, | ||
| 261 | Effect.gen(function* () { | ||
| 262 | const dataDir = yield* withTempData; | ||
| 263 | |||
| 264 | // No daemon: launch the whole fleet simultaneously so they race the | ||
| 265 | // start-lock + double-check. Exactly one should start a server; the rest | ||
| 266 | // must bridge to it. A broken guard trips the singleton or a port collision. | ||
| 267 | const reports = yield* Effect.promise(() => | ||
| 268 | Promise.all(Array.from({ length: CONCURRENCY }, (_, id) => runOneClient(id, dataDir, CALLS))), | ||
| 269 | ); | ||
| 270 | summarize("cold-start-race", reports); | ||
| 271 | |||
| 272 | for (const report of reports) { | ||
| 273 | expect(report.ok, `client #${report.id} lost the race: ${report.error ?? ""}`).toBe(true); | ||
| 274 | report.results.forEach((text, i) => { | ||
| 275 | expect(text, `client #${report.id} call ${i}`).toContain(String(report.id * 1000 + i)); | ||
| 276 | }); | ||
| 277 | } | ||
| 278 | |||
| 279 | // Exactly one elected owner: a single server.json manifest, not N competing | ||
| 280 | // servers. (The bridges own no DB and write no manifest of their own.) | ||
| 281 | const controlDir = join(dataDir, "server-control"); | ||
| 282 | const serverManifests = (() => { | ||
| 283 | // oxlint-disable-next-line executor/no-try-catch-or-throw -- boundary: dir read | ||
| 284 | try { | ||
| 285 | return readdirSync(controlDir).filter((f) => f === "server.json"); | ||
| 286 | } catch { | ||
| 287 | return []; | ||
| 288 | } | ||
| 289 | })(); | ||
| 290 | expect(serverManifests.length, "exactly one elected server manifest").toBe(1); | ||
| 291 | }).pipe(Effect.scoped), | ||
| 292 | ); | ||
| 293 | |||
| 294 | scenario( | ||
| 295 | "Local CLI MCP · killing the daemon under load tears bridges down without hanging", | ||
| 296 | { timeout: 180_000 }, | ||
| 297 | Effect.gen(function* () { | ||
| 298 | const dataDir = yield* withTempData; | ||
| 299 | const daemon = yield* startForegroundDaemon(dataDir); | ||
| 300 | |||
| 301 | // Attach a bridge, prove it works, then SIGKILL the daemon out from under it | ||
| 302 | // and confirm a subsequent call fails fast (the bridge tore down) rather than | ||
| 303 | // hanging until the test times out. | ||
| 304 | const transport = new StdioClientTransport({ | ||
| 305 | command: "bun", | ||
| 306 | args: ["run", "dev:cli", "mcp", "--scope", testScope], | ||
| 307 | cwd: repoRoot, | ||
| 308 | env: { ...process.env, EXECUTOR_DATA_DIR: dataDir }, | ||
| 309 | stderr: "pipe", | ||
| 310 | }); | ||
| 311 | const client = new Client({ name: "stress-kill", version: "1.0.0" }); | ||
| 312 | yield* Effect.acquireRelease( | ||
| 313 | Effect.promise(() => client.connect(transport)), | ||
| 314 | () => Effect.promise(() => transport.close().catch(() => undefined)), | ||
| 315 | ); | ||
| 316 | |||
| 317 | const first = yield* Effect.promise(() => | ||
| 318 | client.callTool({ name: "execute", arguments: { code: "return 1 + 1" } }), | ||
| 319 | ); | ||
| 320 | expect((first.content as Array<{ text: string }>)[0]?.text, "bridge works pre-kill").toContain( | ||
| 321 | "2", | ||
| 322 | ); | ||
| 323 | |||
| 324 | daemon.proc.kill("SIGKILL"); | ||
| 325 | yield* Effect.promise(() => Promise.race([daemon.proc.exited, Bun.sleep(3000)])); | ||
| 326 | |||
| 327 | // The next call must settle (reject) quickly — a 10s bound well under the | ||
| 328 | // scenario timeout catches a hang. | ||
| 329 | const settled = yield* Effect.promise(() => | ||
| 330 | Promise.race([ | ||
| 331 | client | ||
| 332 | .callTool({ name: "execute", arguments: { code: "return 3" } }) | ||
| 333 | .then(() => "resolved" as const) | ||
| 334 | .catch(() => "rejected" as const), | ||
| 335 | Bun.sleep(10_000).then(() => "timeout" as const), | ||
| 336 | ]), | ||
| 337 | ); | ||
| 338 | // eslint-disable-next-line no-console | ||
| 339 | console.log(`[stress:kill-under-load] post-kill call → ${settled}`); | ||
| 340 | expect(settled, "a call after the daemon dies must not hang").not.toBe("timeout"); | ||
| 341 | }).pipe(Effect.scoped), | ||
| 342 | ); | ||