Fix the local e2e suite: toolkit MCP DB lock, Bun-only spawn, stale auth selector

Rhys Sullivan committed 3 months ago

Commit 41e3ecd1 parent f674fb84 files · +90 −65Browse files

Unified · Split

4 files

apps/local/src/executor.ts +37 −18
apps/local/src/main.ts +5 −0
e2e/local/auth.test.ts +5 −3
e2e/local/cli-mcp-daemon-attach-stress.test.ts +43 −44

 import executorConfig from "../executor.config";
 import { localAnalytics } from "./analytics";
 import { localDataMigrations } from "./db/data-migrations";
-import { openOwnedLocalDatabase } from "./db/owned-database";
+import { openOwnedLocalDatabase, type OwnedLocalDatabase } from "./db/owned-database";
 interface ResolvedStorage {
   readonly dataDir: string;
 export interface LocalExecutorOptions {
   readonly activeToolkitSlug?: string;
+  /**
+   * Reuse an already-open owned database instead of opening (and locking) the
+   * data dir again. A toolkit-scoped MCP session differs from the default one
+   * only in its plugin set, so it must ride the running server's DB handle:
+   * `openOwnedLocalDatabase` takes an EXCLUSIVE lock, and a second open from
+   * inside the same process contends with the lock this process already holds.
+   * The borrowed handle is NOT closed when the derived executor disposes —
+   * whoever opened it still owns its lifetime.
+   */
+  readonly borrowedDb?: OwnedLocalDatabase;
 }
 const loadLocalPlugins = (options: LocalExecutorOptions = {}) =>
 interface LocalExecutorBundle {
   readonly executor: Executor<LocalPlugins>;
   readonly plugins: LocalPlugins;
+  /** The owned DB this bundle opened (or borrowed). Surfaced so a
+   *  toolkit-scoped executor can ride the SAME handle instead of contending
+   *  with this process's own exclusive data-dir lock. */
+  readonly db: OwnedLocalDatabase;
   /** Where this daemon's web UI is reachable, resolved once at boot. Surfaced
    *  so callers building user-facing links (MCP artifact deep links) use the
    *  same origin the executor itself was configured with. */
       const tenantId = makeTenantId(cwd);
       const tables = collectTables();
-      const owned = yield* Effect.acquireRelease(
+      // A borrowed handle is owned by its opener, so it is used as-is and left
-        Effect.tryPromise({
+      // open on release; only a handle opened here is closed here.
-          try: () =>
+      const owned = options.borrowedDb
-            openOwnedLocalDatabase({
+        ? options.borrowedDb
-              dataDir: storage.dataDir,
+        : yield* Effect.acquireRelease(
-              tables,
+            Effect.tryPromise({
-              namespace: localNamespace,
+              try: () =>
-              tenantId,
+                openOwnedLocalDatabase({
+                  dataDir: storage.dataDir,
+                  tables,
+                  namespace: localNamespace,
+                  tenantId,
+                }),
+              catch: (cause) =>
+                new LocalExecutorCreateError({
+                  message: CREATE_SQLITE_ERROR_MESSAGE,
+                  cause,
+                }),
             }),
-          catch: (cause) =>
+            (database) => Effect.promise(() => database.close()).pipe(Effect.ignore),
-            new LocalExecutorCreateError({
+          );
-              message: CREATE_SQLITE_ERROR_MESSAGE,
-              cause,
-            }),
-        }),
-        (database) => Effect.promise(() => database.close()).pipe(Effect.ignore),
-      );
       const sqlite = owned.db;
       const migration = owned.migration;
           );
       }
-      return { executor, plugins, webBaseUrl };
+      return { executor, plugins, webBaseUrl, db: owned };
     }),
   );
 };
     executor: bundle.executor,
     plugins: bundle.plugins,
     webBaseUrl: bundle.webBaseUrl,
+    db: bundle.db,
     dispose: async () => {
       await Effect.runPromise(Effect.ignore(bundle.executor.close()));
       await ignorePromiseFailure("disposeRuntime", () => runtime.dispose());

apps/local/src/main.ts+5−0

             },
           };
         }
+        // Borrow the running server's DB handle: this process already holds the
+        // data dir's exclusive ownership lock, so opening it a second time here
+        // fails against ourselves. The toolkit executor differs only in its
+        // plugin set, and the borrowed handle stays open when it disposes.
         const handle = await createExecutorHandle({
           activeToolkitSlug: resource.slug,
+          borrowedDb: (await getExecutorBundle()).db,
         });
         const toolkitEngine = withExecutionAnalytics(
           createExecutionEngine({

e2e/local/auth.test.ts+5−3

           await page.goto(url, { waitUntil: "domcontentloaded" });
           await page.getByRole("link", { name: "Secrets" }).first().waitFor({ timeout: 30_000 });
           // Integrations actually LOAD (the built-in Executor integration) — proves
-          // auth + data, not just the static shell.
+          // auth + data, not just the static shell. Matched on the row's stable
-          await page.getByText("built-in").first().waitFor({ timeout: 30_000 });
+          // testid: the list renders each integration's name + slug, never the
+          // literal "built-in" (that string is only an internal `kind`).
+          await page.getByTestId("integration-entry-executor").first().waitFor({ timeout: 30_000 });
           // The token is moved out of the URL and persisted to localStorage.
           expect(new URL(page.url()).searchParams.has("_token")).toBe(false);
           const stored = await page.evaluate(() => localStorage.getItem("executor.authToken"));
           await page.getByRole("button", { name: "Connect" }).click();
           await page.getByRole("link", { name: "Secrets" }).first().waitFor({ timeout: 30_000 });
           // The reconnect fully restores — integrations LOAD, not a stale 401.
-          await page.getByText("built-in").first().waitFor({ timeout: 30_000 });
+          await page.getByTestId("integration-entry-executor").first().waitFor({ timeout: 30_000 });
         });
       }),
     );

e2e/local/cli-mcp-daemon-attach-stress.test.ts+43−44

 import { Client } from "@modelcontextprotocol/sdk/client/index.js";
 import { StdioClientTransport } from "@modelcontextprotocol/sdk/client/stdio.js";
 import { Effect } from "effect";
+import { spawn, type ChildProcessWithoutNullStreams } from "node:child_process";
 import { mkdtempSync, readdirSync, rmSync } from "node:fs";
 import { tmpdir } from "node:os";
 import { join } from "node:path";
 import { fileURLToPath } from "node:url";
-import type { Subprocess } from "bun";
 import { scenario } from "../src/scenario";
 // Generous: a dev-mode daemon boots a Vite dev server, slow under machine load.
 const readyTimeoutMs = 150_000;
-type DaemonProc = Subprocess<"ignore", "pipe", "pipe">;
+// vitest runs this suite under NODE, not bun, so the daemon is spawned with
+// node:child_process (the rest of the e2e harness does the same). `Bun.spawn`
+// here threw `ReferenceError: Bun is not defined` on every run.
+type DaemonProc = ChildProcessWithoutNullStreams;
 const waitForDaemonReady = (
   proc: DaemonProc,
     let stdoutBuffer = "";
     let stderrBuffer = "";
     let settled = false;
-    const decoder = new TextDecoder();
-    const stdout = proc.stdout.getReader();
-    const stderr = proc.stderr.getReader();
     const deadline = setTimeout(() => {
       if (settled) return;
       settled = true;
       // oxlint-disable-next-line executor/no-promise-reject, executor/no-error-constructor -- boundary: captured daemon stderr
       rejectReady(new Error(`daemon did not announce ready: ${stderrBuffer}`));
     }, readyTimeoutMs);
-    void (async () => {
+    proc.stderr.on("data", (chunk: Buffer) => {
-      while (true) {
+      stderrBuffer += chunk.toString();
-        const { value, done } = await stderr.read();
+    });
-        if (done) return;
+    proc.stdout.on("data", (chunk: Buffer) => {
-        stderrBuffer += decoder.decode(value);
+      if (settled) return;
-      }
+      stdoutBuffer += chunk.toString();
-    })();
+      const match = /Daemon ready on http:\/\/(?:\[[^\]]+\]|[^:\s]+):(\d+)/.exec(stdoutBuffer);
-    void (async () => {
+      if (match) {
-      while (true) {
+        settled = true;
-        const { value, done } = await stdout.read();
+        clearTimeout(deadline);
-        if (done) {
+        resolveReady({ port: Number(match[1]), stderr: () => stderrBuffer });
-          if (!settled) {
-            settled = true;
-            clearTimeout(deadline);
-            // oxlint-disable-next-line executor/no-promise-reject, executor/no-error-constructor -- boundary: captured daemon stderr
-            rejectReady(new Error(`daemon stdout closed before ready: ${stderrBuffer}`));
-          }
-          return;
-        }
-        stdoutBuffer += decoder.decode(value);
-        const match = /Daemon ready on http:\/\/(?:\[[^\]]+\]|[^:\s]+):(\d+)/.exec(stdoutBuffer);
-        if (match) {
-          settled = true;
-          clearTimeout(deadline);
-          resolveReady({ port: Number(match[1]), stderr: () => stderrBuffer });
-          return;
-        }
       }
-    })();
+    });
+    proc.stdout.on("close", () => {
+      if (settled) return;
+      settled = true;
+      clearTimeout(deadline);
+      // oxlint-disable-next-line executor/no-promise-reject, executor/no-error-constructor -- boundary: captured daemon stderr
+      rejectReady(new Error(`daemon stdout closed before ready: ${stderrBuffer}`));
+    });
   });
 const spawnDaemon = (dataDir: string): DaemonProc =>
-  Bun.spawn(
+  spawn(
+    "bun",
     [
-      "bun",
       "run",
       "dev:cli",
       "daemon",
     {
       cwd: repoRoot,
       env: { ...process.env, EXECUTOR_DATA_DIR: dataDir },
-      stdin: "ignore",
+      stdio: ["ignore", "pipe", "pipe"],
-      stdout: "pipe",
-      stderr: "pipe",
     },
-  );
+  ) as DaemonProc;
+const exited = (proc: DaemonProc): Promise<void> =>
+  proc.exitCode !== null || proc.signalCode !== null
+    ? Promise.resolve()
+    : new Promise((resolve) => proc.once("exit", () => resolve()));
 const stopProc = async (proc: DaemonProc): Promise<void> => {
-  if (proc.exitCode !== null) return;
+  if (proc.exitCode !== null || proc.signalCode !== null) return;
   proc.kill("SIGTERM");
-  await Promise.race([proc.exited, Bun.sleep(3000)]);
+  await Promise.race([exited(proc), new Promise<void>((resolve) => setTimeout(resolve, 3000))]);
-  if (proc.exitCode === null) proc.kill("SIGKILL");
+  if (proc.exitCode === null && proc.signalCode === null) proc.kill("SIGKILL");
 };
 const startForegroundDaemon = (dataDir: string) =>
     );
     daemon.proc.kill("SIGKILL");
-    yield* Effect.promise(() => Promise.race([daemon.proc.exited, Bun.sleep(3000)]));
+    yield* Effect.promise(() =>
+      Promise.race([
+        exited(daemon.proc),
+        new Promise<void>((resolve) => setTimeout(resolve, 3000)),
+      ]),
+    );
     // The next call must settle (reject) quickly — a 10s bound well under the
     // scenario timeout catches a hang.
           .callTool({ name: "execute", arguments: { code: "return 3" } })
           .then(() => "resolved" as const)
           .catch(() => "rejected" as const),
-        Bun.sleep(10_000).then(() => "timeout" as const),
+        new Promise<"timeout">((resolve) => setTimeout(() => resolve("timeout"), 10_000)),
       ]),
     );
     // eslint-disable-next-line no-console

Fix the local e2e suite: toolkit MCP DB lock, Bun-only spawn, stale auth selector

Keyboard shortcuts

General

Issues and pull requests

Issues, pull requests and Inbox

Repository code