// Real OpenCode codemode discovers and invokes tools through a two-tool MCP surface.// The server retains 10,200 tools; the replay model scripts discovery and a real upstream call.import { randomBytes } from "node:crypto";import { createServer } from "node:http";import { join } from "node:path"; import { expect } from "@effect/vitest";import { Effect } from "effect";import { AuthTemplateSlug, ConnectionName, IntegrationSlug } from "@executor-js/sdk/shared"; import { serveReplayBrain } from "../src/clients/replay-brain";import { scenario } from "../src/scenario";import { Api, Cli, OpenCode, RunDir, Target } from "../src/services";import { catalogApi, seedLargeCatalog } from "../scenarios/support/large-catalog"; const SERVER_NAME = "executor"; // 322 (the real Vercel fixture) + 12 × 823 synthetic = 10,198, plus the two// callable operations below = exactly 10,200 searchable tools.const SYNTHETIC_INTEGRATIONS = 12;const OPS_PER_INTEGRATION = 823;const EXPECTED_TOOL_COUNT = 10_200; // OpenCode's MCP connect timeout (`DEFAULT_TIMEOUT` in its mcp service). The// Catalog loading has to fit inside it, or the server reads// "failed" and codemode sees nothing.const OPENCODE_CONNECT_TIMEOUT_MS = 30_000; const unique = (prefix: string) => `${prefix}_${randomBytes(4).toString("hex")}`; /** The one integration that is actually callable: a read and a write against * the recording upstream. Everything else in the catalog is discovery mass. */const notesSpec = (baseUrl: string): string => JSON.stringify({ openapi: "3.0.3", info: { title: "Notes API", version: "1.0.0" }, servers: [{ url: baseUrl }], paths: { "/notes": { get: { operationId: "listNotes", summary: "List every note in the notebook", responses: { "200": { description: "ok" } }, }, post: { operationId: "createNote", summary: "Create a note in the notebook", requestBody: { required: true, content: { "application/json": { schema: { type: "object", properties: { text: { type: "string" } }, required: ["text"], }, }, }, }, responses: { "200": { description: "ok" } }, }, }, }, }); interface RecordedRequest { readonly method: string; readonly authorization: string | undefined;} const serveRecordingUpstream = Effect.acquireRelease( Effect.callback<{ readonly url: string; readonly requests: RecordedRequest[]; close: () => void; }>((resume) => { const requests: RecordedRequest[] = []; const server = createServer((request, response) => { request.on("data", () => undefined); request.on("end", () => { requests.push({ method: request.method ?? "", authorization: request.headers.authorization, }); response.writeHead(200, { "content-type": "application/json" }); response.end(JSON.stringify({ notes: [{ id: "note_0", text: "existing note" }] })); }); }); server.listen(0, "127.0.0.1", () => { const address = server.address(); const port = typeof address === "object" && address ? address.port : 0; resume( Effect.succeed({ url: `http://127.0.0.1:${port}`, requests, close: () => { server.close(); server.closeAllConnections(); }, }), ); }); }), (upstream) => Effect.sync(() => upstream.close()),); scenario( "Passthrough · the real OpenCode binary runs its own codemode over 10,200 Executor tools", { timeout: 900_000 }, Effect.scoped( Effect.gen(function* () { const target = yield* Target; const opencode = yield* OpenCode; const runDir = yield* RunDir; const cli = yield* Cli; const { client: makeClient } = yield* Api; const identity = yield* target.newIdentity(); const email = identity.credentials?.email ?? identity.label; const client = yield* makeClient(catalogApi, identity); const upstream = yield* serveRecordingUpstream; const notesSlug = unique("notes"); // --- Seed: 10,198 discovery-mass tools + 2 callable ones. --- const seeded = yield* seedLargeCatalog(client, { syntheticIntegrations: SYNTHETIC_INTEGRATIONS, opsPerIntegration: OPS_PER_INTEGRATION, }); const cleanup = Effect.gen(function* () { yield* client.connections .remove({ params: { owner: "org", integration: IntegrationSlug.make(notesSlug), name: ConnectionName.make("main"), }, }) .pipe(Effect.ignore); yield* client.openapi.removeSpec({ params: { slug: notesSlug } }).pipe(Effect.ignore); yield* seeded.cleanup; }); yield* Effect.ensuring( Effect.gen(function* () { yield* client.openapi.addSpec({ payload: { spec: { kind: "blob", value: notesSpec(upstream.url) }, slug: notesSlug, baseUrl: upstream.url, authenticationTemplate: [ { slug: "apiKey", type: "apiKey", headers: { authorization: ["Bearer ", { type: "variable", name: "token" }] }, }, ], }, }); yield* client.connections.create({ payload: { owner: "org", name: ConnectionName.make("main"), integration: IntegrationSlug.make(notesSlug), template: AuthTemplateSlug.make("apiKey"), value: "tok_notes", }, }); const visible = (yield* client.tools.list({ query: {} })).filter( (tool) => tool.static !== true, ); expect(visible.length, "the catalog is exactly the advertised size").toBe( EXPECTED_TOOL_COUNT, ); // --- The replay brain: two scripted turns of OpenCode codemode. --- // Turn 0: discover the notes tool through Executor search. // Turn 1: call the path search returned. Turn 2: summarize, stop. let discoveredId: string | undefined; const brain = yield* serveReplayBrain((ctx) => { // OpenCode also asks the model for a session title, with no tools // offered. Answer it with text and keep it out of the script. if (ctx.toolNames.length === 0) return { text: "Notebook" }; if (ctx.lastRole === "user") { return { text: "Searching the connected tools.", tool: { name: "execute", args: { code: `return await tools.${SERVER_NAME}.search({ query: "${notesSlug} listNotes", limit: 5 });`, }, }, }; } if (discoveredId === undefined) { const result = ctx.lastToolResult ?? ""; // Search returns an opaque tool ID for the next invoke call. const match = new RegExp(`"id":\\s*"([^"\\n]*${notesSlug}[^"]*listNotes)"`).exec( result, ); if (!match) { throw new Error(`search did not surface the notes tool: ${result.slice(0, 600)}`); } discoveredId = match[1]!; return { text: "Found it. Listing the notes.", tool: { name: "execute", args: { code: `return await tools.${SERVER_NAME}.invoke({tool: ${JSON.stringify(discoveredId)}, arguments: {}});`, }, }, }; } return { text: "The notebook has one existing note." }; }); const passthroughUrl = new URL("/mcp?mode=passthrough", target.baseUrl).toString(); const home = opencode.makeHome(SERVER_NAME, passthroughUrl, { chatBrainUrl: brain.baseUrl, }); const env = { ...home.env, OPENCODE_EXPERIMENTAL_CODE_MODE: "true", PS1: "$ ", BASH_SILENCE_DEPRECATION_WARNING: "1", }; // First-run database migration happens off camera. yield* Effect.sync(() => opencode.warmUp(home)); let connectMs = -1; yield* cli.session( ["bash", "--norc"], async (term) => { await term.screen.waitForText("$", { timeoutMs: 10_000 }); const outputAfter = (text: string, line: string): string | null => { const echoed = text.lastIndexOf(line); if (echoed === -1) return null; const after = text.slice(echoed + line.length); return after.trimEnd().endsWith("\n$") ? after : null; }; const sh = async (line: string, timeoutMs: number) => { await term.keyboard.type(line); await term.keyboard.press("Enter"); const snapshot = await term.screen.waitUntil( (current) => outputAfter(current.text, line) !== null, { timeoutMs }, ); return outputAfter(snapshot.text, line) ?? ""; }; // OpenCode's own OAuth against the target: discovery, DCR, PKCE. const consent = opencode.completeOAuthConsent(home, email, home.openedUrls().length); const auth = await sh(`opencode mcp auth ${SERVER_NAME}`, 90_000); await consent; expect(auth, "opencode mcp auth completes").not.toContain("failed"); // The load-bearing connect: catalog loading and two MCP definitions // inside OpenCode's own 30s connect timeout. const startedAt = Date.now(); const listed = await sh("opencode mcp list", 120_000); connectMs = Date.now() - startedAt; expect( listed, `OpenCode connects to the 10,200-tool passthrough endpoint (took ${connectMs}ms)`, ).toContain("connected"); // A real agent turn: OpenCode's codemode over our catalog. const ran = await sh(`opencode run "List the notes in my notebook"`, 300_000); expect(ran, "the run did not error").not.toContain("UnknownError"); }, { cwd: home.projectDir, env, record: join(runDir, "terminal.cast"), viewport: { cols: 100, rows: 40 }, }, ); // --- What OpenCode showed the model, and what came back. --- expect(brain.errors(), "the scripted brain hit no surprises").toEqual([]); // Only the turns that carried tools are the agent loop; the // title request is OpenCode housekeeping. const requests = brain.requests().filter((request) => request.toolNames.length > 0); expect(requests.length, "three model turns: search, call, summary").toBe(3); // Codemode: the model sees ONE execute tool, not 10,200 functions. const offered = requests[0]!.toolNames; expect(offered, "OpenCode offers its codemode execute tool").toContain("execute"); expect( offered.filter((name) => name.startsWith(`${SERVER_NAME}_`)), "no MCP tool is flattened into the model's tool list", ).toEqual([]); expect( offered.length, "the model's tool list stays small in front of a 10,200-tool server", ).toBeLessThan(40); // Discovery returned the full underlying tool ID. expect(discoveredId, "search returned a tool ID").toBeDefined(); expect(discoveredId, "the ID names the integration tool").toContain(notesSlug); // The call went over the wire with the connection's credential and // the payload came back through OpenCode's interpreter. const lastToolResult = [...requests[2]!.messages] .reverse() .find((message) => message.role === "tool")?.content; expect(lastToolResult, "the executed program returned the upstream payload").toContain( "existing note", ); const upstreamGet = upstream.requests.find((request) => request.method === "GET"); expect(upstreamGet, "the GET reached the upstream").toBeDefined(); expect(upstreamGet?.authorization, "the connection's credential was applied").toBe( "Bearer tok_notes", ); // Recorded for the run report; the hard bound is OpenCode's own // connect timeout, which `mcp list` above already proved. expect(connectMs, "connect stays inside OpenCode's timeout").toBeLessThan( OPENCODE_CONNECT_TIMEOUT_MS * 4, ); }), cleanup, ); }), ),);
// Real OpenCode codemode discovers and invokes tools through a two-tool MCP surface.
// The server retains 10,200 tools; the replay model scripts discovery and a real upstream call.
import { randomBytes } from "node:crypto";
import { createServer } from "node:http";
import { join } from "node:path";
import { expect } from "@effect/vitest";
import { Effect } from "effect";
import { AuthTemplateSlug, ConnectionName, IntegrationSlug } from "@executor-js/sdk/shared";
import { serveReplayBrain } from "../src/clients/replay-brain";
import { scenario } from "../src/scenario";
import { Api, Cli, OpenCode, RunDir, Target } from "../src/services";
import { catalogApi, seedLargeCatalog } from "../scenarios/support/large-catalog";
const SERVER_NAME = "executor";
// 322 (the real Vercel fixture) + 12 × 823 synthetic = 10,198, plus the two
// callable operations below = exactly 10,200 searchable tools.
const SYNTHETIC_INTEGRATIONS = 12;
const OPS_PER_INTEGRATION = 823;
const EXPECTED_TOOL_COUNT = 10_200;
// OpenCode's MCP connect timeout (`DEFAULT_TIMEOUT` in its mcp service). The
// Catalog loading has to fit inside it, or the server reads
// "failed" and codemode sees nothing.
const OPENCODE_CONNECT_TIMEOUT_MS = 30_000;
const unique = (prefix: string) => `${prefix}_${randomBytes(4).toString("hex")}`;
/** The one integration that is actually callable: a read and a write against
* the recording upstream. Everything else in the catalog is discovery mass. */
const notesSpec = (baseUrl: string): string =>
JSON.stringify({
openapi: "3.0.3",
info: { title: "Notes API", version: "1.0.0" },
servers: [{ url: baseUrl }],
paths: {
"/notes": {
get: {
operationId: "listNotes",
summary: "List every note in the notebook",
responses: { "200": { description: "ok" } },
},
post: {
operationId: "createNote",
summary: "Create a note in the notebook",
requestBody: {
required: true,
content: {
"application/json": {
schema: {
type: "object",
properties: { text: { type: "string" } },
required: ["text"],
},
},
},
},
responses: { "200": { description: "ok" } },
},
},
},
});
interface RecordedRequest {
readonly method: string;
readonly authorization: string | undefined;
}
const serveRecordingUpstream = Effect.acquireRelease(
Effect.callback<{
readonly url: string;
readonly requests: RecordedRequest[];
close: () => void;
}>((resume) => {
const requests: RecordedRequest[] = [];
const server = createServer((request, response) => {
request.on("data", () => undefined);
request.on("end", () => {
requests.push({
method: request.method ?? "",
authorization: request.headers.authorization,
});
response.writeHead(200, { "content-type": "application/json" });
response.end(JSON.stringify({ notes: [{ id: "note_0", text: "existing note" }] }));
});
});
server.listen(0, "127.0.0.1", () => {
const address = server.address();
const port = typeof address === "object" && address ? address.port : 0;
resume(
Effect.succeed({
url: `http://127.0.0.1:${port}`,
requests,
close: () => {
server.close();
server.closeAllConnections();
},
}),
);
});
}),
(upstream) => Effect.sync(() => upstream.close()),
);
scenario(
"Passthrough · the real OpenCode binary runs its own codemode over 10,200 Executor tools",
{ timeout: 900_000 },
Effect.scoped(
Effect.gen(function* () {
const target = yield* Target;
const opencode = yield* OpenCode;
const runDir = yield* RunDir;
const cli = yield* Cli;
const { client: makeClient } = yield* Api;
const identity = yield* target.newIdentity();
const email = identity.credentials?.email ?? identity.label;
const client = yield* makeClient(catalogApi, identity);
const upstream = yield* serveRecordingUpstream;
const notesSlug = unique("notes");
// --- Seed: 10,198 discovery-mass tools + 2 callable ones. ---
const seeded = yield* seedLargeCatalog(client, {
syntheticIntegrations: SYNTHETIC_INTEGRATIONS,
opsPerIntegration: OPS_PER_INTEGRATION,
});
const cleanup = Effect.gen(function* () {
yield* client.connections
.remove({
params: {
owner: "org",
integration: IntegrationSlug.make(notesSlug),
name: ConnectionName.make("main"),
},
})
.pipe(Effect.ignore);
yield* client.openapi.removeSpec({ params: { slug: notesSlug } }).pipe(Effect.ignore);
yield* seeded.cleanup;
});
yield* Effect.ensuring(
Effect.gen(function* () {
yield* client.openapi.addSpec({
payload: {
spec: { kind: "blob", value: notesSpec(upstream.url) },
slug: notesSlug,
baseUrl: upstream.url,
authenticationTemplate: [
{
slug: "apiKey",
type: "apiKey",
headers: { authorization: ["Bearer ", { type: "variable", name: "token" }] },
},
],
},
});
yield* client.connections.create({
payload: {
owner: "org",
name: ConnectionName.make("main"),
integration: IntegrationSlug.make(notesSlug),
template: AuthTemplateSlug.make("apiKey"),
value: "tok_notes",
},
});
const visible = (yield* client.tools.list({ query: {} })).filter(
(tool) => tool.static !== true,
);
expect(visible.length, "the catalog is exactly the advertised size").toBe(
EXPECTED_TOOL_COUNT,
);
// --- The replay brain: two scripted turns of OpenCode codemode. ---
// Turn 0: discover the notes tool through Executor search.
// Turn 1: call the path search returned. Turn 2: summarize, stop.
let discoveredId: string | undefined;
const brain = yield* serveReplayBrain((ctx) => {
// OpenCode also asks the model for a session title, with no tools
// offered. Answer it with text and keep it out of the script.
if (ctx.toolNames.length === 0) return { text: "Notebook" };
if (ctx.lastRole === "user") {
return {
text: "Searching the connected tools.",
tool: {
name: "execute",
args: {
code: `return await tools.${SERVER_NAME}.search({ query: "${notesSlug} listNotes", limit: 5 });`,
},
},
};
}
if (discoveredId === undefined) {
const result = ctx.lastToolResult ?? "";
// Search returns an opaque tool ID for the next invoke call.
const match = new RegExp(`"id":\\s*"([^"\\n]*${notesSlug}[^"]*listNotes)"`).exec(
result,
);
if (!match) {
throw new Error(`search did not surface the notes tool: ${result.slice(0, 600)}`);
}
discoveredId = match[1]!;
return {
text: "Found it. Listing the notes.",
tool: {
name: "execute",
args: {
code: `return await tools.${SERVER_NAME}.invoke({tool: ${JSON.stringify(discoveredId)}, arguments: {}});`,
},
},
};
}
return { text: "The notebook has one existing note." };
});
const passthroughUrl = new URL("/mcp?mode=passthrough", target.baseUrl).toString();
const home = opencode.makeHome(SERVER_NAME, passthroughUrl, {
chatBrainUrl: brain.baseUrl,
});
const env = {
...home.env,
OPENCODE_EXPERIMENTAL_CODE_MODE: "true",
PS1: "$ ",
BASH_SILENCE_DEPRECATION_WARNING: "1",
};
// First-run database migration happens off camera.
yield* Effect.sync(() => opencode.warmUp(home));
let connectMs = -1;
yield* cli.session(
["bash", "--norc"],
async (term) => {
await term.screen.waitForText("$", { timeoutMs: 10_000 });
const outputAfter = (text: string, line: string): string | null => {
const echoed = text.lastIndexOf(line);
if (echoed === -1) return null;
const after = text.slice(echoed + line.length);
return after.trimEnd().endsWith("\n$") ? after : null;
};
const sh = async (line: string, timeoutMs: number) => {
await term.keyboard.type(line);
await term.keyboard.press("Enter");
const snapshot = await term.screen.waitUntil(
(current) => outputAfter(current.text, line) !== null,
{ timeoutMs },
);
return outputAfter(snapshot.text, line) ?? "";
};
// OpenCode's own OAuth against the target: discovery, DCR, PKCE.
const consent = opencode.completeOAuthConsent(home, email, home.openedUrls().length);
const auth = await sh(`opencode mcp auth ${SERVER_NAME}`, 90_000);
await consent;
expect(auth, "opencode mcp auth completes").not.toContain("failed");
// The load-bearing connect: catalog loading and two MCP definitions
// inside OpenCode's own 30s connect timeout.
const startedAt = Date.now();
const listed = await sh("opencode mcp list", 120_000);
connectMs = Date.now() - startedAt;
expect(
listed,
`OpenCode connects to the 10,200-tool passthrough endpoint (took ${connectMs}ms)`,
).toContain("connected");
// A real agent turn: OpenCode's codemode over our catalog.
const ran = await sh(`opencode run "List the notes in my notebook"`, 300_000);
expect(ran, "the run did not error").not.toContain("UnknownError");
},
{
cwd: home.projectDir,
env,
record: join(runDir, "terminal.cast"),
viewport: { cols: 100, rows: 40 },
},
);
// --- What OpenCode showed the model, and what came back. ---
expect(brain.errors(), "the scripted brain hit no surprises").toEqual([]);
// Only the turns that carried tools are the agent loop; the
// title request is OpenCode housekeeping.
const requests = brain.requests().filter((request) => request.toolNames.length > 0);
expect(requests.length, "three model turns: search, call, summary").toBe(3);
// Codemode: the model sees ONE execute tool, not 10,200 functions.
const offered = requests[0]!.toolNames;
expect(offered, "OpenCode offers its codemode execute tool").toContain("execute");
expect(
offered.filter((name) => name.startsWith(`${SERVER_NAME}_`)),
"no MCP tool is flattened into the model's tool list",
).toEqual([]);
expect(
offered.length,
"the model's tool list stays small in front of a 10,200-tool server",
).toBeLessThan(40);
// Discovery returned the full underlying tool ID.
expect(discoveredId, "search returned a tool ID").toBeDefined();
expect(discoveredId, "the ID names the integration tool").toContain(notesSlug);
// The call went over the wire with the connection's credential and
// the payload came back through OpenCode's interpreter.
const lastToolResult = [...requests[2]!.messages]
.reverse()
.find((message) => message.role === "tool")?.content;
expect(lastToolResult, "the executed program returned the upstream payload").toContain(
"existing note",
);
const upstreamGet = upstream.requests.find((request) => request.method === "GET");
expect(upstreamGet, "the GET reached the upstream").toBeDefined();
expect(upstreamGet?.authorization, "the connection's credential was applied").toBe(
"Bearer tok_notes",
);
// Recorded for the run report; the hard bound is OpenCode's own
// connect timeout, which `mcp list` above already proved.
expect(connectMs, "connect stays inside OpenCode's timeout").toBeLessThan(
OPENCODE_CONNECT_TIMEOUT_MS * 4,
);
}),
cleanup,
);
}),
),
);