// Cloud: a toolkit MCP endpoint must connect in time that does NOT scale with // the size of the surrounding catalog. // // When a client (OpenCode, Claude Code, mcporter) connects to // /mcp/toolkits/, the server builds the `execute` tool's description, // which lists the workspace's connections, which (for a toolkit session) runs // the policy engine over the WHOLE catalog to decide tool visibility. A // per-tool policy resolution there is an N+1 that scales with total catalog // size, not toolkit size: a workspace with thousands of tools across a dozen // integrations pushes the connect past the MCP client's connect timeout, and the // toolkit appears permanently "failed" even though nothing is broken. // // This scenario seeds a production-shaped catalog (one real OpenAPI spec plus // enough synthetic integrations to look like a working workspace, ~3,300 tools over // 11 integrations) and asserts the catalog size adds only a small, bounded cost to // connect. The control is a fresh identity with a near-empty catalog: it pays // the same OAuth + MCP handshake, so the DIFFERENCE isolates the catalog-walk // cost the fix removes. Before the fix this delta is tens of seconds (and in // production, a hard timeout); after it, it is sub-second. import { expect } from "@effect/vitest"; import { Effect } from "effect"; import { AuthTemplateSlug, ConnectionName, IntegrationSlug } from "@executor-js/sdk/shared"; import { scenario } from "../src/scenario"; import { Api, Mcp, Target } from "../src/services"; import type { McpSurface } from "../src/surfaces/mcp"; import type { Identity } from "../src/target"; import { catalogApi, seedLargeCatalog, unique, type SeededCatalog, } from "../scenarios/support/large-catalog"; const toolkitUrl = (baseUrl: string, slug: string): string => new URL(`/mcp/toolkits/${slug}`, baseUrl).toString(); // The extra wall-clock a ~3,300-tool catalog is allowed to add to a toolkit // connect over a near-empty one. Post-fix the catalog walk is a couple of // batched reads (~tens of ms); pre-fix it is 2 uncached reads PER TOOL, tens // of seconds. 10s sits an order of magnitude below the regression and well // above OAuth/scheduling jitter, so it is decisive without being flaky. const MAX_CATALOG_CONNECT_OVERHEAD_MS = 10_000; // One-operation OpenAPI spec for the control identity: a real toolkit session, // minimal catalog, so its connect time is "the handshake without the walk". const tinySpec = (baseUrl: string): string => JSON.stringify({ openapi: "3.0.3", info: { title: "Tiny API", version: "1.0.0" }, servers: [{ url: baseUrl }], paths: { "/ping/{id}": { get: { operationId: "getPing", security: [{ apiKey: [] }], parameters: [{ name: "id", in: "path", required: true, schema: { type: "string" } }], responses: { "200": { description: "ok" } }, }, }, }, components: { securitySchemes: { apiKey: { type: "apiKey", in: "header", name: "x-tok" } } }, }); /** Time how long a fresh MCP client takes to connect to a toolkit endpoint and * read its advertised tools (the path that pays the catalog-walk cost). */ const timeToolkitConnect = (mcp: McpSurface, identity: Identity, url: string) => Effect.gen(function* () { const session = mcp.session(identity, { url }); const startedAt = Date.now(); const defs = yield* session.describeTools(); const elapsedMs = Date.now() - startedAt; return { elapsedMs, toolNames: defs.map((d) => d.name) }; }); scenario( "Toolkits ยท connect time does not scale with catalog size", { timeout: 240_000 }, Effect.scoped( Effect.gen(function* () { const target = yield* Target; const mcp = yield* Mcp; const { client: makeClient } = yield* Api; // --- control: a fresh identity with a near-empty catalog --------------- const controlIdentity = yield* target.newIdentity(); const controlClient = yield* makeClient(catalogApi, controlIdentity); const controlIntegration = unique("tiny"); const controlToolkitName = unique("control-kit"); const control = yield* Effect.gen(function* () { yield* controlClient.openapi.addSpec({ payload: { spec: { kind: "blob", value: tinySpec("https://tiny.example") }, slug: IntegrationSlug.make(controlIntegration), baseUrl: "https://tiny.example", authenticationTemplate: [ { slug: "apiKey", type: "apiKey", headers: { "x-tok": [{ type: "variable", name: "token" }] }, }, ], }, }); yield* controlClient.connections.create({ payload: { owner: "org", name: ConnectionName.make("conn0"), integration: IntegrationSlug.make(controlIntegration), template: AuthTemplateSlug.make("apiKey"), value: "unused-token", }, }); const toolkit = yield* controlClient.toolkits.create({ payload: { owner: "org", name: controlToolkitName }, }); yield* controlClient.toolkits.createConnection({ params: { toolkitId: toolkit.id }, payload: { pattern: `${controlIntegration}.org.conn0.*` }, }); return yield* timeToolkitConnect( mcp, controlIdentity, toolkitUrl(target.baseUrl, toolkit.slug), ); }).pipe( Effect.ensuring( controlClient.openapi .removeSpec({ params: { slug: controlIntegration } }) .pipe(Effect.ignore), ), ); expect(control.toolNames, "control toolkit advertises the execute tool").toContain("execute"); // --- subject: a fresh identity with a large, production-shaped catalog - const bigIdentity = yield* target.newIdentity(); const bigClient = yield* makeClient(catalogApi, bigIdentity); let seededCatalog: SeededCatalog | undefined; yield* Effect.gen(function* () { // 1 real integration (Vercel, 322 ops) + 10 synthetic = 11 integrations, ~2,200 // tools. Sized so the pre-fix N+1 connect (~27s here) still completes // under the MCP client's connect timeout, so the regression surfaces as // a failed ASSERTION on the overhead rather than a connect crash. const seeded = yield* seedLargeCatalog(bigClient, { includeRealSpec: true, syntheticIntegrations: 10, opsPerIntegration: 190, }); seededCatalog = seeded; expect( seeded.toolCount, "the seeded catalog is large enough to expose the N+1 (thousands of tools)", ).toBeGreaterThan(2_000); expect( seeded.integrationSlugs.length, "the catalog spans a production-like number of integrations", ).toBeGreaterThanOrEqual(11); const big = yield* timeToolkitConnect( mcp, bigIdentity, toolkitUrl(target.baseUrl, seeded.toolkitSlug), ); expect(big.toolNames, "large-catalog toolkit advertises the execute tool").toContain( "execute", ); const overhead = big.elapsedMs - control.elapsedMs; expect( overhead, `catalog of ${seeded.toolCount} tools across ${seeded.integrationSlugs.length} integrations ` + `added ${overhead}ms to connect (big ${big.elapsedMs}ms vs control ${control.elapsedMs}ms); ` + `the per-tool policy N+1 would make this scale into tens of seconds`, ).toBeLessThan(MAX_CATALOG_CONNECT_OVERHEAD_MS); }).pipe( Effect.ensuring( Effect.suspend(() => seededCatalog?.cleanup ?? Effect.void).pipe(Effect.ignore), ), ); }), ), );