// Seeds a production-shaped catalog for the toolkit-policy performance// scenarios, plus the pure builders behind it.//// Why a large catalog at all: a toolkit MCP endpoint resolves tool visibility// by running the policy engine over the WHOLE catalog on connect (the// getDescription -> connections.list -> toolsList path), not just the toolkit's// own connections. A per-tool resolution there is an N+1 that scales with total// catalog size, so a realistic workspace (a real spec plus enough integrations to// look like one) is what surfaces the regression.//// `catalogApi` is exported so scenarios build their client from the SAME// composition `seedLargeCatalog` is typed against — that keeps the seeding// fully typed (no structural-client gymnastics) while staying DRY across the// deterministic guard and the OpenCode recording.import { randomBytes } from "node:crypto";import { readFileSync } from "node:fs";import { fileURLToPath } from "node:url"; import { Effect } from "effect";import type { HttpApi, HttpApiClient } from "effect/unstable/httpapi";import { composePluginApi } from "@executor-js/api/server";import { openApiHttpPlugin } from "@executor-js/plugin-openapi/api";import { toolkitsPlugin } from "@executor-js/plugin-toolkits/server";import { AuthTemplateSlug, ConnectionName, IntegrationSlug } from "@executor-js/sdk/shared"; export const unique = (prefix: string): string => `${prefix}_${randomBytes(4).toString("hex")}`; const VERCEL_SPEC_PATH = fileURLToPath( new URL("../../../packages/plugins/openapi/fixtures/vercel.json", import.meta.url),); /** The real Vercel OpenAPI fixture (322 operations) as a JSON string. */export const vercelSpecText = (): string => readFileSync(VERCEL_SPEC_PATH, "utf-8"); /** An OpenAPI 3 doc with `ops` independent GET operations, shaped like a real * REST surface (path params, an apiKey scheme, JSON responses). */export const syntheticSpec = (title: string, ops: number, baseUrl: string): string => { const paths: Record<string, unknown> = {}; for (let i = 0; i < ops; i++) { paths[`/resource${i}/{id}`] = { get: { operationId: `get_resource_${i}`, summary: `Fetch resource ${i}`, security: [{ apiKey: [] }], parameters: [{ name: "id", in: "path", required: true, schema: { type: "string" } }], responses: { "200": { description: "ok", content: { "application/json": { schema: { type: "object", properties: { id: { type: "string" } } }, }, }, }, }, }, }; } return JSON.stringify({ openapi: "3.0.3", info: { title, version: "1.0.0" }, servers: [{ url: baseUrl }], paths, components: { securitySchemes: { apiKey: { type: "apiKey", in: "header", name: "x-tok" } } }, });}; export interface SeedIntegration { readonly slug: string; readonly specText: string; readonly baseUrl: string; readonly description?: string;} export interface SeedPlan { readonly integrations: ReadonlyArray<SeedIntegration>; /** The first integration's slug — the one the toolkit is scoped to. */ readonly firstSlug: string; /** Connection pattern the toolkit binds (the first integration, org, conn0). */ readonly toolkitConnectionPattern: string;} export interface SeedOptions { /** Synthetic integrations on top of the real spec. Default 10. */ readonly syntheticIntegrations?: number; /** Operations per synthetic integration. Default 300. */ readonly opsPerIntegration?: number; /** Include the real Vercel fixture (322 ops) as one integration. Default true. */ readonly includeRealSpec?: boolean;} /** * Plan a large, production-shaped catalog. With the defaults (1 real + 10 * synthetic integrations, 300 ops each) it is ~3,300 tools across 11 integrations, where * the per-tool policy N+1 turns a toolkit connect from sub-second into a 30s+ * client timeout. Each integration gets exactly one org connection at `conn0`. */export const planLargeCatalog = (options: SeedOptions = {}): SeedPlan => { const syntheticIntegrations = options.syntheticIntegrations ?? 10; const opsPerIntegration = options.opsPerIntegration ?? 300; const includeRealSpec = options.includeRealSpec ?? true; const integrations: SeedIntegration[] = []; if (includeRealSpec) { integrations.push({ slug: unique("vercel"), specText: vercelSpecText(), baseUrl: "https://api.vercel.com", description: "Vercel API", }); } for (let s = 0; s < syntheticIntegrations; s++) { integrations.push({ slug: unique("svc"), specText: syntheticSpec(`Service ${s}`, opsPerIntegration, "https://service.example"), baseUrl: "https://service.example", }); } const firstSlug = integrations[0]!.slug; return { integrations, firstSlug, toolkitConnectionPattern: `${firstSlug}.org.conn0.*`, };}; // ---------------------------------------------------------------------------// Typed seeding. Scenarios build their client from `catalogApi` so the client// type here matches theirs exactly.// --------------------------------------------------------------------------- /** The plugin API the seeding (and the scenarios) speak: OpenAPI specs + * toolkits. Build the scenario's client from THIS so types line up. */export const catalogApi = composePluginApi([openApiHttpPlugin(), toolkitsPlugin()] as const); type GroupsOf<A> = A extends HttpApi.HttpApi<infer _Id, infer Groups> ? Groups : never;export type CatalogClient = HttpApiClient.Client<GroupsOf<typeof catalogApi>>; export interface SeededCatalog { /** The org toolkit slug to point an MCP client at: /mcp/toolkits/<slug>. */ readonly toolkitSlug: string; readonly toolkitId: string; /** Total tools in the identity's catalog after seeding (the N+1 multiplier). */ readonly toolCount: number; /** Integration slugs created, for assertions / debugging. */ readonly integrationSlugs: ReadonlyArray<string>; /** Remove every spec + toolkit this seeder created. */ readonly cleanup: Effect.Effect<void>;} /** * Build the planned catalog under the current identity (one org connection per * integration) and a toolkit scoped to the first integration, then report the total tool * count and a finalizer. The toolkit's own surface is small; the connect cost * the scenarios guard comes from the policy engine walking the whole catalog. */export const seedLargeCatalog = ( client: CatalogClient, options: SeedOptions = {},): Effect.Effect<SeededCatalog, unknown> => Effect.gen(function* () { const plan = planLargeCatalog(options); for (const integration of plan.integrations) { yield* client.openapi.addSpec({ payload: { spec: { kind: "blob", value: integration.specText }, slug: IntegrationSlug.make(integration.slug), baseUrl: integration.baseUrl, ...(integration.description ? { description: integration.description } : {}), authenticationTemplate: [ { slug: "apiKey", type: "apiKey", headers: { "x-tok": [{ type: "variable", name: "token" }] }, }, ], }, }); yield* client.connections.create({ payload: { owner: "org", name: ConnectionName.make("conn0"), integration: IntegrationSlug.make(integration.slug), template: AuthTemplateSlug.make("apiKey"), value: "unused-token", }, }); } const allTools = yield* client.tools.list({ query: {} }); const toolCount = allTools.length; const toolkit = yield* client.toolkits.create({ payload: { owner: "org", name: unique("perf-kit") }, }); yield* client.toolkits.createConnection({ params: { toolkitId: toolkit.id }, payload: { pattern: plan.toolkitConnectionPattern }, }); const integrationSlugs = plan.integrations.map((integration) => integration.slug); const cleanup = Effect.gen(function* () { yield* client.toolkits.remove({ params: { toolkitId: toolkit.id } }).pipe(Effect.ignore); yield* Effect.forEach( integrationSlugs, (slug) => client.openapi.removeSpec({ params: { slug } }).pipe(Effect.ignore), { discard: true }, ); }).pipe(Effect.ignore); return { toolkitSlug: toolkit.slug, toolkitId: toolkit.id, toolCount, integrationSlugs, cleanup, }; });
// Seeds a production-shaped catalog for the toolkit-policy performance
// scenarios, plus the pure builders behind it.
//
// Why a large catalog at all: a toolkit MCP endpoint resolves tool visibility
// by running the policy engine over the WHOLE catalog on connect (the
// getDescription -> connections.list -> toolsList path), not just the toolkit's
// own connections. A per-tool resolution there is an N+1 that scales with total
// catalog size, so a realistic workspace (a real spec plus enough integrations to
// look like one) is what surfaces the regression.
//
// `catalogApi` is exported so scenarios build their client from the SAME
// composition `seedLargeCatalog` is typed against — that keeps the seeding
// fully typed (no structural-client gymnastics) while staying DRY across the
// deterministic guard and the OpenCode recording.
import { randomBytes } from "node:crypto";
import { readFileSync } from "node:fs";
import { fileURLToPath } from "node:url";
import { Effect } from "effect";
import type { HttpApi, HttpApiClient } from "effect/unstable/httpapi";
import { composePluginApi } from "@executor-js/api/server";
import { openApiHttpPlugin } from "@executor-js/plugin-openapi/api";
import { toolkitsPlugin } from "@executor-js/plugin-toolkits/server";
import { AuthTemplateSlug, ConnectionName, IntegrationSlug } from "@executor-js/sdk/shared";
export const unique = (prefix: string): string => `${prefix}_${randomBytes(4).toString("hex")}`;
const VERCEL_SPEC_PATH = fileURLToPath(
new URL("../../../packages/plugins/openapi/fixtures/vercel.json", import.meta.url),
);
/** The real Vercel OpenAPI fixture (322 operations) as a JSON string. */
export const vercelSpecText = (): string => readFileSync(VERCEL_SPEC_PATH, "utf-8");
/** An OpenAPI 3 doc with `ops` independent GET operations, shaped like a real
* REST surface (path params, an apiKey scheme, JSON responses). */
export const syntheticSpec = (title: string, ops: number, baseUrl: string): string => {
const paths: Record<string, unknown> = {};
for (let i = 0; i < ops; i++) {
paths[`/resource${i}/{id}`] = {
get: {
operationId: `get_resource_${i}`,
summary: `Fetch resource ${i}`,
security: [{ apiKey: [] }],
parameters: [{ name: "id", in: "path", required: true, schema: { type: "string" } }],
responses: {
"200": {
description: "ok",
content: {
"application/json": {
schema: { type: "object", properties: { id: { type: "string" } } },
},
},
},
},
},
};
}
return JSON.stringify({
openapi: "3.0.3",
info: { title, version: "1.0.0" },
servers: [{ url: baseUrl }],
paths,
components: { securitySchemes: { apiKey: { type: "apiKey", in: "header", name: "x-tok" } } },
});
};
export interface SeedIntegration {
readonly slug: string;
readonly specText: string;
readonly baseUrl: string;
readonly description?: string;
}
export interface SeedPlan {
readonly integrations: ReadonlyArray<SeedIntegration>;
/** The first integration's slug — the one the toolkit is scoped to. */
readonly firstSlug: string;
/** Connection pattern the toolkit binds (the first integration, org, conn0). */
readonly toolkitConnectionPattern: string;
}
export interface SeedOptions {
/** Synthetic integrations on top of the real spec. Default 10. */
readonly syntheticIntegrations?: number;
/** Operations per synthetic integration. Default 300. */
readonly opsPerIntegration?: number;
/** Include the real Vercel fixture (322 ops) as one integration. Default true. */
readonly includeRealSpec?: boolean;
}
/**
* Plan a large, production-shaped catalog. With the defaults (1 real + 10
* synthetic integrations, 300 ops each) it is ~3,300 tools across 11 integrations, where
* the per-tool policy N+1 turns a toolkit connect from sub-second into a 30s+
* client timeout. Each integration gets exactly one org connection at `conn0`.
*/
export const planLargeCatalog = (options: SeedOptions = {}): SeedPlan => {
const syntheticIntegrations = options.syntheticIntegrations ?? 10;
const opsPerIntegration = options.opsPerIntegration ?? 300;
const includeRealSpec = options.includeRealSpec ?? true;
const integrations: SeedIntegration[] = [];
if (includeRealSpec) {
integrations.push({
slug: unique("vercel"),
specText: vercelSpecText(),
baseUrl: "https://api.vercel.com",
description: "Vercel API",
});
}
for (let s = 0; s < syntheticIntegrations; s++) {
integrations.push({
slug: unique("svc"),
specText: syntheticSpec(`Service ${s}`, opsPerIntegration, "https://service.example"),
baseUrl: "https://service.example",
});
}
const firstSlug = integrations[0]!.slug;
return {
integrations,
firstSlug,
toolkitConnectionPattern: `${firstSlug}.org.conn0.*`,
};
};
// ---------------------------------------------------------------------------
// Typed seeding. Scenarios build their client from `catalogApi` so the client
// type here matches theirs exactly.
// ---------------------------------------------------------------------------
/** The plugin API the seeding (and the scenarios) speak: OpenAPI specs +
* toolkits. Build the scenario's client from THIS so types line up. */
export const catalogApi = composePluginApi([openApiHttpPlugin(), toolkitsPlugin()] as const);
type GroupsOf<A> = A extends HttpApi.HttpApi<infer _Id, infer Groups> ? Groups : never;
export type CatalogClient = HttpApiClient.Client<GroupsOf<typeof catalogApi>>;
export interface SeededCatalog {
/** The org toolkit slug to point an MCP client at: /mcp/toolkits/<slug>. */
readonly toolkitSlug: string;
readonly toolkitId: string;
/** Total tools in the identity's catalog after seeding (the N+1 multiplier). */
readonly toolCount: number;
/** Integration slugs created, for assertions / debugging. */
readonly integrationSlugs: ReadonlyArray<string>;
/** Remove every spec + toolkit this seeder created. */
readonly cleanup: Effect.Effect<void>;
}
/**
* Build the planned catalog under the current identity (one org connection per
* integration) and a toolkit scoped to the first integration, then report the total tool
* count and a finalizer. The toolkit's own surface is small; the connect cost
* the scenarios guard comes from the policy engine walking the whole catalog.
*/
export const seedLargeCatalog = (
client: CatalogClient,
options: SeedOptions = {},
): Effect.Effect<SeededCatalog, unknown> =>
Effect.gen(function* () {
const plan = planLargeCatalog(options);
for (const integration of plan.integrations) {
yield* client.openapi.addSpec({
payload: {
spec: { kind: "blob", value: integration.specText },
slug: IntegrationSlug.make(integration.slug),
baseUrl: integration.baseUrl,
...(integration.description ? { description: integration.description } : {}),
authenticationTemplate: [
{
slug: "apiKey",
type: "apiKey",
headers: { "x-tok": [{ type: "variable", name: "token" }] },
},
],
},
});
yield* client.connections.create({
payload: {
owner: "org",
name: ConnectionName.make("conn0"),
integration: IntegrationSlug.make(integration.slug),
template: AuthTemplateSlug.make("apiKey"),
value: "unused-token",
},
});
}
const allTools = yield* client.tools.list({ query: {} });
const toolCount = allTools.length;
const toolkit = yield* client.toolkits.create({
payload: { owner: "org", name: unique("perf-kit") },
});
yield* client.toolkits.createConnection({
params: { toolkitId: toolkit.id },
payload: { pattern: plan.toolkitConnectionPattern },
});
const integrationSlugs = plan.integrations.map((integration) => integration.slug);
const cleanup = Effect.gen(function* () {
yield* client.toolkits.remove({ params: { toolkitId: toolkit.id } }).pipe(Effect.ignore);
yield* Effect.forEach(
integrationSlugs,
(slug) => client.openapi.removeSpec({ params: { slug } }).pipe(Effect.ignore),
{ discard: true },
);
}).pipe(Effect.ignore);
return {
toolkitSlug: toolkit.slug,
toolkitId: toolkit.id,
toolCount,
integrationSlugs,
cleanup,
};
});