Running executor generate against an instance writes one self-contained TypeScript file: a dependency-free Proxy-based runtime client plus full input/output types for every visible tool, so client.github.org.main.issues.create({ title }) is typed end to end and executes through the server's /api/executions endpoint (auth, policies, approvals included). New pieces: - tools.export SDK surface + GET /tools/export: the bulk schema-bearing catalog read (schemas + trimmed shared $defs, grouped per connection, policy-filtered), one request regardless of catalog size. - compileToolChunkTypeScript: compiles many tools per compiler pass. Per-tool passes pay fixed overhead thousands of times and one whole-catalog pass is super-linear (30s+ at 10k tools); 200-tool chunks generate a 10k-tool catalog in ~400ms. - generateToolProxySource: assembles the generated file (type-only namespaces per connection, ExecutorTools path tree, embedded runtime). - executor generate CLI command with --output/--integration/--connection/ --include-static, reusing the server-target and auth machinery. Tested down to the wire: generated source is typechecked with the real compiler (valid calls accept, invalid reject), the transpiled runtime is imported and run against a fake fetch (completed/paused/injection paths), and a scale test ingests a 10,000-operation OpenAPI spec through addSpec and proves export + generate + strict typecheck hold up with regression tripwires on time.
Showing the changes to packages/plugins/openapi (1 of 14 files). Show all
Unified · Split
| @@ -0,0 +1,213 @@ | ||
| 1 | +// --------------------------------------------------------------------------- | |
| 2 | +// `executor generate` at catalog scale: 10,000 tools from one OpenAPI spec. | |
| 3 | +// | |
| 4 | +// The typed-proxy pipeline must hold up on instances with five-digit tool | |
| 5 | +// counts, end to end through the REAL ingestion path (addSpec compiles the | |
| 6 | +// spec, a connection persists the tool rows) rather than hand-built fixtures: | |
| 7 | +// - `tools.export` returns every tool in one read, | |
| 8 | +// - `generateToolProxySource` emits the file without blowing time or memory | |
| 9 | +// (chunked schema compilation: one whole-catalog compiler pass is | |
| 10 | +// super-linear and takes 30s+ at this size), | |
| 11 | +// - the generated source is valid strict TypeScript, verified with the real | |
| 12 | +// compiler, and spot-checked types resolve to the right shapes. | |
| 13 | +// | |
| 14 | +// Budgets are deliberately loose (CI machines vary) but tight enough to catch | |
| 15 | +// a regression to per-tool or whole-catalog compilation. | |
| 16 | +// --------------------------------------------------------------------------- | |
| 17 | + | |
| 18 | +import { describe, expect, it } from "@effect/vitest"; | |
| 19 | +import { Effect } from "effect"; | |
| 20 | +import { FetchHttpClient } from "effect/unstable/http"; | |
| 21 | +import * as ts from "typescript"; | |
| 22 | + | |
| 23 | +import { | |
| 24 | + AuthTemplateSlug, | |
| 25 | + ConnectionName, | |
| 26 | + IntegrationSlug, | |
| 27 | + createExecutor, | |
| 28 | + generateToolProxySource, | |
| 29 | +} from "@executor-js/sdk"; | |
| 30 | +import { makeTestConfig, memoryCredentialsPlugin } from "@executor-js/sdk/testing"; | |
| 31 | + | |
| 32 | +import { openApiPlugin } from "./plugin"; | |
| 33 | + | |
| 34 | +const TOOL_COUNT = 10_000; | |
| 35 | + | |
| 36 | +// Build a 10k-operation spec with realistic shape variety: per-operation | |
| 37 | +// parameter schemas, shared component refs, and enough distinct field names | |
| 38 | +// that deduplication cannot collapse the work. | |
| 39 | +const buildScaleSpec = (toolCount: number): string => { | |
| 40 | + const paths: Record<string, unknown> = {}; | |
| 41 | + for (let index = 0; index < toolCount; index += 1) { | |
| 42 | + paths[`/resources${index % 100}/r${index}`] = { | |
| 43 | + get: { | |
| 44 | + operationId: `res.op${index}`, | |
| 45 | + summary: `Operation ${index}`, | |
| 46 | + parameters: [ | |
| 47 | + { name: "id", in: "query", required: true, schema: { type: "string" } }, | |
| 48 | + { name: `filter${index % 250}`, in: "query", schema: { type: "string" } }, | |
| 49 | + { name: "limit", in: "query", schema: { type: "integer" } }, | |
| 50 | + ], | |
| 51 | + responses: { | |
| 52 | + "200": { | |
| 53 | + description: "ok", | |
| 54 | + content: { | |
| 55 | + "application/json": { | |
| 56 | + schema: { | |
| 57 | + type: "object", | |
| 58 | + properties: { | |
| 59 | + item: { $ref: "#/components/schemas/Item" }, | |
| 60 | + page: { $ref: "#/components/schemas/Page" }, | |
| 61 | + total: { type: "integer" }, | |
| 62 | + }, | |
| 63 | + }, | |
| 64 | + }, | |
| 65 | + }, | |
| 66 | + }, | |
| 67 | + }, | |
| 68 | + }, | |
| 69 | + }; | |
| 70 | + } | |
| 71 | + // @effect-diagnostics-next-line preferSchemaOverJson:off | |
| 72 | + return JSON.stringify({ | |
| 73 | + openapi: "3.0.0", | |
| 74 | + info: { title: "Scale", version: "1.0.0" }, | |
| 75 | + servers: [{ url: "https://scale.example.test" }], | |
| 76 | + security: [{ apiKey: [] }], | |
| 77 | + paths, | |
| 78 | + components: { | |
| 79 | + securitySchemes: { | |
| 80 | + apiKey: { type: "apiKey", in: "header", name: "x-api-key" }, | |
| 81 | + }, | |
| 82 | + schemas: { | |
| 83 | + Item: { | |
| 84 | + type: "object", | |
| 85 | + properties: { | |
| 86 | + id: { type: "string" }, | |
| 87 | + name: { type: "string" }, | |
| 88 | + tags: { type: "array", items: { type: "string" } }, | |
| 89 | + }, | |
| 90 | + required: ["id"], | |
| 91 | + }, | |
| 92 | + Page: { | |
| 93 | + type: "object", | |
| 94 | + properties: { cursor: { type: "string" }, hasMore: { type: "boolean" } }, | |
| 95 | + }, | |
| 96 | + }, | |
| 97 | + }, | |
| 98 | + }); | |
| 99 | +}; | |
| 100 | + | |
| 101 | +const typecheck = (source: string, extraSource: string): readonly string[] => { | |
| 102 | + const fileName = "generated.ts"; | |
| 103 | + const fullSource = `${source}\n${extraSource}`; | |
| 104 | + const options: ts.CompilerOptions = { | |
| 105 | + strict: true, | |
| 106 | + noEmit: true, | |
| 107 | + skipLibCheck: true, | |
| 108 | + target: ts.ScriptTarget.ES2022, | |
| 109 | + module: ts.ModuleKind.ESNext, | |
| 110 | + lib: ["lib.es2022.d.ts", "lib.dom.d.ts"], | |
| 111 | + }; | |
| 112 | + const host = ts.createCompilerHost(options); | |
| 113 | + const originalGetSourceFile = host.getSourceFile.bind(host); | |
| 114 | + const originalReadFile = host.readFile.bind(host); | |
| 115 | + const originalFileExists = host.fileExists.bind(host); | |
| 116 | + host.getSourceFile = (candidate, languageVersion, onError, shouldCreateNewSourceFile) => | |
| 117 | + candidate === fileName | |
| 118 | + ? ts.createSourceFile(candidate, fullSource, languageVersion, true) | |
| 119 | + : originalGetSourceFile(candidate, languageVersion, onError, shouldCreateNewSourceFile); | |
| 120 | + host.readFile = (candidate) => | |
| 121 | + candidate === fileName ? fullSource : originalReadFile(candidate); | |
| 122 | + host.fileExists = (candidate) => candidate === fileName || originalFileExists(candidate); | |
| 123 | + | |
| 124 | + const program = ts.createProgram([fileName], options, host); | |
| 125 | + return ts | |
| 126 | + .getPreEmitDiagnostics(program) | |
| 127 | + .map((diagnostic) => ts.flattenDiagnosticMessageText(diagnostic.messageText, "\n")); | |
| 128 | +}; | |
| 129 | + | |
| 130 | +describe("typed proxy generation at 10k-tool scale", () => { | |
| 131 | + it.effect( | |
| 132 | + "exports, generates, and typechecks a 10,000-tool catalog", | |
| 133 | + () => | |
| 134 | + Effect.scoped( | |
| 135 | + Effect.gen(function* () { | |
| 136 | + const executor = yield* createExecutor( | |
| 137 | + makeTestConfig({ | |
| 138 | + plugins: [ | |
| 139 | + openApiPlugin({ httpClientLayer: FetchHttpClient.layer }), | |
| 140 | + memoryCredentialsPlugin(), | |
| 141 | + ] as const, | |
| 142 | + }), | |
| 143 | + ); | |
| 144 | + | |
| 145 | + const added = yield* executor.openapi.addSpec({ | |
| 146 | + spec: { kind: "blob", value: buildScaleSpec(TOOL_COUNT) }, | |
| 147 | + slug: "scale", | |
| 148 | + }); | |
| 149 | + expect(added.toolCount).toBe(TOOL_COUNT); | |
| 150 | + | |
| 151 | + yield* executor.connections.create({ | |
| 152 | + owner: "org", | |
| 153 | + name: ConnectionName.make("main"), | |
| 154 | + integration: IntegrationSlug.make("scale"), | |
| 155 | + template: AuthTemplateSlug.make("apikey-0"), | |
| 156 | + value: "token", | |
| 157 | + }); | |
| 158 | + | |
| 159 | + const exportStart = performance.now(); | |
| 160 | + const exported = yield* executor.tools.export({ | |
| 161 | + integration: IntegrationSlug.make("scale"), | |
| 162 | + }); | |
| 163 | + const exportMs = performance.now() - exportStart; | |
| 164 | + const exportedCount = exported.connections.reduce( | |
| 165 | + (sum, connection) => sum + connection.tools.length, | |
| 166 | + 0, | |
| 167 | + ); | |
| 168 | + expect(exportedCount).toBe(TOOL_COUNT); | |
| 169 | + | |
| 170 | + const generateStart = performance.now(); | |
| 171 | + const generated = generateToolProxySource(exported); | |
| 172 | + const generateMs = performance.now() - generateStart; | |
| 173 | + expect(generated.toolCount).toBe(TOOL_COUNT); | |
| 174 | + | |
| 175 | + // Regression tripwires, not benchmarks: whole-catalog single-pass | |
| 176 | + // compilation measured 30s+ here and per-tool passes are far worse; | |
| 177 | + // the chunked path runs in well under a second. 15s/30s absorb slow | |
| 178 | + // CI machines while still failing on a complexity regression. | |
| 179 | + expect(exportMs).toBeLessThan(15_000); | |
| 180 | + expect(generateMs).toBeLessThan(30_000); | |
| 181 | + | |
| 182 | + // Every tool surfaced in the generated interface exactly once. The | |
| 183 | + // plugin derives paths from the URL (`/resources77/r7777` → | |
| 184 | + // `resources77.resOp7777`), so count the leaf entries. | |
| 185 | + const opMatches = generated.source.match(/resOp\d+: ExecutorToolFn</g) ?? []; | |
| 186 | + expect(opMatches.length).toBe(TOOL_COUNT); | |
| 187 | + // Shared component schemas stay named refs, not 10k inlined copies. | |
| 188 | + expect(generated.source).toContain("export type Item ="); | |
| 189 | + | |
| 190 | + // The whole 10k-tool file typechecks under strict mode, and a | |
| 191 | + // consumer gets real types out of an arbitrary tool in the middle. | |
| 192 | + const diagnostics = typecheck( | |
| 193 | + generated.source, | |
| 194 | + ` | |
| 195 | + const client = createExecutorClient(); | |
| 196 | + async function main() { | |
| 197 | + const outcome = await client.scale.org.main.resources77.resOp7777({ id: "x" }); | |
| 198 | + if (outcome.ok) { | |
| 199 | + const total: number | undefined = outcome.data.total; | |
| 200 | + const itemId: string | undefined = outcome.data.item?.id; | |
| 201 | + void total; | |
| 202 | + void itemId; | |
| 203 | + } | |
| 204 | + } | |
| 205 | + void main; | |
| 206 | + `, | |
| 207 | + ); | |
| 208 | + expect(diagnostics).toEqual([]); | |
| 209 | + }), | |
| 210 | + ), | |
| 211 | + { timeout: 180_000 }, | |
| 212 | + ); | |
| 213 | +}); | |