Running executor generate against an instance writes one self-contained TypeScript file: a dependency-free Proxy-based runtime client plus full input/output types for every visible tool, so client.github.org.main.issues.create({ title }) is typed end to end and executes through the server's /api/executions endpoint (auth, policies, approvals included). New pieces: - tools.export SDK surface + GET /tools/export: the bulk schema-bearing catalog read (schemas + trimmed shared $defs, grouped per connection, policy-filtered), one request regardless of catalog size. - compileToolChunkTypeScript: compiles many tools per compiler pass. Per-tool passes pay fixed overhead thousands of times and one whole-catalog pass is super-linear (30s+ at 10k tools); 200-tool chunks generate a 10k-tool catalog in ~400ms. - generateToolProxySource: assembles the generated file (type-only namespaces per connection, ExecutorTools path tree, embedded runtime). - executor generate CLI command with --output/--integration/--connection/ --include-static, reusing the server-target and auth machinery. Tested down to the wire: generated source is typechecked with the real compiler (valid calls accept, invalid reject), the transpiled runtime is imported and run against a fake fetch (completed/paused/injection paths), and a scale test ingests a 10,000-operation OpenAPI spec through addSpec and proves export + generate + strict typecheck hold up with regression tripwires on time.
Showing the changes to packages/plugins/openapi (1 of 14 files). Show all
Unified · Split
| @@ -0,0 +1,213 @@ | |||
| 1 | // --------------------------------------------------------------------------- | ||
| 2 | // `executor generate` at catalog scale: 10,000 tools from one OpenAPI spec. | ||
| 3 | // | ||
| 4 | // The typed-proxy pipeline must hold up on instances with five-digit tool | ||
| 5 | // counts, end to end through the REAL ingestion path (addSpec compiles the | ||
| 6 | // spec, a connection persists the tool rows) rather than hand-built fixtures: | ||
| 7 | // - `tools.export` returns every tool in one read, | ||
| 8 | // - `generateToolProxySource` emits the file without blowing time or memory | ||
| 9 | // (chunked schema compilation: one whole-catalog compiler pass is | ||
| 10 | // super-linear and takes 30s+ at this size), | ||
| 11 | // - the generated source is valid strict TypeScript, verified with the real | ||
| 12 | // compiler, and spot-checked types resolve to the right shapes. | ||
| 13 | // | ||
| 14 | // Budgets are deliberately loose (CI machines vary) but tight enough to catch | ||
| 15 | // a regression to per-tool or whole-catalog compilation. | ||
| 16 | // --------------------------------------------------------------------------- | ||
| 17 | |||
| 18 | import { describe, expect, it } from "@effect/vitest"; | ||
| 19 | import { Effect } from "effect"; | ||
| 20 | import { FetchHttpClient } from "effect/unstable/http"; | ||
| 21 | import * as ts from "typescript"; | ||
| 22 | |||
| 23 | import { | ||
| 24 | AuthTemplateSlug, | ||
| 25 | ConnectionName, | ||
| 26 | IntegrationSlug, | ||
| 27 | createExecutor, | ||
| 28 | generateToolProxySource, | ||
| 29 | } from "@executor-js/sdk"; | ||
| 30 | import { makeTestConfig, memoryCredentialsPlugin } from "@executor-js/sdk/testing"; | ||
| 31 | |||
| 32 | import { openApiPlugin } from "./plugin"; | ||
| 33 | |||
| 34 | const TOOL_COUNT = 10_000; | ||
| 35 | |||
| 36 | // Build a 10k-operation spec with realistic shape variety: per-operation | ||
| 37 | // parameter schemas, shared component refs, and enough distinct field names | ||
| 38 | // that deduplication cannot collapse the work. | ||
| 39 | const buildScaleSpec = (toolCount: number): string => { | ||
| 40 | const paths: Record<string, unknown> = {}; | ||
| 41 | for (let index = 0; index < toolCount; index += 1) { | ||
| 42 | paths[`/resources${index % 100}/r${index}`] = { | ||
| 43 | get: { | ||
| 44 | operationId: `res.op${index}`, | ||
| 45 | summary: `Operation ${index}`, | ||
| 46 | parameters: [ | ||
| 47 | { name: "id", in: "query", required: true, schema: { type: "string" } }, | ||
| 48 | { name: `filter${index % 250}`, in: "query", schema: { type: "string" } }, | ||
| 49 | { name: "limit", in: "query", schema: { type: "integer" } }, | ||
| 50 | ], | ||
| 51 | responses: { | ||
| 52 | "200": { | ||
| 53 | description: "ok", | ||
| 54 | content: { | ||
| 55 | "application/json": { | ||
| 56 | schema: { | ||
| 57 | type: "object", | ||
| 58 | properties: { | ||
| 59 | item: { $ref: "#/components/schemas/Item" }, | ||
| 60 | page: { $ref: "#/components/schemas/Page" }, | ||
| 61 | total: { type: "integer" }, | ||
| 62 | }, | ||
| 63 | }, | ||
| 64 | }, | ||
| 65 | }, | ||
| 66 | }, | ||
| 67 | }, | ||
| 68 | }, | ||
| 69 | }; | ||
| 70 | } | ||
| 71 | // @effect-diagnostics-next-line preferSchemaOverJson:off | ||
| 72 | return JSON.stringify({ | ||
| 73 | openapi: "3.0.0", | ||
| 74 | info: { title: "Scale", version: "1.0.0" }, | ||
| 75 | servers: [{ url: "https://scale.example.test" }], | ||
| 76 | security: [{ apiKey: [] }], | ||
| 77 | paths, | ||
| 78 | components: { | ||
| 79 | securitySchemes: { | ||
| 80 | apiKey: { type: "apiKey", in: "header", name: "x-api-key" }, | ||
| 81 | }, | ||
| 82 | schemas: { | ||
| 83 | Item: { | ||
| 84 | type: "object", | ||
| 85 | properties: { | ||
| 86 | id: { type: "string" }, | ||
| 87 | name: { type: "string" }, | ||
| 88 | tags: { type: "array", items: { type: "string" } }, | ||
| 89 | }, | ||
| 90 | required: ["id"], | ||
| 91 | }, | ||
| 92 | Page: { | ||
| 93 | type: "object", | ||
| 94 | properties: { cursor: { type: "string" }, hasMore: { type: "boolean" } }, | ||
| 95 | }, | ||
| 96 | }, | ||
| 97 | }, | ||
| 98 | }); | ||
| 99 | }; | ||
| 100 | |||
| 101 | const typecheck = (source: string, extraSource: string): readonly string[] => { | ||
| 102 | const fileName = "generated.ts"; | ||
| 103 | const fullSource = `${source}\n${extraSource}`; | ||
| 104 | const options: ts.CompilerOptions = { | ||
| 105 | strict: true, | ||
| 106 | noEmit: true, | ||
| 107 | skipLibCheck: true, | ||
| 108 | target: ts.ScriptTarget.ES2022, | ||
| 109 | module: ts.ModuleKind.ESNext, | ||
| 110 | lib: ["lib.es2022.d.ts", "lib.dom.d.ts"], | ||
| 111 | }; | ||
| 112 | const host = ts.createCompilerHost(options); | ||
| 113 | const originalGetSourceFile = host.getSourceFile.bind(host); | ||
| 114 | const originalReadFile = host.readFile.bind(host); | ||
| 115 | const originalFileExists = host.fileExists.bind(host); | ||
| 116 | host.getSourceFile = (candidate, languageVersion, onError, shouldCreateNewSourceFile) => | ||
| 117 | candidate === fileName | ||
| 118 | ? ts.createSourceFile(candidate, fullSource, languageVersion, true) | ||
| 119 | : originalGetSourceFile(candidate, languageVersion, onError, shouldCreateNewSourceFile); | ||
| 120 | host.readFile = (candidate) => | ||
| 121 | candidate === fileName ? fullSource : originalReadFile(candidate); | ||
| 122 | host.fileExists = (candidate) => candidate === fileName || originalFileExists(candidate); | ||
| 123 | |||
| 124 | const program = ts.createProgram([fileName], options, host); | ||
| 125 | return ts | ||
| 126 | .getPreEmitDiagnostics(program) | ||
| 127 | .map((diagnostic) => ts.flattenDiagnosticMessageText(diagnostic.messageText, "\n")); | ||
| 128 | }; | ||
| 129 | |||
| 130 | describe("typed proxy generation at 10k-tool scale", () => { | ||
| 131 | it.effect( | ||
| 132 | "exports, generates, and typechecks a 10,000-tool catalog", | ||
| 133 | () => | ||
| 134 | Effect.scoped( | ||
| 135 | Effect.gen(function* () { | ||
| 136 | const executor = yield* createExecutor( | ||
| 137 | makeTestConfig({ | ||
| 138 | plugins: [ | ||
| 139 | openApiPlugin({ httpClientLayer: FetchHttpClient.layer }), | ||
| 140 | memoryCredentialsPlugin(), | ||
| 141 | ] as const, | ||
| 142 | }), | ||
| 143 | ); | ||
| 144 | |||
| 145 | const added = yield* executor.openapi.addSpec({ | ||
| 146 | spec: { kind: "blob", value: buildScaleSpec(TOOL_COUNT) }, | ||
| 147 | slug: "scale", | ||
| 148 | }); | ||
| 149 | expect(added.toolCount).toBe(TOOL_COUNT); | ||
| 150 | |||
| 151 | yield* executor.connections.create({ | ||
| 152 | owner: "org", | ||
| 153 | name: ConnectionName.make("main"), | ||
| 154 | integration: IntegrationSlug.make("scale"), | ||
| 155 | template: AuthTemplateSlug.make("apikey-0"), | ||
| 156 | value: "token", | ||
| 157 | }); | ||
| 158 | |||
| 159 | const exportStart = performance.now(); | ||
| 160 | const exported = yield* executor.tools.export({ | ||
| 161 | integration: IntegrationSlug.make("scale"), | ||
| 162 | }); | ||
| 163 | const exportMs = performance.now() - exportStart; | ||
| 164 | const exportedCount = exported.connections.reduce( | ||
| 165 | (sum, connection) => sum + connection.tools.length, | ||
| 166 | 0, | ||
| 167 | ); | ||
| 168 | expect(exportedCount).toBe(TOOL_COUNT); | ||
| 169 | |||
| 170 | const generateStart = performance.now(); | ||
| 171 | const generated = generateToolProxySource(exported); | ||
| 172 | const generateMs = performance.now() - generateStart; | ||
| 173 | expect(generated.toolCount).toBe(TOOL_COUNT); | ||
| 174 | |||
| 175 | // Regression tripwires, not benchmarks: whole-catalog single-pass | ||
| 176 | // compilation measured 30s+ here and per-tool passes are far worse; | ||
| 177 | // the chunked path runs in well under a second. 15s/30s absorb slow | ||
| 178 | // CI machines while still failing on a complexity regression. | ||
| 179 | expect(exportMs).toBeLessThan(15_000); | ||
| 180 | expect(generateMs).toBeLessThan(30_000); | ||
| 181 | |||
| 182 | // Every tool surfaced in the generated interface exactly once. The | ||
| 183 | // plugin derives paths from the URL (`/resources77/r7777` → | ||
| 184 | // `resources77.resOp7777`), so count the leaf entries. | ||
| 185 | const opMatches = generated.source.match(/resOp\d+: ExecutorToolFn</g) ?? []; | ||
| 186 | expect(opMatches.length).toBe(TOOL_COUNT); | ||
| 187 | // Shared component schemas stay named refs, not 10k inlined copies. | ||
| 188 | expect(generated.source).toContain("export type Item ="); | ||
| 189 | |||
| 190 | // The whole 10k-tool file typechecks under strict mode, and a | ||
| 191 | // consumer gets real types out of an arbitrary tool in the middle. | ||
| 192 | const diagnostics = typecheck( | ||
| 193 | generated.source, | ||
| 194 | ` | ||
| 195 | const client = createExecutorClient(); | ||
| 196 | async function main() { | ||
| 197 | const outcome = await client.scale.org.main.resources77.resOp7777({ id: "x" }); | ||
| 198 | if (outcome.ok) { | ||
| 199 | const total: number | undefined = outcome.data.total; | ||
| 200 | const itemId: string | undefined = outcome.data.item?.id; | ||
| 201 | void total; | ||
| 202 | void itemId; | ||
| 203 | } | ||
| 204 | } | ||
| 205 | void main; | ||
| 206 | `, | ||
| 207 | ); | ||
| 208 | expect(diagnostics).toEqual([]); | ||
| 209 | }), | ||
| 210 | ), | ||
| 211 | { timeout: 180_000 }, | ||
| 212 | ); | ||
| 213 | }); | ||