Add executor generate: typed TypeScript client for the tool catalog

Running executor generate against an instance writes one self-contained TypeScript file: a dependency-free Proxy-based runtime client plus full input/output types for every visible tool, so client.github.org.main.issues.create({ title }) is typed end to end and executes through the server's /api/executions endpoint (auth, policies, approvals included). New pieces: - tools.export SDK surface + GET /tools/export: the bulk schema-bearing catalog read (schemas + trimmed shared $defs, grouped per connection, policy-filtered), one request regardless of catalog size. - compileToolChunkTypeScript: compiles many tools per compiler pass. Per-tool passes pay fixed overhead thousands of times and one whole-catalog pass is super-linear (30s+ at 10k tools); 200-tool chunks generate a 10k-tool catalog in ~400ms. - generateToolProxySource: assembles the generated file (type-only namespaces per connection, ExecutorTools path tree, embedded runtime). - executor generate CLI command with --output/--integration/--connection/ --include-static, reusing the server-target and auth machinery. Tested down to the wire: generated source is typechecked with the real compiler (valid calls accept, invalid reject), the transpiled runtime is imported and run against a fake fetch (completed/paused/injection paths), and a scale test ingests a 10,000-operation OpenAPI spec through addSpec and proves export + generate + strict typecheck hold up with regression tripwires on time.

Rhys Sullivan committed 4 months ago

Commit dabe51f1 parent e39dbcf14 files · +1733 −5Browse files

Unified · Split

14 files

.changeset/typed-api-proxy-generate.md +16 −0
apps/cli/src/main.ts +103 −0
apps/docs/local/cli.mdx +25 −0
packages/core/api/src/handlers/tools.ts +15 −0
packages/core/api/src/tools/api.ts +12 −0
packages/core/execution/src/promise.ts +2 −0
packages/core/sdk/src/executor.ts +131 −1
packages/core/sdk/src/index.ts +21 −2
packages/core/sdk/src/schema-types.ts +108 −0
packages/core/sdk/src/shared.ts +7 −1
packages/core/sdk/src/typegen.test.ts +516 −0
packages/core/sdk/src/typegen.ts +528 −0
packages/core/sdk/src/types.ts +36 −1
packages/plugins/openapi/src/sdk/typegen-scale.test.ts +213 −0

.changeset/typed-api-proxy-generate.md+16−0

@@ -0,0 +1,16 @@
	1	+---
	2	+"@executor-js/sdk": minor
	3	+"executor": minor
	4	+---
	5	+
	6	+Add `executor generate`: emit a typed TypeScript client for an instance's tool catalog.
	7	+
	8	+Running `executor generate` against a server writes a single self-contained
	9	+file (default `executor.gen.ts`) with a dependency-free runtime client and
	10	+full input/output types for every visible tool, so
	11	+`client.github.org.main.issues.create({ title })` is typed end to end and
	12	+calls go through the server's execution endpoint (auth, policies, and
	13	+approvals included). New `GET /tools/export` endpoint and
	14	+`executor.tools.export()` SDK surface return the whole schema-bearing catalog
	15	+in one read; generation compiles schemas in chunks and stays fast at
	16	+10,000-tool scale.

apps/cli/src/main.ts+103−0

@@ -69,6 +69,8 @@ import type { JSONRPCMessage } from "@modelcontextprotocol/sdk/types.js"; · 68 unmodified lines
69	69
70	70	import { ExecutorApi, checkForUpdate } from "@executor-js/api";
71	71	import {
	72	+ ConnectionName,
	73	+ IntegrationSlug,
72	74	getExecutorServerAuthorizationHeader,
73	75	normalizeExecutorServerConnection,
74	76	normalizeExecutorServerOrigin,
@@ -77,6 +79,7 @@ import { · 2 unmodified lines
77	79	type ExecutorServerConnection,
78	80	type ExecutorServerConnectionInput,
79	81	} from "@executor-js/sdk/shared";
	82	+import { generateToolProxySource } from "@executor-js/sdk/core";
80	83	import {
81	84	decodeAccessTokenClaims,
82	85	discoverCliLogin,
@@ -2710,6 +2713,105 @@ const mcpCommand = Command.make( · 2627 unmodified lines
2710	2713	}),
2711	2714	).pipe(Command.withDescription("Start an MCP server over stdio"));
2712	2715
	2716	+// ---------------------------------------------------------------------------
	2717	+// Generate — emit a typed TypeScript client for the instance's tool catalog
	2718	+// ---------------------------------------------------------------------------
	2719	+
	2720	+const generateCommand = Command.make(
	2721	+ "generate",
	2722	+ {
	2723	+ output: Options.string("output").pipe(
	2724	+ Options.withAlias("o"),
	2725	+ Options.withDefault("executor.gen.ts"),
	2726	+ Options.withDescription("Path of the generated TypeScript file."),
	2727	+ ),
	2728	+ integration: Options.string("integration").pipe(
	2729	+ Options.optional,
	2730	+ Options.withDescription("Only generate tools from this integration slug."),
	2731	+ ),
	2732	+ connection: Options.string("connection").pipe(
	2733	+ Options.optional,
	2734	+ Options.withDescription("Only generate tools from this connection name."),
	2735	+ ),
	2736	+ includeStatic: Options.boolean("include-static").pipe(
	2737	+ Options.withDefault(false),
	2738	+ Options.withDescription(
	2739	+ "Include Executor's built-in static tools (executor., search, describe.).",
	2740	+ ),
	2741	+ ),
	2742	+ baseUrl: serverBaseUrl,
	2743	+ server: serverProfile,
	2744	+ scope,
	2745	+ },
	2746	+ ({ output, integration, connection, includeStatic, baseUrl, server, scope }) =>
	2747	+ Effect.gen(function* () {
	2748	+ applyScope(scope);
	2749	+ const target = serverTargetFromOptions({ baseUrl, server });
	2750	+ const serverConnection = yield* resolveExecutorServerConnection(target);
	2751	+ const client = yield* makeApiClient(serverConnection);
	2752	+
	2753	+ const catalog = yield* client.tools
	2754	+ .export({
	2755	+ query: {
	2756	+ ...(Option.isSome(integration)
	2757	+ ? { integration: IntegrationSlug.make(integration.value) }
	2758	+ : {}),
	2759	+ ...(Option.isSome(connection)
	2760	+ ? { connection: ConnectionName.make(connection.value) }
	2761	+ : {}),
	2762	+ },
	2763	+ })
	2764	+ .pipe(Effect.mapError(toError));
	2765	+
	2766	+ const filtered = includeStatic
	2767	+ ? catalog
	2768	+ : {
	2769	+ connections: catalog.connections
	2770	+ .map((entry) => ({
	2771	+ ...entry,
	2772	+ tools: entry.tools.filter((tool) => tool.static !== true),
	2773	+ }))
	2774	+ .filter((entry) => entry.tools.length > 0),
	2775	+ };
	2776	+
	2777	+ const generated = yield* Effect.try({
	2778	+ try: () => generateToolProxySource(filtered),
	2779	+ catch: (cause) =>
	2780	+ cause instanceof Error
	2781	+ ? cause
	2782	+ : new Error(`Failed to generate TypeScript source: ${String(cause)}`),
	2783	+ });
	2784	+
	2785	+ if (generated.toolCount === 0) {
	2786	+ return yield* Effect.fail(
	2787	+ new Error(
	2788	+ [
	2789	+ "No tools matched, nothing to generate.",
	2790	+ `Server: ${serverConnection.origin}`,
	2791	+ `Check filters, or add an integration first: ${cliPrefix} web`,
	2792	+ ].join("\n"),
	2793	+ ),
	2794	+ );
	2795	+ }
	2796	+
	2797	+ const fs = yield* FileSystem.FileSystem;
	2798	+ const path = yield* PlatformPath.Path;
	2799	+ const outputPath = path.isAbsolute(output) ? output : path.join(process.cwd(), output);
	2800	+ yield* fs.makeDirectory(path.dirname(outputPath), { recursive: true });
	2801	+ yield* fs.writeFileString(outputPath, generated.source);
	2802	+
	2803	+ console.log(
	2804	+ `Generated ${outputPath} (${generated.toolCount} tools, ${generated.connectionCount} connections).`,
	2805	+ );
	2806	+ console.log("");
	2807	+ console.log("Use it:");
	2808	+ console.log(` import { createExecutorClient } from "./${path.basename(output, ".ts")}";`);
	2809	+ console.log(` const executor = createExecutorClient();`);
	2810	+ }).pipe(Effect.mapError(toError)),
	2811	+).pipe(
	2812	+ Command.withDescription("Generate a typed TypeScript client for this instance's tool catalog"),
	2813	+);
	2814	+
2713	2815	// ---------------------------------------------------------------------------
2714	2816	// Service — register the daemon with the OS so it survives app-quit + restart
2715	2817	// ---------------------------------------------------------------------------
@@ -3112,6 +3214,7 @@ const root = Command.make("executor").pipe( · 396 unmodified lines
3112	3214	daemonCommand,
3113	3215	serviceCommand,
3114	3216	mcpCommand,
	3217	+ generateCommand,
3115	3218	openCommand,
3116	3219	docsCommand,
3117	3220	] as const),

apps/docs/local/cli.mdx+25−0

@@ -109,3 +109,28 @@ resume it: · 108 unmodified lines
109	109	```bash
110	110	executor resume --execution-id exec_123
111	111	```
	112	+
	113	+## Generate a typed client
	114	+
	115	+`executor generate` writes a single self-contained TypeScript file with full
	116	+input/output types for every tool in your catalog, plus a dependency-free
	117	+runtime client. Drop it into any TypeScript project and every call is typed
	118	+end to end, while still executing through Executor (auth, policies, and
	119	+approvals included):
	120	+
	121	+```bash
	122	+executor generate # writes executor.gen.ts
	123	+executor generate -o src/executor.ts # custom output path
	124	+executor generate --integration github # only one integration
	125	+```
	126	+
	127	+```ts
	128	+import { createExecutorClient } from "./executor.gen";
	129	+
	130	+const executor = createExecutorClient(); // EXECUTOR_API_KEY / EXECUTOR_AUTH_TOKEN
	131	+const created = await executor.github.org.main.issues.create({ title: "Hi" });
	132	+if (created.ok) console.log(created.data.number);
	133	+```
	134	+
	135	+Calls that pause for approval throw `ExecutorPausedError` with the approval
	136	+URL. Re-run `executor generate` whenever your catalog changes.

packages/core/api/src/handlers/tools.ts+15−0

@@ -49,5 +49,20 @@ export const ToolsHandlers = HttpApiBuilder.group(ExecutorApi, "tools", (handler · 48 unmodified lines
49	49	return schema;
50	50	}),
51	51	),
	52	+ )
	53	+ .handle("export", ({ query }) =>
	54	+ capture(
	55	+ Effect.gen(function* () {
	56	+ const executor = yield* ExecutorService;
	57	+ return yield* executor.tools.export({
	58	+ integration: query.integration,
	59	+ owner: query.owner,
	60	+ connection: query.connection,
	61	+ query: query.query,
	62	+ includeAnnotations: query.includeAnnotations === "true",
	63	+ includeBlocked: query.includeBlocked === "true",
	64	+ });
	65	+ }),
	66	+ ),
52	67	),
53	68	);

packages/core/api/src/tools/api.ts+12−0

@@ -16,6 +16,7 @@ import { · 15 unmodified lines
16	16	InternalError,
17	17	Owner,
18	18	ToolAddress,
	19	+ ToolCatalogExport,
19	20	ToolNotFoundError,
20	21	ToolSchemaView,
21	22	} from "@executor-js/sdk/shared";
@@ -82,4 +83,15 @@ export const ToolsApi = HttpApiGroup.make("tools") · 60 unmodified lines
82	83	success: ToolSchemaView,
83	84	error: [InternalError, ToolNotFound],
84	85	}),
	86	+ )
	87	+ .add(
	88	+ // Bulk schema-bearing read for codegen (`executor generate`): every
	89	+ // visible tool's input/output JSON schema grouped per connection with the
	90	+ // connection's shared `$defs`. One request regardless of catalog size —
	91	+ // per-tool `schema` round trips do not scale to 10k-tool catalogs.
	92	+ HttpApiEndpoint.get("export", "/tools/export", {
	93	+ query: ListToolsQuery,
	94	+ success: ToolCatalogExport,
	95	+ error: InternalError,
	96	+ }),
85	97	);

packages/core/execution/src/promise.ts+2−0

@@ -109,6 +109,8 @@ const wrapPromiseExecutor = (pe: PromiseExecutor): EffectExecutor => { · 108 unmodified lines
109	109	fromPromise(() => pe.tools.list(filter)),
110	110	schema: (address: Parameters<PromiseExecutor["tools"]["schema"]>[0]) =>
111	111	fromPromise(() => pe.tools.schema(address)),
	112	+ export: (filter?: Parameters<PromiseExecutor["tools"]["export"]>[0]) =>
	113	+ fromPromise(() => pe.tools.export(filter)),
112	114	},
113	115	providers: {
114	116	list: () => fromPromise(() => pe.providers.list()),

packages/core/sdk/src/executor.ts+131−1

@@ -136,7 +136,13 @@ import { · 135 unmodified lines
136	136	ORG_SUBJECT,
137	137	type ExecutorOwnerPolicyContext,
138	138	} from "./owner-policy";
139		−import { ToolSchemaView, type IntegrationDetectionResult } from "./types";
	139	+import {
	140	+ ToolCatalogExport,
	141	+ ToolSchemaView,
	142	+ type IntegrationDetectionResult,
	143	+ type ToolCatalogConnectionExport,
	144	+ type ToolCatalogToolExport,
	145	+} from "./types";
140	146	import { type Tool, type ToolAnnotations, type ToolDef, type ToolListFilter } from "./tool";
141	147	import { buildToolTypeScriptPreview } from "./schema-types";
142	148	import { collectReferencedDefinitions } from "./schema-refs";
@@ -304,6 +310,9 @@ export type Executor<TPlugins extends readonly AnyPlugin[] = readonly []> = { · 161 unmodified lines
304	310	readonly tools: {
305	311	readonly list: (filter?: ToolListFilter) => Effect.Effect<readonly Tool[], StorageFailure>;
306	312	readonly schema: (address: ToolAddress) => Effect.Effect<ToolSchemaView \| null, StorageFailure>;
	313	+ /** Bulk schema-bearing read for codegen: every visible tool's input/output
	314	+ * JSON schema, grouped per connection with trimmed shared `$defs`. */
	315	+ readonly export: (filter?: ToolListFilter) => Effect.Effect<ToolCatalogExport, StorageFailure>;
307	316	};
308	317
309	318	readonly providers: {
@@ -2752,6 +2761,126 @@ export const createExecutor = <const TPlugins extends readonly AnyPlugin[] = rea · 2442 unmodified lines
2752	2761	});
2753	2762	});
2754	2763
	2764	+ // The bulk schema-bearing read behind `executor generate`. One pass over
	2765	+ // full tool rows (schemas included, unlike the projected `toolsList`),
	2766	+ // policy-filtered like `toolsList`, grouped per connection with that
	2767	+ // connection's `$defs` trimmed to the referenced subset. Kept as a single
	2768	+ // surface instead of N `toolSchema` calls: a 10k-tool catalog must not pay
	2769	+ // 10k round trips or 10k per-tool definition queries.
	2770	+ const toolsExport = (
	2771	+ filter?: ToolListFilter,
	2772	+ ): Effect.Effect<ToolCatalogExport, StorageFailure> =>
	2773	+ Effect.gen(function* () {
	2774	+ yield* syncStaleConnectionTools;
	2775	+ const rows = yield* core.findMany("tool", {
	2776	+ where: (b: AnyCb) =>
	2777	+ b.and(
	2778	+ filter?.integration === undefined
	2779	+ ? true
	2780	+ : b("integration", "=", String(filter.integration)),
	2781	+ filter?.owner === undefined ? true : b("owner", "=", filter.owner),
	2782	+ filter?.connection === undefined
	2783	+ ? true
	2784	+ : b("connection", "=", String(filter.connection)),
	2785	+ ),
	2786	+ });
	2787	+ const includeBlocked = filter?.includeBlocked ?? false;
	2788	+ const policyRules = yield* listActivePolicyRuleSet();
	2789	+
	2790	+ type ConnectionGroup = {
	2791	+ readonly owner: Owner;
	2792	+ readonly integration: IntegrationSlug;
	2793	+ readonly connection: ConnectionName;
	2794	+ readonly tools: ToolCatalogToolExport[];
	2795	+ };
	2796	+ const groups = new Map<string, ConnectionGroup>();
	2797	+ const groupFor = (tool: Tool): ConnectionGroup => {
	2798	+ const key = `${tool.owner}${tool.integration}${tool.connection}`;
	2799	+ let group = groups.get(key);
	2800	+ if (!group) {
	2801	+ group = {
	2802	+ owner: tool.owner,
	2803	+ integration: tool.integration,
	2804	+ connection: tool.connection,
	2805	+ tools: [],
	2806	+ };
	2807	+ groups.set(key, group);
	2808	+ }
	2809	+ return group;
	2810	+ };
	2811	+
	2812	+ const visible = (tool: Tool): Effect.Effect<boolean, StorageFailure> =>
	2813	+ Effect.gen(function* () {
	2814	+ if (!matchesToolFilter(tool, filter)) return false;
	2815	+ if (includeBlocked) return true;
	2816	+ const effective = yield* resolvePolicyFromRuleSet(
	2817	+ normalizedPolicyId(tool),
	2818	+ policyRules,
	2819	+ tool.annotations?.requiresApproval,
	2820	+ );
	2821	+ return effective.action !== "block";
	2822	+ });
	2823	+
	2824	+ for (const row of rows) {
	2825	+ const tool = rowToTool(row);
	2826	+ if (!(yield* visible(tool))) continue;
	2827	+ groupFor(tool).tools.push({
	2828	+ address: tool.address,
	2829	+ name: String(tool.name),
	2830	+ description: tool.description,
	2831	+ inputSchema: tool.inputSchema,
	2832	+ outputSchema: tool.outputSchema,
	2833	+ });
	2834	+ }
	2835	+ for (const entry of staticTools.values()) {
	2836	+ const tool = staticToolToTool(entry);
	2837	+ if (!(yield* visible(tool))) continue;
	2838	+ groupFor(tool).tools.push({
	2839	+ address: tool.address,
	2840	+ name: String(tool.name),
	2841	+ description: tool.description,
	2842	+ inputSchema: tool.inputSchema,
	2843	+ outputSchema: tool.outputSchema,
	2844	+ static: true,
	2845	+ });
	2846	+ }
	2847	+
	2848	+ const connectionsOut: ToolCatalogConnectionExport[] = [];
	2849	+ for (const group of groups.values()) {
	2850	+ // Static pseudo-connections have no definition rows; skip the query.
	2851	+ const isStatic = group.tools.every((tool) => tool.static === true);
	2852	+ let definitions: Record<string, unknown> \| undefined;
	2853	+ if (!isStatic) {
	2854	+ const definitionRows = yield* core.findMany("definition", {
	2855	+ where: (b: AnyCb) =>
	2856	+ b.and(
	2857	+ byOwner(group.owner)(b),
	2858	+ b("integration", "=", String(group.integration)),
	2859	+ b("connection", "=", String(group.connection)),
	2860	+ ),
	2861	+ });
	2862	+ const defs = new Map<string, unknown>();
	2863	+ for (const def of definitionRows) defs.set(def.name, decodeJsonColumn(def.schema));
	2864	+ const referenced = collectReferencedDefinitions(
	2865	+ group.tools.flatMap((tool) => [tool.inputSchema, tool.outputSchema]),
	2866	+ defs,
	2867	+ );
	2868	+ if (Object.keys(referenced).length > 0) {
	2869	+ definitions = referenced;
	2870	+ }
	2871	+ }
	2872	+ connectionsOut.push({
	2873	+ owner: group.owner,
	2874	+ integration: group.integration,
	2875	+ connection: group.connection,
	2876	+ tools: group.tools,
	2877	+ ...(definitions !== undefined ? { definitions } : {}),
	2878	+ });
	2879	+ }
	2880	+
	2881	+ return ToolCatalogExport.make({ connections: connectionsOut });
	2882	+ });
	2883	+
2755	2884	// ------------------------------------------------------------------
2756	2885	// Providers
2757	2886	// ------------------------------------------------------------------
@@ -3465,6 +3594,7 @@ export const createExecutor = <const TPlugins extends readonly AnyPlugin[] = rea · 707 unmodified lines
3465	3594	tools: {
3466	3595	list: toolsList,
3467	3596	schema: toolSchema,
	3597	+ export: toolsExport,
3468	3598	},
3469	3599	providers: {
3470	3600	list: providersList,

packages/core/sdk/src/index.ts+21−2

@@ -99,7 +99,13 @@ export type { Tool, ToolDef, ToolListFilter, ToolAnnotations } from "./tool"; · 98 unmodified lines
99	99	export type { CredentialProvider, ProviderEntry } from "./provider";
100	100
101	101	// Public projections / detection.
102		−export { ToolSchemaView, IntegrationDetectionResult } from "./types";
	102	+export {
	103	+ ToolCatalogConnectionExport,
	104	+ ToolCatalogExport,
	105	+ ToolCatalogToolExport,
	106	+ ToolSchemaView,
	107	+ IntegrationDetectionResult,
	108	+} from "./types";
103	109
104	110	// Core schema.
105	111	export {
@@ -329,7 +335,20 @@ export { · 223 unmodified lines
329	335	} from "./config";
330	336
331	337	// The one TS-preview generator plugins assert against.
332		−export { buildToolTypeScriptPreview } from "./schema-types";
	338	+export {
	339	+ buildToolTypeScriptPreview,
	340	+ compileToolChunkTypeScript,
	341	+ type ToolChunkSchemaEntry,
	342	+ type ToolChunkTypeScript,
	343	+ type ToolChunkTypeScriptEntry,
	344	+} from "./schema-types";
	345	+
	346	+// Typed tool-proxy source generation (the `executor generate` backend).
	347	+export {
	348	+ generateToolProxySource,
	349	+ type GeneratedToolProxy,
	350	+ type GenerateToolProxyOptions,
	351	+} from "./typegen";
333	352
334	353	// Wire-level HTTP error schemas usable by plugin HttpApiGroup definitions.
335	354	export { InternalError } from "./api-errors";

packages/core/sdk/src/schema-types.ts+108−0

@@ -789,6 +789,114 @@ export type ToolTypeScriptPreview = { · 788 unmodified lines
789	789	typeScriptDefinitions?: Record<string, string>;
790	790	};
791	791
	792	+// ---------------------------------------------------------------------------
	793	+// Chunked tool compilation — many tools per compiler pass.
	794	+//
	795	+// `buildToolTypeScriptPreview` compiles ONE tool per pass; at catalog scale
	796	+// (the `executor generate` typegen walks every tool, potentially 10k+) the
	797	+// per-pass overhead dominates and compile time grows super-linearly with the
	798	+// number of declarations in a single pass. This entry point compiles a CHUNK
	799	+// of tools in one pass by wrapping every tool's input/output as a property of
	800	+// one root object, then splits the compiled root interface back into per-tool
	801	+// type strings. Callers pick the chunk size; ~200 keeps each pass linear.
	802	+// ---------------------------------------------------------------------------
	803	+
	804	+export type ToolChunkSchemaEntry = {
	805	+ readonly key: string;
	806	+ readonly inputSchema?: unknown;
	807	+ readonly outputSchema?: unknown;
	808	+};
	809	+
	810	+export type ToolChunkTypeScriptEntry = {
	811	+ readonly inputTypeScript?: string;
	812	+ readonly outputTypeScript?: string;
	813	+};
	814	+
	815	+export type ToolChunkTypeScript = {
	816	+ readonly tools: ReadonlyMap<string, ToolChunkTypeScriptEntry>;
	817	+ readonly definitions: Record<string, string>;
	818	+};
	819	+
	820	+const CHUNK_INPUT_SUFFIX = "__in";
	821	+const CHUNK_OUTPUT_SUFFIX = "__out";
	822	+
	823	+/**
	824	+ * Compile a chunk of tools' input/output schemas in one compiler pass.
	825	+ *
	826	+ * `key`s must be unique valid identifiers within the chunk (the caller indexes
	827	+ * them, e.g. `t0`, `t1`, …). Shared `$defs` are compiled once per chunk and
	828	+ * returned as named `definitions`; the per-tool type strings reference them by
	829	+ * name. Throws when the underlying compiler rejects a schema — callers decide
	830	+ * whether to retry tool-by-tool or degrade to `unknown`.
	831	+ */
	832	+export const compileToolChunkTypeScript = (
	833	+ tools: ReadonlyArray<ToolChunkSchemaEntry>,
	834	+ defs: ReadonlyMap<string, unknown>,
	835	+ options: TypeScriptRenderOptions = {},
	836	+): ToolChunkTypeScript => {
	837	+ const properties: Array<readonly [string, unknown]> = [];
	838	+ for (const tool of tools) {
	839	+ if (tool.inputSchema !== undefined) {
	840	+ properties.push([`${tool.key}${CHUNK_INPUT_SUFFIX}`, tool.inputSchema]);
	841	+ }
	842	+ if (tool.outputSchema !== undefined) {
	843	+ properties.push([`${tool.key}${CHUNK_OUTPUT_SUFFIX}`, tool.outputSchema]);
	844	+ }
	845	+ }
	846	+ if (properties.length === 0) {
	847	+ return {
	848	+ tools: new Map(tools.map((tool) => [tool.key, {}])),
	849	+ definitions: {},
	850	+ };
	851	+ }
	852	+
	853	+ const wrappedSchema = buildWrappedObjectSchema(properties, defs);
	854	+ const source = compile(wrappedSchema, ROOT_WRAPPER_NAME, compilerOptionsFrom(options));
	855	+ const declarations = parseGeneratedDeclarations(source);
	856	+ const rootDeclaration = declarations.find(
	857	+ (declaration) => declaration.name === ROOT_WRAPPER_NAME,
	858	+ );
	859	+ const body = rootDeclaration?.kind === "interface" ? rootDeclaration.body : "";
	860	+
	861	+ // Single forward pass: the compiler emits root properties in insertion
	862	+ // order, so a running offset keeps extraction linear in the body size and
	863	+ // immune to one key being a substring of a later one.
	864	+ let offset = 0;
	865	+ const extractNext = (propertyName: string): string \| null => {
	866	+ const propertyIndex = body.indexOf(propertyName, offset);
	867	+ if (propertyIndex < 0) {
	868	+ return null;
	869	+ }
	870	+ const colonIndex = body.indexOf(":", propertyIndex + propertyName.length);
	871	+ if (colonIndex < 0) {
	872	+ return null;
	873	+ }
	874	+ const end = findTypeAliasEnd(body, colonIndex + 1);
	875	+ if (end < 0) {
	876	+ return null;
	877	+ }
	878	+ offset = end;
	879	+ return body.slice(colonIndex + 1, end).trim();
	880	+ };
	881	+
	882	+ const compiled = new Map<string, ToolChunkTypeScriptEntry>();
	883	+ for (const tool of tools) {
	884	+ const input =
	885	+ tool.inputSchema !== undefined ? extractNext(`${tool.key}${CHUNK_INPUT_SUFFIX}`) : null;
	886	+ const output =
	887	+ tool.outputSchema !== undefined ? extractNext(`${tool.key}${CHUNK_OUTPUT_SUFFIX}`) : null;
	888	+ compiled.set(tool.key, {
	889	+ ...(input ? { inputTypeScript: compactTypeScript(input) } : {}),
	890	+ ...(output ? { outputTypeScript: compactTypeScript(output) } : {}),
	891	+ });
	892	+ }
	893	+
	894	+ return {
	895	+ tools: compiled,
	896	+ definitions: getDefinitionsFromDeclarations(declarations),
	897	+ };
	898	+};
	899	+
792	900	export const buildToolTypeScriptPreview = async (input: {
793	901	inputSchema?: unknown;
794	902	outputSchema?: unknown;

packages/core/sdk/src/shared.ts+7−1

@@ -97,7 +97,13 @@ export { · 96 unmodified lines
97	97	export type { ToolPolicyAction } from "./core-schema";
98	98
99	99	// Schema-side views + onboarding autodetect.
100		−export { ToolSchemaView, IntegrationDetectionResult } from "./types";
	100	+export {
	101	+ ToolCatalogConnectionExport,
	102	+ ToolCatalogExport,
	103	+ ToolCatalogToolExport,
	104	+ ToolSchemaView,
	105	+ IntegrationDetectionResult,
	106	+} from "./types";
101	107
102	108	export {
103	109	decodeOAuthCallbackState,

packages/core/sdk/src/typegen.test.ts+516−0

@@ -0,0 +1,516 @@
	1	+// ---------------------------------------------------------------------------
	2	+// generateToolProxySource: the `executor generate` backend.
	3	+//
	4	+// Covered here:
	5	+// - the generated source is valid strict TypeScript (checked with the real
	6	+// compiler) and its types reject wrong inputs,
	7	+// - the generated runtime client invokes tools through /api/executions and
	8	+// unwraps completed/paused/error responses (transpiled and imported, run
	9	+// against a fake fetch),
	10	+// - naming: hyphenated tool names, colliding sanitized names, description
	11	+// text that tries to escape its JSDoc comment,
	12	+// - resilience: a schema the compiler rejects degrades that one tool to
	13	+// `unknown` without poisoning its chunk,
	14	+// - `executor.tools.export` (the data source): schemas + trimmed shared
	15	+// $defs per connection, policy-filtered, static tools flagged.
	16	+// ---------------------------------------------------------------------------
	17	+
	18	+import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
	19	+import { tmpdir } from "node:os";
	20	+import { join } from "node:path";
	21	+import { pathToFileURL } from "node:url";
	22	+import { describe, expect, it } from "@effect/vitest";
	23	+import { Effect } from "effect";
	24	+import * as ts from "typescript";
	25	+
	26	+import { AuthTemplateSlug, ConnectionName, IntegrationSlug, ToolAddress, ToolName } from "./ids";
	27	+import { definePlugin } from "./plugin";
	28	+import { makeTestExecutor, memoryCredentialsPlugin } from "./testing";
	29	+import { generateToolProxySource } from "./typegen";
	30	+import type { ToolCatalogExport } from "./types";
	31	+
	32	+// ---------------------------------------------------------------------------
	33	+// Helpers
	34	+// ---------------------------------------------------------------------------
	35	+
	36	+const typecheck = (source: string, extraSource = ""): readonly string[] => {
	37	+ const fileName = "generated.ts";
	38	+ const fullSource = extraSource.length > 0 ? `${source}\n${extraSource}` : source;
	39	+ const options: ts.CompilerOptions = {
	40	+ strict: true,
	41	+ noEmit: true,
	42	+ skipLibCheck: true,
	43	+ target: ts.ScriptTarget.ES2022,
	44	+ module: ts.ModuleKind.ESNext,
	45	+ lib: ["lib.es2022.d.ts", "lib.dom.d.ts"],
	46	+ };
	47	+ const host = ts.createCompilerHost(options);
	48	+ const originalGetSourceFile = host.getSourceFile.bind(host);
	49	+ const originalReadFile = host.readFile.bind(host);
	50	+ const originalFileExists = host.fileExists.bind(host);
	51	+ host.getSourceFile = (candidate, languageVersion, onError, shouldCreateNewSourceFile) =>
	52	+ candidate === fileName
	53	+ ? ts.createSourceFile(candidate, fullSource, languageVersion, true)
	54	+ : originalGetSourceFile(candidate, languageVersion, onError, shouldCreateNewSourceFile);
	55	+ host.readFile = (candidate) =>
	56	+ candidate === fileName ? fullSource : originalReadFile(candidate);
	57	+ host.fileExists = (candidate) => candidate === fileName \|\| originalFileExists(candidate);
	58	+
	59	+ const program = ts.createProgram([fileName], options, host);
	60	+ return ts.getPreEmitDiagnostics(program).map((diagnostic) => {
	61	+ const message = ts.flattenDiagnosticMessageText(diagnostic.messageText, "\n");
	62	+ const position =
	63	+ diagnostic.file && diagnostic.start !== undefined
	64	+ ? diagnostic.file.getLineAndCharacterOfPosition(diagnostic.start)
	65	+ : null;
	66	+ return position
	67	+ ? `${diagnostic.file!.fileName}:${position.line + 1}:${position.character + 1} ${message}`
	68	+ : message;
	69	+ });
	70	+};
	71	+
	72	+/** Transpile the generated TypeScript and import it as a real module. */
	73	+const importGenerated = async (source: string): Promise<Record<string, unknown>> => {
	74	+ const transpiled = ts.transpileModule(source, {
	75	+ compilerOptions: {
	76	+ target: ts.ScriptTarget.ES2022,
	77	+ module: ts.ModuleKind.ESNext,
	78	+ },
	79	+ }).outputText;
	80	+ const dir = mkdtempSync(join(tmpdir(), "executor-typegen-"));
	81	+ const file = join(dir, "generated.mjs");
	82	+ writeFileSync(file, transpiled);
	83	+ // oxlint-disable-next-line executor/no-try-catch-or-throw -- test boundary: temp-dir cleanup around a dynamic import of the generated module
	84	+ try {
	85	+ return (await import(pathToFileURL(file).href)) as Record<string, unknown>;
	86	+ } finally {
	87	+ rmSync(dir, { recursive: true, force: true });
	88	+ }
	89	+};
	90	+
	91	+type ConnectionExport = ToolCatalogExport["connections"][number];
	92	+type ToolExport = ConnectionExport["tools"][number];
	93	+
	94	+const catalog = (connections: readonly ConnectionExport[]): ToolCatalogExport => ({ connections });
	95	+
	96	+const connectionExport = (input: {
	97	+ owner: "org" \| "user";
	98	+ integration: string;
	99	+ connection: string;
	100	+ definitions?: Record<string, unknown>;
	101	+ tools: ReadonlyArray<{
	102	+ address: string;
	103	+ name: string;
	104	+ description?: string;
	105	+ inputSchema?: unknown;
	106	+ outputSchema?: unknown;
	107	+ static?: boolean;
	108	+ }>;
	109	+}): ConnectionExport => ({
	110	+ owner: input.owner,
	111	+ integration: IntegrationSlug.make(input.integration),
	112	+ connection: ConnectionName.make(input.connection),
	113	+ ...(input.definitions !== undefined ? { definitions: input.definitions } : {}),
	114	+ tools: input.tools.map(
	115	+ (tool): ToolExport => ({
	116	+ ...tool,
	117	+ address: ToolAddress.make(tool.address),
	118	+ }),
	119	+ ),
	120	+});
	121	+
	122	+const githubConnection = connectionExport({
	123	+ owner: "org",
	124	+ integration: "github",
	125	+ connection: "main",
	126	+ definitions: {
	127	+ User: {
	128	+ type: "object",
	129	+ properties: { id: { type: "string" }, login: { type: "string" } },
	130	+ required: ["id"],
	131	+ },
	132	+ },
	133	+ tools: [
	134	+ {
	135	+ address: "tools.github.org.main.issues.create",
	136	+ name: "issues.create",
	137	+ description: "Create an issue",
	138	+ inputSchema: {
	139	+ type: "object",
	140	+ properties: { title: { type: "string" }, assignee: { $ref: "#/$defs/User" } },
	141	+ required: ["title"],
	142	+ },
	143	+ outputSchema: {
	144	+ type: "object",
	145	+ properties: { number: { type: "number" }, user: { $ref: "#/$defs/User" } },
	146	+ },
	147	+ },
	148	+ {
	149	+ address: "tools.github.org.main.issues.list",
	150	+ name: "issues.list",
	151	+ description: "List issues */ } escape attempt",
	152	+ inputSchema: {
	153	+ type: "object",
	154	+ properties: { state: { enum: ["open", "closed"] } },
	155	+ },
	156	+ },
	157	+ {
	158	+ address: "tools.github.org.main.repos.get",
	159	+ name: "repos.get",
	160	+ description: "No schemas at all",
	161	+ },
	162	+ ],
	163	+});
	164	+
	165	+// ---------------------------------------------------------------------------
	166	+// Generation
	167	+// ---------------------------------------------------------------------------
	168	+
	169	+describe("generateToolProxySource", { timeout: 60_000 }, () => {
	170	+ it("emits strict-TypeScript-valid source with per-tool types", () => {
	171	+ const generated = generateToolProxySource(catalog([githubConnection]));
	172	+ expect(generated.toolCount).toBe(3);
	173	+ expect(generated.connectionCount).toBe(1);
	174	+
	175	+ const diagnostics = typecheck(generated.source);
	176	+ expect(diagnostics).toEqual([]);
	177	+
	178	+ expect(generated.source).toContain("export interface ExecutorTools");
	179	+ expect(generated.source).toContain("issues_create_Input");
	180	+ // Shared $defs become named, reused types instead of inlined copies.
	181	+ expect(generated.source).toContain("export type User =");
	182	+ });
	183	+
	184	+ it("generated types accept correct calls and reject wrong ones", () => {
	185	+ const generated = generateToolProxySource(catalog([githubConnection]));
	186	+
	187	+ const validConsumer = `
	188	+ const client = createExecutorClient({ baseUrl: "http://localhost:4788" });
	189	+ async function main() {
	190	+ const created = await client.github.org.main.issues.create({ title: "hi" });
	191	+ if (created.ok) {
	192	+ const n: number \| undefined = created.data.number;
	193	+ void n;
	194	+ } else {
	195	+ const code: string = created.error.code;
	196	+ void code;
	197	+ }
	198	+ await client.github.org.main.repos.get();
	199	+ await client.$call("github.org.main.repos.get", {});
	200	+ }
	201	+ void main;
	202	+ `;
	203	+ expect(typecheck(generated.source, validConsumer)).toEqual([]);
	204	+
	205	+ const invalidConsumer = `
	206	+ const client = createExecutorClient();
	207	+ async function main() {
	208	+ // title is required and must be a string
	209	+ await client.github.org.main.issues.create({ title: 42 });
	210	+ }
	211	+ void main;
	212	+ `;
	213	+ expect(typecheck(generated.source, invalidConsumer)).not.toEqual([]);
	214	+ });
	215	+
	216	+ it("keeps hyphenated path segments callable and dedupes colliding type names", () => {
	217	+ const generated = generateToolProxySource(
	218	+ catalog([
	219	+ connectionExport({
	220	+ owner: "user",
	221	+ integration: "linear",
	222	+ connection: "personal",
	223	+ tools: [
	224	+ {
	225	+ address: "tools.linear.user.personal.issue-create",
	226	+ name: "issue-create",
	227	+ inputSchema: { type: "object", properties: { title: { type: "string" } } },
	228	+ },
	229	+ {
	230	+ // Sanitizes to the same identifier as issue-create.
	231	+ address: "tools.linear.user.personal.issue_create",
	232	+ name: "issue_create",
	233	+ inputSchema: { type: "object", properties: { key: { type: "string" } } },
	234	+ },
	235	+ ],
	236	+ }),
	237	+ ]),
	238	+ );
	239	+
	240	+ expect(typecheck(generated.source)).toEqual([]);
	241	+ expect(generated.source).toContain('"issue-create":');
	242	+ // Both tools keep distinct input types despite the name collision.
	243	+ expect(generated.source).toContain("issue_create_Input");
	244	+ expect(generated.source).toContain("issue_create_Input_2");
	245	+ });
	246	+
	247	+ it("degrades a compiler-rejected schema to unknown without poisoning its chunk", () => {
	248	+ const generated = generateToolProxySource(
	249	+ catalog([
	250	+ connectionExport({
	251	+ owner: "org",
	252	+ integration: "demo",
	253	+ connection: "main",
	254	+ tools: [
	255	+ {
	256	+ address: "tools.demo.org.main.good",
	257	+ name: "good",
	258	+ inputSchema: { type: "object", properties: { ok: { type: "boolean" } } },
	259	+ },
	260	+ {
	261	+ address: "tools.demo.org.main.broken",
	262	+ name: "broken",
	263	+ // $ref to a definition that does not exist: the vendored
	264	+ // compiler throws on dangling refs.
	265	+ inputSchema: { $ref: "#/$defs/DoesNotExist" },
	266	+ },
	267	+ ],
	268	+ }),
	269	+ ]),
	270	+ { chunkSize: 50 },
	271	+ );
	272	+
	273	+ expect(typecheck(generated.source)).toEqual([]);
	274	+ // The good tool keeps its real type; the broken one degrades to unknown.
	275	+ expect(generated.source).toContain("good_Input = { ok?: boolean; }");
	276	+ expect(generated.source).toContain("broken_Input = unknown");
	277	+ });
	278	+
	279	+ it("returns an empty interface for an empty catalog", () => {
	280	+ const generated = generateToolProxySource(catalog([]));
	281	+ expect(generated.toolCount).toBe(0);
	282	+ expect(typecheck(generated.source)).toEqual([]);
	283	+ });
	284	+});
	285	+
	286	+// ---------------------------------------------------------------------------
	287	+// Generated runtime client
	288	+// ---------------------------------------------------------------------------
	289	+
	290	+type FetchCall = { url: string; init: { headers: Record<string, string>; body: string } };
	291	+
	292	+const makeFakeFetch = (respond: (call: FetchCall) => unknown) => {
	293	+ const calls: FetchCall[] = [];
	294	+ const fetchImpl: typeof globalThis.fetch = async (url, init) => {
	295	+ const call: FetchCall = {
	296	+ url: String(url),
	297	+ init: {
	298	+ headers: (init?.headers ?? {}) as Record<string, string>,
	299	+ body: String(init?.body ?? ""),
	300	+ },
	301	+ };
	302	+ calls.push(call);
	303	+ return new Response(JSON.stringify(respond(call)), { status: 200 });
	304	+ };
	305	+ return { calls, fetchImpl };
	306	+};
	307	+
	308	+describe("generated runtime client", { timeout: 60_000 }, () => {
	309	+ it("invokes tools through /api/executions and unwraps the outcome", async () => {
	310	+ const generated = generateToolProxySource(catalog([githubConnection]));
	311	+ const module = await importGenerated(generated.source);
	312	+ const createClient = module.createExecutorClient as (
	313	+ options: Record<string, unknown>,
	314	+ ) => unknown;
	315	+
	316	+ const { calls, fetchImpl } = makeFakeFetch(() => ({
	317	+ status: "completed",
	318	+ text: "",
	319	+ isError: false,
	320	+ structured: { result: { ok: true, data: { number: 7 } } },
	321	+ }));
	322	+
	323	+ // oxlint-disable-next-line executor/no-double-cast -- test boundary: the client is a dynamically imported Proxy; the cast pins the path this test dials
	324	+ const client = createClient({
	325	+ baseUrl: "http://example.test:4788/",
	326	+ token: "tok_123",
	327	+ fetch: fetchImpl,
	328	+ }) as unknown as {
	329	+ github: {
	330	+ org: {
	331	+ main: {
	332	+ issues: { create: (input: unknown) => Promise<{ ok: boolean; data?: unknown }> };
	333	+ };
	334	+ };
	335	+ };
	336	+ };
	337	+
	338	+ const outcome = await client.github.org.main.issues.create({ title: "hi" });
	339	+ expect(outcome).toEqual({ ok: true, data: { number: 7 } });
	340	+
	341	+ expect(calls).toHaveLength(1);
	342	+ expect(calls[0]!.url).toBe("http://example.test:4788/api/executions");
	343	+ expect(calls[0]!.init.headers.authorization).toBe("Bearer tok_123");
	344	+ // oxlint-disable-next-line executor/no-json-parse -- test boundary: asserting on the raw request body the generated client produced
	345	+ const body = JSON.parse(calls[0]!.init.body) as { code: string };
	346	+ expect(body.code).toContain('["github"]["org"]["main"]["issues"]["create"]');
	347	+ expect(body.code).toContain('{"title":"hi"}');
	348	+ });
	349	+
	350	+ it("throws ExecutorPausedError with the approval url on paused executions", async () => {
	351	+ const generated = generateToolProxySource(catalog([githubConnection]));
	352	+ const module = await importGenerated(generated.source);
	353	+ const createClient = module.createExecutorClient as (options: Record<string, unknown>) => {
	354	+ $call: (path: string, input?: unknown) => Promise<unknown>;
	355	+ };
	356	+
	357	+ const { fetchImpl } = makeFakeFetch(() => ({
	358	+ status: "paused",
	359	+ text: "Approval required",
	360	+ structured: { executionId: "exec_42" },
	361	+ }));
	362	+
	363	+ const client = createClient({ baseUrl: "http://example.test:4788", fetch: fetchImpl });
	364	+ const failure = await client.$call("github.org.main.issues.create", { title: "hi" }).then(
	365	+ () => null,
	366	+ (error: unknown) => error as Record<string, unknown>,
	367	+ );
	368	+
	369	+ expect(failure).not.toBeNull();
	370	+ expect(failure!.name).toBe("ExecutorPausedError");
	371	+ expect(failure!.executionId).toBe("exec_42");
	372	+ expect(failure!.approvalUrl).toBe("http://example.test:4788/resume/exec_42");
	373	+ });
	374	+
	375	+ it("rejects path segments that could break out of the invoke code", async () => {
	376	+ const generated = generateToolProxySource(catalog([githubConnection]));
	377	+ const module = await importGenerated(generated.source);
	378	+ const createClient = module.createExecutorClient as (options: Record<string, unknown>) => {
	379	+ $call: (path: string, input?: unknown) => Promise<unknown>;
	380	+ };
	381	+
	382	+ const { calls, fetchImpl } = makeFakeFetch(() => ({}));
	383	+ const client = createClient({ fetch: fetchImpl });
	384	+
	385	+ await expect(client.$call('bad"segment];evil()', {})).rejects.toMatchObject({
	386	+ name: "ExecutorRequestError",
	387	+ });
	388	+ expect(calls).toHaveLength(0);
	389	+ });
	390	+});
	391	+
	392	+// ---------------------------------------------------------------------------
	393	+// executor.tools.export: the catalog read behind the CLI
	394	+// ---------------------------------------------------------------------------
	395	+
	396	+const INTEG = IntegrationSlug.make("demo");
	397	+const CONN = ConnectionName.make("main");
	398	+const TEMPLATE = AuthTemplateSlug.make("apiKey");
	399	+
	400	+const demoPlugin = definePlugin(() => ({
	401	+ id: "demo" as const,
	402	+ storage: () => ({}),
	403	+ resolveTools: () =>
	404	+ Effect.succeed({
	405	+ tools: [
	406	+ {
	407	+ name: ToolName.make("inspect"),
	408	+ description: "inspect",
	409	+ inputSchema: {
	410	+ type: "object",
	411	+ properties: { pet: { $ref: "#/$defs/Pet" } },
	412	+ required: ["pet"],
	413	+ },
	414	+ outputSchema: { $ref: "#/$defs/Owner" },
	415	+ },
	416	+ { name: ToolName.make("run"), description: "run" },
	417	+ ],
	418	+ definitions: {
	419	+ Pet: { type: "object", properties: { name: { type: "string" } } },
	420	+ Owner: { type: "object", properties: { pet: { $ref: "#/$defs/Pet" } } },
	421	+ Unreferenced: { type: "object", properties: { value: { type: "string" } } },
	422	+ },
	423	+ }),
	424	+ invokeTool: ({ toolRow }) => Effect.succeed({ ran: toolRow.name }),
	425	+ extension: (ctx) => ({
	426	+ seed: () =>
	427	+ ctx.core.integrations.register({
	428	+ slug: INTEG,
	429	+ description: "Demo",
	430	+ config: {},
	431	+ }),
	432	+ }),
	433	+}))();
	434	+
	435	+const setup = () =>
	436	+ Effect.gen(function* () {
	437	+ const executor = yield* makeTestExecutor({
	438	+ plugins: [memoryCredentialsPlugin(), demoPlugin] as const,
	439	+ });
	440	+ yield* executor.demo.seed();
	441	+ yield* executor.connections.create({
	442	+ owner: "org",
	443	+ name: CONN,
	444	+ integration: INTEG,
	445	+ template: TEMPLATE,
	446	+ value: "token",
	447	+ });
	448	+ return executor;
	449	+ });
	450	+
	451	+describe("tools.export", { timeout: 60_000 }, () => {
	452	+ it.effect("returns schemas grouped per connection with trimmed shared $defs", () =>
	453	+ Effect.gen(function* () {
	454	+ const executor = yield* setup();
	455	+ const exported = yield* executor.tools.export({ integration: INTEG });
	456	+
	457	+ expect(exported.connections).toHaveLength(1);
	458	+ const connection = exported.connections[0]!;
	459	+ expect(String(connection.integration)).toBe("demo");
	460	+ expect(String(connection.connection)).toBe("main");
	461	+ expect(connection.tools.map((tool) => tool.name).sort()).toEqual(["inspect", "run"]);
	462	+
	463	+ const inspect = connection.tools.find((tool) => tool.name === "inspect")!;
	464	+ expect(inspect.inputSchema).toMatchObject({ type: "object" });
	465	+ expect(inspect.outputSchema).toEqual({ $ref: "#/$defs/Owner" });
	466	+
	467	+ // Referenced defs (Pet transitively via Owner) come along; unreferenced
	468	+ // ones are trimmed.
	469	+ expect(Object.keys(connection.definitions ?? {}).sort()).toEqual(["Owner", "Pet"]);
	470	+ }),
	471	+ );
	472	+
	473	+ it.effect("omits blocked tools like tools.list does", () =>
	474	+ Effect.gen(function* () {
	475	+ const executor = yield* setup();
	476	+ yield* executor.policies.create({
	477	+ owner: "org",
	478	+ pattern: "demo.org.main.run",
	479	+ action: "block",
	480	+ });
	481	+
	482	+ const exported = yield* executor.tools.export({ integration: INTEG });
	483	+ const names = exported.connections.flatMap((connection) =>
	484	+ connection.tools.map((tool) => tool.name),
	485	+ );
	486	+ expect(names).toEqual(["inspect"]);
	487	+ }),
	488	+ );
	489	+
	490	+ it.effect("flags static tools and feeds a generatable catalog end to end", () =>
	491	+ Effect.gen(function* () {
	492	+ const executor = yield* makeTestExecutor({
	493	+ plugins: [memoryCredentialsPlugin(), demoPlugin] as const,
	494	+ coreTools: { webBaseUrl: "http://localhost:3000" },
	495	+ });
	496	+ yield* executor.demo.seed();
	497	+ yield* executor.connections.create({
	498	+ owner: "org",
	499	+ name: CONN,
	500	+ integration: INTEG,
	501	+ template: TEMPLATE,
	502	+ value: "token",
	503	+ });
	504	+ const exported = yield* executor.tools.export({ includeBlocked: true });
	505	+
	506	+ const staticTools = exported.connections
	507	+ .flatMap((connection) => connection.tools)
	508	+ .filter((tool) => tool.static === true);
	509	+ expect(staticTools.length).toBeGreaterThan(0);
	510	+
	511	+ const generated = generateToolProxySource(exported);
	512	+ expect(generated.toolCount).toBeGreaterThanOrEqual(2);
	513	+ expect(typecheck(generated.source)).toEqual([]);
	514	+ }),
	515	+ );
	516	+});

packages/core/sdk/src/typegen.ts+528−0

@@ -0,0 +1,528 @@
	1	+// ---------------------------------------------------------------------------
	2	+// Typed tool proxy generation: the `executor generate` backend.
	3	+//
	4	+// Turns a `ToolCatalogExport` (every visible tool's input/output JSON schema,
	5	+// grouped per connection with the connection's shared `$defs`) into ONE
	6	+// self-contained TypeScript source file:
	7	+//
	8	+// - a dependency-free runtime client (Proxy-based, so its size is constant
	9	+// no matter how many tools the catalog holds) that invokes tools through
	10	+// the server's `/api/executions` endpoint,
	11	+// - one non-instantiated (type-only, erasable) namespace per connection
	12	+// carrying that connection's shared definitions and per-tool input/output
	13	+// types,
	14	+// - an `ExecutorTools` interface mirroring the sandbox `tools.*` path tree,
	15	+// so `client.github.org.main.issues.create({...})` is fully typed.
	16	+//
	17	+// Scale is a first-class constraint: catalogs reach 10k+ tools. Schemas are
	18	+// compiled through `compileToolChunkTypeScript` (many tools per compiler pass;
	19	+// one pass per `chunkSize` tools) because per-tool passes pay fixed overhead
	20	+// 10k times and one whole-catalog pass grows super-linearly with declaration
	21	+// count. All source assembly appends to arrays and joins once.
	22	+// ---------------------------------------------------------------------------
	23	+
	24	+import {
	25	+ compileToolChunkTypeScript,
	26	+ type ToolChunkSchemaEntry,
	27	+ type ToolChunkTypeScriptEntry,
	28	+} from "./schema-types";
	29	+import type { ToolCatalogConnectionExport, ToolCatalogExport } from "./types";
	30	+
	31	+export type GenerateToolProxyOptions = {
	32	+ /** Tools compiled per compiler pass. The pass cost grows super-linearly
	33	+ * with declaration count, so keep chunks small; 200 keeps each pass tens
	34	+ * of milliseconds while amortizing per-pass overhead. */
	35	+ readonly chunkSize?: number;
	36	+ /** Client factory name in the generated file. */
	37	+ readonly clientName?: string;
	38	+};
	39	+
	40	+const DEFAULT_CHUNK_SIZE = 200;
	41	+
	42	+// ---------------------------------------------------------------------------
	43	+// Identifier helpers
	44	+// ---------------------------------------------------------------------------
	45	+
	46	+const IDENTIFIER_PATTERN = /^[A-Za-z_$][A-Za-z0-9_$]*$/;
	47	+
	48	+const sanitizeIdentifier = (value: string): string => {
	49	+ const cleaned = value.replace(/[^A-Za-z0-9_$]/g, "_");
	50	+ return /^[0-9]/.test(cleaned) ? `_${cleaned}` : cleaned.length > 0 ? cleaned : "_";
	51	+};
	52	+
	53	+const propertyKey = (segment: string): string =>
	54	+ IDENTIFIER_PATTERN.test(segment) ? segment : JSON.stringify(segment);
	55	+
	56	+const uniqueName = (base: string, used: Set<string>): string => {
	57	+ let candidate = base;
	58	+ let counter = 2;
	59	+ while (used.has(candidate)) {
	60	+ candidate = `${base}_${counter}`;
	61	+ counter += 1;
	62	+ }
	63	+ used.add(candidate);
	64	+ return candidate;
	65	+};
	66	+
	67	+const escapeJsDoc = (value: string): string => value.replace(/\\//g, "\\/");
	68	+
	69	+// ---------------------------------------------------------------------------
	70	+// Per-connection compilation
	71	+// ---------------------------------------------------------------------------
	72	+
	73	+type CompiledTool = {
	74	+ readonly pathSegments: readonly string[];
	75	+ readonly description?: string;
	76	+ /** Fully-qualified type reference (`Ns.Name_Input`) or undefined when the
	77	+ * tool has no input schema (callable with no argument). */
	78	+ readonly inputTypeRef?: string;
	79	+ /** Fully-qualified type reference or "unknown". */
	80	+ readonly outputTypeRef: string;
	81	+};
	82	+
	83	+type CompiledConnection = {
	84	+ readonly namespaceName: string;
	85	+ /** `export type Name = body;` lines inside the namespace. */
	86	+ readonly declarations: readonly string[];
	87	+ readonly tools: readonly CompiledTool[];
	88	+};
	89	+
	90	+/** Sandbox `tools.*` path for a tool: dynamic addresses drop the `tools.`
	91	+ * proxy-root prefix; static addresses are already the callable path. */
	92	+const toolPathSegments = (
	93	+ connection: ToolCatalogConnectionExport,
	94	+ tool: ToolCatalogConnectionExport["tools"][number],
	95	+): readonly string[] => {
	96	+ const address = String(tool.address);
	97	+ const path = address.startsWith("tools.") ? address.slice("tools.".length) : address;
	98	+ return path.split(".");
	99	+};
	100	+
	101	+const compileConnection = (
	102	+ connection: ToolCatalogConnectionExport,
	103	+ namespaceName: string,
	104	+ chunkSize: number,
	105	+): CompiledConnection => {
	106	+ const defs = new Map<string, unknown>(Object.entries(connection.definitions ?? {}));
	107	+ const tools = [...connection.tools].sort((a, b) =>
	108	+ String(a.address).localeCompare(String(b.address)),
	109	+ );
	110	+
	111	+ // Chunked compile: keys are positional (`t<index>`), unique and identifier-
	112	+ // safe by construction. A failing chunk retries tool-by-tool so one broken
	113	+ // schema degrades that tool to `unknown` instead of the whole chunk.
	114	+ const compiled = new Map<string, ToolChunkTypeScriptEntry>();
	115	+ const definitionDecls = new Map<string, string>();
	116	+ const entries: ToolChunkSchemaEntry[] = tools.map((tool, index) => ({
	117	+ key: `t${index}`,
	118	+ ...(tool.inputSchema !== undefined ? { inputSchema: tool.inputSchema } : {}),
	119	+ ...(tool.outputSchema !== undefined ? { outputSchema: tool.outputSchema } : {}),
	120	+ }));
	121	+
	122	+ const absorb = (result: ReturnType<typeof compileToolChunkTypeScript>) => {
	123	+ for (const [key, value] of result.tools) {
	124	+ compiled.set(key, value);
	125	+ }
	126	+ for (const [name, body] of Object.entries(result.definitions)) {
	127	+ if (!definitionDecls.has(name)) {
	128	+ definitionDecls.set(name, body);
	129	+ }
	130	+ }
	131	+ };
	132	+
	133	+ for (let start = 0; start < entries.length; start += chunkSize) {
	134	+ const chunk = entries.slice(start, start + chunkSize);
	135	+ // oxlint-disable-next-line executor/no-try-catch-or-throw -- boundary: the vendored json-schema-to-typescript compiler throws on invalid schemas; a failing chunk retries tool-by-tool
	136	+ try {
	137	+ absorb(compileToolChunkTypeScript(chunk, defs));
	138	+ } catch {
	139	+ for (const entry of chunk) {
	140	+ // oxlint-disable-next-line executor/no-try-catch-or-throw -- boundary: same throwing compiler; a single broken schema degrades to unknown
	141	+ try {
	142	+ absorb(compileToolChunkTypeScript([entry], defs));
	143	+ } catch {
	144	+ compiled.set(entry.key, {
	145	+ ...(entry.inputSchema !== undefined ? { inputTypeScript: "unknown" } : {}),
	146	+ ...(entry.outputSchema !== undefined ? { outputTypeScript: "unknown" } : {}),
	147	+ });
	148	+ }
	149	+ }
	150	+ }
	151	+ }
	152	+
	153	+ // Name per-tool types after the tool, avoiding collisions with definition
	154	+ // names (which the compiled type bodies reference bare) and each other.
	155	+ const usedNames = new Set<string>(definitionDecls.keys());
	156	+ const declarations: string[] = [];
	157	+ for (const [name, body] of [...definitionDecls.entries()].sort(([a], [b]) =>
	158	+ a.localeCompare(b),
	159	+ )) {
	160	+ declarations.push(` export type ${name} = ${body};`);
	161	+ }
	162	+
	163	+ const compiledTools: CompiledTool[] = tools.map((tool, index) => {
	164	+ const entry = compiled.get(`t${index}`) ?? {};
	165	+ const base = sanitizeIdentifier(tool.name);
	166	+ let inputTypeRef: string \| undefined;
	167	+ if (entry.inputTypeScript !== undefined) {
	168	+ const inputName = uniqueName(`${base}_Input`, usedNames);
	169	+ declarations.push(` export type ${inputName} = ${entry.inputTypeScript};`);
	170	+ inputTypeRef = `${namespaceName}.${inputName}`;
	171	+ }
	172	+ let outputTypeRef = "unknown";
	173	+ if (entry.outputTypeScript !== undefined) {
	174	+ const outputName = uniqueName(`${base}_Output`, usedNames);
	175	+ declarations.push(` export type ${outputName} = ${entry.outputTypeScript};`);
	176	+ outputTypeRef = `${namespaceName}.${outputName}`;
	177	+ }
	178	+ return {
	179	+ pathSegments: toolPathSegments(connection, tool),
	180	+ ...(tool.description !== undefined && tool.description.length > 0
	181	+ ? { description: tool.description }
	182	+ : {}),
	183	+ ...(inputTypeRef !== undefined ? { inputTypeRef } : {}),
	184	+ outputTypeRef,
	185	+ };
	186	+ });
	187	+
	188	+ return { namespaceName, declarations, tools: compiledTools };
	189	+};
	190	+
	191	+// ---------------------------------------------------------------------------
	192	+// Path tree
	193	+// ---------------------------------------------------------------------------
	194	+
	195	+type TreeNode = {
	196	+ children: Map<string, TreeNode>;
	197	+ leaf?: CompiledTool;
	198	+};
	199	+
	200	+const insertTool = (root: TreeNode, tool: CompiledTool): void => {
	201	+ let node = root;
	202	+ for (const segment of tool.pathSegments) {
	203	+ let child = node.children.get(segment);
	204	+ if (!child) {
	205	+ child = { children: new Map() };
	206	+ node.children.set(segment, child);
	207	+ }
	208	+ node = child;
	209	+ }
	210	+ node.leaf = tool;
	211	+};
	212	+
	213	+const leafType = (tool: CompiledTool): string =>
	214	+ tool.inputTypeRef !== undefined
	215	+ ? `ExecutorToolFn<${tool.inputTypeRef}, ${tool.outputTypeRef}>`
	216	+ : `ExecutorToolFnNoInput<${tool.outputTypeRef}>`;
	217	+
	218	+const emitNode = (node: TreeNode, indent: string, out: string[]): void => {
	219	+ // A node can be both callable (a tool lives at this exact path) and a
	220	+ // container (deeper tools share the prefix); emit the intersection.
	221	+ const segments = [...node.children.entries()].sort(([a], [b]) => a.localeCompare(b));
	222	+ if (node.leaf) {
	223	+ out.push(leafType(node.leaf));
	224	+ if (segments.length > 0) {
	225	+ out.push(" & ");
	226	+ }
	227	+ }
	228	+ if (segments.length === 0) {
	229	+ return;
	230	+ }
	231	+ out.push("{\n");
	232	+ for (const [segment, child] of segments) {
	233	+ if (child.leaf?.description !== undefined) {
	234	+ out.push(`${indent} /** ${escapeJsDoc(child.leaf.description)} */\n`);
	235	+ }
	236	+ out.push(`${indent} ${propertyKey(segment)}: `);
	237	+ emitNode(child, `${indent} `, out);
	238	+ out.push(";\n");
	239	+ }
	240	+ out.push(`${indent}}`);
	241	+};
	242	+
	243	+// ---------------------------------------------------------------------------
	244	+// Runtime template: embedded verbatim in the generated file so the output
	245	+// has zero dependencies. Keep this plain TypeScript: no imports, no
	246	+// namespaces with runtime meaning, nothing beyond ES2020 + fetch.
	247	+// ---------------------------------------------------------------------------
	248	+
	249	+const runtimeTemplate = (clientName: string): string => `\
	250	+export type ExecutorToolError = {
	251	+ code: string;
	252	+ message: string;
	253	+ status?: number;
	254	+ details?: unknown;
	255	+ retryable?: boolean;
	256	+};
	257	+export type ExecutorToolHttpMeta = { status: number; headers: { [k: string]: string } };
	258	+export type ExecutorToolOutcome<O> =
	259	+ \| { ok: true; data: O; http?: ExecutorToolHttpMeta }
	260	+ \| { ok: false; error: ExecutorToolError };
	261	+export interface ExecutorCallOptions {
	262	+ /** Approve approval-gated tools as the caller instead of pausing. */
	263	+ autoApprove?: boolean;
	264	+ signal?: AbortSignal;
	265	+}
	266	+export type ExecutorToolFn<I, O> = (
	267	+ input: I,
	268	+ options?: ExecutorCallOptions,
	269	+) => Promise<ExecutorToolOutcome<O>>;
	270	+export type ExecutorToolFnNoInput<O> = (
	271	+ input?: Record<string, unknown>,
	272	+ options?: ExecutorCallOptions,
	273	+) => Promise<ExecutorToolOutcome<O>>;
	274	+
	275	+export interface ExecutorClientOptions {
	276	+ /** Executor server origin, e.g. "http://localhost:4788". */
	277	+ baseUrl?: string;
	278	+ /** Bearer token. Defaults to EXECUTOR_API_KEY / EXECUTOR_AUTH_TOKEN. */
	279	+ token?: string;
	280	+ fetch?: typeof globalThis.fetch;
	281	+ headers?: Record<string, string>;
	282	+ /** Approve approval-gated tools for every call from this client. */
	283	+ autoApprove?: boolean;
	284	+}
	285	+
	286	+export class ExecutorRequestError extends Error {
	287	+ readonly status: number \| null;
	288	+ readonly body: string;
	289	+ constructor(message: string, status: number \| null, body: string) {
	290	+ super(message);
	291	+ this.name = "ExecutorRequestError";
	292	+ this.status = status;
	293	+ this.body = body;
	294	+ }
	295	+}
	296	+
	297	+export class ExecutorPausedError extends Error {
	298	+ readonly executionId: string \| null;
	299	+ readonly approvalUrl: string \| null;
	300	+ constructor(message: string, executionId: string \| null, approvalUrl: string \| null) {
	301	+ super(message);
	302	+ this.name = "ExecutorPausedError";
	303	+ this.executionId = executionId;
	304	+ this.approvalUrl = approvalUrl;
	305	+ }
	306	+}
	307	+
	308	+const DEFAULT_BASE_URL = "http://localhost:4788";
	309	+const PATH_SEGMENT_PATTERN = /^[A-Za-z0-9._-]+$/;
	310	+
	311	+const readEnvToken = (): string \| undefined => {
	312	+ const env = (globalThis as { process?: { env?: Record<string, string \| undefined> } }).process
	313	+ ?.env;
	314	+ return env?.EXECUTOR_API_KEY ?? env?.EXECUTOR_AUTH_TOKEN;
	315	+};
	316	+
	317	+const isRecord = (value: unknown): value is Record<string, unknown> =>
	318	+ typeof value === "object" && value !== null && !Array.isArray(value);
	319	+
	320	+const buildInvokeCode = (segments: readonly string[], input: unknown): string => {
	321	+ const access = segments.map((segment) => \`[\${JSON.stringify(segment)}]\`).join("");
	322	+ return [
	323	+ \`const __args = \${JSON.stringify(input ?? {})};\`,
	324	+ \`const __target = tools\${access};\`,
	325	+ \`if (typeof __target !== "function") {\`,
	326	+ \` throw new Error(\${JSON.stringify(\`Tool not found: \${segments.join(".")}\`)});\`,
	327	+ \`}\`,
	328	+ \`return await __target(__args);\`,
	329	+ ].join("\\n");
	330	+};
	331	+
	332	+export interface ExecutorClientHandle {
	333	+ /** Invoke any tool by dotted path, untyped. */
	334	+ $call: (
	335	+ path: string,
	336	+ input?: Record<string, unknown>,
	337	+ options?: ExecutorCallOptions,
	338	+ ) => Promise<ExecutorToolOutcome<unknown>>;
	339	+}
	340	+
	341	+export type ExecutorClient = ExecutorTools & ExecutorClientHandle;
	342	+
	343	+export function ${clientName}(options: ExecutorClientOptions = {}): ExecutorClient {
	344	+ const origin = (options.baseUrl ?? DEFAULT_BASE_URL).replace(/\\/+$/, "");
	345	+ const token = options.token ?? readEnvToken();
	346	+ const fetchImpl = options.fetch ?? globalThis.fetch;
	347	+
	348	+ const invoke = async (
	349	+ segments: readonly string[],
	350	+ input: unknown,
	351	+ callOptions?: ExecutorCallOptions,
	352	+ ): Promise<ExecutorToolOutcome<unknown>> => {
	353	+ for (const segment of segments) {
	354	+ if (!PATH_SEGMENT_PATTERN.test(segment)) {
	355	+ throw new ExecutorRequestError(\`Invalid tool path segment: \${segment}\`, null, "");
	356	+ }
	357	+ }
	358	+ if (input !== undefined && !isRecord(input)) {
	359	+ throw new ExecutorRequestError("Tool input must be a JSON object", null, "");
	360	+ }
	361	+
	362	+ const autoApprove = callOptions?.autoApprove ?? options.autoApprove;
	363	+ const response = await fetchImpl(\`\${origin}/api/executions\`, {
	364	+ method: "POST",
	365	+ headers: {
	366	+ "content-type": "application/json",
	367	+ ...(token ? { authorization: \`Bearer \${token}\` }: {}),
	368	+ ...options.headers,
	369	+ },
	370	+ body: JSON.stringify({
	371	+ code: buildInvokeCode(segments, input),
	372	+ ...(autoApprove !== undefined ? { autoApprove }: {}),
	373	+ }),
	374	+ ...(callOptions?.signal ? { signal: callOptions.signal }: {}),
	375	+ });
	376	+
	377	+ const bodyText = await response.text();
	378	+ if (!response.ok) {
	379	+ throw new ExecutorRequestError(
	380	+ \`Executor request failed with status \${response.status}\`,
	381	+ response.status,
	382	+ bodyText,
	383	+ );
	384	+ }
	385	+
	386	+ let payload: unknown;
	387	+ try {
	388	+ payload = JSON.parse(bodyText);
	389	+ } catch {
	390	+ throw new ExecutorRequestError("Executor returned a non-JSON response", null, bodyText);
	391	+ }
	392	+ if (!isRecord(payload)) {
	393	+ throw new ExecutorRequestError("Executor returned an unexpected response", null, bodyText);
	394	+ }
	395	+
	396	+ if (payload.status === "paused") {
	397	+ const structured = isRecord(payload.structured) ? payload.structured: {};
	398	+ const executionId =
	399	+ typeof structured.executionId === "string" ? structured.executionId: null;
	400	+ throw new ExecutorPausedError(
	401	+ typeof payload.text === "string" ? payload.text: "Execution paused awaiting approval",
	402	+ executionId,
	403	+ executionId ? \`\${origin}/resume/\${encodeURIComponent(executionId)}\`: null,
	404	+ );
	405	+ }
	406	+
	407	+ if (payload.isError === true) {
	408	+ throw new ExecutorRequestError(
	409	+ typeof payload.text === "string" ? payload.text: "Tool execution failed",
	410	+ null,
	411	+ bodyText,
	412	+ );
	413	+ }
	414	+
	415	+ const structured = isRecord(payload.structured) ? payload.structured: {};
	416	+ const result = structured.result;
	417	+ // Dynamic tool results already arrive in the outcome envelope; anything
	418	+ // else (static tools returning raw values) is wrapped as a success.
	419	+ if (isRecord(result) && typeof result.ok === "boolean") {
	420	+ return result as ExecutorToolOutcome<unknown>;
	421	+ }
	422	+ return { ok: true, data: result };
	423	+ };
	424	+
	425	+ const callByPath = (
	426	+ path: string,
	427	+ input?: Record<string, unknown>,
	428	+ callOptions?: ExecutorCallOptions,
	429	+ ) => invoke(path.split("."), input, callOptions);
	430	+
	431	+ const makeNode = (segments: readonly string[]): unknown =>
	432	+ new Proxy(() => undefined, {
	433	+ get: (_target, property) => {
	434	+ if (segments.length === 0 && property === "$call") {
	435	+ return callByPath;
	436	+ }
	437	+ if (typeof property !== "string") {
	438	+ return undefined;
	439	+ }
	440	+ return makeNode([...segments, property]);
	441	+ },
	442	+ apply: (_target, _thisArg, args: unknown[]) => invoke(segments, args[0], args[1] as ExecutorCallOptions \| undefined),
	443	+ });
	444	+
	445	+ return makeNode([]) as ExecutorClient;
	446	+}
	447	+`;
	448	+
	449	+// ---------------------------------------------------------------------------
	450	+// Entry point
	451	+// ---------------------------------------------------------------------------
	452	+
	453	+export type GeneratedToolProxy = {
	454	+ readonly source: string;
	455	+ readonly toolCount: number;
	456	+ readonly connectionCount: number;
	457	+};
	458	+
	459	+export const generateToolProxySource = (
	460	+ catalog: ToolCatalogExport,
	461	+ options: GenerateToolProxyOptions = {},
	462	+): GeneratedToolProxy => {
	463	+ const chunkSize = Math.max(1, options.chunkSize ?? DEFAULT_CHUNK_SIZE);
	464	+ const clientName = options.clientName ?? "createExecutorClient";
	465	+
	466	+ const connections = [...catalog.connections].sort((a, b) => {
	467	+ const left = `${a.integration}${a.owner}${a.connection}`;
	468	+ const right = `${b.integration}${b.owner}${b.connection}`;
	469	+ return left.localeCompare(right);
	470	+ });
	471	+
	472	+ const usedNamespaces = new Set<string>();
	473	+ const compiledConnections = connections.map((connection) =>
	474	+ compileConnection(
	475	+ connection,
	476	+ uniqueName(
	477	+ `Executor_${sanitizeIdentifier(`${connection.integration}_${connection.owner}_${connection.connection}`)}`,
	478	+ usedNamespaces,
	479	+ ),
	480	+ chunkSize,
	481	+ ),
	482	+ );
	483	+
	484	+ const root: TreeNode = { children: new Map() };
	485	+ let toolCount = 0;
	486	+ for (const connection of compiledConnections) {
	487	+ for (const tool of connection.tools) {
	488	+ insertTool(root, tool);
	489	+ toolCount += 1;
	490	+ }
	491	+ }
	492	+
	493	+ const out: string[] = [];
	494	+ out.push(
	495	+ "// Generated by `executor generate`. Do not edit.\n",
	496	+ "// Regenerate with: executor generate\n",
	497	+ "/* eslint-disable */\n",
	498	+ "\n",
	499	+ );
	500	+ out.push(runtimeTemplate(clientName));
	501	+ out.push("\n");
	502	+
	503	+ for (const connection of compiledConnections) {
	504	+ if (connection.declarations.length === 0) {
	505	+ continue;
	506	+ }
	507	+ // Type-only (non-instantiated) namespace: erased at compile time, so the
	508	+ // generated file stays valid under erasableSyntaxOnly and isolatedModules.
	509	+ out.push(`export namespace ${connection.namespaceName} {\n`);
	510	+ for (const declaration of connection.declarations) {
	511	+ out.push(declaration, "\n");
	512	+ }
	513	+ out.push("}\n\n");
	514	+ }
	515	+
	516	+ out.push("export interface ExecutorTools ");
	517	+ const treeOut: string[] = [];
	518	+ emitNode(root, "", treeOut);
	519	+ // The root is never callable; emitNode on an empty tree emits nothing.
	520	+ out.push(treeOut.length > 0 ? treeOut.join("") : "{\n}");
	521	+ out.push("\n");
	522	+
	523	+ return {
	524	+ source: out.join(""),
	525	+ toolCount,
	526	+ connectionCount: compiledConnections.length,
	527	+ };
	528	+};

packages/core/sdk/src/types.ts+36−1

@@ -7,7 +7,7 @@ · 6 unmodified lines
7	7
8	8	import { Schema } from "effect";
9	9
10		−import { ToolAddress } from "./ids";
	10	+import { ConnectionName, IntegrationSlug, Owner, ToolAddress } from "./ids";
11	11
12	12	// ---------------------------------------------------------------------------
13	13	// ToolSchemaView — the full schema-side view of a tool, returned by
@@ -28,6 +28,41 @@ export const ToolSchemaView = Schema.Struct({ · 14 unmodified lines
28	28	});
29	29	export type ToolSchemaView = typeof ToolSchemaView.Type;
30	30
	31	+// ---------------------------------------------------------------------------
	32	+// ToolCatalogExport — the bulk schema-bearing read returned by
	33	+// `executor.tools.export(filter)`. Unlike `tools.list` (metadata-only, schemas
	34	+// deliberately projected out) this carries every tool's input/output JSON
	35	+// schema plus the connection's shared `$defs`, grouped per connection so a
	36	+// consumer (the `executor generate` typegen) can compile self-contained
	37	+// TypeScript without one `tools.schema` round trip per tool.
	38	+// ---------------------------------------------------------------------------
	39	+
	40	+export const ToolCatalogToolExport = Schema.Struct({
	41	+ address: ToolAddress,
	42	+ name: Schema.String,
	43	+ description: Schema.optional(Schema.String),
	44	+ inputSchema: Schema.optional(Schema.Unknown),
	45	+ outputSchema: Schema.optional(Schema.Unknown),
	46	+ static: Schema.optional(Schema.Boolean),
	47	+});
	48	+export type ToolCatalogToolExport = typeof ToolCatalogToolExport.Type;
	49	+
	50	+export const ToolCatalogConnectionExport = Schema.Struct({
	51	+ owner: Owner,
	52	+ integration: IntegrationSlug,
	53	+ connection: ConnectionName,
	54	+ tools: Schema.Array(ToolCatalogToolExport),
	55	+ /** Shared `$defs` referenced by this connection's tool schemas, already
	56	+ * trimmed to the referenced subset. */
	57	+ definitions: Schema.optional(Schema.Record(Schema.String, Schema.Unknown)),
	58	+});
	59	+export type ToolCatalogConnectionExport = typeof ToolCatalogConnectionExport.Type;
	60	+
	61	+export const ToolCatalogExport = Schema.Struct({
	62	+ connections: Schema.Array(ToolCatalogConnectionExport),
	63	+});
	64	+export type ToolCatalogExport = typeof ToolCatalogExport.Type;
	65	+
31	66	// ---------------------------------------------------------------------------
32	67	// Integration detection — optional capability on `PluginSpec.detect`. When a
33	68	// user pastes a URL in the onboarding UI, `executor.integrations.detect(url)`

packages/plugins/openapi/src/sdk/typegen-scale.test.ts+213−0

@@ -0,0 +1,213 @@
	1	+// ---------------------------------------------------------------------------
	2	+// `executor generate` at catalog scale: 10,000 tools from one OpenAPI spec.
	3	+//
	4	+// The typed-proxy pipeline must hold up on instances with five-digit tool
	5	+// counts, end to end through the REAL ingestion path (addSpec compiles the
	6	+// spec, a connection persists the tool rows) rather than hand-built fixtures:
	7	+// - `tools.export` returns every tool in one read,
	8	+// - `generateToolProxySource` emits the file without blowing time or memory
	9	+// (chunked schema compilation: one whole-catalog compiler pass is
	10	+// super-linear and takes 30s+ at this size),
	11	+// - the generated source is valid strict TypeScript, verified with the real
	12	+// compiler, and spot-checked types resolve to the right shapes.
	13	+//
	14	+// Budgets are deliberately loose (CI machines vary) but tight enough to catch
	15	+// a regression to per-tool or whole-catalog compilation.
	16	+// ---------------------------------------------------------------------------
	17	+
	18	+import { describe, expect, it } from "@effect/vitest";
	19	+import { Effect } from "effect";
	20	+import { FetchHttpClient } from "effect/unstable/http";
	21	+import * as ts from "typescript";
	22	+
	23	+import {
	24	+ AuthTemplateSlug,
	25	+ ConnectionName,
	26	+ IntegrationSlug,
	27	+ createExecutor,
	28	+ generateToolProxySource,
	29	+} from "@executor-js/sdk";
	30	+import { makeTestConfig, memoryCredentialsPlugin } from "@executor-js/sdk/testing";
	31	+
	32	+import { openApiPlugin } from "./plugin";
	33	+
	34	+const TOOL_COUNT = 10_000;
	35	+
	36	+// Build a 10k-operation spec with realistic shape variety: per-operation
	37	+// parameter schemas, shared component refs, and enough distinct field names
	38	+// that deduplication cannot collapse the work.
	39	+const buildScaleSpec = (toolCount: number): string => {
	40	+ const paths: Record<string, unknown> = {};
	41	+ for (let index = 0; index < toolCount; index += 1) {
	42	+ paths[`/resources${index % 100}/r${index}`] = {
	43	+ get: {
	44	+ operationId: `res.op${index}`,
	45	+ summary: `Operation ${index}`,
	46	+ parameters: [
	47	+ { name: "id", in: "query", required: true, schema: { type: "string" } },
	48	+ { name: `filter${index % 250}`, in: "query", schema: { type: "string" } },
	49	+ { name: "limit", in: "query", schema: { type: "integer" } },
	50	+ ],
	51	+ responses: {
	52	+ "200": {
	53	+ description: "ok",
	54	+ content: {
	55	+ "application/json": {
	56	+ schema: {
	57	+ type: "object",
	58	+ properties: {
	59	+ item: { $ref: "#/components/schemas/Item" },
	60	+ page: { $ref: "#/components/schemas/Page" },
	61	+ total: { type: "integer" },
	62	+ },
	63	+ },
	64	+ },
	65	+ },
	66	+ },
	67	+ },
	68	+ },
	69	+ };
	70	+ }
	71	+ // @effect-diagnostics-next-line preferSchemaOverJson:off
	72	+ return JSON.stringify({
	73	+ openapi: "3.0.0",
	74	+ info: { title: "Scale", version: "1.0.0" },
	75	+ servers: [{ url: "https://scale.example.test" }],
	76	+ security: [{ apiKey: [] }],
	77	+ paths,
	78	+ components: {
	79	+ securitySchemes: {
	80	+ apiKey: { type: "apiKey", in: "header", name: "x-api-key" },
	81	+ },
	82	+ schemas: {
	83	+ Item: {
	84	+ type: "object",
	85	+ properties: {
	86	+ id: { type: "string" },
	87	+ name: { type: "string" },
	88	+ tags: { type: "array", items: { type: "string" } },
	89	+ },
	90	+ required: ["id"],
	91	+ },
	92	+ Page: {
	93	+ type: "object",
	94	+ properties: { cursor: { type: "string" }, hasMore: { type: "boolean" } },
	95	+ },
	96	+ },
	97	+ },
	98	+ });
	99	+};
	100	+
	101	+const typecheck = (source: string, extraSource: string): readonly string[] => {
	102	+ const fileName = "generated.ts";
	103	+ const fullSource = `${source}\n${extraSource}`;
	104	+ const options: ts.CompilerOptions = {
	105	+ strict: true,
	106	+ noEmit: true,
	107	+ skipLibCheck: true,
	108	+ target: ts.ScriptTarget.ES2022,
	109	+ module: ts.ModuleKind.ESNext,
	110	+ lib: ["lib.es2022.d.ts", "lib.dom.d.ts"],
	111	+ };
	112	+ const host = ts.createCompilerHost(options);
	113	+ const originalGetSourceFile = host.getSourceFile.bind(host);
	114	+ const originalReadFile = host.readFile.bind(host);
	115	+ const originalFileExists = host.fileExists.bind(host);
	116	+ host.getSourceFile = (candidate, languageVersion, onError, shouldCreateNewSourceFile) =>
	117	+ candidate === fileName
	118	+ ? ts.createSourceFile(candidate, fullSource, languageVersion, true)
	119	+ : originalGetSourceFile(candidate, languageVersion, onError, shouldCreateNewSourceFile);
	120	+ host.readFile = (candidate) =>
	121	+ candidate === fileName ? fullSource : originalReadFile(candidate);
	122	+ host.fileExists = (candidate) => candidate === fileName \|\| originalFileExists(candidate);
	123	+
	124	+ const program = ts.createProgram([fileName], options, host);
	125	+ return ts
	126	+ .getPreEmitDiagnostics(program)
	127	+ .map((diagnostic) => ts.flattenDiagnosticMessageText(diagnostic.messageText, "\n"));
	128	+};
	129	+
	130	+describe("typed proxy generation at 10k-tool scale", () => {
	131	+ it.effect(
	132	+ "exports, generates, and typechecks a 10,000-tool catalog",
	133	+ () =>
	134	+ Effect.scoped(
	135	+ Effect.gen(function* () {
	136	+ const executor = yield* createExecutor(
	137	+ makeTestConfig({
	138	+ plugins: [
	139	+ openApiPlugin({ httpClientLayer: FetchHttpClient.layer }),
	140	+ memoryCredentialsPlugin(),
	141	+ ] as const,
	142	+ }),
	143	+ );
	144	+
	145	+ const added = yield* executor.openapi.addSpec({
	146	+ spec: { kind: "blob", value: buildScaleSpec(TOOL_COUNT) },
	147	+ slug: "scale",
	148	+ });
	149	+ expect(added.toolCount).toBe(TOOL_COUNT);
	150	+
	151	+ yield* executor.connections.create({
	152	+ owner: "org",
	153	+ name: ConnectionName.make("main"),
	154	+ integration: IntegrationSlug.make("scale"),
	155	+ template: AuthTemplateSlug.make("apikey-0"),
	156	+ value: "token",
	157	+ });
	158	+
	159	+ const exportStart = performance.now();
	160	+ const exported = yield* executor.tools.export({
	161	+ integration: IntegrationSlug.make("scale"),
	162	+ });
	163	+ const exportMs = performance.now() - exportStart;
	164	+ const exportedCount = exported.connections.reduce(
	165	+ (sum, connection) => sum + connection.tools.length,
	166	+ 0,
	167	+ );
	168	+ expect(exportedCount).toBe(TOOL_COUNT);
	169	+
	170	+ const generateStart = performance.now();
	171	+ const generated = generateToolProxySource(exported);
	172	+ const generateMs = performance.now() - generateStart;
	173	+ expect(generated.toolCount).toBe(TOOL_COUNT);
	174	+
	175	+ // Regression tripwires, not benchmarks: whole-catalog single-pass
	176	+ // compilation measured 30s+ here and per-tool passes are far worse;
	177	+ // the chunked path runs in well under a second. 15s/30s absorb slow
	178	+ // CI machines while still failing on a complexity regression.
	179	+ expect(exportMs).toBeLessThan(15_000);
	180	+ expect(generateMs).toBeLessThan(30_000);
	181	+
	182	+ // Every tool surfaced in the generated interface exactly once. The
	183	+ // plugin derives paths from the URL (`/resources77/r7777` →
	184	+ // `resources77.resOp7777`), so count the leaf entries.
	185	+ const opMatches = generated.source.match(/resOp\d+: ExecutorToolFn</g) ?? [];
	186	+ expect(opMatches.length).toBe(TOOL_COUNT);
	187	+ // Shared component schemas stay named refs, not 10k inlined copies.
	188	+ expect(generated.source).toContain("export type Item =");
	189	+
	190	+ // The whole 10k-tool file typechecks under strict mode, and a
	191	+ // consumer gets real types out of an arbitrary tool in the middle.
	192	+ const diagnostics = typecheck(
	193	+ generated.source,
	194	+ `
	195	+ const client = createExecutorClient();
	196	+ async function main() {
	197	+ const outcome = await client.scale.org.main.resources77.resOp7777({ id: "x" });
	198	+ if (outcome.ok) {
	199	+ const total: number \| undefined = outcome.data.total;
	200	+ const itemId: string \| undefined = outcome.data.item?.id;
	201	+ void total;
	202	+ void itemId;
	203	+ }
	204	+ }
	205	+ void main;
	206	+ `,
	207	+ );
	208	+ expect(diagnostics).toEqual([]);
	209	+ }),
	210	+ ),
	211	+ { timeout: 180_000 },
	212	+ );
	213	+});