History

Improve OpenAPI tool hint display and remove fast-mode prep

Use schema-derived argument preview keys for compact /tools signatures, carry preview metadata through descriptors/discover, and always run full OpenAPI preparation so large sources like Cloudflare stay fully typed.

Rhys Sullivan committed 9 months ago

Commit 5abc0a81 parent d42191d8 files · +190 −77Browse files

Unified · Split

8 files
  1. executor/apps/web/src/lib/types.ts +1 −0
  2. executor/convex/executorNode.ts +2 −3
  3. executor/lib/openapi-cache.test.ts +108 −10
  4. executor/lib/tool_discovery.ts +5 −1
  5. executor/lib/tool_sources.ts +45 −59
  6. executor/lib/type_hints.test.ts +6 −0
  7. executor/lib/type_hints.ts +19 −4
  8. executor/lib/types.ts +4 −0
executor/apps/web/src/lib/types.ts+1−0
@@ -99,6 +99,7 @@ export interface ToolDescriptor { · 98 unmodified lines
9999 returnsType?: string;
100100 strictArgsType?: string;
101101 strictReturnsType?: string;
102+ argPreviewKeys?: string[];
102103 operationId?: string;
103104 }
104105
executor/convex/executorNode.ts+2−3
@@ -338,9 +338,7 @@ async function loadCachedOpenApiSpec( · 337 unmodified lines
338338 }
339339
340340 // 2. Cache miss — prepare from scratch
341− const prepared = await prepareOpenApiSpec(specUrl, sourceName, {
342− mode: "fast",
343− });
341+ const prepared = await prepareOpenApiSpec(specUrl, sourceName);
344342
345343 // Store in file storage + metadata table (best-effort, don't block on failure)
346344 try {
@@ -584,6 +582,7 @@ function toToolDescriptor(tool: ToolDefinition, approval: "auto" | "required"): · 237 unmodified lines
584582 returnsType: tool.metadata?.displayReturnsType ?? tool.metadata?.returnsType,
585583 strictArgsType: tool.metadata?.argsType,
586584 strictReturnsType: tool.metadata?.returnsType,
585+ argPreviewKeys: tool.metadata?.argPreviewKeys,
587586 operationId: tool.metadata?.operationId,
588587 // Note: sourceDts is NOT included — it's too large to send over the wire.
589588 // Monaco fetches .d.ts blobs separately via Convex storage URLs.
executor/lib/openapi-cache.test.ts+108−10
@@ -168,23 +168,20 @@ describe("prepareOpenApiSpec with large specs", () => { · 167 unmodified lines
168168 expect(json.length).toBeGreaterThan(100_000);
169169 });
170170
171− test("fast mode skips bundle and d.ts generation for very large specs", async () => {
171+ test("very large specs still generate d.ts and typed tools", async () => {
172172 const spec = makeLargeSpec(250); // 750 operations
173173 const start = performance.now();
174− const prepared = await prepareOpenApiSpec(spec, "large-fast", {
175− mode: "fast",
176− fastPathOperationThreshold: 600,
177− });
174+ const prepared = await prepareOpenApiSpec(spec, "large-full");
178175 const elapsed = performance.now() - start;
179176
180177 expect(Object.keys(prepared.paths)).toHaveLength(250);
181− expect(prepared.dts).toBeUndefined();
182− expect(prepared.warnings.some((warning: string) => warning.includes("fast mode"))).toBe(true);
178+ expect(prepared.dts).toBeDefined();
179+ expect(prepared.warnings.some((warning: string) => warning.includes("fast mode"))).toBe(false);
183180
184181 const tools = buildOpenApiToolsFromPrepared(
185182 {
186183 type: "openapi",
187− name: "large-fast",
184+ name: "large-full",
188185 spec,
189186 baseUrl: "https://api.example.com/v1",
190187 },
@@ -194,8 +191,8 @@ describe("prepareOpenApiSpec with large specs", () => { · 3 unmodified lines
194191 expect(tools.length).toBeGreaterThan(700);
195192 const deleteTool = tools.find((tool) => tool.metadata?.operationId?.startsWith("delete_"));
196193 expect(deleteTool).toBeDefined();
197− expect(deleteTool?.metadata?.returnsType).toBe("void");
198− expect(elapsed).toBeLessThan(5_000);
194+ expect(deleteTool?.metadata?.displayReturnsType).toContain("ToolOutput<operations[");
195+ expect(elapsed).toBeLessThan(20_000);
199196 });
200197
201198
@@ -246,6 +243,107 @@ describe("buildOpenApiToolsFromPrepared", () => { · 44 unmodified lines
246243 }
247244 });
248245
246+ test("uses compact preview args for large nested request bodies", async () => {
247+ const spec = {
248+ openapi: "3.0.3",
249+ info: { title: "Notion-like", version: "1.0.0" },
250+ servers: [{ url: "https://api.notion.example" }],
251+ paths: {
252+ "/databases": {
253+ post: {
254+ operationId: "createDatabase",
255+ tags: ["databases"],
256+ summary: "Create a database",
257+ requestBody: {
258+ required: true,
259+ content: {
260+ "application/json": {
261+ schema: {
262+ type: "object",
263+ properties: {
264+ parent: {
265+ type: "object",
266+ properties: {
267+ page_id: { type: "string" },
268+ },
269+ },
270+ title: {
271+ type: "array",
272+ items: {
273+ type: "object",
274+ properties: {
275+ text: {
276+ type: "object",
277+ properties: {
278+ content: { type: "string" },
279+ },
280+ },
281+ },
282+ },
283+ },
284+ properties: {
285+ type: "object",
286+ additionalProperties: true,
287+ },
288+ icon: {
289+ type: "object",
290+ properties: { emoji: { type: "string" } },
291+ },
292+ cover: {
293+ type: "object",
294+ properties: {
295+ external: {
296+ type: "object",
297+ properties: { url: { type: "string" } },
298+ },
299+ },
300+ },
301+ },
302+ required: ["parent", "title", "properties"],
303+ },
304+ },
305+ },
306+ },
307+ responses: {
308+ "200": {
309+ description: "ok",
310+ content: {
311+ "application/json": {
312+ schema: {
313+ type: "object",
314+ properties: {
315+ id: { type: "string" },
316+ },
317+ },
318+ },
319+ },
320+ },
321+ },
322+ },
323+ },
324+ },
325+ };
326+
327+ const prepared = await prepareOpenApiSpec(spec, "notion-like");
328+ const tools = buildOpenApiToolsFromPrepared(
329+ { type: "openapi", name: "notion", spec, baseUrl: "https://api.notion.example" },
330+ prepared,
331+ );
332+
333+ const createDbTool = tools.find((tool) => tool.metadata?.operationId === "createDatabase");
334+ expect(createDbTool).toBeDefined();
335+ expect(createDbTool?.metadata?.argPreviewKeys).toEqual([
336+ "parent",
337+ "title",
338+ "properties",
339+ "icon",
340+ "cover",
341+ ]);
342+ expect(createDbTool?.metadata?.displayArgsType).toBe(
343+ "{ parent: ...; title: ...; properties: ...; icon: ...; cover: ... }",
344+ );
345+ });
346+
249347 test("resolves shared parameter refs and maps 204 responses to void", async () => {
250348 const spec: Record<string, unknown> = {
251349 openapi: "3.0.3",
executor/lib/tool_discovery.ts+5−1
@@ -14,6 +14,7 @@ interface DiscoverIndexEntry { · 13 unmodified lines
1414 source: string;
1515 argsType: string;
1616 returnsType: string;
17+ argPreviewKeys: string[];
1718 searchText: string;
1819 normalizedPath: string;
1920 normalizedSearchText: string;
@@ -71,7 +72,7 @@ function buildExampleCall(entry: DiscoverIndexEntry): string { · 51 unmodified lines
7172 return `await tools.${entry.path}({});`;
7273 }
7374
74− const keys = extractTopLevelTypeKeys(entry.argsType);
75+ const keys = entry.argPreviewKeys.length > 0 ? entry.argPreviewKeys : extractTopLevelTypeKeys(entry.argsType);
7576 if (keys.length > 0) {
7677 const argsSnippet = keys.slice(0, 3)
7778 .map((key) => `${key}: ${key.toLowerCase().includes("input") ? "{ /* ... */ }" : "..."}`)
@@ -97,6 +98,9 @@ function buildIndex(tools: ToolDefinition[]): DiscoverIndexEntry[] { · 19 unmodified lines
9798 source: tool.source ?? "local",
9899 argsType: normalizeType(tool.metadata?.argsType),
99100 returnsType: normalizeType(tool.metadata?.returnsType),
101+ argPreviewKeys: Array.isArray(tool.metadata?.argPreviewKeys)
102+ ? tool.metadata.argPreviewKeys.filter((value): value is string => typeof value === "string")
103+ : [],
100104 searchText,
101105 normalizedPath: normalizeSearchToken(tool.path),
102106 normalizedSearchText: normalizeSearchToken(searchText),
executor/lib/tool_sources.ts+45−59
@@ -6,7 +6,7 @@ import { SSEClientTransport } from "@modelcontextprotocol/sdk/client/sse.js"; · 5 unmodified lines
66 import { StreamableHTTPClientTransport } from "@modelcontextprotocol/sdk/client/streamableHttp.js";
77 import openapiTS, { astToString } from "openapi-typescript";
88 import { Kind, parse, type FragmentDefinitionNode, type SelectionSetNode } from "graphql";
9−import { compactArgTypeHint, compactReturnTypeHint } from "./type_hints";
9+import { compactArgKeysHint, compactArgTypeHint, compactReturnTypeHint } from "./type_hints";
1010 import type { ToolApprovalMode, ToolCredentialSpec, ToolDefinition, ToolTypeMetadata } from "./types";
1111 import { asRecord } from "./utils";
1212
@@ -572,6 +572,7 @@ async function loadMcpTools(config: McpToolSourceConfig): Promise<ToolDefinition · 559 unmodified lines
572572 return tools.map((tool) => {
573573 const toolName = String(tool.name ?? "tool");
574574 const inputSchema = asRecord(tool.inputSchema);
575+ const argPreviewKeys = Object.keys(asRecord(inputSchema.properties)).filter((key) => key.length > 0);
575576 const argsType = jsonSchemaTypeHintFallback(inputSchema);
576577 const returnsType = "unknown";
577578 return {
@@ -584,6 +585,7 @@ async function loadMcpTools(config: McpToolSourceConfig): Promise<ToolDefinition · 6 unmodified lines
584585 returnsType,
585586 displayArgsType: compactArgTypeHint(argsType),
586587 displayReturnsType: compactReturnTypeHint(returnsType),
588+ ...(argPreviewKeys.length > 0 ? { argPreviewKeys } : {}),
587589 },
588590 _runSpec: {
589591 kind: "mcp" as const,
@@ -693,7 +695,6 @@ function compactOpenApiPaths( · 103 unmodified lines
693695 componentSchemas?: Record<string, unknown>,
694696 componentResponses?: Record<string, unknown>,
695697 componentRequestBodies?: Record<string, unknown>,
696− typeHintMode: "full" | "fast" = "full",
697698 ): Record<string, unknown> {
698699 const paths = asRecord(pathsValue);
699700 const methods = ["get", "post", "put", "delete", "patch", "head", "options"] as const;
@@ -778,9 +779,6 @@ function compactOpenApiPaths( · 78 unmodified lines
778779 for (const [status, responseValue] of Object.entries(responses)) {
779780 if (!status.startsWith("2")) continue;
780781 responseStatus = status;
781− if (typeHintMode === "fast" && !hasGeneratedTypes) {
782− break;
783− }
784782 const resolvedResponse = resolveResponseRef(asRecord(responseValue), compResponses);
785783 responseSchema = resolveSchemaRef(
786784 getPreferredResponseSchema(resolvedResponse),
@@ -816,11 +814,14 @@ function compactOpenApiPaths( · 29 unmodified lines
816814 ? jsonSchemaTypeHintFallback(combinedSchema, 0, compSchemas)
817815 : "{}";
818816 compactOperation._returnsTypeHint = responseTypeHintFromSchema(responseSchema, responseStatus, compSchemas);
819− } else if (typeHintMode === "fast") {
820− const mergedParameters = operationParameters.concat(sharedParameters);
821− const hasInputSchema = mergedParameters.length > 0 || Object.keys(rawRequestBodySchema).length > 0;
822− compactOperation._argsTypeHint = hasInputSchema ? "Record<string, unknown>" : "{}";
823− compactOperation._returnsTypeHint = responseStatus === "204" ? "void" : "unknown";
817+ compactOperation._usesGeneratedTypes = true;
818+ const previewKeys = [
819+ ...mergedParameters.map((param) => String(param.name ?? "")).filter((name) => name.length > 0),
820+ ...Object.keys(asRecord(requestBodySchema.properties)),
821+ ];
822+ if (previewKeys.length > 0) {
823+ compactOperation._argPreviewKeys = [...new Set(previewKeys)];
824+ }
824825 } else {
825826 // Keep full schemas for the fallback path
826827 if (Object.keys(requestBodySchema).length > 0) {
@@ -868,34 +869,9 @@ export interface PreparedOpenApiSpec { · 41 unmodified lines
868869 warnings: string[];
869870 }
870871
871−export interface PrepareOpenApiSpecOptions {
872− mode?: "full" | "fast";
873− fastPathOperationThreshold?: number;
874−}
875−
876−const DEFAULT_FAST_PATH_OPERATION_THRESHOLD = 600;
877−
878−function countOpenApiOperations(pathsValue: unknown): number {
879− const paths = asRecord(pathsValue);
880− const methods = new Set(["get", "post", "put", "delete", "patch", "head", "options"]);
881− let count = 0;
882−
883− for (const pathValue of Object.values(paths)) {
884− const pathObject = asRecord(pathValue);
885− for (const key of Object.keys(pathObject)) {
886− if (methods.has(key)) {
887− count += 1;
888− }
889− }
890− }
891−
892− return count;
893−}
894−
895872 export async function prepareOpenApiSpec(
896873 spec: string | Record<string, unknown>,
897874 sourceName = "openapi",
898− options?: PrepareOpenApiSpecOptions,
899875 ): Promise<PreparedOpenApiSpec> {
900876 const parser = SwaggerParser as unknown as {
901877 bundle(spec: unknown): Promise<unknown>;
@@ -920,32 +896,19 @@ export async function prepareOpenApiSpec( · 18 unmodified lines
920896 parsed = spec;
921897 }
922898
923− const operationCount = countOpenApiOperations(parsed.paths);
924− const mode = options?.mode ?? "full";
925− const fastPathOperationThreshold =
926− options?.fastPathOperationThreshold ?? DEFAULT_FAST_PATH_OPERATION_THRESHOLD;
927− const useFastPath = mode === "fast" && operationCount >= fastPathOperationThreshold;
928−
929899 // ── Step 2: Generate .d.ts and bundle in parallel ─────────────────────
930900 // Both now operate on the in-memory object — no additional HTTP fetches.
931− let dts: string | null = null;
901+ let dts: string | null;
932902 let bundled: Record<string, unknown>;
933− if (useFastPath) {
903+ const dtsPromise = generateOpenApiDts(parsed);
904+ try {
905+ bundled = (await parser.bundle(parsed)) as Record<string, unknown>;
906+ } catch (bundleError) {
907+ const bundleMessage = bundleError instanceof Error ? bundleError.message : String(bundleError);
908+ warnings.push(`OpenAPI bundle failed for '${sourceName}', using parse-only mode: ${bundleMessage}`);
934909 bundled = parsed;
935− warnings.push(
936− `OpenAPI source '${sourceName}' loaded in fast mode (${operationCount} operations): skipped bundle + d.ts generation for latency`,
937− );
938− } else {
939− const dtsPromise = generateOpenApiDts(parsed);
940− try {
941− bundled = (await parser.bundle(parsed)) as Record<string, unknown>;
942− } catch (bundleError) {
943− const bundleMessage = bundleError instanceof Error ? bundleError.message : String(bundleError);
944− warnings.push(`OpenAPI bundle failed for '${sourceName}', using parse-only mode: ${bundleMessage}`);
945− bundled = parsed;
946− }
947− dts = await dtsPromise;
948910 }
911+ dts = await dtsPromise;
949912
950913 // ── Step 3: Extract operation IDs for compaction ───────────────────────
951914 // If we have a .d.ts, we know which operations have full generated types.
@@ -966,7 +929,6 @@ export async function prepareOpenApiSpec( · 14 unmodified lines
966929 asRecord(asRecord(bundled.components).schemas),
967930 asRecord(asRecord(bundled.components).responses),
968931 asRecord(asRecord(bundled.components).requestBodies),
969− useFastPath ? "fast" : "full",
970932 ),
971933 dts: dts ?? undefined,
972934 warnings,
@@ -1028,6 +990,9 @@ export function buildOpenApiToolsFromPrepared( · 55 unmodified lines
1028990 // Pre-computed during compaction when .d.ts types exist (schemas stripped to save space).
1029991 let argsType: string;
1030992 let returnsType: string;
993+ let argPreviewKeys: string[] = Array.isArray(operation._argPreviewKeys)
994+ ? operation._argPreviewKeys.filter((value): value is string => typeof value === "string")
995+ : [];
1031996 if (typeof operation._argsTypeHint === "string" && typeof operation._returnsTypeHint === "string") {
1032997 argsType = operation._argsTypeHint as string;
1033998 returnsType = operation._returnsTypeHint as string;
@@ -1064,8 +1029,25 @@ export function buildOpenApiToolsFromPrepared( · 30 unmodified lines
10641029
10651030 argsType = hasInputSchema ? jsonSchemaTypeHintFallback(combinedSchema) : "{}";
10661031 returnsType = responseTypeHintFromSchema(responseSchema, responseStatus);
1032+ if (argPreviewKeys.length === 0) {
1033+ argPreviewKeys = [
1034+ ...parameters.map((param) => param.name),
1035+ ...Object.keys(asRecord(requestBodySchema.properties)),
1036+ ].filter((name, index, all) => name.length > 0 && all.indexOf(name) === index);
1037+ }
10671038 }
10681039
1040+ const usesGeneratedTypes = Boolean(operation._usesGeneratedTypes);
1041+ const operationIdKey = JSON.stringify(operationIdRaw);
1042+ const displayArgsType = argPreviewKeys.length > 0
1043+ ? compactArgKeysHint(argPreviewKeys)
1044+ : usesGeneratedTypes
1045+ ? `ToolInput<operations[${operationIdKey}]>`
1046+ : compactArgTypeHint(argsType);
1047+ const displayReturnsType = usesGeneratedTypes
1048+ ? `ToolOutput<operations[${operationIdKey}]>`
1049+ : compactReturnTypeHint(returnsType);
1050+
10691051 const approval = config.overrides?.[operationIdRaw]?.approval
10701052 ?? (readMethods.has(method)
10711053 ? config.defaultReadApproval ?? "auto"
@@ -1088,8 +1070,9 @@ export function buildOpenApiToolsFromPrepared( · 16 unmodified lines
10881070 metadata: {
10891071 argsType,
10901072 returnsType,
1091− displayArgsType: compactArgTypeHint(argsType),
1092− displayReturnsType: compactReturnTypeHint(returnsType),
1073+ displayArgsType,
1074+ displayReturnsType,
1075+ ...(argPreviewKeys.length > 0 ? { argPreviewKeys } : {}),
10931076 operationId: operationIdRaw,
10941077 // Only attach sourceDts to the first tool to avoid duplicating the full .d.ts
10951078 ...(sourceDts && !sourceDtsEmitted ? { sourceDts } : {}),
@@ -1617,6 +1600,7 @@ async function loadGraphqlTools(config: GraphqlToolSourceConfig): Promise<ToolDe · 521 unmodified lines
16171600 returnsType: "{ data: unknown; errors: unknown[] }",
16181601 displayArgsType: "{ query: string; variables?: ... }",
16191602 displayReturnsType: "{ data: ...; errors: unknown[] }",
1603+ argPreviewKeys: ["query", "variables"],
16201604 },
16211605 credential: credentialSpec,
16221606 // Tag as graphql source so invokeTool knows to do dynamic path extraction
@@ -1681,6 +1665,7 @@ async function loadGraphqlTools(config: GraphqlToolSourceConfig): Promise<ToolDe · 58 unmodified lines
16811665 const approval = config.overrides?.[field.name]?.approval ?? defaultApproval;
16821666 const argsType = gqlFieldArgsTypeHint(field.args, typeMap);
16831667 const returnsType = `{ data: ${gqlTypeToHint(field.type, typeMap)}; errors: unknown[] }`;
1668+ const argPreviewKeys = field.args.map((arg) => arg.name).filter((name) => name.length > 0);
16841669
16851670 // Build the example query for the description
16861671 const exampleQuery = buildFieldQuery(operationType, field.name, field.args, field.type, typeMap);
@@ -1701,6 +1686,7 @@ async function loadGraphqlTools(config: GraphqlToolSourceConfig): Promise<ToolDe · 14 unmodified lines
17011686 returnsType,
17021687 displayArgsType: compactArgTypeHint(argsType),
17031688 displayReturnsType: "{ data: ...; errors: unknown[] }",
1689+ ...(argPreviewKeys.length > 0 ? { argPreviewKeys } : {}),
17041690 },
17051691 _runSpec: {
17061692 kind: "graphql_field" as const,
executor/lib/type_hints.test.ts+6−0
@@ -1,5 +1,6 @@
11 import { expect, test } from "bun:test";
22 import {
3+ compactArgKeysHint,
34 compactArgTypeHint,
45 compactDescriptionLine,
56 compactReturnTypeHint,
@@ -18,6 +19,11 @@ test("compactArgTypeHint keeps only top-level previews", async () => { · 12 unmodified lines
1819 expect(compact).toBe("{ parent: ...; title: ...; icon: ...; properties: ... }");
1920 });
2021
22+test("compactArgKeysHint caps key list with ellipsis", async () => {
23+ const compact = compactArgKeysHint(["a", "b", "c", "d", "e", "f", "g", "h"]);
24+ expect(compact).toBe("{ a: ...; b: ...; c: ...; d: ...; e: ...; f: ...; ... }");
25+});
26+
2127 test("compactReturnTypeHint collapses graphql envelopes", async () => {
2228 const compact = compactReturnTypeHint("{ data: { issue: { id: string; title: string } }; errors: unknown[] }");
2329 expect(compact).toBe("{ data: ...; errors: unknown[] }");
executor/lib/type_hints.ts+19−4
@@ -50,14 +50,29 @@ export function extractTopLevelTypeKeys(typeHint: string): string[] { · 49 unmodified lines
5050 return keys;
5151 }
5252
53+export function compactArgKeysHint(keys: string[]): string {
54+ const normalized = keys
55+ .map((key) => key.trim())
56+ .filter((key) => key.length > 0);
57+
58+ if (normalized.length === 0) return "{}";
59+
60+ const unique: string[] = [];
61+ for (const key of normalized) {
62+ if (!unique.includes(key)) unique.push(key);
63+ }
64+
65+ const maxKeys = 6;
66+ const shown = unique.slice(0, maxKeys).map((key) => `${key}: ...`);
67+ const suffix = unique.length > maxKeys ? "; ..." : "";
68+ return `{ ${shown.join("; ")}${suffix} }`;
69+}
70+
5371 export function compactArgTypeHint(argsType: string): string {
5472 if (argsType === "{}") return "{}";
5573 const keys = extractTopLevelTypeKeys(argsType);
5674 if (keys.length > 0) {
57− const maxKeys = 4;
58− const shown = keys.slice(0, maxKeys).map((key) => `${key}: ...`);
59− const suffix = keys.length > maxKeys ? "; ..." : "";
60− return `{ ${shown.join("; ")}${suffix} }`;
75+ return compactArgKeysHint(keys);
6176 }
6277 return truncateInline(argsType, 120);
6378 }
executor/lib/types.ts+4−0
@@ -103,6 +103,8 @@ export interface ToolDescriptor { · 102 unmodified lines
103103 strictArgsType?: string;
104104 /** Full-fidelity return type hint for strict editor/typecheck usage. */
105105 strictReturnsType?: string;
106+ /** Top-level argument key preview for examples/UI. */
107+ argPreviewKeys?: string[];
106108 operationId?: string;
107109 }
108110
@@ -225,6 +227,8 @@ export interface ToolTypeMetadata { · 116 unmodified lines
225227 displayArgsType?: string;
226228 /** Compact return hint intended for UI/discovery display. */
227229 displayReturnsType?: string;
230+ /** Top-level argument keys for examples/UI without parsing type strings. */
231+ argPreviewKeys?: string[];
228232 /** Raw operationId from the OpenAPI spec (used to generate typechecker wrapper). */
229233 operationId?: string;
230234 /**