Split strict typechecking hints from compact display signatures, add shared .d.ts caching, and use GraphQL AST parsing so tool UX stays clean while policy/typecheck paths remain accurate.
Unified · Split
| @@ -58,9 +58,13 @@ function emitToolMethod(tool: ToolDescriptor, dtsSources: Set<string>): string { · 57 unmodified lines | |||
| 58 | } | 58 | } |
| 59 | 59 | ||
| 60 | // Fallback for MCP, GraphQL, builtins — use argsType/returnsType strings | 60 | // Fallback for MCP, GraphQL, builtins — use argsType/returnsType strings |
| 61 | const hasArgsType = Boolean(tool.argsType?.trim()); | 61 | const strictArgsType = tool.strictArgsType?.trim(); |
| 62 | const argsType = hasArgsType ? tool.argsType!.trim() : "Record<string, unknown>"; | 62 | const strictReturnsType = tool.strictReturnsType?.trim(); |
| 63 | const returnsType = tool.returnsType?.trim() || "unknown"; | 63 | const fallbackArgsType = tool.argsType?.trim(); |
| 64 | const fallbackReturnsType = tool.returnsType?.trim(); | ||
| 65 | const hasArgsType = Boolean(strictArgsType || fallbackArgsType); | ||
| 66 | const argsType = strictArgsType || fallbackArgsType || "Record<string, unknown>"; | ||
| 67 | const returnsType = strictReturnsType || fallbackReturnsType || "unknown"; | ||
| 64 | const inputParam = !hasArgsType || argsType === "{}" | 68 | const inputParam = !hasArgsType || argsType === "{}" |
| 65 | ? `input?: ${argsType}` | 69 | ? `input?: ${argsType}` |
| 66 | : `input: ${argsType}`; | 70 | : `input: ${argsType}`; |
| @@ -97,6 +97,8 @@ export interface ToolDescriptor { · 96 unmodified lines | |||
| 97 | source?: string; | 97 | source?: string; |
| 98 | argsType?: string; | 98 | argsType?: string; |
| 99 | returnsType?: string; | 99 | returnsType?: string; |
| 100 | strictArgsType?: string; | ||
| 101 | strictReturnsType?: string; | ||
| 100 | operationId?: string; | 102 | operationId?: string; |
| 101 | } | 103 | } |
| 102 | 104 | ||
| @@ -0,0 +1,74 @@ | |||
| 1 | { | ||
| 2 | "lockfileVersion": 1, | ||
| 3 | "configVersion": 1, | ||
| 4 | "workspaces": { | ||
| 5 | "": { | ||
| 6 | "name": "executor", | ||
| 7 | "dependencies": { | ||
| 8 | "convex": "^1.31.7", | ||
| 9 | "graphql": "^16.12.0", | ||
| 10 | }, | ||
| 11 | }, | ||
| 12 | }, | ||
| 13 | "packages": { | ||
| 14 | "@esbuild/aix-ppc64": ["@esbuild/aix-ppc64@0.27.0", "", { "os": "aix", "cpu": "ppc64" }, "sha512-KuZrd2hRjz01y5JK9mEBSD3Vj3mbCvemhT466rSuJYeE/hjuBrHfjjcjMdTm/sz7au+++sdbJZJmuBwQLuw68A=="], | ||
| 15 | |||
| 16 | "@esbuild/android-arm": ["@esbuild/android-arm@0.27.0", "", { "os": "android", "cpu": "arm" }, "sha512-j67aezrPNYWJEOHUNLPj9maeJte7uSMM6gMoxfPC9hOg8N02JuQi/T7ewumf4tNvJadFkvLZMlAq73b9uwdMyQ=="], | ||
| 17 | |||
| 18 | "@esbuild/android-arm64": ["@esbuild/android-arm64@0.27.0", "", { "os": "android", "cpu": "arm64" }, "sha512-CC3vt4+1xZrs97/PKDkl0yN7w8edvU2vZvAFGD16n9F0Cvniy5qvzRXjfO1l94efczkkQE6g1x0i73Qf5uthOQ=="], | ||
| 19 | |||
| 20 | "@esbuild/android-x64": ["@esbuild/android-x64@0.27.0", "", { "os": "android", "cpu": "x64" }, "sha512-wurMkF1nmQajBO1+0CJmcN17U4BP6GqNSROP8t0X/Jiw2ltYGLHpEksp9MpoBqkrFR3kv2/te6Sha26k3+yZ9Q=="], | ||
| 21 | |||
| 22 | "@esbuild/darwin-arm64": ["@esbuild/darwin-arm64@0.27.0", "", { "os": "darwin", "cpu": "arm64" }, "sha512-uJOQKYCcHhg07DL7i8MzjvS2LaP7W7Pn/7uA0B5S1EnqAirJtbyw4yC5jQ5qcFjHK9l6o/MX9QisBg12kNkdHg=="], | ||
| 23 | |||
| 24 | "@esbuild/darwin-x64": ["@esbuild/darwin-x64@0.27.0", "", { "os": "darwin", "cpu": "x64" }, "sha512-8mG6arH3yB/4ZXiEnXof5MK72dE6zM9cDvUcPtxhUZsDjESl9JipZYW60C3JGreKCEP+p8P/72r69m4AZGJd5g=="], | ||
| 25 | |||
| 26 | "@esbuild/freebsd-arm64": ["@esbuild/freebsd-arm64@0.27.0", "", { "os": "freebsd", "cpu": "arm64" }, "sha512-9FHtyO988CwNMMOE3YIeci+UV+x5Zy8fI2qHNpsEtSF83YPBmE8UWmfYAQg6Ux7Gsmd4FejZqnEUZCMGaNQHQw=="], | ||
| 27 | |||
| 28 | "@esbuild/freebsd-x64": ["@esbuild/freebsd-x64@0.27.0", "", { "os": "freebsd", "cpu": "x64" }, "sha512-zCMeMXI4HS/tXvJz8vWGexpZj2YVtRAihHLk1imZj4efx1BQzN76YFeKqlDr3bUWI26wHwLWPd3rwh6pe4EV7g=="], | ||
| 29 | |||
| 30 | "@esbuild/linux-arm": ["@esbuild/linux-arm@0.27.0", "", { "os": "linux", "cpu": "arm" }, "sha512-t76XLQDpxgmq2cNXKTVEB7O7YMb42atj2Re2Haf45HkaUpjM2J0UuJZDuaGbPbamzZ7bawyGFUkodL+zcE+jvQ=="], | ||
| 31 | |||
| 32 | "@esbuild/linux-arm64": ["@esbuild/linux-arm64@0.27.0", "", { "os": "linux", "cpu": "arm64" }, "sha512-AS18v0V+vZiLJyi/4LphvBE+OIX682Pu7ZYNsdUHyUKSoRwdnOsMf6FDekwoAFKej14WAkOef3zAORJgAtXnlQ=="], | ||
| 33 | |||
| 34 | "@esbuild/linux-ia32": ["@esbuild/linux-ia32@0.27.0", "", { "os": "linux", "cpu": "ia32" }, "sha512-Mz1jxqm/kfgKkc/KLHC5qIujMvnnarD9ra1cEcrs7qshTUSksPihGrWHVG5+osAIQ68577Zpww7SGapmzSt4Nw=="], | ||
| 35 | |||
| 36 | "@esbuild/linux-loong64": ["@esbuild/linux-loong64@0.27.0", "", { "os": "linux", "cpu": "none" }, "sha512-QbEREjdJeIreIAbdG2hLU1yXm1uu+LTdzoq1KCo4G4pFOLlvIspBm36QrQOar9LFduavoWX2msNFAAAY9j4BDg=="], | ||
| 37 | |||
| 38 | "@esbuild/linux-mips64el": ["@esbuild/linux-mips64el@0.27.0", "", { "os": "linux", "cpu": "none" }, "sha512-sJz3zRNe4tO2wxvDpH/HYJilb6+2YJxo/ZNbVdtFiKDufzWq4JmKAiHy9iGoLjAV7r/W32VgaHGkk35cUXlNOg=="], | ||
| 39 | |||
| 40 | "@esbuild/linux-ppc64": ["@esbuild/linux-ppc64@0.27.0", "", { "os": "linux", "cpu": "ppc64" }, "sha512-z9N10FBD0DCS2dmSABDBb5TLAyF1/ydVb+N4pi88T45efQ/w4ohr/F/QYCkxDPnkhkp6AIpIcQKQ8F0ANoA2JA=="], | ||
| 41 | |||
| 42 | "@esbuild/linux-riscv64": ["@esbuild/linux-riscv64@0.27.0", "", { "os": "linux", "cpu": "none" }, "sha512-pQdyAIZ0BWIC5GyvVFn5awDiO14TkT/19FTmFcPdDec94KJ1uZcmFs21Fo8auMXzD4Tt+diXu1LW1gHus9fhFQ=="], | ||
| 43 | |||
| 44 | "@esbuild/linux-s390x": ["@esbuild/linux-s390x@0.27.0", "", { "os": "linux", "cpu": "s390x" }, "sha512-hPlRWR4eIDDEci953RI1BLZitgi5uqcsjKMxwYfmi4LcwyWo2IcRP+lThVnKjNtk90pLS8nKdroXYOqW+QQH+w=="], | ||
| 45 | |||
| 46 | "@esbuild/linux-x64": ["@esbuild/linux-x64@0.27.0", "", { "os": "linux", "cpu": "x64" }, "sha512-1hBWx4OUJE2cab++aVZ7pObD6s+DK4mPGpemtnAORBvb5l/g5xFGk0vc0PjSkrDs0XaXj9yyob3d14XqvnQ4gw=="], | ||
| 47 | |||
| 48 | "@esbuild/netbsd-arm64": ["@esbuild/netbsd-arm64@0.27.0", "", { "os": "none", "cpu": "arm64" }, "sha512-6m0sfQfxfQfy1qRuecMkJlf1cIzTOgyaeXaiVaaki8/v+WB+U4hc6ik15ZW6TAllRlg/WuQXxWj1jx6C+dfy3w=="], | ||
| 49 | |||
| 50 | "@esbuild/netbsd-x64": ["@esbuild/netbsd-x64@0.27.0", "", { "os": "none", "cpu": "x64" }, "sha512-xbbOdfn06FtcJ9d0ShxxvSn2iUsGd/lgPIO2V3VZIPDbEaIj1/3nBBe1AwuEZKXVXkMmpr6LUAgMkLD/4D2PPA=="], | ||
| 51 | |||
| 52 | "@esbuild/openbsd-arm64": ["@esbuild/openbsd-arm64@0.27.0", "", { "os": "openbsd", "cpu": "arm64" }, "sha512-fWgqR8uNbCQ/GGv0yhzttj6sU/9Z5/Sv/VGU3F5OuXK6J6SlriONKrQ7tNlwBrJZXRYk5jUhuWvF7GYzGguBZQ=="], | ||
| 53 | |||
| 54 | "@esbuild/openbsd-x64": ["@esbuild/openbsd-x64@0.27.0", "", { "os": "openbsd", "cpu": "x64" }, "sha512-aCwlRdSNMNxkGGqQajMUza6uXzR/U0dIl1QmLjPtRbLOx3Gy3otfFu/VjATy4yQzo9yFDGTxYDo1FfAD9oRD2A=="], | ||
| 55 | |||
| 56 | "@esbuild/openharmony-arm64": ["@esbuild/openharmony-arm64@0.27.0", "", { "os": "none", "cpu": "arm64" }, "sha512-nyvsBccxNAsNYz2jVFYwEGuRRomqZ149A39SHWk4hV0jWxKM0hjBPm3AmdxcbHiFLbBSwG6SbpIcUbXjgyECfA=="], | ||
| 57 | |||
| 58 | "@esbuild/sunos-x64": ["@esbuild/sunos-x64@0.27.0", "", { "os": "sunos", "cpu": "x64" }, "sha512-Q1KY1iJafM+UX6CFEL+F4HRTgygmEW568YMqDA5UV97AuZSm21b7SXIrRJDwXWPzr8MGr75fUZPV67FdtMHlHA=="], | ||
| 59 | |||
| 60 | "@esbuild/win32-arm64": ["@esbuild/win32-arm64@0.27.0", "", { "os": "win32", "cpu": "arm64" }, "sha512-W1eyGNi6d+8kOmZIwi/EDjrL9nxQIQ0MiGqe/AWc6+IaHloxHSGoeRgDRKHFISThLmsewZ5nHFvGFWdBYlgKPg=="], | ||
| 61 | |||
| 62 | "@esbuild/win32-ia32": ["@esbuild/win32-ia32@0.27.0", "", { "os": "win32", "cpu": "ia32" }, "sha512-30z1aKL9h22kQhilnYkORFYt+3wp7yZsHWus+wSKAJR8JtdfI76LJ4SBdMsCopTR3z/ORqVu5L1vtnHZWVj4cQ=="], | ||
| 63 | |||
| 64 | "@esbuild/win32-x64": ["@esbuild/win32-x64@0.27.0", "", { "os": "win32", "cpu": "x64" }, "sha512-aIitBcjQeyOhMTImhLZmtxfdOcuNRpwlPNmlFKPcHQYPhEssw75Cl1TSXJXpMkzaua9FUetx/4OQKq7eJul5Cg=="], | ||
| 65 | |||
| 66 | "convex": ["convex@1.31.7", "", { "dependencies": { "esbuild": "0.27.0", "prettier": "^3.0.0" }, "peerDependencies": { "@auth0/auth0-react": "^2.0.1", "@clerk/clerk-react": "^4.12.8 || ^5.0.0", "react": "^18.0.0 || ^19.0.0-0 || ^19.0.0" }, "optionalPeers": ["@auth0/auth0-react", "@clerk/clerk-react", "react"], "bin": { "convex": "bin/main.js" } }, "sha512-PtNMe1mAIOvA8Yz100QTOaIdgt2rIuWqencVXrb4McdhxBHZ8IJ1eXTnrgCC9HydyilGT1pOn+KNqT14mqn9fQ=="], | ||
| 67 | |||
| 68 | "esbuild": ["esbuild@0.27.0", "", { "optionalDependencies": { "@esbuild/aix-ppc64": "0.27.0", "@esbuild/android-arm": "0.27.0", "@esbuild/android-arm64": "0.27.0", "@esbuild/android-x64": "0.27.0", "@esbuild/darwin-arm64": "0.27.0", "@esbuild/darwin-x64": "0.27.0", "@esbuild/freebsd-arm64": "0.27.0", "@esbuild/freebsd-x64": "0.27.0", "@esbuild/linux-arm": "0.27.0", "@esbuild/linux-arm64": "0.27.0", "@esbuild/linux-ia32": "0.27.0", "@esbuild/linux-loong64": "0.27.0", "@esbuild/linux-mips64el": "0.27.0", "@esbuild/linux-ppc64": "0.27.0", "@esbuild/linux-riscv64": "0.27.0", "@esbuild/linux-s390x": "0.27.0", "@esbuild/linux-x64": "0.27.0", "@esbuild/netbsd-arm64": "0.27.0", "@esbuild/netbsd-x64": "0.27.0", "@esbuild/openbsd-arm64": "0.27.0", "@esbuild/openbsd-x64": "0.27.0", "@esbuild/openharmony-arm64": "0.27.0", "@esbuild/sunos-x64": "0.27.0", "@esbuild/win32-arm64": "0.27.0", "@esbuild/win32-ia32": "0.27.0", "@esbuild/win32-x64": "0.27.0" }, "bin": { "esbuild": "bin/esbuild" } }, "sha512-jd0f4NHbD6cALCyGElNpGAOtWxSq46l9X/sWB0Nzd5er4Kz2YTm+Vl0qKFT9KUJvD8+fiO8AvoHhFvEatfVixA=="], | ||
| 69 | |||
| 70 | "graphql": ["graphql@16.12.0", "", {}, "sha512-DKKrynuQRne0PNpEbzuEdHlYOMksHSUI8Zc9Unei5gTsMNA2/vMpoMz/yKba50pejK56qj98qM0SjYxAKi13gQ=="], | ||
| 71 | |||
| 72 | "prettier": ["prettier@3.8.1", "", { "bin": { "prettier": "bin/prettier.cjs" } }, "sha512-UOnG6LftzbdaHZcKoPFtOcCKztrQ57WkHDeRD9t/PTQtmT0NHSeWWepj6pS0z/N7+08BHFDQVUrfmfMRcZwbMg=="], | ||
| 73 | } | ||
| 74 | } | ||
| @@ -12,13 +12,14 @@ import { actorIdForAccount } from "../lib/identity"; · 11 unmodified lines | |||
| 12 | import { runCodeWithAdapter } from "../lib/runtimes/runtime_core"; | 12 | import { runCodeWithAdapter } from "../lib/runtimes/runtime_core"; |
| 13 | import { createDiscoverTool } from "../lib/tool_discovery"; | 13 | import { createDiscoverTool } from "../lib/tool_discovery"; |
| 14 | import { | 14 | import { |
| 15 | buildOpenApiToolsFromPrepared, | 15 | compileExternalToolSource, |
| 16 | loadExternalTools, | 16 | compileOpenApiToolSourceFromPrepared, |
| 17 | materializeCompiledToolSource, | ||
| 18 | materializeWorkspaceSnapshot, | ||
| 17 | parseGraphqlOperationPaths, | 19 | parseGraphqlOperationPaths, |
| 18 | prepareOpenApiSpec, | 20 | prepareOpenApiSpec, |
| 19 | rehydrateTools, | ||
| 20 | serializeTools, | ||
| 21 | type PreparedOpenApiSpec, | 21 | type PreparedOpenApiSpec, |
| 22 | type CompiledToolSourceArtifact, | ||
| 22 | type ExternalToolSourceConfig, | 23 | type ExternalToolSourceConfig, |
| 23 | type McpToolSourceConfig, | 24 | type McpToolSourceConfig, |
| 24 | type OpenApiToolSourceConfig, | 25 | type OpenApiToolSourceConfig, |
| @@ -337,7 +338,9 @@ async function loadCachedOpenApiSpec( · 312 unmodified lines | |||
| 337 | } | 338 | } |
| 338 | 339 | ||
| 339 | // 2. Cache miss — prepare from scratch | 340 | // 2. Cache miss — prepare from scratch |
| 340 | const prepared = await prepareOpenApiSpec(specUrl, sourceName); | 341 | const prepared = await prepareOpenApiSpec(specUrl, sourceName, { |
| 342 | mode: "fast", | ||
| 343 | }); | ||
| 341 | 344 | ||
| 342 | // Store in file storage + metadata table (best-effort, don't block on failure) | 345 | // Store in file storage + metadata table (best-effort, don't block on failure) |
| 343 | try { | 346 | try { |
| @@ -358,28 +361,37 @@ async function loadCachedOpenApiSpec( · 14 unmodified lines | |||
| 358 | return prepared; | 361 | return prepared; |
| 359 | } | 362 | } |
| 360 | 363 | ||
| 361 | async function loadSourceTools( | 364 | async function loadSourceArtifact( |
| 362 | ctx: ActionCtx, | 365 | ctx: ActionCtx, |
| 363 | source: ExternalToolSourceConfig, | 366 | source: ExternalToolSourceConfig, |
| 364 | ): Promise<{ tools: ToolDefinition[]; warnings: string[] }> { | 367 | ): Promise<{ artifact?: CompiledToolSourceArtifact; warnings: string[] }> { |
| 365 | if (source.type === "openapi" && typeof source.spec === "string") { | 368 | if (source.type === "openapi" && typeof source.spec === "string") { |
| 366 | try { | 369 | try { |
| 367 | const prepared = await loadCachedOpenApiSpec(ctx, source.spec, source.name); | 370 | const prepared = await loadCachedOpenApiSpec(ctx, source.spec, source.name); |
| 368 | const tools = buildOpenApiToolsFromPrepared(source, prepared); | 371 | const artifact = compileOpenApiToolSourceFromPrepared(source, prepared); |
| 369 | const warnings = (prepared.warnings ?? []).map( | 372 | const warnings = (prepared.warnings ?? []).map( |
| 370 | (warning) => `Source '${source.name}': ${warning}`, | 373 | (warning) => `Source '${source.name}': ${warning}`, |
| 371 | ); | 374 | ); |
| 372 | return { tools, warnings }; | 375 | return { artifact, warnings }; |
| 373 | } catch (error) { | 376 | } catch (error) { |
| 374 | const message = error instanceof Error ? error.message : String(error); | 377 | const message = error instanceof Error ? error.message : String(error); |
| 375 | return { | 378 | return { |
| 376 | tools: [], | 379 | artifact: undefined, |
| 377 | warnings: [`Failed to load openapi source '${source.name}': ${message}`], | 380 | warnings: [`Failed to load openapi source '${source.name}': ${message}`], |
| 378 | }; | 381 | }; |
| 379 | } | 382 | } |
| 380 | } | 383 | } |
| 381 | 384 | ||
| 382 | return await loadExternalTools([source]); | 385 | try { |
| 386 | const artifact = await compileExternalToolSource(source); | ||
| 387 | return { artifact, warnings: [] }; | ||
| 388 | } catch (error) { | ||
| 389 | const message = error instanceof Error ? error.message : String(error); | ||
| 390 | return { | ||
| 391 | artifact: undefined, | ||
| 392 | warnings: [`Failed to load ${source.type} source '${source.name}': ${message}`], | ||
| 393 | }; | ||
| 394 | } | ||
| 383 | } | 395 | } |
| 384 | 396 | ||
| 385 | interface WorkspaceToolsResult { | 397 | interface WorkspaceToolsResult { |
| @@ -404,10 +416,15 @@ async function getWorkspaceTools(ctx: ActionCtx, workspaceId: Id<"workspaces">): · 18 unmodified lines | |||
| 404 | const blob = await ctx.storage.get(cacheEntry.storageId); | 416 | const blob = await ctx.storage.get(cacheEntry.storageId); |
| 405 | if (blob) { | 417 | if (blob) { |
| 406 | const snapshot = JSON.parse(await blob.text()) as WorkspaceToolSnapshot; | 418 | const snapshot = JSON.parse(await blob.text()) as WorkspaceToolSnapshot; |
| 407 | const rehydrated = rehydrateTools(snapshot.tools, baseTools); | 419 | const restored = materializeWorkspaceSnapshot(snapshot); |
| 408 | 420 | ||
| 409 | const merged = new Map<string, ToolDefinition>(); | 421 | const merged = new Map<string, ToolDefinition>(); |
| 410 | for (const tool of rehydrated) { | 422 | for (const tool of baseTools.values()) { |
| 423 | if (tool.path === "discover") continue; | ||
| 424 | merged.set(tool.path, tool); | ||
| 425 | } | ||
| 426 | for (const tool of restored) { | ||
| 427 | if (tool.path === "discover") continue; | ||
| 411 | merged.set(tool.path, tool); | 428 | merged.set(tool.path, tool); |
| 412 | } | 429 | } |
| 413 | // Recreate discover tool (it captures the full tool list in a closure) | 430 | // Recreate discover tool (it captures the full tool list in a closure) |
| @@ -436,8 +453,11 @@ async function getWorkspaceTools(ctx: ActionCtx, workspaceId: Id<"workspaces">): · 22 unmodified lines | |||
| 436 | } | 453 | } |
| 437 | } | 454 | } |
| 438 | 455 | ||
| 439 | const loadedSources = await Promise.all(configs.map((config) => loadSourceTools(ctx, config))); | 456 | const loadedSources = await Promise.all(configs.map((config) => loadSourceArtifact(ctx, config))); |
| 440 | const externalTools = loadedSources.flatMap((loaded) => loaded.tools); | 457 | const externalArtifacts = loadedSources |
| 458 | .map((loaded) => loaded.artifact) | ||
| 459 | .filter((artifact): artifact is CompiledToolSourceArtifact => Boolean(artifact)); | ||
| 460 | const externalTools = externalArtifacts.flatMap((artifact) => materializeCompiledToolSource(artifact)); | ||
| 441 | warnings.push(...loadedSources.flatMap((loaded) => loaded.warnings)); | 461 | warnings.push(...loadedSources.flatMap((loaded) => loaded.warnings)); |
| 442 | 462 | ||
| 443 | const merged = new Map<string, ToolDefinition>(); | 463 | const merged = new Map<string, ToolDefinition>(); |
| @@ -460,10 +480,12 @@ async function getWorkspaceTools(ctx: ActionCtx, workspaceId: Id<"workspaces">): · 16 unmodified lines | |||
| 460 | // Extract and store .d.ts blobs per source (too large for action responses) | 480 | // Extract and store .d.ts blobs per source (too large for action responses) |
| 461 | const seenDtsSources = new Set<string>(); | 481 | const seenDtsSources = new Set<string>(); |
| 462 | const dtsEntries: { sourceKey: string; content: string }[] = []; | 482 | const dtsEntries: { sourceKey: string; content: string }[] = []; |
| 463 | for (const tool of allTools) { | 483 | for (const artifact of externalArtifacts) { |
| 464 | if (tool.metadata?.sourceDts && tool.source && !seenDtsSources.has(tool.source)) { | 484 | for (const tool of artifact.tools) { |
| 465 | seenDtsSources.add(tool.source); | 485 | if (tool.metadata?.sourceDts && tool.source && !seenDtsSources.has(tool.source)) { |
| 466 | dtsEntries.push({ sourceKey: tool.source, content: tool.metadata.sourceDts }); | 486 | seenDtsSources.add(tool.source); |
| 487 | dtsEntries.push({ sourceKey: tool.source, content: tool.metadata.sourceDts }); | ||
| 488 | } | ||
| 467 | } | 489 | } |
| 468 | } | 490 | } |
| 469 | 491 | ||
| @@ -477,16 +499,22 @@ async function getWorkspaceTools(ctx: ActionCtx, workspaceId: Id<"workspaces">): · 7 unmodified lines | |||
| 477 | ); | 499 | ); |
| 478 | dtsStorageIds = storedDts; | 500 | dtsStorageIds = storedDts; |
| 479 | 501 | ||
| 480 | // Strip sourceDts from serialized tools (it's stored separately) | 502 | // Strip sourceDts from artifacts (it's stored separately) |
| 503 | const sanitizedArtifacts: CompiledToolSourceArtifact[] = externalArtifacts.map((artifact) => ({ | ||
| 504 | ...artifact, | ||
| 505 | tools: artifact.tools.map((tool) => { | ||
| 506 | if (!tool.metadata?.sourceDts) return tool; | ||
| 507 | const metadata = { ...tool.metadata }; | ||
| 508 | delete (metadata as Record<string, unknown>).sourceDts; | ||
| 509 | return { ...tool, metadata }; | ||
| 510 | }), | ||
| 511 | })); | ||
| 512 | |||
| 481 | const snapshot: WorkspaceToolSnapshot = { | 513 | const snapshot: WorkspaceToolSnapshot = { |
| 482 | tools: serializeTools(allTools), | 514 | version: "v2", |
| 515 | externalArtifacts: sanitizedArtifacts, | ||
| 483 | warnings, | 516 | warnings, |
| 484 | }; | 517 | }; |
| 485 | for (const st of snapshot.tools) { | ||
| 486 | if (st.metadata?.sourceDts) { | ||
| 487 | delete (st.metadata as Record<string, unknown>).sourceDts; | ||
| 488 | } | ||
| 489 | } | ||
| 490 | 518 | ||
| 491 | const json = JSON.stringify(snapshot); | 519 | const json = JSON.stringify(snapshot); |
| 492 | const blob = new Blob([json], { type: "application/json" }); | 520 | const blob = new Blob([json], { type: "application/json" }); |
| @@ -552,8 +580,10 @@ function toToolDescriptor(tool: ToolDefinition, approval: "auto" | "required"): · 59 unmodified lines | |||
| 552 | description: tool.description, | 580 | description: tool.description, |
| 553 | approval, | 581 | approval, |
| 554 | source: tool.source, | 582 | source: tool.source, |
| 555 | argsType: tool.metadata?.argsType, | 583 | argsType: tool.metadata?.displayArgsType ?? tool.metadata?.argsType, |
| 556 | returnsType: tool.metadata?.returnsType, | 584 | returnsType: tool.metadata?.displayReturnsType ?? tool.metadata?.returnsType, |
| 585 | strictArgsType: tool.metadata?.argsType, | ||
| 586 | strictReturnsType: tool.metadata?.returnsType, | ||
| 557 | operationId: tool.metadata?.operationId, | 587 | operationId: tool.metadata?.operationId, |
| 558 | // Note: sourceDts is NOT included — it's too large to send over the wire. | 588 | // Note: sourceDts is NOT included — it's too large to send over the wire. |
| 559 | // Monaco fetches .d.ts blobs separately via Convex storage URLs. | 589 | // Monaco fetches .d.ts blobs separately via Convex storage URLs. |
| @@ -3,51 +3,10 @@ · 2 unmodified lines | |||
| 3 | import { v } from "convex/values"; | 3 | import { v } from "convex/values"; |
| 4 | import { internal } from "./_generated/api"; | 4 | import { internal } from "./_generated/api"; |
| 5 | import { internalAction } from "./_generated/server"; | 5 | import { internalAction } from "./_generated/server"; |
| 6 | import { loadSourceDtsByUrlCached } from "../lib/dts_loader"; | ||
| 6 | import { generateToolDeclarations, typecheckCode } from "../lib/typechecker"; | 7 | import { generateToolDeclarations, typecheckCode } from "../lib/typechecker"; |
| 7 | import type { ToolDescriptor } from "../lib/types"; | 8 | import type { ToolDescriptor } from "../lib/types"; |
| 8 | 9 | ||
| 9 | const dtsUrlCache = new Map<string, Promise<string | null>>(); | ||
| 10 | |||
| 11 | async function loadSourceDtsByUrl(dtsUrls: Record<string, string>): Promise<Record<string, string>> { | ||
| 12 | const entries = Object.entries(dtsUrls); | ||
| 13 | if (entries.length === 0) { | ||
| 14 | return {}; | ||
| 15 | } | ||
| 16 | |||
| 17 | const results = await Promise.all(entries.map(async ([sourceKey, url]) => { | ||
| 18 | if (!url) return [sourceKey, null] as const; | ||
| 19 | |||
| 20 | if (!dtsUrlCache.has(url)) { | ||
| 21 | dtsUrlCache.set(url, (async () => { | ||
| 22 | try { | ||
| 23 | const response = await fetch(url); | ||
| 24 | if (!response.ok) { | ||
| 25 | console.warn(`[executor] failed to fetch source .d.ts from ${url}: HTTP ${response.status}`); | ||
| 26 | return null; | ||
| 27 | } | ||
| 28 | return await response.text(); | ||
| 29 | } catch (error) { | ||
| 30 | console.warn( | ||
| 31 | `[executor] failed to fetch source .d.ts from ${url}: ${error instanceof Error ? error.message : String(error)}`, | ||
| 32 | ); | ||
| 33 | return null; | ||
| 34 | } | ||
| 35 | })()); | ||
| 36 | } | ||
| 37 | |||
| 38 | const content = await dtsUrlCache.get(url)!; | ||
| 39 | return [sourceKey, content] as const; | ||
| 40 | })); | ||
| 41 | |||
| 42 | const sourceDtsBySource: Record<string, string> = {}; | ||
| 43 | for (const [sourceKey, dts] of results) { | ||
| 44 | if (dts) { | ||
| 45 | sourceDtsBySource[sourceKey] = dts; | ||
| 46 | } | ||
| 47 | } | ||
| 48 | return sourceDtsBySource; | ||
| 49 | } | ||
| 50 | |||
| 51 | export const typecheckRunCodeInternal = internalAction({ | 10 | export const typecheckRunCodeInternal = internalAction({ |
| 52 | args: { | 11 | args: { |
| 53 | code: v.string(), | 12 | code: v.string(), |
| @@ -67,7 +26,7 @@ export const typecheckRunCodeInternal = internalAction({ · 13 unmodified lines | |||
| 67 | dtsUrls?: Record<string, string>; | 26 | dtsUrls?: Record<string, string>; |
| 68 | }; | 27 | }; |
| 69 | 28 | ||
| 70 | const sourceDtsBySource = await loadSourceDtsByUrl(result.dtsUrls ?? {}); | 29 | const sourceDtsBySource = await loadSourceDtsByUrlCached(result.dtsUrls ?? {}); |
| 71 | const declarations = generateToolDeclarations(result.tools, { | 30 | const declarations = generateToolDeclarations(result.tools, { |
| 72 | sourceDtsBySource, | 31 | sourceDtsBySource, |
| 73 | }); | 32 | }); |
| @@ -0,0 +1,63 @@ | |||
| 1 | import { expect, test } from "bun:test"; | ||
| 2 | import { clearDtsUrlCache, loadSourceDtsByUrlCached } from "./dts_loader"; | ||
| 3 | |||
| 4 | test("loadSourceDtsByUrlCached reuses cached URL fetches", async () => { | ||
| 5 | let requestCount = 0; | ||
| 6 | const server = Bun.serve({ | ||
| 7 | port: 0, | ||
| 8 | fetch: () => { | ||
| 9 | requestCount += 1; | ||
| 10 | return new Response("export interface operations { ping: {} }", { | ||
| 11 | headers: { "content-type": "text/plain" }, | ||
| 12 | }); | ||
| 13 | }, | ||
| 14 | }); | ||
| 15 | |||
| 16 | try { | ||
| 17 | clearDtsUrlCache(); | ||
| 18 | const url = `http://127.0.0.1:${server.port}/types.d.ts`; | ||
| 19 | |||
| 20 | const first = await loadSourceDtsByUrlCached({ | ||
| 21 | "openapi:one": url, | ||
| 22 | "openapi:two": url, | ||
| 23 | }); | ||
| 24 | const second = await loadSourceDtsByUrlCached({ | ||
| 25 | "openapi:one": url, | ||
| 26 | }); | ||
| 27 | |||
| 28 | expect(first["openapi:one"]).toContain("interface operations"); | ||
| 29 | expect(first["openapi:two"]).toContain("interface operations"); | ||
| 30 | expect(second["openapi:one"]).toContain("interface operations"); | ||
| 31 | expect(requestCount).toBe(1); | ||
| 32 | } finally { | ||
| 33 | server.stop(true); | ||
| 34 | clearDtsUrlCache(); | ||
| 35 | } | ||
| 36 | }); | ||
| 37 | |||
| 38 | test("loadSourceDtsByUrlCached respects ttl expiration", async () => { | ||
| 39 | let requestCount = 0; | ||
| 40 | const server = Bun.serve({ | ||
| 41 | port: 0, | ||
| 42 | fetch: () => { | ||
| 43 | requestCount += 1; | ||
| 44 | return new Response("export interface operations { ping: {} }", { | ||
| 45 | headers: { "content-type": "text/plain" }, | ||
| 46 | }); | ||
| 47 | }, | ||
| 48 | }); | ||
| 49 | |||
| 50 | try { | ||
| 51 | clearDtsUrlCache(); | ||
| 52 | const url = `http://127.0.0.1:${server.port}/types.d.ts`; | ||
| 53 | |||
| 54 | await loadSourceDtsByUrlCached({ "openapi:one": url }, { ttlMs: 5 }); | ||
| 55 | await Bun.sleep(20); | ||
| 56 | await loadSourceDtsByUrlCached({ "openapi:one": url }, { ttlMs: 5 }); | ||
| 57 | |||
| 58 | expect(requestCount).toBe(2); | ||
| 59 | } finally { | ||
| 60 | server.stop(true); | ||
| 61 | clearDtsUrlCache(); | ||
| 62 | } | ||
| 63 | }); | ||
| @@ -0,0 +1,67 @@ | |||
| 1 | const DEFAULT_DTS_CACHE_TTL_MS = 5 * 60_000; | ||
| 2 | |||
| 3 | interface DtsCacheEntry { | ||
| 4 | expiresAt: number; | ||
| 5 | promise: Promise<string | null>; | ||
| 6 | } | ||
| 7 | |||
| 8 | const dtsUrlCache = new Map<string, DtsCacheEntry>(); | ||
| 9 | |||
| 10 | async function fetchDtsText(url: string): Promise<string | null> { | ||
| 11 | try { | ||
| 12 | const response = await fetch(url); | ||
| 13 | if (!response.ok) { | ||
| 14 | console.warn(`[executor] failed to fetch source .d.ts from ${url}: HTTP ${response.status}`); | ||
| 15 | return null; | ||
| 16 | } | ||
| 17 | return await response.text(); | ||
| 18 | } catch (error) { | ||
| 19 | console.warn( | ||
| 20 | `[executor] failed to fetch source .d.ts from ${url}: ${error instanceof Error ? error.message : String(error)}`, | ||
| 21 | ); | ||
| 22 | return null; | ||
| 23 | } | ||
| 24 | } | ||
| 25 | |||
| 26 | function getCachedDtsPromise(url: string, ttlMs: number): Promise<string | null> { | ||
| 27 | const now = Date.now(); | ||
| 28 | const cached = dtsUrlCache.get(url); | ||
| 29 | if (cached && cached.expiresAt > now) { | ||
| 30 | return cached.promise; | ||
| 31 | } | ||
| 32 | |||
| 33 | const promise = fetchDtsText(url).catch(() => null); | ||
| 34 | dtsUrlCache.set(url, { | ||
| 35 | expiresAt: now + Math.max(1, ttlMs), | ||
| 36 | promise, | ||
| 37 | }); | ||
| 38 | return promise; | ||
| 39 | } | ||
| 40 | |||
| 41 | export async function loadSourceDtsByUrlCached( | ||
| 42 | dtsUrls: Record<string, string>, | ||
| 43 | options?: { ttlMs?: number }, | ||
| 44 | ): Promise<Record<string, string>> { | ||
| 45 | const entries = Object.entries(dtsUrls); | ||
| 46 | if (entries.length === 0) { | ||
| 47 | return {}; | ||
| 48 | } | ||
| 49 | |||
| 50 | const ttlMs = options?.ttlMs ?? DEFAULT_DTS_CACHE_TTL_MS; | ||
| 51 | const results = await Promise.all(entries.map(async ([sourceKey, url]) => { | ||
| 52 | if (!url) return [sourceKey, null] as const; | ||
| 53 | return [sourceKey, await getCachedDtsPromise(url, ttlMs)] as const; | ||
| 54 | })); | ||
| 55 | |||
| 56 | const sourceDtsBySource: Record<string, string> = {}; | ||
| 57 | for (const [sourceKey, dts] of results) { | ||
| 58 | if (dts) { | ||
| 59 | sourceDtsBySource[sourceKey] = dts; | ||
| 60 | } | ||
| 61 | } | ||
| 62 | return sourceDtsBySource; | ||
| 63 | } | ||
| 64 | |||
| 65 | export function clearDtsUrlCache(): void { | ||
| 66 | dtsUrlCache.clear(); | ||
| 67 | } | ||
| @@ -1,6 +1,7 @@ | |||
| 1 | import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js"; | 1 | import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js"; |
| 2 | import { WebStandardStreamableHTTPServerTransport } from "@modelcontextprotocol/sdk/server/webStandardStreamableHttp.js"; | 2 | import { WebStandardStreamableHTTPServerTransport } from "@modelcontextprotocol/sdk/server/webStandardStreamableHttp.js"; |
| 3 | import { z } from "zod"; | 3 | import { z } from "zod"; |
| 4 | import { loadSourceDtsByUrlCached } from "./dts_loader"; | ||
| 4 | import { generateToolDeclarations, generateToolInventory, typecheckCode } from "./typechecker"; | 5 | import { generateToolDeclarations, generateToolInventory, typecheckCode } from "./typechecker"; |
| 5 | import type { LiveTaskEvent } from "./events"; | 6 | import type { LiveTaskEvent } from "./events"; |
| 6 | import type { | 7 | import type { |
| @@ -92,48 +93,6 @@ function listTopLevelToolKeys(tools: ToolDescriptor[]): string[] { · 85 unmodified lines | |||
| 92 | return [...keys].sort(); | 93 | return [...keys].sort(); |
| 93 | } | 94 | } |
| 94 | 95 | ||
| 95 | const dtsUrlCache = new Map<string, Promise<string | null>>(); | ||
| 96 | |||
| 97 | async function loadSourceDtsByUrl(dtsUrls: Record<string, string>): Promise<Record<string, string>> { | ||
| 98 | const entries = Object.entries(dtsUrls); | ||
| 99 | if (entries.length === 0) { | ||
| 100 | return {}; | ||
| 101 | } | ||
| 102 | |||
| 103 | const results = await Promise.all(entries.map(async ([sourceKey, url]) => { | ||
| 104 | if (!url) return [sourceKey, null] as const; | ||
| 105 | |||
| 106 | if (!dtsUrlCache.has(url)) { | ||
| 107 | dtsUrlCache.set(url, (async () => { | ||
| 108 | try { | ||
| 109 | const response = await fetch(url); | ||
| 110 | if (!response.ok) { | ||
| 111 | console.warn(`[executor] failed to fetch source .d.ts from ${url}: HTTP ${response.status}`); | ||
| 112 | return null; | ||
| 113 | } | ||
| 114 | return await response.text(); | ||
| 115 | } catch (error) { | ||
| 116 | console.warn( | ||
| 117 | `[executor] failed to fetch source .d.ts from ${url}: ${error instanceof Error ? error.message : String(error)}`, | ||
| 118 | ); | ||
| 119 | return null; | ||
| 120 | } | ||
| 121 | })()); | ||
| 122 | } | ||
| 123 | |||
| 124 | const content = await dtsUrlCache.get(url)!; | ||
| 125 | return [sourceKey, content] as const; | ||
| 126 | })); | ||
| 127 | |||
| 128 | const sourceDtsBySource: Record<string, string> = {}; | ||
| 129 | for (const [sourceKey, dts] of results) { | ||
| 130 | if (dts) { | ||
| 131 | sourceDtsBySource[sourceKey] = dts; | ||
| 132 | } | ||
| 133 | } | ||
| 134 | return sourceDtsBySource; | ||
| 135 | } | ||
| 136 | |||
| 137 | function summarizeTask(task: TaskRecord): string { | 96 | function summarizeTask(task: TaskRecord): string { |
| 138 | const maxStreamPreviewChars = 30_000; | 97 | const maxStreamPreviewChars = 30_000; |
| 139 | const lines = [ | 98 | const lines = [ |
| @@ -469,7 +428,7 @@ function createRunCodeTool( · 329 unmodified lines | |||
| 469 | clientId: context.clientId, | 428 | clientId: context.clientId, |
| 470 | }); | 429 | }); |
| 471 | toolsForContext = tools; | 430 | toolsForContext = tools; |
| 472 | sourceDtsBySource = await loadSourceDtsByUrl(dtsUrls ?? {}); | 431 | sourceDtsBySource = await loadSourceDtsByUrlCached(dtsUrls ?? {}); |
| 473 | } else { | 432 | } else { |
| 474 | toolsForContext = await service.listTools({ | 433 | toolsForContext = await service.listTools({ |
| 475 | workspaceId: context.workspaceId, | 434 | workspaceId: context.workspaceId, |
| @@ -669,14 +628,13 @@ function createDelegatingService(ref: { current: McpExecutorService }): McpExecu · 193 unmodified lines | |||
| 669 | }; | 628 | }; |
| 670 | } | 629 | } |
| 671 | 630 | ||
| 672 | const mcpSessionRuntimes = new Map< | 631 | type McpSessionRuntime = { |
| 673 | string, | 632 | transport: WebStandardStreamableHTTPServerTransport; |
| 674 | { | 633 | mcp: McpServer; |
| 675 | transport: WebStandardStreamableHTTPServerTransport; | 634 | serviceRef: { current: McpExecutorService }; |
| 676 | mcp: McpServer; | 635 | }; |
| 677 | serviceRef: { current: McpExecutorService }; | 636 | |
| 678 | } | 637 | const mcpSessionRuntimes: Record<string, McpSessionRuntime> = Object.create(null); |
| 679 | >(); | ||
| 680 | 638 | ||
| 681 | async function handleStatelessMcpRequest( | 639 | async function handleStatelessMcpRequest( |
| 682 | service: McpExecutorService, | 640 | service: McpExecutorService, |
| @@ -710,7 +668,7 @@ export async function handleMcpRequest( · 27 unmodified lines | |||
| 710 | const runtimeSessionId = request.headers.get("mcp-session-id") ?? undefined; | 668 | const runtimeSessionId = request.headers.get("mcp-session-id") ?? undefined; |
| 711 | 669 | ||
| 712 | if (runtimeSessionId) { | 670 | if (runtimeSessionId) { |
| 713 | const runtime = mcpSessionRuntimes.get(runtimeSessionId); | 671 | const runtime = mcpSessionRuntimes[runtimeSessionId]; |
| 714 | if (runtime) { | 672 | if (runtime) { |
| 715 | runtime.serviceRef.current = service; | 673 | runtime.serviceRef.current = service; |
| 716 | return await runtime.transport.handleRequest(request); | 674 | return await runtime.transport.handleRequest(request); |
| @@ -730,23 +688,23 @@ export async function handleMcpRequest( · 13 unmodified lines | |||
| 730 | sessionIdGenerator: () => crypto.randomUUID(), | 688 | sessionIdGenerator: () => crypto.randomUUID(), |
| 731 | enableJsonResponse: true, | 689 | enableJsonResponse: true, |
| 732 | onsessioninitialized: (sessionId) => { | 690 | onsessioninitialized: (sessionId) => { |
| 733 | mcpSessionRuntimes.set(sessionId, { | 691 | mcpSessionRuntimes[sessionId] = { |
| 734 | transport, | 692 | transport, |
| 735 | mcp, | 693 | mcp, |
| 736 | serviceRef, | 694 | serviceRef, |
| 737 | }); | 695 | }; |
| 738 | }, | 696 | }, |
| 739 | onsessionclosed: async (sessionId) => { | 697 | onsessionclosed: async (sessionId) => { |
| 740 | if (!sessionId) { | 698 | if (!sessionId) { |
| 741 | return; | 699 | return; |
| 742 | } | 700 | } |
| 743 | 701 | ||
| 744 | const runtime = mcpSessionRuntimes.get(sessionId); | 702 | const runtime = mcpSessionRuntimes[sessionId]; |
| 745 | if (!runtime) { | 703 | if (!runtime) { |
| 746 | return; | 704 | return; |
| 747 | } | 705 | } |
| 748 | 706 | ||
| 749 | mcpSessionRuntimes.delete(sessionId); | 707 | delete mcpSessionRuntimes[sessionId]; |
| 750 | await runtime.mcp.close().catch(() => {}); | 708 | await runtime.mcp.close().catch(() => {}); |
| 751 | }, | 709 | }, |
| 752 | }); | 710 | }); |
| @@ -168,6 +168,36 @@ describe("prepareOpenApiSpec with large specs", () => { · 167 unmodified lines | |||
| 168 | expect(json.length).toBeGreaterThan(100_000); | 168 | expect(json.length).toBeGreaterThan(100_000); |
| 169 | }); | 169 | }); |
| 170 | 170 | ||
| 171 | test("fast mode skips bundle and d.ts generation for very large specs", async () => { | ||
| 172 | const spec = makeLargeSpec(250); // 750 operations | ||
| 173 | const start = performance.now(); | ||
| 174 | const prepared = await prepareOpenApiSpec(spec, "large-fast", { | ||
| 175 | mode: "fast", | ||
| 176 | fastPathOperationThreshold: 600, | ||
| 177 | }); | ||
| 178 | const elapsed = performance.now() - start; | ||
| 179 | |||
| 180 | expect(Object.keys(prepared.paths)).toHaveLength(250); | ||
| 181 | expect(prepared.dts).toBeUndefined(); | ||
| 182 | expect(prepared.warnings.some((warning: string) => warning.includes("fast mode"))).toBe(true); | ||
| 183 | |||
| 184 | const tools = buildOpenApiToolsFromPrepared( | ||
| 185 | { | ||
| 186 | type: "openapi", | ||
| 187 | name: "large-fast", | ||
| 188 | spec, | ||
| 189 | baseUrl: "https://api.example.com/v1", | ||
| 190 | }, | ||
| 191 | prepared, | ||
| 192 | ); | ||
| 193 | |||
| 194 | expect(tools.length).toBeGreaterThan(700); | ||
| 195 | const deleteTool = tools.find((tool) => tool.metadata?.operationId?.startsWith("delete_")); | ||
| 196 | expect(deleteTool).toBeDefined(); | ||
| 197 | expect(deleteTool?.metadata?.returnsType).toBe("void"); | ||
| 198 | expect(elapsed).toBeLessThan(5_000); | ||
| 199 | }); | ||
| 200 | |||
| 171 | 201 | ||
| 172 | }); | 202 | }); |
| 173 | 203 | ||
| @@ -194,6 +224,8 @@ describe("buildOpenApiToolsFromPrepared", () => { · 20 unmodified lines | |||
| 194 | expect(tool.metadata).toBeDefined(); | 224 | expect(tool.metadata).toBeDefined(); |
| 195 | expect(tool.metadata!.argsType).toBeDefined(); | 225 | expect(tool.metadata!.argsType).toBeDefined(); |
| 196 | expect(tool.metadata!.returnsType).toBeDefined(); | 226 | expect(tool.metadata!.returnsType).toBeDefined(); |
| 227 | expect(tool.metadata!.displayArgsType).toBeDefined(); | ||
| 228 | expect(tool.metadata!.displayReturnsType).toBeDefined(); | ||
| 197 | } | 229 | } |
| 198 | 230 | ||
| 199 | // GET operations should default to "auto" approval | 231 | // GET operations should default to "auto" approval |
| @@ -1,6 +1,11 @@ | |||
| 1 | import { expect, test } from "bun:test"; | 1 | import { expect, test } from "bun:test"; |
| 2 | import type { Id } from "../convex/_generated/dataModel"; | 2 | import type { Id } from "../convex/_generated/dataModel"; |
| 3 | import { loadExternalTools } from "./tool_sources"; | 3 | import { |
| 4 | compileExternalToolSource, | ||
| 5 | loadExternalTools, | ||
| 6 | materializeCompiledToolSource, | ||
| 7 | parseGraphqlOperationPaths, | ||
| 8 | } from "./tool_sources"; | ||
| 4 | import type { ExternalToolSourceConfig } from "./tool_sources"; | 9 | import type { ExternalToolSourceConfig } from "./tool_sources"; |
| 5 | 10 | ||
| 6 | const TEST_WORKSPACE_ID = "w" as Id<"workspaces">; | 11 | const TEST_WORKSPACE_ID = "w" as Id<"workspaces">; |
| @@ -238,6 +243,112 @@ test("loadExternalTools tolerates OpenAPI specs with broken internal refs", asyn · 231 unmodified lines | |||
| 238 | expect(toolPaths).toContain("intercom_like.conversations.create_conversation"); | 243 | expect(toolPaths).toContain("intercom_like.conversations.create_conversation"); |
| 239 | }); | 244 | }); |
| 240 | 245 | ||
| 246 | test("compiled source artifacts can be materialized and executed", async () => { | ||
| 247 | const server = Bun.serve({ | ||
| 248 | port: 0, | ||
| 249 | fetch: (req) => { | ||
| 250 | const url = new URL(req.url); | ||
| 251 | if (url.pathname === "/health" && req.method === "GET") { | ||
| 252 | return Response.json({ ok: true, probe: url.searchParams.get("probe") }); | ||
| 253 | } | ||
| 254 | return new Response("not found", { status: 404 }); | ||
| 255 | }, | ||
| 256 | }); | ||
| 257 | |||
| 258 | const spec: Record<string, unknown> = { | ||
| 259 | openapi: "3.0.3", | ||
| 260 | info: { title: "Compiled", version: "1.0.0" }, | ||
| 261 | servers: [{ url: `http://127.0.0.1:${server.port}` }], | ||
| 262 | paths: { | ||
| 263 | "/health": { | ||
| 264 | get: { | ||
| 265 | operationId: "healthCheck", | ||
| 266 | tags: ["health"], | ||
| 267 | parameters: [ | ||
| 268 | { | ||
| 269 | name: "probe", | ||
| 270 | in: "query", | ||
| 271 | required: false, | ||
| 272 | schema: { type: "string" }, | ||
| 273 | }, | ||
| 274 | ], | ||
| 275 | responses: { | ||
| 276 | "200": { | ||
| 277 | description: "ok", | ||
| 278 | content: { | ||
| 279 | "application/json": { | ||
| 280 | schema: { | ||
| 281 | type: "object", | ||
| 282 | properties: { | ||
| 283 | ok: { type: "boolean" }, | ||
| 284 | probe: { type: "string" }, | ||
| 285 | }, | ||
| 286 | }, | ||
| 287 | }, | ||
| 288 | }, | ||
| 289 | }, | ||
| 290 | }, | ||
| 291 | }, | ||
| 292 | }, | ||
| 293 | }, | ||
| 294 | }; | ||
| 295 | |||
| 296 | try { | ||
| 297 | const artifact = await compileExternalToolSource({ | ||
| 298 | type: "openapi", | ||
| 299 | name: "compiled-api", | ||
| 300 | spec, | ||
| 301 | baseUrl: `http://127.0.0.1:${server.port}`, | ||
| 302 | }); | ||
| 303 | |||
| 304 | expect(artifact.version).toBe("v1"); | ||
| 305 | expect(artifact.sourceType).toBe("openapi"); | ||
| 306 | expect(artifact.tools).toHaveLength(1); | ||
| 307 | expect(artifact.tools[0]?.runSpec.kind).toBe("openapi"); | ||
| 308 | |||
| 309 | const tools = materializeCompiledToolSource(artifact); | ||
| 310 | expect(tools).toHaveLength(1); | ||
| 311 | expect(tools[0]?.path).toBe("compiled_api.health.check"); | ||
| 312 | |||
| 313 | const result = await tools[0]!.run( | ||
| 314 | { probe: "status" }, | ||
| 315 | { taskId: "t", workspaceId: TEST_WORKSPACE_ID, isToolAllowed: () => true }, | ||
| 316 | ); | ||
| 317 | |||
| 318 | expect(result).toEqual({ ok: true, probe: "status" }); | ||
| 319 | } finally { | ||
| 320 | server.stop(true); | ||
| 321 | } | ||
| 322 | }); | ||
| 323 | |||
| 324 | test("parseGraphqlOperationPaths handles aliases and fragments", async () => { | ||
| 325 | const parsed = parseGraphqlOperationPaths( | ||
| 326 | "linear", | ||
| 327 | ` | ||
| 328 | query TeamsAndViewer { | ||
| 329 | currentTeams: teams { nodes { id } } | ||
| 330 | ...ViewerFields | ||
| 331 | } | ||
| 332 | |||
| 333 | fragment ViewerFields on Query { | ||
| 334 | viewer { id } | ||
| 335 | } | ||
| 336 | `, | ||
| 337 | ); | ||
| 338 | |||
| 339 | expect(parsed.operationType).toBe("query"); | ||
| 340 | expect(parsed.fieldPaths).toEqual([ | ||
| 341 | "linear.query.teams", | ||
| 342 | "linear.query.viewer", | ||
| 343 | ]); | ||
| 344 | }); | ||
| 345 | |||
| 346 | test("parseGraphqlOperationPaths returns empty paths for invalid queries", async () => { | ||
| 347 | const parsed = parseGraphqlOperationPaths("linear", "query { teams("); | ||
| 348 | expect(parsed.operationType).toBe("query"); | ||
| 349 | expect(parsed.fieldPaths).toEqual([]); | ||
| 350 | }); | ||
| 351 | |||
| 241 | test("graphql helper tools generate valid selection sets and envelope responses", async () => { | 352 | test("graphql helper tools generate valid selection sets and envelope responses", async () => { |
| 242 | const capturedQueries: string[] = []; | 353 | const capturedQueries: string[] = []; |
| 243 | const capturedVariables: Array<Record<string, unknown> | undefined> = []; | 354 | const capturedVariables: Array<Record<string, unknown> | undefined> = []; |
| @@ -1,4 +1,10 @@ | |||
| 1 | import type { ToolDefinition } from "./types"; | 1 | import type { ToolDefinition } from "./types"; |
| 2 | import { | ||
| 3 | compactArgTypeHint, | ||
| 4 | compactDescriptionLine, | ||
| 5 | compactReturnTypeHint, | ||
| 6 | extractTopLevelTypeKeys, | ||
| 7 | } from "./type_hints"; | ||
| 2 | 8 | ||
| 3 | interface DiscoverIndexEntry { | 9 | interface DiscoverIndexEntry { |
| 4 | path: string; | 10 | path: string; |
| @@ -56,52 +62,6 @@ function getPathAliases(path: string): string[] { · 51 unmodified lines | |||
| 56 | return [...aliases].slice(0, 4); | 62 | return [...aliases].slice(0, 4); |
| 57 | } | 63 | } |
| 58 | 64 | ||
| 59 | function extractTopLevelArgKeys(argsType: string): string[] { | ||
| 60 | const text = argsType.trim(); | ||
| 61 | if (!text.startsWith("{") || !text.endsWith("}")) return []; | ||
| 62 | |||
| 63 | const inner = text.slice(1, -1); | ||
| 64 | const keys: string[] = []; | ||
| 65 | let segment = ""; | ||
| 66 | let depthCurly = 0; | ||
| 67 | let depthSquare = 0; | ||
| 68 | let depthParen = 0; | ||
| 69 | let depthAngle = 0; | ||
| 70 | |||
| 71 | const flushSegment = () => { | ||
| 72 | const part = segment.trim(); | ||
| 73 | segment = ""; | ||
| 74 | if (!part) return; | ||
| 75 | const colon = part.indexOf(":"); | ||
| 76 | if (colon <= 0) return; | ||
| 77 | const rawKey = part.slice(0, colon).trim(); | ||
| 78 | const cleanedKey = rawKey.replace(/[?"']/g, "").trim(); | ||
| 79 | if (!cleanedKey || !/^[a-zA-Z_][a-zA-Z0-9_]*$/.test(cleanedKey)) return; | ||
| 80 | if (!keys.includes(cleanedKey)) keys.push(cleanedKey); | ||
| 81 | }; | ||
| 82 | |||
| 83 | for (const char of inner) { | ||
| 84 | if (char === "{" ) depthCurly += 1; | ||
| 85 | else if (char === "}" ) depthCurly = Math.max(0, depthCurly - 1); | ||
| 86 | else if (char === "[") depthSquare += 1; | ||
| 87 | else if (char === "]") depthSquare = Math.max(0, depthSquare - 1); | ||
| 88 | else if (char === "(") depthParen += 1; | ||
| 89 | else if (char === ")") depthParen = Math.max(0, depthParen - 1); | ||
| 90 | else if (char === "<") depthAngle += 1; | ||
| 91 | else if (char === ">") depthAngle = Math.max(0, depthAngle - 1); | ||
| 92 | |||
| 93 | if (char === ";" && depthCurly === 0 && depthSquare === 0 && depthParen === 0 && depthAngle === 0) { | ||
| 94 | flushSegment(); | ||
| 95 | continue; | ||
| 96 | } | ||
| 97 | |||
| 98 | segment += char; | ||
| 99 | } | ||
| 100 | |||
| 101 | flushSegment(); | ||
| 102 | return keys; | ||
| 103 | } | ||
| 104 | |||
| 105 | function buildExampleCall(entry: DiscoverIndexEntry): string { | 65 | function buildExampleCall(entry: DiscoverIndexEntry): string { |
| 106 | if (entry.path.endsWith(".graphql")) { | 66 | if (entry.path.endsWith(".graphql")) { |
| 107 | return `await tools.${entry.path}({ query: "query { __typename }", variables: {} });`; | 67 | return `await tools.${entry.path}({ query: "query { __typename }", variables: {} });`; |
| @@ -111,7 +71,7 @@ function buildExampleCall(entry: DiscoverIndexEntry): string { · 3 unmodified lines | |||
| 111 | return `await tools.${entry.path}({});`; | 71 | return `await tools.${entry.path}({});`; |
| 112 | } | 72 | } |
| 113 | 73 | ||
| 114 | const keys = extractTopLevelArgKeys(entry.argsType); | 74 | const keys = extractTopLevelTypeKeys(entry.argsType); |
| 115 | if (keys.length > 0) { | 75 | if (keys.length > 0) { |
| 116 | const argsSnippet = keys.slice(0, 3) | 76 | const argsSnippet = keys.slice(0, 3) |
| 117 | .map((key) => `${key}: ${key.toLowerCase().includes("input") ? "{ /* ... */ }" : "..."}`) | 77 | .map((key) => `${key}: ${key.toLowerCase().includes("input") ? "{ /* ... */ }" : "..."}`) |
| @@ -122,40 +82,6 @@ function buildExampleCall(entry: DiscoverIndexEntry): string { · 4 unmodified lines | |||
| 122 | return `await tools.${entry.path}({ /* ... */ });`; | 82 | return `await tools.${entry.path}({ /* ... */ });`; |
| 123 | } | 83 | } |
| 124 | 84 | ||
| 125 | function truncateInline(value: string, maxLength: number): string { | ||
| 126 | const normalized = value.replace(/\s+/g, " ").trim(); | ||
| 127 | if (normalized.length <= maxLength) return normalized; | ||
| 128 | return `${normalized.slice(0, Math.max(16, maxLength - 3)).trim()}...`; | ||
| 129 | } | ||
| 130 | |||
| 131 | function compactArgTypeHint(argsType: string): string { | ||
| 132 | if (argsType === "{}") return "{}"; | ||
| 133 | const keys = extractTopLevelArgKeys(argsType); | ||
| 134 | if (keys.length > 0) { | ||
| 135 | const maxKeys = 4; | ||
| 136 | const shown = keys.slice(0, maxKeys).map((key) => `${key}: ...`); | ||
| 137 | const suffix = keys.length > maxKeys ? "; ..." : ""; | ||
| 138 | return `{ ${shown.join("; ")}${suffix} }`; | ||
| 139 | } | ||
| 140 | return truncateInline(argsType, 120); | ||
| 141 | } | ||
| 142 | |||
| 143 | function compactReturnTypeHint(returnsType: string): string { | ||
| 144 | const normalized = returnsType.replace(/\s+/g, " ").trim(); | ||
| 145 | if (normalized.startsWith("{ data:") && normalized.includes("errors:")) { | ||
| 146 | return "{ data: ...; errors: unknown[] }"; | ||
| 147 | } | ||
| 148 | if (normalized.endsWith("[]") && normalized.length > 90) { | ||
| 149 | return "Array<...>"; | ||
| 150 | } | ||
| 151 | return truncateInline(normalized, 130); | ||
| 152 | } | ||
| 153 | |||
| 154 | function compactDescription(description: string): string { | ||
| 155 | const firstLine = description.split("\n")[0] ?? description; | ||
| 156 | return truncateInline(firstLine, 180); | ||
| 157 | } | ||
| 158 | |||
| 159 | function buildIndex(tools: ToolDefinition[]): DiscoverIndexEntry[] { | 85 | function buildIndex(tools: ToolDefinition[]): DiscoverIndexEntry[] { |
| 160 | return tools | 86 | return tools |
| 161 | .filter((tool) => tool.path !== "discover") | 87 | .filter((tool) => tool.path !== "discover") |
| @@ -345,7 +271,7 @@ export function createDiscoverTool(tools: ToolDefinition[]): ToolDefinition { · 183 unmodified lines | |||
| 345 | aliases: entry.aliases, | 271 | aliases: entry.aliases, |
| 346 | source: entry.source, | 272 | source: entry.source, |
| 347 | approval: entry.approval, | 273 | approval: entry.approval, |
| 348 | description: compact ? compactDescription(entry.description) : entry.description, | 274 | description: compact ? compactDescriptionLine(entry.description) : entry.description, |
| 349 | signature: formatSignature(entry, depth, compact), | 275 | signature: formatSignature(entry, depth, compact), |
| 350 | exampleCall: buildExampleCall(entry), | 276 | exampleCall: buildExampleCall(entry), |
| 351 | })); | 277 | })); |
| @@ -5,6 +5,8 @@ import { Client } from "@modelcontextprotocol/sdk/client/index.js"; · 4 unmodified lines | |||
| 5 | import { SSEClientTransport } from "@modelcontextprotocol/sdk/client/sse.js"; | 5 | import { SSEClientTransport } from "@modelcontextprotocol/sdk/client/sse.js"; |
| 6 | import { StreamableHTTPClientTransport } from "@modelcontextprotocol/sdk/client/streamableHttp.js"; | 6 | import { StreamableHTTPClientTransport } from "@modelcontextprotocol/sdk/client/streamableHttp.js"; |
| 7 | import openapiTS, { astToString } from "openapi-typescript"; | 7 | import openapiTS, { astToString } from "openapi-typescript"; |
| 8 | import { Kind, parse, type FragmentDefinitionNode, type SelectionSetNode } from "graphql"; | ||
| 9 | import { compactArgTypeHint, compactReturnTypeHint } from "./type_hints"; | ||
| 8 | import type { ToolApprovalMode, ToolCredentialSpec, ToolDefinition, ToolTypeMetadata } from "./types"; | 10 | import type { ToolApprovalMode, ToolCredentialSpec, ToolDefinition, ToolTypeMetadata } from "./types"; |
| 9 | import { asRecord } from "./utils"; | 11 | import { asRecord } from "./utils"; |
| 10 | 12 | ||
| @@ -570,14 +572,18 @@ async function loadMcpTools(config: McpToolSourceConfig): Promise<ToolDefinition · 559 unmodified lines | |||
| 570 | return tools.map((tool) => { | 572 | return tools.map((tool) => { |
| 571 | const toolName = String(tool.name ?? "tool"); | 573 | const toolName = String(tool.name ?? "tool"); |
| 572 | const inputSchema = asRecord(tool.inputSchema); | 574 | const inputSchema = asRecord(tool.inputSchema); |
| 575 | const argsType = jsonSchemaTypeHintFallback(inputSchema); | ||
| 576 | const returnsType = "unknown"; | ||
| 573 | return { | 577 | return { |
| 574 | path: `${sanitizeSegment(config.name)}.${sanitizeSegment(toolName)}`, | 578 | path: `${sanitizeSegment(config.name)}.${sanitizeSegment(toolName)}`, |
| 575 | source: `mcp:${config.name}`, | 579 | source: `mcp:${config.name}`, |
| 576 | approval: config.overrides?.[toolName]?.approval ?? config.defaultApproval ?? "auto", | 580 | approval: config.overrides?.[toolName]?.approval ?? config.defaultApproval ?? "auto", |
| 577 | description: String(tool.description ?? `MCP tool ${toolName}`), | 581 | description: String(tool.description ?? `MCP tool ${toolName}`), |
| 578 | metadata: { | 582 | metadata: { |
| 579 | argsType: jsonSchemaTypeHintFallback(inputSchema), | 583 | argsType, |
| 580 | returnsType: "unknown", | 584 | returnsType, |
| 585 | displayArgsType: compactArgTypeHint(argsType), | ||
| 586 | displayReturnsType: compactReturnTypeHint(returnsType), | ||
| 581 | }, | 587 | }, |
| 582 | _runSpec: { | 588 | _runSpec: { |
| 583 | kind: "mcp" as const, | 589 | kind: "mcp" as const, |
| @@ -687,6 +693,7 @@ function compactOpenApiPaths( · 103 unmodified lines | |||
| 687 | componentSchemas?: Record<string, unknown>, | 693 | componentSchemas?: Record<string, unknown>, |
| 688 | componentResponses?: Record<string, unknown>, | 694 | componentResponses?: Record<string, unknown>, |
| 689 | componentRequestBodies?: Record<string, unknown>, | 695 | componentRequestBodies?: Record<string, unknown>, |
| 696 | typeHintMode: "full" | "fast" = "full", | ||
| 690 | ): Record<string, unknown> { | 697 | ): Record<string, unknown> { |
| 691 | const paths = asRecord(pathsValue); | 698 | const paths = asRecord(pathsValue); |
| 692 | const methods = ["get", "post", "put", "delete", "patch", "head", "options"] as const; | 699 | const methods = ["get", "post", "put", "delete", "patch", "head", "options"] as const; |
| @@ -762,22 +769,23 @@ function compactOpenApiPaths( · 69 unmodified lines | |||
| 762 | { | 769 | { |
| 763 | const requestBody = resolveRequestBodyRef(asRecord(operation.requestBody), compRequestBodies); | 770 | const requestBody = resolveRequestBodyRef(asRecord(operation.requestBody), compRequestBodies); |
| 764 | const requestBodyContent = asRecord(requestBody.content); | 771 | const requestBodyContent = asRecord(requestBody.content); |
| 765 | const requestBodySchema = resolveSchemaRef( | 772 | const rawRequestBodySchema = getPreferredContentSchema(requestBodyContent); |
| 766 | getPreferredContentSchema(requestBodyContent), | 773 | const requestBodySchema = resolveSchemaRef(rawRequestBodySchema, compSchemas); |
| 767 | compSchemas, | ||
| 768 | ); | ||
| 769 | 774 | ||
| 770 | const responses = asRecord(operation.responses); | 775 | const responses = asRecord(operation.responses); |
| 771 | let responseSchema: Record<string, unknown> = {}; | 776 | let responseSchema: Record<string, unknown> = {}; |
| 772 | let responseStatus = ""; | 777 | let responseStatus = ""; |
| 773 | for (const [status, responseValue] of Object.entries(responses)) { | 778 | for (const [status, responseValue] of Object.entries(responses)) { |
| 774 | if (!status.startsWith("2")) continue; | 779 | if (!status.startsWith("2")) continue; |
| 780 | responseStatus = status; | ||
| 781 | if (typeHintMode === "fast" && !hasGeneratedTypes) { | ||
| 782 | break; | ||
| 783 | } | ||
| 775 | const resolvedResponse = resolveResponseRef(asRecord(responseValue), compResponses); | 784 | const resolvedResponse = resolveResponseRef(asRecord(responseValue), compResponses); |
| 776 | responseSchema = resolveSchemaRef( | 785 | responseSchema = resolveSchemaRef( |
| 777 | getPreferredResponseSchema(resolvedResponse), | 786 | getPreferredResponseSchema(resolvedResponse), |
| 778 | compSchemas, | 787 | compSchemas, |
| 779 | ); | 788 | ); |
| 780 | responseStatus = status; | ||
| 781 | if (Object.keys(responseSchema).length > 0) break; | 789 | if (Object.keys(responseSchema).length > 0) break; |
| 782 | } | 790 | } |
| 783 | 791 | ||
| @@ -808,6 +816,11 @@ function compactOpenApiPaths( · 24 unmodified lines | |||
| 808 | ? jsonSchemaTypeHintFallback(combinedSchema, 0, compSchemas) | 816 | ? jsonSchemaTypeHintFallback(combinedSchema, 0, compSchemas) |
| 809 | : "{}"; | 817 | : "{}"; |
| 810 | compactOperation._returnsTypeHint = responseTypeHintFromSchema(responseSchema, responseStatus, compSchemas); | 818 | compactOperation._returnsTypeHint = responseTypeHintFromSchema(responseSchema, responseStatus, compSchemas); |
| 819 | } else if (typeHintMode === "fast") { | ||
| 820 | const mergedParameters = operationParameters.concat(sharedParameters); | ||
| 821 | const hasInputSchema = mergedParameters.length > 0 || Object.keys(rawRequestBodySchema).length > 0; | ||
| 822 | compactOperation._argsTypeHint = hasInputSchema ? "Record<string, unknown>" : "{}"; | ||
| 823 | compactOperation._returnsTypeHint = responseStatus === "204" ? "void" : "unknown"; | ||
| 811 | } else { | 824 | } else { |
| 812 | // Keep full schemas for the fallback path | 825 | // Keep full schemas for the fallback path |
| 813 | if (Object.keys(requestBodySchema).length > 0) { | 826 | if (Object.keys(requestBodySchema).length > 0) { |
| @@ -855,9 +868,34 @@ export interface PreparedOpenApiSpec { · 41 unmodified lines | |||
| 855 | warnings: string[]; | 868 | warnings: string[]; |
| 856 | } | 869 | } |
| 857 | 870 | ||
| 871 | export interface PrepareOpenApiSpecOptions { | ||
| 872 | mode?: "full" | "fast"; | ||
| 873 | fastPathOperationThreshold?: number; | ||
| 874 | } | ||
| 875 | |||
| 876 | const DEFAULT_FAST_PATH_OPERATION_THRESHOLD = 600; | ||
| 877 | |||
| 878 | function countOpenApiOperations(pathsValue: unknown): number { | ||
| 879 | const paths = asRecord(pathsValue); | ||
| 880 | const methods = new Set(["get", "post", "put", "delete", "patch", "head", "options"]); | ||
| 881 | let count = 0; | ||
| 882 | |||
| 883 | for (const pathValue of Object.values(paths)) { | ||
| 884 | const pathObject = asRecord(pathValue); | ||
| 885 | for (const key of Object.keys(pathObject)) { | ||
| 886 | if (methods.has(key)) { | ||
| 887 | count += 1; | ||
| 888 | } | ||
| 889 | } | ||
| 890 | } | ||
| 891 | |||
| 892 | return count; | ||
| 893 | } | ||
| 894 | |||
| 858 | export async function prepareOpenApiSpec( | 895 | export async function prepareOpenApiSpec( |
| 859 | spec: string | Record<string, unknown>, | 896 | spec: string | Record<string, unknown>, |
| 860 | sourceName = "openapi", | 897 | sourceName = "openapi", |
| 898 | options?: PrepareOpenApiSpecOptions, | ||
| 861 | ): Promise<PreparedOpenApiSpec> { | 899 | ): Promise<PreparedOpenApiSpec> { |
| 862 | const parser = SwaggerParser as unknown as { | 900 | const parser = SwaggerParser as unknown as { |
| 863 | bundle(spec: unknown): Promise<unknown>; | 901 | bundle(spec: unknown): Promise<unknown>; |
| @@ -882,21 +920,33 @@ export async function prepareOpenApiSpec( · 18 unmodified lines | |||
| 882 | parsed = spec; | 920 | parsed = spec; |
| 883 | } | 921 | } |
| 884 | 922 | ||
| 923 | const operationCount = countOpenApiOperations(parsed.paths); | ||
| 924 | const mode = options?.mode ?? "full"; | ||
| 925 | const fastPathOperationThreshold = | ||
| 926 | options?.fastPathOperationThreshold ?? DEFAULT_FAST_PATH_OPERATION_THRESHOLD; | ||
| 927 | const useFastPath = mode === "fast" && operationCount >= fastPathOperationThreshold; | ||
| 928 | |||
| 885 | // ── Step 2: Generate .d.ts and bundle in parallel ───────────────────── | 929 | // ── Step 2: Generate .d.ts and bundle in parallel ───────────────────── |
| 886 | // Both now operate on the in-memory object — no additional HTTP fetches. | 930 | // Both now operate on the in-memory object — no additional HTTP fetches. |
| 887 | const dtsPromise = generateOpenApiDts(parsed); | 931 | let dts: string | null = null; |
| 888 | |||
| 889 | let bundled: Record<string, unknown>; | 932 | let bundled: Record<string, unknown>; |
| 890 | try { | 933 | if (useFastPath) { |
| 891 | bundled = (await parser.bundle(parsed)) as Record<string, unknown>; | ||
| 892 | } catch (bundleError) { | ||
| 893 | const bundleMessage = bundleError instanceof Error ? bundleError.message : String(bundleError); | ||
| 894 | warnings.push(`OpenAPI bundle failed for '${sourceName}', using parse-only mode: ${bundleMessage}`); | ||
| 895 | bundled = parsed; | 934 | bundled = parsed; |
| 935 | warnings.push( | ||
| 936 | `OpenAPI source '${sourceName}' loaded in fast mode (${operationCount} operations): skipped bundle + d.ts generation for latency`, | ||
| 937 | ); | ||
| 938 | } else { | ||
| 939 | const dtsPromise = generateOpenApiDts(parsed); | ||
| 940 | try { | ||
| 941 | bundled = (await parser.bundle(parsed)) as Record<string, unknown>; | ||
| 942 | } catch (bundleError) { | ||
| 943 | const bundleMessage = bundleError instanceof Error ? bundleError.message : String(bundleError); | ||
| 944 | warnings.push(`OpenAPI bundle failed for '${sourceName}', using parse-only mode: ${bundleMessage}`); | ||
| 945 | bundled = parsed; | ||
| 946 | } | ||
| 947 | dts = await dtsPromise; | ||
| 896 | } | 948 | } |
| 897 | 949 | ||
| 898 | const dts = await dtsPromise; | ||
| 899 | |||
| 900 | // ── Step 3: Extract operation IDs for compaction ─────────────────────── | 950 | // ── Step 3: Extract operation IDs for compaction ─────────────────────── |
| 901 | // If we have a .d.ts, we know which operations have full generated types. | 951 | // If we have a .d.ts, we know which operations have full generated types. |
| 902 | // compactOpenApiPaths uses this to strip request/response schemas from | 952 | // compactOpenApiPaths uses this to strip request/response schemas from |
| @@ -916,6 +966,7 @@ export async function prepareOpenApiSpec( · 13 unmodified lines | |||
| 916 | asRecord(asRecord(bundled.components).schemas), | 966 | asRecord(asRecord(bundled.components).schemas), |
| 917 | asRecord(asRecord(bundled.components).responses), | 967 | asRecord(asRecord(bundled.components).responses), |
| 918 | asRecord(asRecord(bundled.components).requestBodies), | 968 | asRecord(asRecord(bundled.components).requestBodies), |
| 969 | useFastPath ? "fast" : "full", | ||
| 919 | ), | 970 | ), |
| 920 | dts: dts ?? undefined, | 971 | dts: dts ?? undefined, |
| 921 | warnings, | 972 | warnings, |
| @@ -1037,6 +1088,8 @@ export function buildOpenApiToolsFromPrepared( · 115 unmodified lines | |||
| 1037 | metadata: { | 1088 | metadata: { |
| 1038 | argsType, | 1089 | argsType, |
| 1039 | returnsType, | 1090 | returnsType, |
| 1091 | displayArgsType: compactArgTypeHint(argsType), | ||
| 1092 | displayReturnsType: compactReturnTypeHint(returnsType), | ||
| 1040 | operationId: operationIdRaw, | 1093 | operationId: operationIdRaw, |
| 1041 | // Only attach sourceDts to the first tool to avoid duplicating the full .d.ts | 1094 | // Only attach sourceDts to the first tool to avoid duplicating the full .d.ts |
| 1042 | ...(sourceDts && !sourceDtsEmitted ? { sourceDts } : {}), | 1095 | ...(sourceDts && !sourceDtsEmitted ? { sourceDts } : {}), |
| @@ -1441,65 +1494,77 @@ function printGqlType(ref: GqlTypeRef): string { · 398 unmodified lines | |||
| 1441 | 1494 | ||
| 1442 | /** | 1495 | /** |
| 1443 | * Parse a GraphQL query string to extract the operation type and root field names. | 1496 | * Parse a GraphQL query string to extract the operation type and root field names. |
| 1444 | * This is intentionally simple — no full parser needed, just enough for policy routing. | 1497 | * Uses GraphQL AST parsing so aliases/fragments are handled correctly for policy routing. |
| 1445 | */ | 1498 | */ |
| 1446 | export function parseGraphqlOperationPaths( | 1499 | export function parseGraphqlOperationPaths( |
| 1447 | sourceName: string, | 1500 | sourceName: string, |
| 1448 | queryString: string, | 1501 | queryString: string, |
| 1449 | ): { operationType: "query" | "mutation" | "subscription"; fieldPaths: string[] } { | 1502 | ): { operationType: "query" | "mutation" | "subscription"; fieldPaths: string[] } { |
| 1450 | const trimmed = queryString.trim(); | 1503 | const trimmed = queryString.trim(); |
| 1504 | if (!trimmed) { | ||
| 1505 | return { operationType: "query", fieldPaths: [] }; | ||
| 1506 | } | ||
| 1451 | 1507 | ||
| 1452 | // Determine operation type | ||
| 1453 | let operationType: "query" | "mutation" | "subscription" = "query"; | 1508 | let operationType: "query" | "mutation" | "subscription" = "query"; |
| 1454 | if (/^mutation\b/i.test(trimmed)) operationType = "mutation"; | 1509 | |
| 1455 | else if (/^subscription\b/i.test(trimmed)) operationType = "subscription"; | 1510 | try { |
| 1456 | 1511 | const document = parse(trimmed, { noLocation: true }); | |
| 1457 | // Find the first { ... } block and extract top-level field names | 1512 | const firstOperation = document.definitions.find((definition) => definition.kind === Kind.OPERATION_DEFINITION); |
| 1458 | const braceStart = trimmed.indexOf("{"); | 1513 | if (!firstOperation || firstOperation.kind !== Kind.OPERATION_DEFINITION) { |
| 1459 | if (braceStart === -1) return { operationType, fieldPaths: [] }; | 1514 | return { operationType, fieldPaths: [] }; |
| 1460 | 1515 | } | |
| 1461 | // Walk the content inside the first braces, extract field names at depth 0 | 1516 | |
| 1462 | const content = trimmed.slice(braceStart + 1); | 1517 | operationType = firstOperation.operation; |
| 1463 | const fieldPaths: string[] = []; | 1518 | |
| 1464 | let depth = 0; | 1519 | const fragmentByName = new Map<string, FragmentDefinitionNode>( |
| 1465 | let current = ""; | 1520 | document.definitions |
| 1466 | 1521 | .filter((definition) => definition.kind === Kind.FRAGMENT_DEFINITION) | |
| 1467 | for (const char of content) { | 1522 | .map((definition) => [definition.name.value, definition]), |
| 1468 | if (char === "{") { | 1523 | ); |
| 1469 | if (depth === 0 && current.trim()) { | 1524 | |
| 1470 | // Grab the field name (before any args in parens) | 1525 | const fieldNames = new Set<string>(); |
| 1471 | const fieldName = current.trim().split(/[\s(]/)[0]; | 1526 | |
| 1472 | if (fieldName && !fieldName.startsWith("__")) { | 1527 | const collectSelectionSet = ( |
| 1473 | fieldPaths.push(`${sanitizeSegment(sourceName)}.${operationType}.${sanitizeSegment(fieldName)}`); | 1528 | selectionSet: SelectionSetNode, |
| 1529 | visitedFragments: Set<string>, | ||
| 1530 | ) => { | ||
| 1531 | for (const selection of selectionSet.selections) { | ||
| 1532 | if (selection.kind === Kind.FIELD) { | ||
| 1533 | const fieldName = selection.name.value; | ||
| 1534 | if (!fieldName.startsWith("__")) { | ||
| 1535 | fieldNames.add(fieldName); | ||
| 1536 | } | ||
| 1537 | continue; | ||
| 1474 | } | 1538 | } |
| 1475 | } | 1539 | |
| 1476 | depth++; | 1540 | if (selection.kind === Kind.INLINE_FRAGMENT) { |
| 1477 | current = ""; | 1541 | collectSelectionSet(selection.selectionSet, visitedFragments); |
| 1478 | } else if (char === "}") { | 1542 | continue; |
| 1479 | if (depth === 0) { | ||
| 1480 | // End of top-level block — grab last field if any | ||
| 1481 | const fieldName = current.trim().split(/[\s(]/)[0]; | ||
| 1482 | if (fieldName && !fieldName.startsWith("__")) { | ||
| 1483 | fieldPaths.push(`${sanitizeSegment(sourceName)}.${operationType}.${sanitizeSegment(fieldName)}`); | ||
| 1484 | } | 1543 | } |
| 1485 | break; | 1544 | |
| 1486 | } | 1545 | if (selection.kind === Kind.FRAGMENT_SPREAD) { |
| 1487 | depth--; | 1546 | const fragmentName = selection.name.value; |
| 1488 | current = ""; | 1547 | if (visitedFragments.has(fragmentName)) continue; |
| 1489 | } else if (depth === 0) { | 1548 | const fragment = fragmentByName.get(fragmentName); |
| 1490 | if (char === "\n" || char === ",") { | 1549 | if (!fragment) continue; |
| 1491 | const fieldName = current.trim().split(/[\s(]/)[0]; | 1550 | |
| 1492 | if (fieldName && !fieldName.startsWith("__")) { | 1551 | const nextVisited = new Set(visitedFragments); |
| 1493 | fieldPaths.push(`${sanitizeSegment(sourceName)}.${operationType}.${sanitizeSegment(fieldName)}`); | 1552 | nextVisited.add(fragmentName); |
| 1553 | collectSelectionSet(fragment.selectionSet, nextVisited); | ||
| 1494 | } | 1554 | } |
| 1495 | current = ""; | ||
| 1496 | } else { | ||
| 1497 | current += char; | ||
| 1498 | } | 1555 | } |
| 1499 | } | 1556 | }; |
| 1500 | } | ||
| 1501 | 1557 | ||
| 1502 | return { operationType, fieldPaths }; | 1558 | collectSelectionSet(firstOperation.selectionSet, new Set()); |
| 1559 | |||
| 1560 | return { | ||
| 1561 | operationType, | ||
| 1562 | fieldPaths: [...fieldNames] | ||
| 1563 | .map((fieldName) => `${sanitizeSegment(sourceName)}.${operationType}.${sanitizeSegment(fieldName)}`), | ||
| 1564 | }; | ||
| 1565 | } catch { | ||
| 1566 | return { operationType, fieldPaths: [] }; | ||
| 1567 | } | ||
| 1503 | } | 1568 | } |
| 1504 | 1569 | ||
| 1505 | async function loadGraphqlTools(config: GraphqlToolSourceConfig): Promise<ToolDefinition[]> { | 1570 | async function loadGraphqlTools(config: GraphqlToolSourceConfig): Promise<ToolDefinition[]> { |
| @@ -1550,6 +1615,8 @@ async function loadGraphqlTools(config: GraphqlToolSourceConfig): Promise<ToolDe · 44 unmodified lines | |||
| 1550 | metadata: { | 1615 | metadata: { |
| 1551 | argsType: "{ query: string; variables?: Record<string, unknown> }", | 1616 | argsType: "{ query: string; variables?: Record<string, unknown> }", |
| 1552 | returnsType: "{ data: unknown; errors: unknown[] }", | 1617 | returnsType: "{ data: unknown; errors: unknown[] }", |
| 1618 | displayArgsType: "{ query: string; variables?: ... }", | ||
| 1619 | displayReturnsType: "{ data: ...; errors: unknown[] }", | ||
| 1553 | }, | 1620 | }, |
| 1554 | credential: credentialSpec, | 1621 | credential: credentialSpec, |
| 1555 | // Tag as graphql source so invokeTool knows to do dynamic path extraction | 1622 | // Tag as graphql source so invokeTool knows to do dynamic path extraction |
| @@ -1612,6 +1679,8 @@ async function loadGraphqlTools(config: GraphqlToolSourceConfig): Promise<ToolDe · 56 unmodified lines | |||
| 1612 | 1679 | ||
| 1613 | const fieldPath = `${sourceName}.${operationType}.${sanitizeSegment(field.name)}`; | 1680 | const fieldPath = `${sourceName}.${operationType}.${sanitizeSegment(field.name)}`; |
| 1614 | const approval = config.overrides?.[field.name]?.approval ?? defaultApproval; | 1681 | const approval = config.overrides?.[field.name]?.approval ?? defaultApproval; |
| 1682 | const argsType = gqlFieldArgsTypeHint(field.args, typeMap); | ||
| 1683 | const returnsType = `{ data: ${gqlTypeToHint(field.type, typeMap)}; errors: unknown[] }`; | ||
| 1615 | 1684 | ||
| 1616 | // Build the example query for the description | 1685 | // Build the example query for the description |
| 1617 | const exampleQuery = buildFieldQuery(operationType, field.name, field.args, field.type, typeMap); | 1686 | const exampleQuery = buildFieldQuery(operationType, field.name, field.args, field.type, typeMap); |
| @@ -1628,8 +1697,10 @@ async function loadGraphqlTools(config: GraphqlToolSourceConfig): Promise<ToolDe · 10 unmodified lines | |||
| 1628 | approval, | 1697 | approval, |
| 1629 | credential: credentialSpec, | 1698 | credential: credentialSpec, |
| 1630 | metadata: { | 1699 | metadata: { |
| 1631 | argsType: gqlFieldArgsTypeHint(field.args, typeMap), | 1700 | argsType, |
| 1632 | returnsType: `{ data: ${gqlTypeToHint(field.type, typeMap)}; errors: unknown[] }`, | 1701 | returnsType, |
| 1702 | displayArgsType: compactArgTypeHint(argsType), | ||
| 1703 | displayReturnsType: "{ data: ...; errors: unknown[] }", | ||
| 1633 | }, | 1704 | }, |
| 1634 | _runSpec: { | 1705 | _runSpec: { |
| 1635 | kind: "graphql_field" as const, | 1706 | kind: "graphql_field" as const, |
| @@ -1682,27 +1753,63 @@ export function parseToolSourcesFromEnv(raw: string | undefined): ExternalToolSo · 46 unmodified lines | |||
| 1682 | return parsed as ExternalToolSourceConfig[]; | 1753 | return parsed as ExternalToolSourceConfig[]; |
| 1683 | } | 1754 | } |
| 1684 | 1755 | ||
| 1756 | export interface CompiledToolSourceArtifact { | ||
| 1757 | version: "v1"; | ||
| 1758 | sourceType: ExternalToolSourceConfig["type"]; | ||
| 1759 | sourceName: string; | ||
| 1760 | tools: SerializedTool[]; | ||
| 1761 | } | ||
| 1762 | |||
| 1763 | async function loadSourceToolDefinitions(source: ExternalToolSourceConfig): Promise<ToolDefinition[]> { | ||
| 1764 | if (source.type === "mcp") { | ||
| 1765 | return await loadMcpTools(source); | ||
| 1766 | } | ||
| 1767 | if (source.type === "openapi") { | ||
| 1768 | return await loadOpenApiTools(source); | ||
| 1769 | } | ||
| 1770 | if (source.type === "graphql") { | ||
| 1771 | return await loadGraphqlTools(source); | ||
| 1772 | } | ||
| 1773 | return []; | ||
| 1774 | } | ||
| 1775 | |||
| 1776 | export async function compileExternalToolSource(source: ExternalToolSourceConfig): Promise<CompiledToolSourceArtifact> { | ||
| 1777 | const tools = await loadSourceToolDefinitions(source); | ||
| 1778 | return { | ||
| 1779 | version: "v1", | ||
| 1780 | sourceType: source.type, | ||
| 1781 | sourceName: source.name, | ||
| 1782 | tools: serializeTools(tools), | ||
| 1783 | }; | ||
| 1784 | } | ||
| 1785 | |||
| 1786 | export function compileOpenApiToolSourceFromPrepared( | ||
| 1787 | source: OpenApiToolSourceConfig, | ||
| 1788 | prepared: PreparedOpenApiSpec, | ||
| 1789 | ): CompiledToolSourceArtifact { | ||
| 1790 | const tools = buildOpenApiToolsFromPrepared(source, prepared); | ||
| 1791 | return { | ||
| 1792 | version: "v1", | ||
| 1793 | sourceType: source.type, | ||
| 1794 | sourceName: source.name, | ||
| 1795 | tools: serializeTools(tools), | ||
| 1796 | }; | ||
| 1797 | } | ||
| 1798 | |||
| 1799 | export function materializeCompiledToolSource(artifact: CompiledToolSourceArtifact): ToolDefinition[] { | ||
| 1800 | return rehydrateTools(artifact.tools, new Map()); | ||
| 1801 | } | ||
| 1802 | |||
| 1685 | export async function loadExternalTools(sources: ExternalToolSourceConfig[]): Promise<{ tools: ToolDefinition[]; warnings: string[] }> { | 1803 | export async function loadExternalTools(sources: ExternalToolSourceConfig[]): Promise<{ tools: ToolDefinition[]; warnings: string[] }> { |
| 1686 | const results = await Promise.allSettled( | 1804 | const results = await Promise.allSettled(sources.map((source) => compileExternalToolSource(source))); |
| 1687 | sources.map(async (source) => { | ||
| 1688 | if (source.type === "mcp") { | ||
| 1689 | return await loadMcpTools(source); | ||
| 1690 | } else if (source.type === "openapi") { | ||
| 1691 | return await loadOpenApiTools(source); | ||
| 1692 | } else if (source.type === "graphql") { | ||
| 1693 | return await loadGraphqlTools(source); | ||
| 1694 | } | ||
| 1695 | return []; | ||
| 1696 | }), | ||
| 1697 | ); | ||
| 1698 | 1805 | ||
| 1699 | const loaded: ToolDefinition[] = []; | 1806 | const artifacts: CompiledToolSourceArtifact[] = []; |
| 1700 | const warnings: string[] = []; | 1807 | const warnings: string[] = []; |
| 1701 | 1808 | ||
| 1702 | for (let i = 0; i < results.length; i++) { | 1809 | for (let i = 0; i < results.length; i++) { |
| 1703 | const result = results[i]!; | 1810 | const result = results[i]!; |
| 1704 | if (result.status === "fulfilled") { | 1811 | if (result.status === "fulfilled") { |
| 1705 | loaded.push(...result.value); | 1812 | artifacts.push(result.value); |
| 1706 | } else { | 1813 | } else { |
| 1707 | const source = sources[i]!; | 1814 | const source = sources[i]!; |
| 1708 | const message = result.reason instanceof Error ? result.reason.message : String(result.reason); | 1815 | const message = result.reason instanceof Error ? result.reason.message : String(result.reason); |
| @@ -1711,7 +1818,8 @@ export async function loadExternalTools(sources: ExternalToolSourceConfig[]): Pr · 2 unmodified lines | |||
| 1711 | } | 1818 | } |
| 1712 | } | 1819 | } |
| 1713 | 1820 | ||
| 1714 | return { tools: loaded, warnings }; | 1821 | const tools = artifacts.flatMap((artifact) => materializeCompiledToolSource(artifact)); |
| 1822 | return { tools, warnings }; | ||
| 1715 | } | 1823 | } |
| 1716 | 1824 | ||
| 1717 | // ── Workspace tool cache serialization ────────────────────────────────────── | 1825 | // ── Workspace tool cache serialization ────────────────────────────────────── |
| @@ -1771,10 +1879,17 @@ export interface SerializedTool { · 53 unmodified lines | |||
| 1771 | } | 1879 | } |
| 1772 | 1880 | ||
| 1773 | export interface WorkspaceToolSnapshot { | 1881 | export interface WorkspaceToolSnapshot { |
| 1774 | tools: SerializedTool[]; | 1882 | version: "v2"; |
| 1883 | externalArtifacts: CompiledToolSourceArtifact[]; | ||
| 1775 | warnings: string[]; | 1884 | warnings: string[]; |
| 1776 | } | 1885 | } |
| 1777 | 1886 | ||
| 1887 | export function materializeWorkspaceSnapshot( | ||
| 1888 | snapshot: WorkspaceToolSnapshot, | ||
| 1889 | ): ToolDefinition[] { | ||
| 1890 | return snapshot.externalArtifacts.flatMap((artifact) => materializeCompiledToolSource(artifact)); | ||
| 1891 | } | ||
| 1892 | |||
| 1778 | /** Serialize tools for cache storage. Strips `run` closures, stores reconstruction data. */ | 1893 | /** Serialize tools for cache storage. Strips `run` closures, stores reconstruction data. */ |
| 1779 | export function serializeTools(tools: ToolDefinition[]): SerializedTool[] { | 1894 | export function serializeTools(tools: ToolDefinition[]): SerializedTool[] { |
| 1780 | return tools.map((tool) => ({ | 1895 | return tools.map((tool) => ({ |
| @@ -0,0 +1,29 @@ | |||
| 1 | import { expect, test } from "bun:test"; | ||
| 2 | import { | ||
| 3 | compactArgTypeHint, | ||
| 4 | compactDescriptionLine, | ||
| 5 | compactReturnTypeHint, | ||
| 6 | extractTopLevelTypeKeys, | ||
| 7 | } from "./type_hints"; | ||
| 8 | |||
| 9 | test("extractTopLevelTypeKeys handles nested object literals", async () => { | ||
| 10 | const keys = extractTopLevelTypeKeys("{ parent: { title: string }; icon?: { emoji: string }; children: string[] }"); | ||
| 11 | expect(keys).toEqual(["parent", "icon", "children"]); | ||
| 12 | }); | ||
| 13 | |||
| 14 | test("compactArgTypeHint keeps only top-level previews", async () => { | ||
| 15 | const compact = compactArgTypeHint( | ||
| 16 | "{ parent: { database_id: string }; title: Array<{ text: { content: string } }>; icon?: { emoji: string }; properties?: Record<string, unknown> }", | ||
| 17 | ); | ||
| 18 | expect(compact).toBe("{ parent: ...; title: ...; icon: ...; properties: ... }"); | ||
| 19 | }); | ||
| 20 | |||
| 21 | test("compactReturnTypeHint collapses graphql envelopes", async () => { | ||
| 22 | const compact = compactReturnTypeHint("{ data: { issue: { id: string; title: string } }; errors: unknown[] }"); | ||
| 23 | expect(compact).toBe("{ data: ...; errors: unknown[] }"); | ||
| 24 | }); | ||
| 25 | |||
| 26 | test("compactDescriptionLine trims to a single concise line", async () => { | ||
| 27 | const compact = compactDescriptionLine("Create a database in Notion.\nThis endpoint supports parent/page context."); | ||
| 28 | expect(compact).toBe("Create a database in Notion."); | ||
| 29 | }); | ||
| @@ -0,0 +1,79 @@ | |||
| 1 | function truncateInline(value: string, maxLength: number): string { | ||
| 2 | const normalized = value.replace(/\s+/g, " ").trim(); | ||
| 3 | if (normalized.length <= maxLength) return normalized; | ||
| 4 | return `${normalized.slice(0, Math.max(16, maxLength - 3)).trim()}...`; | ||
| 5 | } | ||
| 6 | |||
| 7 | export function extractTopLevelTypeKeys(typeHint: string): string[] { | ||
| 8 | const text = typeHint.trim(); | ||
| 9 | if (!text.startsWith("{") || !text.endsWith("}")) return []; | ||
| 10 | |||
| 11 | const inner = text.slice(1, -1); | ||
| 12 | const keys: string[] = []; | ||
| 13 | let segment = ""; | ||
| 14 | let depthCurly = 0; | ||
| 15 | let depthSquare = 0; | ||
| 16 | let depthParen = 0; | ||
| 17 | let depthAngle = 0; | ||
| 18 | |||
| 19 | const flushSegment = () => { | ||
| 20 | const part = segment.trim(); | ||
| 21 | segment = ""; | ||
| 22 | if (!part) return; | ||
| 23 | const colon = part.indexOf(":"); | ||
| 24 | if (colon <= 0) return; | ||
| 25 | const rawKey = part.slice(0, colon).trim(); | ||
| 26 | const cleanedKey = rawKey.replace(/[?"']/g, "").trim(); | ||
| 27 | if (!cleanedKey || !/^[a-zA-Z_][a-zA-Z0-9_]*$/.test(cleanedKey)) return; | ||
| 28 | if (!keys.includes(cleanedKey)) keys.push(cleanedKey); | ||
| 29 | }; | ||
| 30 | |||
| 31 | for (const char of inner) { | ||
| 32 | if (char === "{") depthCurly += 1; | ||
| 33 | else if (char === "}") depthCurly = Math.max(0, depthCurly - 1); | ||
| 34 | else if (char === "[") depthSquare += 1; | ||
| 35 | else if (char === "]") depthSquare = Math.max(0, depthSquare - 1); | ||
| 36 | else if (char === "(") depthParen += 1; | ||
| 37 | else if (char === ")") depthParen = Math.max(0, depthParen - 1); | ||
| 38 | else if (char === "<") depthAngle += 1; | ||
| 39 | else if (char === ">") depthAngle = Math.max(0, depthAngle - 1); | ||
| 40 | |||
| 41 | if (char === ";" && depthCurly === 0 && depthSquare === 0 && depthParen === 0 && depthAngle === 0) { | ||
| 42 | flushSegment(); | ||
| 43 | continue; | ||
| 44 | } | ||
| 45 | |||
| 46 | segment += char; | ||
| 47 | } | ||
| 48 | |||
| 49 | flushSegment(); | ||
| 50 | return keys; | ||
| 51 | } | ||
| 52 | |||
| 53 | export function compactArgTypeHint(argsType: string): string { | ||
| 54 | if (argsType === "{}") return "{}"; | ||
| 55 | const keys = extractTopLevelTypeKeys(argsType); | ||
| 56 | if (keys.length > 0) { | ||
| 57 | const maxKeys = 4; | ||
| 58 | const shown = keys.slice(0, maxKeys).map((key) => `${key}: ...`); | ||
| 59 | const suffix = keys.length > maxKeys ? "; ..." : ""; | ||
| 60 | return `{ ${shown.join("; ")}${suffix} }`; | ||
| 61 | } | ||
| 62 | return truncateInline(argsType, 120); | ||
| 63 | } | ||
| 64 | |||
| 65 | export function compactReturnTypeHint(returnsType: string): string { | ||
| 66 | const normalized = returnsType.replace(/\s+/g, " ").trim(); | ||
| 67 | if (normalized.startsWith("{ data:") && normalized.includes("errors:")) { | ||
| 68 | return "{ data: ...; errors: unknown[] }"; | ||
| 69 | } | ||
| 70 | if (normalized.endsWith("[]") && normalized.length > 90) { | ||
| 71 | return "Array<...>"; | ||
| 72 | } | ||
| 73 | return truncateInline(normalized, 130); | ||
| 74 | } | ||
| 75 | |||
| 76 | export function compactDescriptionLine(description: string): string { | ||
| 77 | const firstLine = description.split("\n")[0] ?? description; | ||
| 78 | return truncateInline(firstLine, 180); | ||
| 79 | } | ||
| @@ -139,9 +139,13 @@ export function generateToolDeclarations( · 138 unmodified lines | |||
| 139 | const opKey = JSON.stringify(tool.operationId); | 139 | const opKey = JSON.stringify(tool.operationId); |
| 140 | lines.push(`${pad}${key}(input: ToolInput<operations[${opKey}]>): Promise<ToolOutput<operations[${opKey}]>>;`); | 140 | lines.push(`${pad}${key}(input: ToolInput<operations[${opKey}]>): Promise<ToolOutput<operations[${opKey}]>>;`); |
| 141 | } else { | 141 | } else { |
| 142 | const hasArgsType = Boolean(tool.argsType?.trim()); | 142 | const strictArgsType = tool.strictArgsType?.trim(); |
| 143 | const args = safeTypeExpression(tool.argsType, "Record<string, unknown>"); | 143 | const strictReturnsType = tool.strictReturnsType?.trim(); |
| 144 | const returns = safeTypeExpression(tool.returnsType, "unknown"); | 144 | const effectiveArgs = strictArgsType || tool.argsType; |
| 145 | const effectiveReturns = strictReturnsType || tool.returnsType; | ||
| 146 | const hasArgsType = Boolean(effectiveArgs?.trim()); | ||
| 147 | const args = safeTypeExpression(effectiveArgs, "Record<string, unknown>"); | ||
| 148 | const returns = safeTypeExpression(effectiveReturns, "unknown"); | ||
| 145 | const inputParam = !hasArgsType || args === "{}" | 149 | const inputParam = !hasArgsType || args === "{}" |
| 146 | ? `input?: ${args}` | 150 | ? `input?: ${args}` |
| 147 | : `input: ${args}`; | 151 | : `input: ${args}`; |
| @@ -95,8 +95,14 @@ export interface ToolDescriptor { · 94 unmodified lines | |||
| 95 | description: string; | 95 | description: string; |
| 96 | approval: ToolApprovalMode; | 96 | approval: ToolApprovalMode; |
| 97 | source?: string; | 97 | source?: string; |
| 98 | /** Compact display type hint for tool arguments. */ | ||
| 98 | argsType?: string; | 99 | argsType?: string; |
| 100 | /** Compact display type hint for tool return value. */ | ||
| 99 | returnsType?: string; | 101 | returnsType?: string; |
| 102 | /** Full-fidelity args type hint for strict editor/typecheck usage. */ | ||
| 103 | strictArgsType?: string; | ||
| 104 | /** Full-fidelity return type hint for strict editor/typecheck usage. */ | ||
| 105 | strictReturnsType?: string; | ||
| 100 | operationId?: string; | 106 | operationId?: string; |
| 101 | } | 107 | } |
| 102 | 108 | ||
| @@ -215,6 +221,10 @@ export interface ToolTypeMetadata { · 112 unmodified lines | |||
| 215 | argsType?: string; | 221 | argsType?: string; |
| 216 | /** Lightweight TS type hint for return value (for LLM prompt / discover tool). */ | 222 | /** Lightweight TS type hint for return value (for LLM prompt / discover tool). */ |
| 217 | returnsType?: string; | 223 | returnsType?: string; |
| 224 | /** Compact args hint intended for UI/discovery display. */ | ||
| 225 | displayArgsType?: string; | ||
| 226 | /** Compact return hint intended for UI/discovery display. */ | ||
| 227 | displayReturnsType?: string; | ||
| 218 | /** Raw operationId from the OpenAPI spec (used to generate typechecker wrapper). */ | 228 | /** Raw operationId from the OpenAPI spec (used to generate typechecker wrapper). */ |
| 219 | operationId?: string; | 229 | operationId?: string; |
| 220 | /** | 230 | /** |
| @@ -9,8 +9,10 @@ import type { Id } from "../convex/_generated/dataModel"; · 8 unmodified lines | |||
| 9 | import { | 9 | import { |
| 10 | prepareOpenApiSpec, | 10 | prepareOpenApiSpec, |
| 11 | buildOpenApiToolsFromPrepared, | 11 | buildOpenApiToolsFromPrepared, |
| 12 | materializeWorkspaceSnapshot, | ||
| 12 | serializeTools, | 13 | serializeTools, |
| 13 | rehydrateTools, | 14 | rehydrateTools, |
| 15 | type CompiledToolSourceArtifact, | ||
| 14 | type SerializedTool, | 16 | type SerializedTool, |
| 15 | type WorkspaceToolSnapshot, | 17 | type WorkspaceToolSnapshot, |
| 16 | } from "./tool_sources"; | 18 | } from "./tool_sources"; |
| @@ -361,7 +363,7 @@ describe("serializeTools + rehydrateTools round-trip", () => { · 344 unmodified lines | |||
| 361 | } | 363 | } |
| 362 | }); | 364 | }); |
| 363 | 365 | ||
| 364 | test("full WorkspaceToolSnapshot round-trip", async () => { | 366 | test("WorkspaceToolSnapshot v2 round-trip", async () => { |
| 365 | const prepared = await prepareOpenApiSpec(SMALL_SPEC, "widgets"); | 367 | const prepared = await prepareOpenApiSpec(SMALL_SPEC, "widgets"); |
| 366 | const tools = buildOpenApiToolsFromPrepared( | 368 | const tools = buildOpenApiToolsFromPrepared( |
| 367 | { | 369 | { |
| @@ -373,11 +375,16 @@ describe("serializeTools + rehydrateTools round-trip", () => { · 5 unmodified lines | |||
| 373 | prepared, | 375 | prepared, |
| 374 | ); | 376 | ); |
| 375 | 377 | ||
| 376 | const baseTools = makeBaseTools(); | 378 | const artifact: CompiledToolSourceArtifact = { |
| 377 | const allTools = [...baseTools.values(), ...tools]; | 379 | version: "v1", |
| 380 | sourceType: "openapi", | ||
| 381 | sourceName: "widgets", | ||
| 382 | tools: serializeTools(tools), | ||
| 383 | }; | ||
| 378 | 384 | ||
| 379 | const snapshot: WorkspaceToolSnapshot = { | 385 | const snapshot: WorkspaceToolSnapshot = { |
| 380 | tools: serializeTools(allTools), | 386 | version: "v2", |
| 387 | externalArtifacts: [artifact], | ||
| 381 | warnings: ["test warning"], | 388 | warnings: ["test warning"], |
| 382 | }; | 389 | }; |
| 383 | 390 | ||
| @@ -388,25 +395,19 @@ describe("serializeTools + rehydrateTools round-trip", () => { · 4 unmodified lines | |||
| 388 | const restored = JSON.parse(text) as WorkspaceToolSnapshot; | 395 | const restored = JSON.parse(text) as WorkspaceToolSnapshot; |
| 389 | 396 | ||
| 390 | expect(restored.warnings).toEqual(["test warning"]); | 397 | expect(restored.warnings).toEqual(["test warning"]); |
| 391 | expect(restored.tools.length).toBe(allTools.length); | 398 | expect(restored.version).toBe("v2"); |
| 399 | expect(restored.externalArtifacts).toHaveLength(1); | ||
| 392 | 400 | ||
| 393 | const rehydrated = rehydrateTools(restored.tools, baseTools); | 401 | const restoredTools = materializeWorkspaceSnapshot(restored); |
| 394 | expect(rehydrated.length).toBe(allTools.length); | 402 | expect(restoredTools.length).toBe(tools.length); |
| 395 | 403 | expect(restoredTools.some((tool) => tool.path === "echo")).toBe(false); | |
| 396 | // Builtin echo should work | ||
| 397 | const echoTool = rehydrated.find((t) => t.path === "echo")!; | ||
| 398 | const result = await echoTool.run( | ||
| 399 | { message: "round-trip" }, | ||
| 400 | { taskId: "t", workspaceId: TEST_WORKSPACE_ID, isToolAllowed: () => true }, | ||
| 401 | ); | ||
| 402 | expect(result).toBe("round-trip"); | ||
| 403 | 404 | ||
| 404 | // OpenAPI tools should have correct metadata | 405 | // OpenAPI tools should have correct metadata |
| 405 | const listTool = rehydrated.find((t) => t.metadata?.operationId === "listWidgets")!; | 406 | const listTool = restoredTools.find((t) => t.metadata?.operationId === "listWidgets")!; |
| 406 | expect(listTool.approval).toBe("auto"); // GET = auto | 407 | expect(listTool.approval).toBe("auto"); // GET = auto |
| 407 | expect(listTool.metadata!.argsType).toBeDefined(); | 408 | expect(listTool.metadata!.argsType).toBeDefined(); |
| 408 | 409 | ||
| 409 | const createTool = rehydrated.find((t) => t.metadata?.operationId === "createWidget")!; | 410 | const createTool = restoredTools.find((t) => t.metadata?.operationId === "createWidget")!; |
| 410 | expect(createTool.approval).toBe("required"); // POST = required | 411 | expect(createTool.approval).toBe("required"); // POST = required |
| 411 | }); | 412 | }); |
| 412 | 413 | ||
| @@ -452,8 +453,16 @@ describe("serializeTools + rehydrateTools round-trip", () => { · 39 unmodified lines | |||
| 452 | prepared, | 453 | prepared, |
| 453 | ); | 454 | ); |
| 454 | 455 | ||
| 455 | const snapshot: WorkspaceToolSnapshot = { | 456 | const artifact: CompiledToolSourceArtifact = { |
| 457 | version: "v1", | ||
| 458 | sourceType: "openapi", | ||
| 459 | sourceName: "big-api", | ||
| 456 | tools: serializeTools(tools), | 460 | tools: serializeTools(tools), |
| 461 | }; | ||
| 462 | |||
| 463 | const snapshot: WorkspaceToolSnapshot = { | ||
| 464 | version: "v2", | ||
| 465 | externalArtifacts: [artifact], | ||
| 457 | warnings: [], | 466 | warnings: [], |
| 458 | }; | 467 | }; |
| 459 | 468 | ||
| @@ -14,6 +14,7 @@ · 13 unmodified lines | |||
| 14 | "build:release": "bun run scripts/build-release.ts" | 14 | "build:release": "bun run scripts/build-release.ts" |
| 15 | }, | 15 | }, |
| 16 | "dependencies": { | 16 | "dependencies": { |
| 17 | "convex": "^1.31.7" | 17 | "convex": "^1.31.7", |
| 18 | "graphql": "^16.12.0" | ||
| 18 | } | 19 | } |
| 19 | } | 20 | } |