History

Refactor type hinting functions to improve handling of object intersections and unions. Introduce a new utility to strip outer parentheses from type hints, enhancing the normalization process. Update tests to cover new functionality and ensure accurate type hint generation.

Rhys Sullivan committed 9 months ago

Commit 4873bf21 parent cb248f815 files · +184 −437Browse files

Unified · Split

15 files
  1. executor/apps/web/src/components/tasks/code/editor-monaco.ts +1 −0
  2. executor/apps/web/src/hooks/use/workspace-tools.ts +1 −2
  3. executor/packages/core/src/openapi/schema-hints.test.ts +25 −0
  4. executor/packages/core/src/openapi/schema-hints.ts +5 −1
  5. executor/packages/core/src/tool-sources.ts +0 −45
  6. executor/packages/core/src/type-hints.test.ts +79 −0
  7. executor/packages/core/src/type-hints.ts +67 −4
  8. executor/packages/core/src/workspace/tool-cache.test.ts +0 −233
  9. executor/packages/database/convex/_generated/api.d.ts +0 −21
  10. executor/packages/database/convex/executorNode.github-profile.e2e.test.ts +2 −3
  11. executor/packages/database/convex/schema.ts +0 −21
  12. executor/packages/database/convex/workspaceToolCache.ts +0 −70
  13. executor/packages/database/src/accounts/delete-current-account.ts +0 −12
  14. executor/packages/database/src/runtime/tool_source_loading.ts +1 −1
  15. executor/packages/database/src/runtime/workspace_tools.ts +3 −24
executor/apps/web/src/components/tasks/code/editor-monaco.ts+1−0
@@ -1,6 +1,7 @@
1import type { EditorProps, Monaco } from "@monaco-editor/react";1import type { EditorProps, Monaco } from "@monaco-editor/react";
2 2
3const DIAGNOSTIC_CODES_TO_IGNORE = [3const DIAGNOSTIC_CODES_TO_IGNORE = [
4 1108,
4 1375,5 1375,
5 1378,6 1378,
6 2307,7 2307,
executor/apps/web/src/hooks/use/workspace-tools.ts+1−2
@@ -22,10 +22,9 @@ interface WorkspaceToolsQueryResult { · 21 unmodified lines
22 /** URL to a workspace-wide Monaco `.d.ts` bundle (may be undefined). */22 /** URL to a workspace-wide Monaco `.d.ts` bundle (may be undefined). */
23 typesUrl?: string;23 typesUrl?: string;
24 debug?: {24 debug?: {
25 mode: "cache-fresh" | "cache-stale" | "rebuild" | "registry";25 mode: "rebuild" | "registry";
26 includeDts: boolean;26 includeDts: boolean;
27 sourceTimeoutMs: number | null;27 sourceTimeoutMs: number | null;
28 skipCacheRead: boolean;
29 sourceCount: number;28 sourceCount: number;
30 normalizedSourceCount: number;29 normalizedSourceCount: number;
31 cacheHit: boolean;30 cacheHit: boolean;
executor/packages/core/src/openapi/schema-hints.test.ts+25−0
@@ -78,6 +78,31 @@ test("jsonSchemaTypeHintFallback parenthesizes union inside intersection", () => · 77 unmodified lines
78 expect(hint).toContain("| ");78 expect(hint).toContain("| ");
79});79});
80 80
81test("jsonSchemaTypeHintFallback avoids unnecessary parens around object intersections", () => {
82 const hint = jsonSchemaTypeHintFallback({
83 allOf: [
84 {
85 type: "object",
86 properties: {
87 org: { type: "string" },
88 },
89 required: ["org"],
90 },
91 {
92 type: "object",
93 properties: {
94 enabled_repositories: {
95 enum: ["all", "selected", "none"],
96 },
97 },
98 required: ["enabled_repositories"],
99 },
100 ],
101 });
102
103 expect(hint).toBe('{ org: string } & { enabled_repositories: "all" | "selected" | "none" }');
104});
105
81test("jsonSchemaTypeHintFallback inlines small component schema refs at depth threshold", () => {106test("jsonSchemaTypeHintFallback inlines small component schema refs at depth threshold", () => {
82 const componentSchemas = {107 const componentSchemas = {
83 Pagination: {108 Pagination: {
executor/packages/core/src/openapi/schema-hints.ts+5−1
@@ -240,6 +240,10 @@ function joinUnion(parts: string[]): string { · 239 unmodified lines
240 .join(" | ");240 .join(" | ");
241}241}
242 242
243function hasTopLevelUnion(typeHint: string): boolean {
244 return splitTopLevelBy(typeHint, "|").length > 1;
245}
246
243function isObjectSchema(schema: JsonSchema): boolean {247function isObjectSchema(schema: JsonSchema): boolean {
244 const type = typeof schema.type === "string" ? schema.type : undefined;248 const type = typeof schema.type === "string" ? schema.type : undefined;
245 if (type === "object") return true;249 if (type === "object") return true;
@@ -773,7 +777,7 @@ function joinIntersection(parts: string[]): string { · 527 unmodified lines
773 const expanded = parts.flatMap((part) => splitTopLevelBy(part, "&"));777 const expanded = parts.flatMap((part) => splitTopLevelBy(part, "&"));
774 const unique = dedupeTypeParts(expanded).filter((part) => part !== "unknown");778 const unique = dedupeTypeParts(expanded).filter((part) => part !== "unknown");
775 if (unique.length === 0) return "unknown";779 if (unique.length === 0) return "unknown";
776 const wrapped = unique.map((part) => (part.includes(" | ") ? `(${part})` : part));780 const wrapped = unique.map((part) => (hasTopLevelUnion(part) ? `(${part})` : part));
777 return wrapped.length === 1 ? wrapped[0]! : wrapped.join(" & ");781 return wrapped.length === 1 ? wrapped[0]! : wrapped.join(" & ");
778}782}
779 783
executor/packages/core/src/tool-sources.ts+0−45
@@ -41,12 +41,6 @@ const compiledToolSourceArtifactSchema = z.object({ · 40 unmodified lines
41 tools: z.array(z.unknown()),41 tools: z.array(z.unknown()),
42});42});
43 43
44const workspaceToolSnapshotSchema = z.object({
45 version: z.literal("v2"),
46 externalArtifacts: z.array(z.unknown()),
47 warnings: z.array(z.string()),
48});
49
50export function parseCompiledToolSourceArtifact(value: unknown): Result<CompiledToolSourceArtifact, Error> {44export function parseCompiledToolSourceArtifact(value: unknown): Result<CompiledToolSourceArtifact, Error> {
51 const parsedArtifact = compiledToolSourceArtifactSchema.safeParse(value);45 const parsedArtifact = compiledToolSourceArtifactSchema.safeParse(value);
52 if (!parsedArtifact.success) {46 if (!parsedArtifact.success) {
@@ -70,28 +64,6 @@ export function parseCompiledToolSourceArtifact(value: unknown): Result<Compiled · 17 unmodified lines
70 });64 });
71}65}
72 66
73export function parseWorkspaceToolSnapshot(value: unknown): Result<WorkspaceToolSnapshot, Error> {
74 const parsedSnapshot = workspaceToolSnapshotSchema.safeParse(value);
75 if (!parsedSnapshot.success) {
76 return Result.err(new Error(parsedSnapshot.error.message));
77 }
78
79 const externalArtifacts: CompiledToolSourceArtifact[] = [];
80 for (const artifact of parsedSnapshot.data.externalArtifacts) {
81 const parsedArtifact = parseCompiledToolSourceArtifact(artifact);
82 if (parsedArtifact.isErr()) {
83 return Result.err(new Error(`Invalid workspace snapshot artifact: ${parsedArtifact.error.message}`));
84 }
85 externalArtifacts.push(parsedArtifact.value);
86 }
87
88 return Result.ok({
89 version: parsedSnapshot.data.version,
90 externalArtifacts,
91 warnings: parsedSnapshot.data.warnings,
92 });
93}
94
95async function loadSourceToolDefinitions(source: ExternalToolSourceConfig): Promise<ToolDefinition[]> {67async function loadSourceToolDefinitions(source: ExternalToolSourceConfig): Promise<ToolDefinition[]> {
96 if (source.type === "mcp") {68 if (source.type === "mcp") {
97 return await loadMcpTools(source);69 return await loadMcpTools(source);
@@ -153,20 +125,3 @@ export async function loadExternalTools(sources: ExternalToolSourceConfig[]): Pr · 55 unmodified lines
153 const tools = artifacts.flatMap((artifact) => materializeCompiledToolSource(artifact));125 const tools = artifacts.flatMap((artifact) => materializeCompiledToolSource(artifact));
154 return { tools, warnings };126 return { tools, warnings };
155}127}
156
157// ── Workspace tool cache serialization ──────────────────────────────────────
158//
159// Serializes ToolDefinition[] (minus `run` closures) into a JSON-safe format.
160// On deserialization, `run` functions are reconstructed from stored metadata.
161
162export interface WorkspaceToolSnapshot {
163 version: "v2";
164 externalArtifacts: CompiledToolSourceArtifact[];
165 warnings: string[];
166}
167
168export function materializeWorkspaceSnapshot(
169 snapshot: WorkspaceToolSnapshot,
170): ToolDefinition[] {
171 return snapshot.externalArtifacts.flatMap((artifact) => materializeCompiledToolSource(artifact));
172}
executor/packages/core/src/type-hints.test.ts+79−0
@@ -2,8 +2,10 @@ import { expect, test } from "bun:test"; · 1 unmodified line
2import {2import {
3 compactArgKeysHint,3 compactArgKeysHint,
4 compactArgTypeHint,4 compactArgTypeHint,
5 compactArgDisplayHint,
5 compactDescriptionLine,6 compactDescriptionLine,
6 compactReturnTypeHint,7 compactReturnTypeHint,
8 llmExpandedArgShapeHint,
7 extractTopLevelTypeKeys,9 extractTopLevelTypeKeys,
8} from "./type-hints";10} from "./type-hints";
9 11
@@ -26,11 +28,61 @@ test("compactArgTypeHint flattens simple object intersections", async () => { · 16 unmodified lines
26 expect(compact).toBe("{ owner: string; repo: string; runner_id: number; labels: string[] }");28 expect(compact).toBe("{ owner: string; repo: string; runner_id: number; labels: string[] }");
27});29});
28 30
31test("compactArgTypeHint handles parenthesized object intersections", async () => {
32 const compact = compactArgTypeHint(
33 '{ org: string } & ({ enabled_repositories: "all" | "selected" | "none" })',
34 );
35 expect(compact).toBe('{ org: string; enabled_repositories: "all" | "selected" | "none" }');
36});
37
38test("compactArgTypeHint flattens intersection chains of parenthesized objects", async () => {
39 const compact = compactArgTypeHint(
40 '({ org: string }) & ({ owner: string }) & ({ repo: string })',
41 );
42 expect(compact).toBe('{ org: string; owner: string; repo: string }');
43});
44
45test("compactArgTypeHint strips quotes when flattening quoted keys", async () => {
46 const compact = compactArgTypeHint(
47 '{ org: string } & ({ "enabled_repositories": "all" | "selected" | "none" })',
48 );
49 expect(compact).toBe('{ org: string; enabled_repositories: "all" | "selected" | "none" }');
50});
51
52test("compactArgTypeHint unwraps fully parenthesized intersections", async () => {
53 const compact = compactArgTypeHint('(({ org: string }) & ({ owner: string }))');
54 expect(compact).toBe('{ org: string; owner: string }');
55});
56
57test("compactArgTypeHint does not flatten mixed object and scalar intersections", async () => {
58 const compact = compactArgTypeHint('{ org: string } & ({ enabled_repositories: "all" | "selected" | "none" }) & string');
59 expect(compact).toBe('{ org: string } & ({ enabled_repositories: "all" | "selected" | "none" }) & string');
60});
61
62test("compactArgTypeHint truncates long object-and-array intersections", async () => {
63 const compact = compactArgTypeHint('{ "x-artifact-client-ci"?: string; "x-artifact-client-interactive"?: number; teamId?: string; slug?: string } & (({ sessionId: string; source: "LOCAL" | "REMOTE"; event: "HIT" | "MISS"; hash: string; duration?: number }[])');
64 expect(compact.startsWith('{ "x-artifact-client-ci"?: string; "x-artifact-client-interactive"?: number; teamId?: string; slug?: string } & (({ s')).toBe(true);
65 expect(compact.endsWith('...')).toBe(true);
66 expect(compact.length).toBeLessThanOrEqual(120);
67});
68
69test("compactArgTypeHint truncates long intersected object lists", async () => {
70 const compact = compactArgTypeHint('{ a: string } & ({ b: string }) & ({ c: string }) & ({ d: string }) & ({ e: string }) & ({ f: string }) & ({ g: string }) & ({ h: string }) & ({ i: string })');
71 expect(compact).toMatch(/^\{ a: string \} & \(\{ b: string \}\) \& \(\{ c: string \}\) \& \(\{ d: string \}\) \& \(\{ e: string \}\) \& \(\{ f: string \}\) \& \(\{ g: strin\.\.\.\)?$/);
72 expect(compact.length).toBeLessThanOrEqual(120);
73});
74
29test("compactArgTypeHint dedupes trivial unions while preserving concrete types", async () => {75test("compactArgTypeHint dedupes trivial unions while preserving concrete types", async () => {
30 const compact = compactArgTypeHint("{ app_id: string | string; account_id: string }");76 const compact = compactArgTypeHint("{ app_id: string | string; account_id: string }");
31 expect(compact).toBe("{ app_id: string; account_id: string }");77 expect(compact).toBe("{ app_id: string; account_id: string }");
32});78});
33 79
80test("compactArgDisplayHint uses preview keys when compacted to key-list", async () => {
81 const argsType = `{ ${Array.from({ length: 20 }, (_, i) => `long_key_name_${i}: { nested: { nested: string } }`).join("; ")} }`;
82 const compact = compactArgDisplayHint(argsType, ["owner", "repo", "cursor", "per_page"]);
83 expect(compact).toBe("{ owner: ...; repo: ...; cursor: ...; per_page: ... }");
84});
85
34test("compactArgKeysHint caps key list with ellipsis", async () => {86test("compactArgKeysHint caps key list with ellipsis", async () => {
35 const compact = compactArgKeysHint(["a", "b", "c", "d", "e", "f", "g", "h"]);87 const compact = compactArgKeysHint(["a", "b", "c", "d", "e", "f", "g", "h"]);
36 expect(compact).toBe("{ a: ...; b: ...; c: ...; d: ...; e: ...; f: ...; ... }");88 expect(compact).toBe("{ a: ...; b: ...; c: ...; d: ...; e: ...; f: ...; ... }");
@@ -50,7 +102,34 @@ test("compactReturnTypeHint flattens simple object intersections", async () => { · 13 unmodified lines
50 );102 );
51});103});
52 104
105test("compactReturnTypeHint keeps graphql envelope compaction", async () => {
106 const compact = compactReturnTypeHint('{ data: { id: string } } & ({ errors: { code: number } })');
107 expect(compact).toBe('{ data: ...; errors: unknown[] }');
108});
109
110test("compactReturnTypeHint unwraps fully parenthesized envelopes", async () => {
111 const compact = compactReturnTypeHint('(({ data: { id: string } }) & ({ errors: { code: number } }))');
112 expect(compact).toBe('{ data: ...; errors: unknown[] }');
113});
114
115test("compactReturnTypeHint truncates object-and-array intersections", async () => {
116 const compact = compactReturnTypeHint('{ "x-artifact-client-ci"?: string; "x-artifact-client-interactive"?: number; teamId?: string; slug?: string } & (({ sessionId: string; source: "LOCAL" | "REMOTE"; event: "HIT" | "MISS"; hash: string; duration?: number }[])');
117 expect(compact.startsWith('{ "x-artifact-client-ci"?: string; "x-artifact-client-interactive"?: number; teamId?: string; slug?: string } & (')).toBe(true);
118 expect(compact.endsWith('...')).toBe(true);
119 expect(compact.length).toBeLessThanOrEqual(130);
120});
121
122test("compactReturnTypeHint truncates huge trailing array hints", async () => {
123 const compact = compactReturnTypeHint(`${"x".repeat(95)}[]`);
124 expect(compact).toBe("Array<...>");
125});
126
53test("compactDescriptionLine trims to a single concise line", async () => {127test("compactDescriptionLine trims to a single concise line", async () => {
54 const compact = compactDescriptionLine("Create a database in Notion.\nThis endpoint supports parent/page context.");128 const compact = compactDescriptionLine("Create a database in Notion.\nThis endpoint supports parent/page context.");
55 expect(compact).toBe("Create a database in Notion.");129 expect(compact).toBe("Create a database in Notion.");
56});130});
131
132test("llmExpandedArgShapeHint keeps concise inline object signatures", async () => {
133 const compact = llmExpandedArgShapeHint("{ owner: string; repo: string }");
134 expect(compact).toBe("{ owner: string; repo: string }");
135});
executor/packages/core/src/type-hints.ts+67−4
@@ -53,6 +53,68 @@ function dedupeStrings(values: string[]): string[] { · 52 unmodified lines
53 return unique;53 return unique;
54}54}
55 55
56function stripOuterObjectParens(typeHint: string): string {
57 let current = typeHint.trim();
58
59 while (current.startsWith("(") && current.endsWith(")")) {
60 let depth = 0;
61 let inSingle = false;
62 let inDouble = false;
63 let escaped = false;
64 let closesAtEnd = false;
65
66 for (let i = 0; i < current.length; i += 1) {
67 const char = current[i]!;
68
69 if (escaped) {
70 escaped = false;
71 continue;
72 }
73
74 if (char === "\\" && (inSingle || inDouble)) {
75 escaped = true;
76 continue;
77 }
78
79 if (!inDouble && char === "'") {
80 inSingle = !inSingle;
81 continue;
82 }
83
84 if (!inSingle && char === "\"") {
85 inDouble = !inDouble;
86 continue;
87 }
88
89 if (inSingle || inDouble) {
90 continue;
91 }
92
93 if (char === "(") {
94 depth += 1;
95 } else if (char === ")") {
96 depth -= 1;
97 }
98
99 if (depth === 0 && i < current.length - 1) {
100 closesAtEnd = false;
101 break;
102 }
103 if (depth === 0 && i === current.length - 1) {
104 closesAtEnd = true;
105 }
106 }
107
108 if (!closesAtEnd || depth !== 0) {
109 break;
110 }
111
112 current = current.slice(1, -1).trim();
113 }
114
115 return current;
116}
117
56function normalizeSimpleUnion(typeExpression: string): string {118function normalizeSimpleUnion(typeExpression: string): string {
57 const parts = splitTopLevelBy(typeExpression, "|")119 const parts = splitTopLevelBy(typeExpression, "|")
58 .map((part) => part.trim())120 .map((part) => part.trim())
@@ -111,7 +173,8 @@ function normalizeIntersectionObjectHint( · 52 unmodified lines
111 const keyToType = new Map<string, string>();173 const keyToType = new Map<string, string>();
112 174
113 for (const part of parts) {175 for (const part of parts) {
114 const trimmed = part.trim();176 const normalizedPart = stripOuterObjectParens(part);
177 const trimmed = normalizedPart.trim();
115 if (!trimmed.startsWith("{") || !trimmed.endsWith("}")) return null;178 if (!trimmed.startsWith("{") || !trimmed.endsWith("}")) return null;
116 179
117 const inner = trimmed.slice(1, -1);180 const inner = trimmed.slice(1, -1);
@@ -223,7 +286,7 @@ export function compactArgKeysHint(keys: string[]): string { · 105 unmodified lines
223 286
224export function compactArgTypeHint(argsType: string): string {287export function compactArgTypeHint(argsType: string): string {
225 if (argsType === "{}") return "{}";288 if (argsType === "{}") return "{}";
226 const normalized = argsType.replace(/\s+/g, " ").trim();289 const normalized = stripOuterObjectParens(argsType.replace(/\s+/g, " ").trim());
227 290
228 const flattenedIntersection = normalizeIntersectionObjectHint(normalized, { maxLength: 160 });291 const flattenedIntersection = normalizeIntersectionObjectHint(normalized, { maxLength: 160 });
229 if (flattenedIntersection) {292 if (flattenedIntersection) {
@@ -271,8 +334,8 @@ export function compactArgDisplayHint(argsType: string, argPreviewKeys: string[] · 41 unmodified lines
271}334}
272 335
273export function compactReturnTypeHint(returnsType: string): string {336export function compactReturnTypeHint(returnsType: string): string {
274 const normalized = returnsType.replace(/\s+/g, " ").trim();337 const normalized = stripOuterObjectParens(returnsType.replace(/\s+/g, " ").trim());
275 if (normalized.startsWith("{ data:") && normalized.includes("errors:")) {338 if (normalized.includes("{ data:") && normalized.includes("errors:")) {
276 return "{ data: ...; errors: unknown[] }";339 return "{ data: ...; errors: unknown[] }";
277 }340 }
278 const flattenedIntersection = normalizeIntersectionObjectHint(normalized, {341 const flattenedIntersection = normalizeIntersectionObjectHint(normalized, {
executor/packages/core/src/workspace/tool-cache.test.ts+0−233
@@ -10,12 +10,8 @@ import type { Id } from "../../../database/convex/_generated/dataModel"; · 9 unmodified lines
10import {10import {
11 prepareOpenApiSpec,11 prepareOpenApiSpec,
12 buildOpenApiToolsFromPrepared,12 buildOpenApiToolsFromPrepared,
13 materializeWorkspaceSnapshot,
14 parseWorkspaceToolSnapshot,
15 serializeTools,13 serializeTools,
16 rehydrateTools,14 rehydrateTools,
17 type CompiledToolSourceArtifact,
18 type WorkspaceToolSnapshot,
19} from "../tool-sources";15} from "../tool-sources";
20import type { ToolDefinition } from "../types";16import type { ToolDefinition } from "../types";
21 17
@@ -386,121 +382,6 @@ describe("serializeTools + rehydrateTools round-trip", () => { · 364 unmodified lines
386 }382 }
387 });383 });
388 384
389 test("WorkspaceToolSnapshot v2 round-trip", async () => {
390 const prepared = await prepareOpenApiSpec(SMALL_SPEC, "widgets");
391 const tools = buildOpenApiToolsFromPrepared(
392 {
393 type: "openapi",
394 name: "widgets",
395 spec: SMALL_SPEC,
396 baseUrl: "https://api.example.com",
397 },
398 prepared,
399 );
400
401 const artifact: CompiledToolSourceArtifact = {
402 version: "v1",
403 sourceType: "openapi",
404 sourceName: "widgets",
405 tools: serializeTools(tools),
406 };
407
408 const snapshot: WorkspaceToolSnapshot = {
409 version: "v2",
410 externalArtifacts: [artifact],
411 warnings: ["test warning"],
412 };
413
414 // Simulate storage: serialize → blob → read → deserialize
415 const json = JSON.stringify(snapshot);
416 const blob = new Blob([json], { type: "application/json" });
417 const text = await blob.text();
418 const restored = parseWorkspaceToolSnapshot(JSON.parse(text));
419 expect(restored.isOk()).toBe(true);
420 if (restored.isErr()) {
421 throw restored.error;
422 }
423
424 expect(restored.value.warnings).toEqual(["test warning"]);
425 expect(restored.value.version).toBe("v2");
426 expect(restored.value.externalArtifacts).toHaveLength(1);
427
428 const restoredTools = materializeWorkspaceSnapshot(restored.value);
429 expect(restoredTools.length).toBe(tools.length);
430 expect(restoredTools.some((tool) => tool.path === "echo")).toBe(false);
431
432 // OpenAPI tools should have correct metadata
433 const listTool = restoredTools.find((t) => t.typing?.typedRef?.operationId === "listWidgets")!;
434 expect(listTool.approval).toBe("auto"); // GET = auto
435 expect(listTool.typing!.inputSchema).toBeDefined();
436
437 const createTool = restoredTools.find((t) => t.typing?.typedRef?.operationId === "createWidget")!;
438 expect(createTool.approval).toBe("required"); // POST = required
439 });
440
441 test("snapshot size is reasonable", async () => {
442 // Build tools from a spec with many operations
443 const paths: Record<string, unknown> = {};
444 for (let i = 0; i < 100; i++) {
445 paths[`/resource_${i}`] = {
446 get: {
447 operationId: `getResource${i}`,
448 tags: [`resource_${i}`],
449 summary: `Get resource ${i}`,
450 parameters: [
451 { name: "id", in: "query", schema: { type: "string" } },
452 ],
453 responses: {
454 "200": {
455 description: "ok",
456 content: {
457 "application/json": {
458 schema: {
459 type: "object",
460 properties: { id: { type: "string" }, name: { type: "string" } },
461 },
462 },
463 },
464 },
465 },
466 },
467 };
468 }
469
470 const spec = {
471 openapi: "3.0.3",
472 info: { title: "Big API", version: "1.0.0" },
473 servers: [{ url: "https://api.example.com" }],
474 paths,
475 };
476
477 const prepared = await prepareOpenApiSpec(spec, "big-api");
478 const tools = buildOpenApiToolsFromPrepared(
479 { type: "openapi", name: "big-api", spec, baseUrl: "https://api.example.com" },
480 prepared,
481 );
482
483 const artifact: CompiledToolSourceArtifact = {
484 version: "v1",
485 sourceType: "openapi",
486 sourceName: "big-api",
487 tools: serializeTools(tools),
488 };
489
490 const snapshot: WorkspaceToolSnapshot = {
491 version: "v2",
492 externalArtifacts: [artifact],
493 warnings: [],
494 };
495
496 const json = JSON.stringify(snapshot);
497 const sizeKB = json.length / 1024;
498 console.log(`100-tool snapshot: ${sizeKB.toFixed(0)}KB`);
499
500 // Should be well under 1MB for 100 tools
501 expect(json.length).toBeLessThan(1_000_000);
502 });
503
504 test("invalid serialized tools fail with explicit erroring placeholder", async () => {385 test("invalid serialized tools fail with explicit erroring placeholder", async () => {
505 const invalidSerialized = [386 const invalidSerialized = [
506 {387 {
@@ -528,117 +409,3 @@ describe("serializeTools + rehydrateTools round-trip", () => { · 21 unmodified lines
528 ).rejects.toThrow("Invalid serialized tool 'broken.tool'");409 ).rejects.toThrow("Invalid serialized tool 'broken.tool'");
529 });410 });
530});411});
531
532describe("workspace tool cache table", () => {
533 async function setupCacheTest() {
534 const { convexTest } = await import("convex-test");
535 const { internal } = await import("../../../database/convex/_generated/api");
536 const schema = (await import("../../../database/convex/schema")).default;
537
538 const t = convexTest(schema, {
539 "./workspaceToolCache.ts": () => import("../../../database/convex/workspaceToolCache"),
540 "./_generated/api.js": () => import("../../../database/convex/_generated/api.js"),
541 });
542
543 const wsId = await t.run(async (ctx) => {
544 const orgId = await ctx.db.insert("organizations", {
545 name: "test-org",
546 slug: "test-org",
547 status: "active" as const,
548 createdAt: Date.now(),
549 updatedAt: Date.now(),
550 });
551 return await ctx.db.insert("workspaces", {
552 name: "test-ws",
553 slug: "test-ws",
554 organizationId: orgId,
555 createdAt: Date.now(),
556 updatedAt: Date.now(),
557 });
558 });
559
560 return { t, internal, wsId };
561 }
562
563 test("getEntry + putEntry round-trip", async () => {
564 const { t, internal, wsId } = await setupCacheTest();
565
566 // Empty cache
567 const miss = await t.query(internal.workspaceToolCache.getEntry, {
568 workspaceId: wsId,
569 signature: "sig_1",
570 });
571 expect(miss).toBeNull();
572
573 // Store
574 const storageId = await t.run(async (ctx) => {
575 const blob = new Blob(['{"tools":[],"warnings":[]}'], { type: "application/json" });
576 return await ctx.storage.store(blob);
577 });
578
579 await t.mutation(internal.workspaceToolCache.putEntry, {
580 workspaceId: wsId,
581 signature: "sig_1",
582 storageId,
583 toolCount: 0,
584 sizeBytes: 27,
585 });
586
587 // Hit
588 const hit = await t.query(internal.workspaceToolCache.getEntry, {
589 workspaceId: wsId,
590 signature: "sig_1",
591 });
592 expect(hit).not.toBeNull();
593 expect(hit!.storageId).toBe(storageId);
594
595 // Wrong signature = stale entry
596 const wrongSig = await t.query(internal.workspaceToolCache.getEntry, {
597 workspaceId: wsId,
598 signature: "sig_2",
599 });
600 expect(wrongSig).not.toBeNull();
601 expect(wrongSig!.isFresh).toBe(false);
602 expect(wrongSig!.storageId).toBe(storageId);
603 });
604
605 test("putEntry replaces old entry and deletes old blob", async () => {
606 const { t, internal, wsId } = await setupCacheTest();
607
608 const storageId1 = await t.run(async (ctx) => {
609 return await ctx.storage.store(new Blob(["old"]));
610 });
611
612 await t.mutation(internal.workspaceToolCache.putEntry, {
613 workspaceId: wsId,
614 signature: "sig_1",
615 storageId: storageId1,
616 toolCount: 5,
617 sizeBytes: 3,
618 });
619
620 const storageId2 = await t.run(async (ctx) => {
621 return await ctx.storage.store(new Blob(["new"]));
622 });
623
624 await t.mutation(internal.workspaceToolCache.putEntry, {
625 workspaceId: wsId,
626 signature: "sig_2",
627 storageId: storageId2,
628 toolCount: 10,
629 sizeBytes: 3,
630 });
631
632 // New entry
633 const entry = await t.query(internal.workspaceToolCache.getEntry, {
634 workspaceId: wsId,
635 signature: "sig_2",
636 });
637 expect(entry!.storageId).toBe(storageId2);
638 expect(entry!.toolCount).toBe(10);
639
640 // Old blob deleted
641 const oldBlob = await t.run(async (ctx) => ctx.storage.get(storageId1));
642 expect(oldBlob).toBeNull();
643 });
644});
executor/packages/database/convex/_generated/api.d.ts+0−21
@@ -1312,27 +1312,6 @@ export declare const internal: { · 1311 unmodified lines
1312 any1312 any
1313 >;1313 >;
1314 };1314 };
1315 workspaceToolCache: {
1316 getEntry: FunctionReference<
1317 "query",
1318 "internal",
1319 { signature: string; workspaceId: Id<"workspaces"> },
1320 any
1321 >;
1322 putEntry: FunctionReference<
1323 "mutation",
1324 "internal",
1325 {
1326 signature: string;
1327 sizeBytes: number;
1328 storageId: Id<"_storage">;
1329 toolCount: number;
1330 typesStorageId?: Id<"_storage">;
1331 workspaceId: Id<"workspaces">;
1332 },
1333 any
1334 >;
1335 };
1336};1315};
1337 1316
1338export declare const components: {1317export declare const components: {
executor/packages/database/convex/executorNode.github-profile.e2e.test.ts+2−3
@@ -11,7 +11,6 @@ function setup() { · 10 unmodified lines
11 "./database.ts": () => import("./database"),11 "./database.ts": () => import("./database"),
12 "./executorNode.ts": () => import("./executorNode"),12 "./executorNode.ts": () => import("./executorNode"),
13 "./workspaceAuthInternal.ts": () => import("./workspaceAuthInternal"),13 "./workspaceAuthInternal.ts": () => import("./workspaceAuthInternal"),
14 "./workspaceToolCache.ts": () => import("./workspaceToolCache"),
15 "./toolRegistry.ts": () => import("./toolRegistry"),14 "./toolRegistry.ts": () => import("./toolRegistry"),
16 "./openApiSpecCache.ts": () => import("./openApiSpecCache"),15 "./openApiSpecCache.ts": () => import("./openApiSpecCache"),
17 "./_generated/api.js": () => import("./_generated/api.js"),16 "./_generated/api.js": () => import("./_generated/api.js"),
@@ -58,9 +57,9 @@ test("convex-test keeps GitHub inventory build warm-cache fast", async () => { · 40 unmodified lines
58 expect(typeof cold.typesUrl).toBe("string");57 expect(typeof cold.typesUrl).toBe("string");
59 expect(typeof warm.typesUrl).toBe("string");58 expect(typeof warm.typesUrl).toBe("string");
60 expect(cold.typesUrl).toBe(warm.typesUrl);59 expect(cold.typesUrl).toBe(warm.typesUrl);
61 expect(warm.debug.skipCacheRead).toBe(false);
62 expect(warm.debug.cacheHit).toBe(true);60 expect(warm.debug.cacheHit).toBe(true);
63 expect(warm.debug.mode).toBe("cache-fresh");61 expect(warm.debug.cacheFresh).toBe(true);
62 expect(warm.debug.mode).toBe("registry");
64 63
65 expect(coldMs).toBeLessThan(12_000);64 expect(coldMs).toBeLessThan(12_000);
66 expect(coldMs).toBeGreaterThan(warmMs * 3);65 expect(coldMs).toBeGreaterThan(warmMs * 3);
executor/packages/database/convex/schema.ts+0−21
@@ -426,27 +426,6 @@ export default defineSchema({ · 425 unmodified lines
426 })426 })
427 .index("by_spec_url_version", ["specUrl", "version"]),427 .index("by_spec_url_version", ["specUrl", "version"]),
428 428
429 // Cached, materialized tool catalog for a workspace.
430 // This stores large artifacts (compiled tool definitions and per-source .d.ts files)
431 // in `_storage` and keeps pointers + metadata here.
432 workspaceToolCache: defineTable({
433 workspaceId: v.id("workspaces"),
434 signature: v.string(),
435 storageId: v.id("_storage"),
436 /** Legacy per-source OpenAPI .d.ts blobs. No longer used; retained for safe schema upgrades. */
437 dtsStorageIds: v.optional(v.array(v.object({
438 sourceKey: v.string(),
439 storageId: v.id("_storage"),
440 sizeBytes: v.number(),
441 }))),
442 /** Workspace-wide Monaco type bundle (.d.ts) stored separately. */
443 typesStorageId: v.optional(v.id("_storage")),
444 toolCount: v.number(),
445 sizeBytes: v.number(),
446 createdAt: v.number(),
447 })
448 .index("by_workspace", ["workspaceId"]),
449
450 // Workspace tool registry state.429 // Workspace tool registry state.
451 // Stores the currently "ready" build id for search + invocation.430 // Stores the currently "ready" build id for search + invocation.
452 workspaceToolRegistryState: defineTable({431 workspaceToolRegistryState: defineTable({
executor/packages/database/convex/workspaceToolCache.ts+0−70
@@ -1,70 +0,0 @@
1import { v } from "convex/values";
2import { internalMutation, internalQuery } from "./_generated/server";
3
4/**
5 * Look up a cached workspace tool snapshot by workspace ID.
6 * Returns the storageId if the signature matches (sources haven't changed).
7 */
8export const getEntry = internalQuery({
9 args: {
10 workspaceId: v.id("workspaces"),
11 signature: v.string(),
12 },
13 handler: async (ctx, args) => {
14 const entry = await ctx.db
15 .query("workspaceToolCache")
16 .withIndex("by_workspace", (q) => q.eq("workspaceId", args.workspaceId))
17 .unique();
18
19 if (!entry) return null;
20
21 return {
22 isFresh: entry.signature === args.signature,
23 storageId: entry.storageId,
24 typesStorageId: entry.typesStorageId,
25 toolCount: entry.toolCount,
26 sizeBytes: entry.sizeBytes,
27 createdAt: entry.createdAt,
28 };
29 },
30});
31
32/**
33 * Write (or replace) a workspace tool cache entry.
34 * Deletes old blobs (main snapshot + .d.ts blobs) if replacing.
35 */
36export const putEntry = internalMutation({
37 args: {
38 workspaceId: v.id("workspaces"),
39 signature: v.string(),
40 storageId: v.id("_storage"),
41 typesStorageId: v.optional(v.id("_storage")),
42 toolCount: v.number(),
43 sizeBytes: v.number(),
44 },
45 handler: async (ctx, args) => {
46 const existing = await ctx.db
47 .query("workspaceToolCache")
48 .withIndex("by_workspace", (q) => q.eq("workspaceId", args.workspaceId))
49 .unique();
50
51 if (existing) {
52 // Delete old main snapshot blob
53 await ctx.storage.delete(existing.storageId).catch(() => {});
54 if (existing.typesStorageId) {
55 await ctx.storage.delete(existing.typesStorageId).catch(() => {});
56 }
57 await ctx.db.delete(existing._id);
58 }
59
60 await ctx.db.insert("workspaceToolCache", {
61 workspaceId: args.workspaceId,
62 signature: args.signature,
63 storageId: args.storageId,
64 typesStorageId: args.typesStorageId,
65 toolCount: args.toolCount,
66 sizeBytes: args.sizeBytes,
67 createdAt: Date.now(),
68 });
69 },
70});
executor/packages/database/src/accounts/delete-current-account.ts+0−12
@@ -63,18 +63,6 @@ async function deleteWorkspaceData( · 62 unmodified lines
63 await ctx.db.delete(source._id);63 await ctx.db.delete(source._id);
64 }64 }
65 65
66 const cachedToolsets = await ctx.db
67 .query("workspaceToolCache")
68 .withIndex("by_workspace", (q) => q.eq("workspaceId", workspaceId))
69 .collect();
70 for (const cacheEntry of cachedToolsets) {
71 await ctx.storage.delete(cacheEntry.storageId).catch(() => {});
72 if (cacheEntry.typesStorageId) {
73 await ctx.storage.delete(cacheEntry.typesStorageId).catch(() => {});
74 }
75 await ctx.db.delete(cacheEntry._id);
76 }
77
78 const workspaceMembers = await ctx.db66 const workspaceMembers = await ctx.db
79 .query("workspaceMembers")67 .query("workspaceMembers")
80 .withIndex("by_workspace", (q) => q.eq("workspaceId", workspaceId))68 .withIndex("by_workspace", (q) => q.eq("workspaceId", workspaceId))
executor/packages/database/src/runtime/tool_source_loading.ts+1−1
@@ -30,7 +30,7 @@ import { normalizeToolSourceConfig } from "../database/tool_source_config"; · 29 unmodified lines
30 30
31const OPENAPI_SPEC_CACHE_TTL_MS = 5 * 60 * 60_000;31const OPENAPI_SPEC_CACHE_TTL_MS = 5 * 60 * 60_000;
32 32
33/** Cache version - bump when tool snapshot/registry/type-hint semantics change. */33/** Cache version - bump when registry build/type-hint semantics change. */
34const TOOL_SOURCE_CACHE_VERSION = "v26";34const TOOL_SOURCE_CACHE_VERSION = "v26";
35const OPENAPI_PREPARED_CACHE_VERSION = "openapi_v2";35const OPENAPI_PREPARED_CACHE_VERSION = "openapi_v2";
36 36
executor/packages/database/src/runtime/workspace_tools.ts+3−24
@@ -232,10 +232,9 @@ interface WorkspaceToolsResult { · 231 unmodified lines
232}232}
233 233
234export interface WorkspaceToolsDebug {234export interface WorkspaceToolsDebug {
235 mode: "cache-fresh" | "cache-stale" | "rebuild" | "registry";235 mode: "rebuild" | "registry";
236 includeDts: boolean;236 includeDts: boolean;
237 sourceTimeoutMs: number | null;237 sourceTimeoutMs: number | null;
238 skipCacheRead: boolean;
239 sourceCount: number;238 sourceCount: number;
240 normalizedSourceCount: number;239 normalizedSourceCount: number;
241 cacheHit: boolean;240 cacheHit: boolean;
@@ -247,8 +246,6 @@ export interface WorkspaceToolsDebug { · 5 unmodified lines
247 246
248interface GetWorkspaceToolsOptions {247interface GetWorkspaceToolsOptions {
249 sourceTimeoutMs?: number;248 sourceTimeoutMs?: number;
250 allowStaleOnMismatch?: boolean;
251 skipCacheRead?: boolean;
252 accountId?: Id<"accounts">;249 accountId?: Id<"accounts">;
253}250}
254 251
@@ -693,20 +690,14 @@ export async function getWorkspaceTools( · 438 unmodified lines
693 const accountId = options.accountId;690 const accountId = options.accountId;
694 const sources = (await listWorkspaceToolSources(ctx, workspaceId))691 const sources = (await listWorkspaceToolSources(ctx, workspaceId))
695 .filter((source) => source.enabled);692 .filter((source) => source.enabled);
696 const skipCacheRead = options.skipCacheRead ?? false;
697 traceStep("listToolSources", listSourcesStartedAt);693 traceStep("listToolSources", listSourcesStartedAt);
698 const registrySignature = registrySignatureForWorkspace(workspaceId, sources);694 const registrySignature = registrySignatureForWorkspace(workspaceId, sources);
699 const debugBase: Omit<WorkspaceToolsDebug, "mode" | "normalizedSourceCount" | "cacheHit" | "cacheFresh" | "timedOutSources" | "durationMs" | "trace"> = {695 const debugBase: Omit<WorkspaceToolsDebug, "mode" | "normalizedSourceCount" | "cacheHit" | "cacheFresh" | "timedOutSources" | "durationMs" | "trace"> = {
700 includeDts,696 includeDts,
701 sourceTimeoutMs: sourceTimeoutMs ?? null,697 sourceTimeoutMs: sourceTimeoutMs ?? null,
702 skipCacheRead,
703 sourceCount: sources.length,698 sourceCount: sources.length,
704 };699 };
705 700
706 if (skipCacheRead) {
707 trace.push("cacheEntryLookup=skipped");
708 }
709
710 const configs: ExternalToolSourceConfig[] = [];701 const configs: ExternalToolSourceConfig[] = [];
711 const warnings: string[] = [];702 const warnings: string[] = [];
712 const normalizeSourcesStartedAt = Date.now();703 const normalizeSourcesStartedAt = Date.now();
@@ -876,7 +867,6 @@ async function getWorkspaceToolsFromRegistry( · 163 unmodified lines
876 const debugBase: Omit<WorkspaceToolsDebug, "mode" | "normalizedSourceCount" | "cacheHit" | "cacheFresh" | "timedOutSources" | "durationMs" | "trace"> = {867 const debugBase: Omit<WorkspaceToolsDebug, "mode" | "normalizedSourceCount" | "cacheHit" | "cacheFresh" | "timedOutSources" | "durationMs" | "trace"> = {
877 includeDts,868 includeDts,
878 sourceTimeoutMs: null,869 sourceTimeoutMs: null,
879 skipCacheRead: false,
880 sourceCount: sources.length,870 sourceCount: sources.length,
881 };871 };
882 872
@@ -1032,9 +1022,6 @@ async function loadWorkspaceToolInventoryForContext( · 149 unmodified lines
1032 includeDetails?: boolean;1022 includeDetails?: boolean;
1033 includeSourceMeta?: boolean;1023 includeSourceMeta?: boolean;
1034 toolPaths?: string[];1024 toolPaths?: string[];
1035 sourceTimeoutMs?: number;
1036 allowStaleOnMismatch?: boolean;
1037 skipCacheRead?: boolean;
1038 } = {},1025 } = {},
1039): Promise<WorkspaceToolInventory> {1026): Promise<WorkspaceToolInventory> {
1040 const includeDetails = options.includeDetails ?? true;1027 const includeDetails = options.includeDetails ?? true;
@@ -1119,9 +1106,6 @@ export async function listToolsForContext( · 78 unmodified lines
1119 includeDetails?: boolean;1106 includeDetails?: boolean;
1120 includeSourceMeta?: boolean;1107 includeSourceMeta?: boolean;
1121 toolPaths?: string[];1108 toolPaths?: string[];
1122 sourceTimeoutMs?: number;
1123 allowStaleOnMismatch?: boolean;
1124 skipCacheRead?: boolean;
1125 } = {},1109 } = {},
1126): Promise<ToolDescriptor[]> {1110): Promise<ToolDescriptor[]> {
1127 const inventory = await loadWorkspaceToolInventoryForContext(ctx, context, {1111 const inventory = await loadWorkspaceToolInventoryForContext(ctx, context, {
@@ -1138,8 +1122,6 @@ export async function rebuildWorkspaceToolInventoryForContext( · 10 unmodified lines
1138 return await getWorkspaceTools(ctx, context.workspaceId, {1122 return await getWorkspaceTools(ctx, context.workspaceId, {
1139 accountId: context.accountId,1123 accountId: context.accountId,
1140 sourceTimeoutMs: 20_000,1124 sourceTimeoutMs: 20_000,
1141 allowStaleOnMismatch: false,
1142 skipCacheRead: false,
1143 });1125 });
1144}1126}
1145 1127
@@ -1150,9 +1132,6 @@ export async function listToolsWithWarningsForContext( · 4 unmodified lines
1150 includeDetails?: boolean;1132 includeDetails?: boolean;
1151 includeSourceMeta?: boolean;1133 includeSourceMeta?: boolean;
1152 toolPaths?: string[];1134 toolPaths?: string[];
1153 sourceTimeoutMs?: number;
1154 allowStaleOnMismatch?: boolean;
1155 skipCacheRead?: boolean;
1156 } = {},1135 } = {},
1157): Promise<{1136): Promise<{
1158 tools: ToolDescriptor[];1137 tools: ToolDescriptor[];