History

fix(v2): paginate tool catalog reads and expand default source set

Rhys Sullivan committed 8 months ago

Commit 4be71f41 parent 686206611 files · +384 −123Browse files

Unified · Split

11 files
  1. apps/convex/convex/_generated/api.d.ts +42 −0
  2. apps/convex/convex/control_plane/service.ts +2 −2
  3. apps/convex/convex/control_plane/tool_registry.ts +97 −23
  4. apps/convex/convex/control_plane/tools.ts +116 −87
  5. apps/convex/convex/convex.config.ts +2 −0
  6. apps/convex/convex/executor.test.ts +10 −9
  7. apps/convex/convex/source_tool_registry.ts +1 −1
  8. apps/convex/package.json +2 −1
  9. apps/web/components/tools/source-management-panel.tsx +48 −0
  10. bun.lock +3 −0
  11. legacy/executor/apps/web/src/lib/catalog-collections.ts +61 −0
apps/convex/convex/_generated/api.d.ts+42−0
@@ -101,6 +101,48 @@ export declare const internal: FilterApi< · 100 unmodified lines
101>;101>;
102 102
103export declare const components: {103export declare const components: {
104 actionCache: {
105 crons: {
106 purge: FunctionReference<
107 "mutation",
108 "internal",
109 { expiresAt?: number },
110 null
111 >;
112 };
113 lib: {
114 get: FunctionReference<
115 "query",
116 "internal",
117 { args: any; name: string; ttl: number | null },
118 { kind: "hit"; value: any } | { expiredEntry?: string; kind: "miss" }
119 >;
120 put: FunctionReference<
121 "mutation",
122 "internal",
123 {
124 args: any;
125 expiredEntry?: string;
126 name: string;
127 ttl: number | null;
128 value: any;
129 },
130 { cacheHit: boolean; deletedExpiredEntry: boolean }
131 >;
132 remove: FunctionReference<
133 "mutation",
134 "internal",
135 { args: any; name: string },
136 null
137 >;
138 removeAll: FunctionReference<
139 "mutation",
140 "internal",
141 { batchSize?: number; before?: number; name?: string },
142 null
143 >;
144 };
145 };
104 workflow: {146 workflow: {
105 event: {147 event: {
106 create: FunctionReference<148 create: FunctionReference<
apps/convex/convex/control_plane/service.ts+2−2
@@ -122,7 +122,7 @@ export const makeConvexControlPlaneService = ( · 121 unmodified lines
122 listWorkspaceTools: (workspaceId) =>122 listWorkspaceTools: (workspaceId) =>
123 Effect.tryPromise({123 Effect.tryPromise({
124 try: () =>124 try: () =>
125 ctx.runQuery(api.controlPlane.listWorkspaceTools, {125 ctx.runAction(api.controlPlane.listWorkspaceTools, {
126 workspaceId,126 workspaceId,
127 }),127 }),
128 catch: (cause) => toSourceStoreError("controlPlane.listWorkspaceTools", cause),128 catch: (cause) => toSourceStoreError("controlPlane.listWorkspaceTools", cause),
@@ -130,7 +130,7 @@ export const makeConvexControlPlaneService = ( · 1 unmodified line
130 listSourceTools: (input) =>130 listSourceTools: (input) =>
131 Effect.tryPromise({131 Effect.tryPromise({
132 try: () =>132 try: () =>
133 ctx.runQuery(api.controlPlane.listSourceTools, {133 ctx.runAction(api.controlPlane.listSourceTools, {
134 workspaceId: input.workspaceId,134 workspaceId: input.workspaceId,
135 sourceId: input.sourceId,135 sourceId: input.sourceId,
136 }),136 }),
apps/convex/convex/control_plane/tool_registry.ts+97−23
@@ -1,12 +1,18 @@
1import { ActionCache } from "@convex-dev/action-cache";
1import { v } from "convex/values";2import { v } from "convex/values";
2import { internal } from "../_generated/api";3import { components, internal } from "../_generated/api";
3 4
4import { action, internalMutation, internalQuery } from "../_generated/server";5import { action, internalAction, internalMutation, internalQuery } from "../_generated/server";
5 6
6const defaultSearchLimit = 50;7const defaultSearchLimit = 50;
7const maxSearchLimit = 5_000;8const maxSearchLimit = 5_000;
8const writeBatchSize = 500;9const writeBatchSize = 500;
9const runtimeInternal = internal as any;10const runtimeInternal = internal as any;
11const listWorkspaceNamespacesActionCache = new ActionCache((components as any).actionCache, {
12 action: runtimeInternal.control_plane.tool_registry.listWorkspaceNamespacesUncached,
13 name: "workspace_namespaces_v1",
14 ttl: 15_000,
15});
10 16
11const stripConvexSystemFields = (17const stripConvexSystemFields = (
12 value: Record<string, unknown>,18 value: Record<string, unknown>,
@@ -1252,41 +1258,97 @@ export const listWorkspaceToolsByNormalizedPath = internalQuery({ · 1239 unmodified lines
1252 },1258 },
1253});1259});
1254 1260
1255export const listWorkspaceNamespaces = internalQuery({1261export const listWorkspaceToolIndexPage = internalQuery({
1256 args: {1262 args: {
1257 workspaceId: v.string(),1263 workspaceId: v.string(),
1258 limit: v.optional(v.number()),1264 cursor: v.optional(v.union(v.string(), v.null())),
1265 pageSize: v.optional(v.number()),
1259 },1266 },
1260 handler: async (ctx, args) => {1267 handler: async (ctx, args) => {
1261 const rows = await buildWorkspaceToolRows(ctx, {1268 const pageSize = Math.max(1, Math.min(500, Math.floor(args.pageSize ?? 200)));
1262 workspaceId: args.workspaceId,1269 const page = await ctx.db
1263 includeDisabled: false,1270 .query("workspaceToolIndex")
1264 });1271 .withIndex("by_workspaceId", (q) => q.eq("workspaceId", args.workspaceId))
1272 .paginate({
1273 cursor: args.cursor ?? null,
1274 numItems: pageSize,
1275 });
1265 1276
1277 const rows = page.page
1278 .filter((row) => row.status === "active")
1279 .map((row) => stripConvexSystemFields(row as unknown as Record<string, unknown>));
1280
1281 return {
1282 rows,
1283 continueCursor: page.continueCursor,
1284 isDone: page.isDone,
1285 };
1286 },
1287});
1288
1289export const listWorkspaceNamespacesUncached = internalAction({
1290 args: {
1291 workspaceId: v.string(),
1292 limit: v.optional(v.number()),
1293 },
1294 handler: async (ctx, args) => {
1266 const grouped = new Map<1295 const grouped = new Map<
1267 string,1296 string,
1268 {1297 {
1269 sourceName: string;1298 sourceName: string;
1270 sourceId: string;1299 sourceId: string;
1271 sourceKind: string;1300 sourceKind: string;
1301 toolCount: number;
1272 paths: Array<string>;1302 paths: Array<string>;
1273 }1303 }
1274 >();1304 >();
1275 1305
1276 for (const row of rows) {1306 let cursor: string | null = null;
1277 const namespace = String(row.namespace);1307 do {
1278 const existing = grouped.get(namespace);1308 const page: {
1279 if (existing) {1309 rows: Array<Record<string, unknown>>;
1280 existing.paths.push(String(row.path));1310 continueCursor: string | null;
1281 continue;1311 isDone: boolean;
1312 } = await ctx.runQuery(runtimeInternal.control_plane.tool_registry.listWorkspaceToolIndexPage, {
1313 workspaceId: args.workspaceId,
1314 cursor,
1315 pageSize: 250,
1316 });
1317
1318 for (const row of page.rows) {
1319 const namespace = String(row.namespace);
1320 const path = String(row.path);
1321 const existing = grouped.get(namespace);
1322 if (existing) {
1323 existing.toolCount += 1;
1324
1325 if (existing.paths.length < 3) {
1326 existing.paths.push(path);
1327 existing.paths.sort((left, right) => left.localeCompare(right));
1328 } else if (path.localeCompare(existing.paths[existing.paths.length - 1] ?? path) < 0) {
1329 existing.paths[existing.paths.length - 1] = path;
1330 existing.paths.sort((left, right) => left.localeCompare(right));
1331 }
1332
1333 continue;
1334 }
1335
1336 grouped.set(namespace, {
1337 sourceName: String(row.sourceName),
1338 sourceId: String(row.sourceId),
1339 sourceKind: String(row.sourceKind),
1340 toolCount: 1,
1341 paths: [path],
1342 });
1282 }1343 }
1283 1344
1284 grouped.set(namespace, {1345 cursor = page.isDone ? null : (page.continueCursor as string | null);
1285 sourceName: String(row.sourceName),1346 } while (cursor);
1286 sourceId: String(row.sourceId),1347
1287 sourceKind: String(row.sourceKind),1348 const limit = normalizeLimit(args.limit);
1288 paths: [String(row.path)],1349
1289 });1350 if (limit <= 0 || grouped.size === 0) {
1351 return [];
1290 }1352 }
1291 1353
1292 const namespaces = [...grouped.entries()]1354 const namespaces = [...grouped.entries()]
@@ -1295,11 +1357,23 @@ export const listWorkspaceNamespaces = internalQuery({ · 2 unmodified lines
1295 source: value.sourceName,1357 source: value.sourceName,
1296 sourceId: value.sourceId,1358 sourceId: value.sourceId,
1297 sourceKind: value.sourceKind,1359 sourceKind: value.sourceKind,
1298 toolCount: value.paths.length,1360 toolCount: value.toolCount,
1299 samplePaths: value.paths.sort((left, right) => left.localeCompare(right)).slice(0, 3),1361 samplePaths: value.paths,
1300 }))1362 }))
1301 .sort((left, right) => left.namespace.localeCompare(right.namespace));1363 .sort((left, right) => left.namespace.localeCompare(right.namespace));
1302 1364
1303 return namespaces.slice(0, normalizeLimit(args.limit));1365 return namespaces.slice(0, limit);
1304 },1366 },
1305});1367});
1368
1369export const listWorkspaceNamespaces = internalAction({
1370 args: {
1371 workspaceId: v.string(),
1372 limit: v.optional(v.number()),
1373 },
1374 handler: async (ctx, args) =>
1375 listWorkspaceNamespacesActionCache.fetch(ctx, {
1376 workspaceId: args.workspaceId,
1377 limit: args.limit,
1378 }),
1379});
apps/convex/convex/control_plane/tools.ts+116−87
@@ -6,10 +6,12 @@ import { · 5 unmodified lines
6} from "@executor-v2/schema";6} from "@executor-v2/schema";
7import { v } from "convex/values";7import { v } from "convex/values";
8import * as Schema from "effect/Schema";8import * as Schema from "effect/Schema";
9import { internal } from "../_generated/api";
9 10
10import { query } from "../_generated/server";11import { action, internalQuery, query } from "../_generated/server";
11 12
12const decodeSource = Schema.decodeUnknownSync(SourceSchema);13const decodeSource = Schema.decodeUnknownSync(SourceSchema);
14const runtimeInternal = internal as any;
13 15
14const stripConvexSystemFields = (16const stripConvexSystemFields = (
15 value: Record<string, unknown>,17 value: Record<string, unknown>,
@@ -163,7 +165,7 @@ const resolveToolRefHintTableJson = async ( · 147 unmodified lines
163 165
164 const row = await ctx.db166 const row = await ctx.db
165 .query("artifactSchemaRefs")167 .query("artifactSchemaRefs")
166 .withIndex("by_artifactId_refKey", (q) =>168 .withIndex("by_artifactId_refKey", (q: any) =>
167 q.eq("artifactId", artifactId).eq("refKey", refKey),169 q.eq("artifactId", artifactId).eq("refKey", refKey),
168 )170 )
169 .unique();171 .unique();
@@ -215,6 +217,28 @@ const toSummary = ( · 45 unmodified lines
215 };217 };
216};218};
217 219
220const summaryFromIndexRow = (row: Record<string, unknown>): SourceToolSummary => {
221 const method = asNullableString(row.method)?.toLowerCase() ?? null;
222 const sourceKind = asNullableString(row.sourceKind);
223 const normalizedSourceKind: Source["kind"] =
224 sourceKind === "graphql" || sourceKind === "mcp" || sourceKind === "internal"
225 ? sourceKind
226 : "openapi";
227
228 return {
229 sourceId: (asNullableString(row.sourceId) ?? "") as Source["id"],
230 sourceName: asNullableString(row.sourceName) ?? "",
231 sourceKind: normalizedSourceKind,
232 toolId: asNullableString(row.toolId) ?? "unknown",
233 name: asNullableString(row.name) ?? "Unnamed Tool",
234 description: asNullableString(row.description),
235 method: isMethod(method) ? method : "post",
236 path: asNullableString(row.path) ?? "",
237 operationPath: asNullableString(row.operationPath ?? null),
238 operationHash: asNullableString(row.operationHash) ?? "",
239 };
240};
241
218const sortTools = (tools: ReadonlyArray<SourceToolSummary>): Array<SourceToolSummary> =>242const sortTools = (tools: ReadonlyArray<SourceToolSummary>): Array<SourceToolSummary> =>
219 [...tools].sort((left, right) => {243 [...tools].sort((left, right) => {
220 const leftSource = left.sourceName.toLowerCase();244 const leftSource = left.sourceName.toLowerCase();
@@ -234,91 +258,97 @@ const sortTools = (tools: ReadonlyArray<SourceToolSummary>): Array<SourceToolSum · 13 unmodified lines
234 return left.toolId.localeCompare(right.toolId);258 return left.toolId.localeCompare(right.toolId);
235 });259 });
236 260
237export const listWorkspaceTools = query({261const workspaceToolsPageSize = 250;
262
263export const listWorkspaceToolSummaryPage = internalQuery({
238 args: {264 args: {
239 workspaceId: v.string(),265 workspaceId: v.string(),
266 sourceId: v.optional(v.string()),
267 cursor: v.optional(v.union(v.string(), v.null())),
268 pageSize: v.optional(v.number()),
240 },269 },
241 handler: async (ctx, args): Promise<Array<SourceToolSummary>> => {270 handler: async (ctx, args) => {
242 const sourceRows = await ctx.db271 const pageSize = Math.max(1, Math.min(500, Math.floor(args.pageSize ?? workspaceToolsPageSize)));
243 .query("sources")272 const paginationOptions = {
244 .withIndex("by_workspaceId", (q) => q.eq("workspaceId", args.workspaceId))273 cursor: args.cursor ?? null,
245 .collect();274 numItems: pageSize,
246 275 };
247 const sources = sourceRows
248 .map((row) => toSource(row as unknown as Record<string, unknown>))
249 .filter((source) => source.enabled);
250 const bindings = await ctx.db
251 .query("sourceArtifactBindings")
252 .withIndex("by_workspaceId", (q) => q.eq("workspaceId", args.workspaceId))
253 .collect();
254 const bindingBySourceId = new Map<string, string>(
255 bindings.map((row) => [String(row.sourceId), String(row.artifactId)] as const),
256 );
257
258 const summaries: Array<SourceToolSummary> = [];
259 276
260 for (const source of sources) {277 const page = args.sourceId
261 const artifactId = bindingBySourceId.get(source.id);278 ? await ctx.db
262 if (!artifactId) {279 .query("workspaceToolIndex")
263 continue;280 .withIndex("by_workspaceId_sourceId", (q: any) =>
264 }281 q.eq("workspaceId", args.workspaceId).eq("sourceId", args.sourceId),
282 )
283 .paginate(paginationOptions)
284 : await ctx.db
285 .query("workspaceToolIndex")
286 .withIndex("by_workspaceId", (q: any) => q.eq("workspaceId", args.workspaceId))
287 .paginate(paginationOptions);
288
289 const summaries = page.page
290 .filter((row) => row.status === "active")
291 .map((row) => summaryFromIndexRow(stripConvexSystemFields(row as unknown as Record<string, unknown>)));
265 292
266 const artifactTools = await ctx.db293 return {
267 .query("artifactTools")294 summaries,
268 .withIndex("by_artifactId", (q) => q.eq("artifactId", artifactId))295 continueCursor: page.continueCursor,
269 .collect();296 isDone: page.isDone,
297 };
298 },
299});
270 300
271 for (const artifactTool of artifactTools) {301export const listWorkspaceTools = action({
272 const row = stripConvexSystemFields(artifactTool as unknown as Record<string, unknown>);302 args: {
273 summaries.push(toSummary(source, row, asNullableString(row.metadataJson)));303 workspaceId: v.string(),
274 }304 },
275 }305 handler: async (ctx, args): Promise<Array<SourceToolSummary>> => {
306 const summaries: Array<SourceToolSummary> = [];
307 let cursor: string | null = null;
308
309 do {
310 const page: {
311 summaries: Array<SourceToolSummary>;
312 continueCursor: string | null;
313 isDone: boolean;
314 } = await ctx.runQuery(runtimeInternal.control_plane.tools.listWorkspaceToolSummaryPage, {
315 workspaceId: args.workspaceId,
316 cursor,
317 pageSize: workspaceToolsPageSize,
318 });
319
320 summaries.push(...page.summaries);
321 cursor = page.isDone ? null : page.continueCursor;
322 } while (cursor);
276 323
277 return sortTools(summaries);324 return sortTools(summaries);
278 },325 },
279});326});
280 327
281export const listSourceTools = query({328export const listSourceTools = action({
282 args: {329 args: {
283 workspaceId: v.string(),330 workspaceId: v.string(),
284 sourceId: v.string(),331 sourceId: v.string(),
285 },332 },
286 handler: async (ctx, args): Promise<Array<SourceToolSummary>> => {333 handler: async (ctx, args): Promise<Array<SourceToolSummary>> => {
287 const sourceRow = await ctx.db
288 .query("sources")
289 .withIndex("by_domainId", (q) => q.eq("id", args.sourceId))
290 .unique();
291
292 if (!sourceRow || sourceRow.workspaceId !== args.workspaceId) {
293 return [];
294 }
295
296 const source = toSource(sourceRow as unknown as Record<string, unknown>);
297 if (!source.enabled) {
298 return [];
299 }
300
301 const binding = await ctx.db
302 .query("sourceArtifactBindings")
303 .withIndex("by_workspaceId_sourceId", (q) =>
304 q.eq("workspaceId", args.workspaceId).eq("sourceId", args.sourceId),
305 )
306 .unique();
307 if (!binding) {
308 return [];
309 }
310
311 const artifactTools = await ctx.db
312 .query("artifactTools")
313 .withIndex("by_artifactId", (q) => q.eq("artifactId", binding.artifactId))
314 .collect();
315
316 const summaries: Array<SourceToolSummary> = [];334 const summaries: Array<SourceToolSummary> = [];
317 335 let cursor: string | null = null;
318 for (const artifactTool of artifactTools) {336
319 const row = stripConvexSystemFields(artifactTool as unknown as Record<string, unknown>);337 do {
320 summaries.push(toSummary(source, row, asNullableString(row.metadataJson)));338 const page: {
321 }339 summaries: Array<SourceToolSummary>;
340 continueCursor: string | null;
341 isDone: boolean;
342 } = await ctx.runQuery(runtimeInternal.control_plane.tools.listWorkspaceToolSummaryPage, {
343 workspaceId: args.workspaceId,
344 sourceId: args.sourceId,
345 cursor,
346 pageSize: workspaceToolsPageSize,
347 });
348
349 summaries.push(...page.summaries);
350 cursor = page.isDone ? null : page.continueCursor;
351 } while (cursor);
322 352
323 return sortTools(summaries);353 return sortTools(summaries);
324 },354 },
@@ -341,44 +371,43 @@ export const getToolDetail = query({ · 16 unmodified lines
341 }371 }
342 372
343 const source = toSource(sourceRow as unknown as Record<string, unknown>);373 const source = toSource(sourceRow as unknown as Record<string, unknown>);
344 const binding = await ctx.db374 const indexedRows = await ctx.db
345 .query("sourceArtifactBindings")375 .query("workspaceToolIndex")
346 .withIndex("by_workspaceId_sourceId", (q) =>376 .withIndex("by_workspaceId_operationHash", (q: any) =>
347 q.eq("workspaceId", args.workspaceId).eq("sourceId", args.sourceId),377 q.eq("workspaceId", args.workspaceId).eq("operationHash", args.operationHash),
348 )378 )
349 .unique();379 .collect();
350 if (!binding) {380 const indexedRow = indexedRows.find((row) => row.sourceId === args.sourceId) ?? null;
381 if (!indexedRow) {
351 return null;382 return null;
352 }383 }
353 384
354 const artifactRows = await ctx.db385 const artifactTool = await ctx.db
355 .query("artifactTools")386 .query("artifactTools")
356 .withIndex("by_artifactId", (q) => q.eq("artifactId", binding.artifactId))387 .withIndex("by_artifactId_toolId", (q: any) =>
357 .collect();388 q.eq("artifactId", indexedRow.artifactId).eq("toolId", indexedRow.toolId),
358 389 )
359 const artifactTool = artifactRows.find((row) => row.operationHash === args.operationHash);390 .unique();
360 if (!artifactTool) {391 if (!artifactTool) {
361 return null;392 return null;
362 }393 }
363 394
364 const artifact = await ctx.db395 const artifact = await ctx.db
365 .query("artifacts")396 .query("artifacts")
366 .withIndex("by_domainId", (q) => q.eq("id", binding.artifactId))397 .withIndex("by_domainId", (q) => q.eq("id", indexedRow.artifactId))
367 .unique();398 .unique();
368 399
369 const resolvedRefHintTableJson = source.kind === "openapi"400 const resolvedRefHintTableJson = source.kind === "openapi"
370 ? await resolveToolRefHintTableJson(401 ? await resolveToolRefHintTableJson(
371 ctx,402 ctx,
372 binding.artifactId,403 indexedRow.artifactId,
373 artifactTool.inputSchemaJson ?? null,404 artifactTool.inputSchemaJson ?? null,
374 artifactTool.outputSchemaJson ?? null,405 artifactTool.outputSchemaJson ?? null,
375 )406 )
376 : null;407 : null;
377 408
378 const summary = toSummary(409 const summary = summaryFromIndexRow(
379 source,410 stripConvexSystemFields(indexedRow as unknown as Record<string, unknown>),
380 stripConvexSystemFields(artifactTool as unknown as Record<string, unknown>),
381 artifactTool.metadataJson ?? null,
382 );411 );
383 412
384 return {413 return {
apps/convex/convex/convex.config.ts+2−0
@@ -1,7 +1,9 @@
1import actionCache from "@convex-dev/action-cache/convex.config";
1import workflow from "@convex-dev/workflow/convex.config.js";2import workflow from "@convex-dev/workflow/convex.config.js";
2import { defineApp } from "convex/server";3import { defineApp } from "convex/server";
3 4
4const app = defineApp();5const app = defineApp();
6app.use(actionCache);
5app.use(workflow);7app.use(workflow);
6 8
7export default app;9export default app;
apps/convex/convex/executor.test.ts+10−9
@@ -21,6 +21,7 @@ const setup = () => · 20 unmodified lines
21 "./control_plane/credentials.ts": () => import("./control_plane/credentials"),21 "./control_plane/credentials.ts": () => import("./control_plane/credentials"),
22 "./control_plane/sources.ts": () => import("./control_plane/sources"),22 "./control_plane/sources.ts": () => import("./control_plane/sources"),
23 "./control_plane/tool_registry.ts": () => import("./control_plane/tool_registry"),23 "./control_plane/tool_registry.ts": () => import("./control_plane/tool_registry"),
24 "./control_plane/tools.ts": () => import("./control_plane/tools"),
24 "./control_plane/openapi_ingest.ts": () => import("./control_plane/openapi_ingest"),25 "./control_plane/openapi_ingest.ts": () => import("./control_plane/openapi_ingest"),
25 "./workflow.ts": () => import("./workflow"),26 "./workflow.ts": () => import("./workflow"),
26 "./controlPlane.ts": () => import("./controlPlane"),27 "./controlPlane.ts": () => import("./controlPlane"),
@@ -650,7 +651,7 @@ describe("Convex executor and control-plane", () => { · 623 unmodified lines
650 );651 );
651 652
652 const workspaceTools = (yield* Effect.tryPromise(() =>653 const workspaceTools = (yield* Effect.tryPromise(() =>
653 t.query(api.controlPlane.listWorkspaceTools, {654 t.action(api.controlPlane.listWorkspaceTools, {
654 workspaceId: "ws_1",655 workspaceId: "ws_1",
655 }),656 }),
656 )) as Array<{657 )) as Array<{
@@ -661,7 +662,7 @@ describe("Convex executor and control-plane", () => { · 4 unmodified lines
661 expect(workspaceTools[0]?.sourceId).toBe("src_1");662 expect(workspaceTools[0]?.sourceId).toBe("src_1");
662 663
663 const sourceTools = (yield* Effect.tryPromise(() =>664 const sourceTools = (yield* Effect.tryPromise(() =>
664 t.query(api.controlPlane.listSourceTools, {665 t.action(api.controlPlane.listSourceTools, {
665 workspaceId: "ws_1",666 workspaceId: "ws_1",
666 sourceId: "src_1",667 sourceId: "src_1",
667 }),668 }),
@@ -788,7 +789,7 @@ describe("Convex executor and control-plane", () => { · 120 unmodified lines
788 );789 );
789 790
790 const graphqlSourceTools = (yield* Effect.tryPromise(() =>791 const graphqlSourceTools = (yield* Effect.tryPromise(() =>
791 t.query(api.controlPlane.listSourceTools, {792 t.action(api.controlPlane.listSourceTools, {
792 workspaceId: "ws_1",793 workspaceId: "ws_1",
793 sourceId: "src_graphql_1",794 sourceId: "src_graphql_1",
794 }),795 }),
@@ -803,7 +804,7 @@ describe("Convex executor and control-plane", () => { · 8 unmodified lines
803 expect(graphqlSourceTools[0]?.method).toBe("post");804 expect(graphqlSourceTools[0]?.method).toBe("post");
804 805
805 const workspaceToolsWithGraphql = (yield* Effect.tryPromise(() =>806 const workspaceToolsWithGraphql = (yield* Effect.tryPromise(() =>
806 t.query(api.controlPlane.listWorkspaceTools, {807 t.action(api.controlPlane.listWorkspaceTools, {
807 workspaceId: "ws_1",808 workspaceId: "ws_1",
808 }),809 }),
809 )) as Array<{810 )) as Array<{
@@ -906,7 +907,7 @@ describe("Convex executor and control-plane", () => { · 96 unmodified lines
906 );907 );
907 908
908 const mcpSourceTools = (yield* Effect.tryPromise(() =>909 const mcpSourceTools = (yield* Effect.tryPromise(() =>
909 t.query(api.controlPlane.listSourceTools, {910 t.action(api.controlPlane.listSourceTools, {
910 workspaceId: "ws_1",911 workspaceId: "ws_1",
911 sourceId: "src_mcp_1",912 sourceId: "src_mcp_1",
912 }),913 }),
@@ -920,7 +921,7 @@ describe("Convex executor and control-plane", () => { · 7 unmodified lines
920 expect(mcpSourceTools[0]?.path).toContain("deepwiki.mcp.search_docs");921 expect(mcpSourceTools[0]?.path).toContain("deepwiki.mcp.search_docs");
921 922
922 const workspaceToolsWithMcp = (yield* Effect.tryPromise(() =>923 const workspaceToolsWithMcp = (yield* Effect.tryPromise(() =>
923 t.query(api.controlPlane.listWorkspaceTools, {924 t.action(api.controlPlane.listWorkspaceTools, {
924 workspaceId: "ws_1",925 workspaceId: "ws_1",
925 }),926 }),
926 )) as Array<{927 )) as Array<{
@@ -930,7 +931,7 @@ describe("Convex executor and control-plane", () => { · 3 unmodified lines
930 expect(workspaceToolsWithMcp.some((tool) => tool.sourceId === "src_mcp_1")).toBe(true);931 expect(workspaceToolsWithMcp.some((tool) => tool.sourceId === "src_mcp_1")).toBe(true);
931 932
932 const sourceToolsFromWrongWorkspace = (yield* Effect.tryPromise(() =>933 const sourceToolsFromWrongWorkspace = (yield* Effect.tryPromise(() =>
933 t.query(api.controlPlane.listSourceTools, {934 t.action(api.controlPlane.listSourceTools, {
934 workspaceId: "ws_2",935 workspaceId: "ws_2",
935 sourceId: "src_1",936 sourceId: "src_1",
936 }),937 }),
@@ -1168,7 +1169,7 @@ describe("Convex executor and control-plane", () => { · 231 unmodified lines
1168 expect(sources[0]?.sourceHash).toBeTypeOf("string");1169 expect(sources[0]?.sourceHash).toBeTypeOf("string");
1169 1170
1170 const sourceTools = (yield* Effect.tryPromise(() =>1171 const sourceTools = (yield* Effect.tryPromise(() =>
1171 t.query(api.controlPlane.listSourceTools, {1172 t.action(api.controlPlane.listSourceTools, {
1172 workspaceId: "ws_cred",1173 workspaceId: "ws_cred",
1173 sourceId: "src_graphql_cred_1",1174 sourceId: "src_graphql_cred_1",
1174 }),1175 }),
@@ -1361,7 +1362,7 @@ describe("Convex executor and control-plane", () => { · 186 unmodified lines
1361 expect(sources[0]?.sourceHash).toBeTypeOf("string");1362 expect(sources[0]?.sourceHash).toBeTypeOf("string");
1362 1363
1363 const sourceTools = (yield* Effect.tryPromise(() =>1364 const sourceTools = (yield* Effect.tryPromise(() =>
1364 t.query(api.controlPlane.listSourceTools, {1365 t.action(api.controlPlane.listSourceTools, {
1365 workspaceId: "ws_mcp_ingest",1366 workspaceId: "ws_mcp_ingest",
1366 sourceId: "src_mcp_ingest_1",1367 sourceId: "src_mcp_ingest_1",
1367 }),1368 }),
apps/convex/convex/source_tool_registry.ts+1−1
@@ -1729,7 +1729,7 @@ export const createConvexSourceToolRegistry = ( · 1728 unmodified lines
1729 1729
1730 const rows = yield* Effect.tryPromise({1730 const rows = yield* Effect.tryPromise({
1731 try: async () =>1731 try: async () =>
1732 (await ctx.runQuery(runtimeInternal.control_plane.tool_registry.listWorkspaceNamespaces, {1732 (await ctx.runAction(runtimeInternal.control_plane.tool_registry.listWorkspaceNamespaces, {
1733 workspaceId,1733 workspaceId,
1734 limit,1734 limit,
1735 })) as Array<{1735 })) as Array<{
apps/convex/package.json+2−1
@@ -9,8 +9,8 @@ · 8 unmodified lines
9 "convex:codegen": "bunx convex codegen"9 "convex:codegen": "bunx convex codegen"
10 },10 },
11 "dependencies": {11 "dependencies": {
12 "@convex-dev/action-cache": "^0.3.0",
12 "@convex-dev/workflow": "^0.3.4",13 "@convex-dev/workflow": "^0.3.4",
13 "@modelcontextprotocol/sdk": "^1.27.1",
14 "@executor-v2/confect": "workspace:*",14 "@executor-v2/confect": "workspace:*",
15 "@executor-v2/domain": "workspace:*",15 "@executor-v2/domain": "workspace:*",
16 "@executor-v2/engine": "workspace:*",16 "@executor-v2/engine": "workspace:*",
@@ -20,6 +20,7 @@ · 3 unmodified lines
20 "@executor-v2/runtime-cloudflare-worker-loader": "workspace:*",20 "@executor-v2/runtime-cloudflare-worker-loader": "workspace:*",
21 "@executor-v2/schema": "workspace:*",21 "@executor-v2/schema": "workspace:*",
22 "@executor-v2/sdk": "workspace:*",22 "@executor-v2/sdk": "workspace:*",
23 "@modelcontextprotocol/sdk": "^1.27.1",
23 "convex": "catalog:",24 "convex": "catalog:",
24 "effect": "catalog:",25 "effect": "catalog:",
25 "jose": "^6.1.0"26 "jose": "^6.1.0"
apps/web/components/tools/source-management-panel.tsx+48−0
@@ -84,6 +84,14 @@ const catalogTemplates: ReadonlyArray<CatalogTemplate> = [ · 83 unmodified lines
84 type: "mcp",84 type: "mcp",
85 endpoint: "https://mcp.neon.tech/mcp",85 endpoint: "https://mcp.neon.tech/mcp",
86 },86 },
87 {
88 id: "axiom-mcp",
89 name: "Axiom MCP",
90 summary: "Query, analyze, and monitor your Axiom datasets and logs via MCP.",
91 providerName: "Axiom",
92 type: "mcp",
93 endpoint: "https://mcp.axiom.co/mcp",
94 },
87 {95 {
88 id: "neon-openapi",96 id: "neon-openapi",
89 name: "Neon API",97 name: "Neon API",
@@ -170,6 +178,46 @@ const catalogTemplates: ReadonlyArray<CatalogTemplate> = [ · 80 unmodified lines
170 type: "openapi",178 type: "openapi",
171 endpoint: "https://developer.atlassian.com/cloud/jira/platform/swagger-v3.v3.json",179 endpoint: "https://developer.atlassian.com/cloud/jira/platform/swagger-v3.v3.json",
172 },180 },
181 {
182 id: "kubernetes-api",
183 name: "Kubernetes API",
184 summary: "Manage pods, deployments, services, and cluster resources.",
185 providerName: "Kubernetes",
186 type: "openapi",
187 endpoint:
188 "https://raw.githubusercontent.com/kubernetes/kubernetes/master/api/openapi-spec/swagger.json",
189 faviconUrl: "https://kubernetes.io",
190 },
191 {
192 id: "square-api",
193 name: "Square API",
194 summary: "Process payments, manage inventory, customers, and merchant operations.",
195 providerName: "Square",
196 type: "openapi",
197 endpoint:
198 "https://raw.githubusercontent.com/square/connect-api-specification/master/api.json",
199 faviconUrl: "https://squareup.com",
200 },
201 {
202 id: "twilio-api",
203 name: "Twilio API",
204 summary: "Send SMS, make calls, and manage communication workflows.",
205 providerName: "Twilio",
206 type: "openapi",
207 endpoint:
208 "https://raw.githubusercontent.com/twilio/twilio-oai/main/spec/json/twilio_api_v2010.json",
209 faviconUrl: "https://twilio.com",
210 },
211 {
212 id: "box-api",
213 name: "Box Platform API",
214 summary: "Manage files, folders, collaborations, and enterprise content workflows.",
215 providerName: "Box",
216 type: "openapi",
217 endpoint:
218 "https://raw.githubusercontent.com/box/box-openapi/main/openapi.json",
219 faviconUrl: "https://box.com",
220 },
173];221];
174 222
175// ---------------------------------------------------------------------------223// ---------------------------------------------------------------------------
bun.lock+3−0
@@ -35,6 +35,7 @@ · 34 unmodified lines
35 "apps/convex": {35 "apps/convex": {
36 "name": "@executor-v2/app-convex",36 "name": "@executor-v2/app-convex",
37 "dependencies": {37 "dependencies": {
38 "@convex-dev/action-cache": "^0.3.0",
38 "@convex-dev/workflow": "^0.3.4",39 "@convex-dev/workflow": "^0.3.4",
39 "@executor-v2/confect": "workspace:*",40 "@executor-v2/confect": "workspace:*",
40 "@executor-v2/domain": "workspace:*",41 "@executor-v2/domain": "workspace:*",
@@ -375,6 +376,8 @@ · 334 unmodified lines
375 376
376 "@cloudflare/workers-types": ["@cloudflare/workers-types@4.20260305.0", "", {}, "sha512-sCgPFnQ03SVpC2OVW8wysONLZW/A8hlp9Mq2ckG/h1oId4kr9NawA6vUiOmOjCWRn2hIohejBYVQ+Vu20rCdKA=="],377 "@cloudflare/workers-types": ["@cloudflare/workers-types@4.20260305.0", "", {}, "sha512-sCgPFnQ03SVpC2OVW8wysONLZW/A8hlp9Mq2ckG/h1oId4kr9NawA6vUiOmOjCWRn2hIohejBYVQ+Vu20rCdKA=="],
377 378
379 "@convex-dev/action-cache": ["@convex-dev/action-cache@0.3.0", "", { "peerDependencies": { "convex": "^1.24.8" } }, "sha512-B3GsOgwPv81whdu1mHaMKBwbIYHhNXsv3Fr5oMjynIOrRy1laGLUo9zRWDKWJgtKXfVnn8X/uxYD9RyWmPIbPQ=="],
380
378 "@convex-dev/workflow": ["@convex-dev/workflow@0.3.4", "", { "dependencies": { "async-channel": "^0.2.0" }, "peerDependencies": { "@convex-dev/workpool": "^0.3.0", "convex": "^1.24.8", "convex-helpers": "^0.1.99" } }, "sha512-uX9qhEbgkgt+touvgLnyKztussT3m5q4c2XH/iBNQxzk8TlL29F1+UeoCxXA6WibWEpOQLeuFA6/QMVP4tEX1w=="],381 "@convex-dev/workflow": ["@convex-dev/workflow@0.3.4", "", { "dependencies": { "async-channel": "^0.2.0" }, "peerDependencies": { "@convex-dev/workpool": "^0.3.0", "convex": "^1.24.8", "convex-helpers": "^0.1.99" } }, "sha512-uX9qhEbgkgt+touvgLnyKztussT3m5q4c2XH/iBNQxzk8TlL29F1+UeoCxXA6WibWEpOQLeuFA6/QMVP4tEX1w=="],
379 382
380 "@convex-dev/workpool": ["@convex-dev/workpool@0.3.2", "", { "peerDependencies": { "convex": "^1.31.7", "convex-helpers": "^0.1.94" } }, "sha512-jNPBHLCmZQXaztbBntZgIoM8JhKDmGwaIezZS/zPmX74PzON0Fgysgd9u+NOjJ04Dx2pN8ppBH9ElpsKxCOBpA=="],383 "@convex-dev/workpool": ["@convex-dev/workpool@0.3.2", "", { "peerDependencies": { "convex": "^1.31.7", "convex-helpers": "^0.1.94" } }, "sha512-jNPBHLCmZQXaztbBntZgIoM8JhKDmGwaIezZS/zPmX74PzON0Fgysgd9u+NOjJ04Dx2pN8ppBH9ElpsKxCOBpA=="],
legacy/executor/apps/web/src/lib/catalog-collections.ts+61−0
@@ -53,6 +53,19 @@ export const HARD_CODED_CATALOG_ITEMS: CatalogCollectionItem[] = [ · 52 unmodified lines
53 rank: 0.2,53 rank: 0.2,
54 addedAt: "2026-02-20",54 addedAt: "2026-02-20",
55 },55 },
56 {
57 id: "axiom-mcp",
58 name: "Axiom MCP",
59 summary: "Query, analyze, and monitor your Axiom datasets and logs via MCP.",
60 specUrl: "https://mcp.axiom.co/mcp",
61 sourceType: "mcp",
62 originUrl: "https://axiom.co",
63 providerName: "Axiom",
64 categories: "observability",
65 version: "latest",
66 rank: 0.25,
67 addedAt: "2026-03-01",
68 },
56 {69 {
57 id: "neon-openapi",70 id: "neon-openapi",
58 name: "Neon API",71 name: "Neon API",
@@ -175,4 +188,52 @@ export const HARD_CODED_CATALOG_ITEMS: CatalogCollectionItem[] = [ · 116 unmodified lines
175 rank: 8,188 rank: 8,
176 addedAt: "2025-11-15",189 addedAt: "2025-11-15",
177 },190 },
191 {
192 id: "kubernetes-api",
193 name: "Kubernetes API",
194 summary: "Manage pods, deployments, services, and cluster resources.",
195 specUrl: "https://raw.githubusercontent.com/kubernetes/kubernetes/master/api/openapi-spec/swagger.json",
196 originUrl: "https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.31/",
197 providerName: "Kubernetes",
198 categories: "infrastructure",
199 version: "latest",
200 rank: 9,
201 addedAt: "2026-03-01",
202 },
203 {
204 id: "square-api",
205 name: "Square API",
206 summary: "Process payments, manage inventory, customers, and merchant operations.",
207 specUrl: "https://raw.githubusercontent.com/square/connect-api-specification/master/api.json",
208 originUrl: "https://developer.squareup.com/reference/square",
209 providerName: "Square",
210 categories: "payments",
211 version: "latest",
212 rank: 10,
213 addedAt: "2026-03-01",
214 },
215 {
216 id: "twilio-api",
217 name: "Twilio API",
218 summary: "Send SMS, make calls, and manage communication workflows.",
219 specUrl: "https://raw.githubusercontent.com/twilio/twilio-oai/main/spec/json/twilio_api_v2010.json",
220 originUrl: "https://www.twilio.com/docs/usage/api",
221 providerName: "Twilio",
222 categories: "communications",
223 version: "v2010",
224 rank: 11,
225 addedAt: "2026-03-01",
226 },
227 {
228 id: "box-api",
229 name: "Box Platform API",
230 summary: "Manage files, folders, collaborations, and enterprise content workflows.",
231 specUrl: "https://raw.githubusercontent.com/box/box-openapi/main/openapi.json",
232 originUrl: "https://developer.box.com/reference/",
233 providerName: "Box",
234 categories: "developer-tools",
235 version: "latest",
236 rank: 12,
237 addedAt: "2026-03-01",
238 },
178];239];