Add Discord API preset and enhance tool discovery logic

- Introduced a new API preset for Discord, including its specifications and authentication requirements. - Updated tool discovery logic to improve scoring and selection of best paths based on namespace hints and intent phrases. - Enhanced documentation for tool discovery to reflect changes in the output structure and scoring criteria. - Added tests to validate the new discovery behavior and ensure correct path selection based on user queries.

Rhys Sullivan committed 9 months ago

Commit b25a4b31 parent 669b5b16 files · +193 −20Browse files

Unified · Split

6 files

executor/apps/web/src/components/tools-view.tsx +9 −0
executor/convex/database.ts +0 −8
executor/lib/mcp_server.ts +1 −1
executor/lib/tool-discovery.test.ts +70 −0
executor/lib/tool_discovery.ts +111 −9
executor/lib/typechecker.ts +2 −2

executor/apps/web/src/components/tools-view.tsx+9−0

@@ -101,6 +101,15 @@ const API_PRESETS: ApiPreset[] = [ · 100 unmodified lines
101	101	baseUrl: "https://slack.com/api",
102	102	authNote: "Requires a bot token as bearer credential",
103	103	},
	104	+ {
	105	+ name: "discord",
	106	+ label: "Discord",
	107	+ description: "Guilds, channels, messages, interactions, webhooks",
	108	+ type: "openapi",
	109	+ spec: "https://raw.githubusercontent.com/discord/discord-api-spec/main/specs/openapi.json",
	110	+ baseUrl: "https://discord.com/api/v10",
	111	+ authNote: "Requires a bot token as bearer credential",
	112	+ },
104	113	{
105	114	name: "stripe",
106	115	label: "Stripe",

executor/convex/database.ts+0−8

@@ -1,6 +1,5 @@
1	1	import { v } from "convex/values";
2	2	import type { Doc } from "./_generated/dataModel";
3		−import { internal } from "./_generated/api";
4	3	import { mutation,
5	4	query,
6	5	type MutationCtx, type QueryCtx } from "./_generated/server";
@@ -504,13 +503,6 @@ export const createApproval = internalMutation({ · 497 unmodified lines
504	503	createdAt: now,
505	504	});
506	505
507		− await ctx.scheduler.runAfter(0, internal.pushNotifications.sendApprovalRequested, {
508		− workspaceId: task.workspaceId,
509		− approvalId: args.id,
510		− taskId: args.taskId,
511		− toolPath: args.toolPath,
512		− });
513		−
514	506	const created = await getApprovalDoc(ctx, args.id);
515	507	if (!created) {
516	508	throw new Error(`Failed to fetch approval ${args.id}`);

executor/lib/mcp_server.ts+1−1

@@ -301,7 +301,7 @@ function buildRunCodeDescription(tools?: ToolDescriptor[]): string { · 300 unmodified lines
301	301	? `\n\nTop-level tool keys: ${topLevelKeys.join(", ")}`
302	302	: "";
303	303	const hasGraphqlTools = toolList.some((tool) => tool.path.endsWith(".graphql"));
304		− const discoverNote = "\n\nTip: use `tools.discover({ query, depth?, limit?, compact? })` first. It returns `{ results, total }`; call the exact `results[i].path` (or copy `results[i].exampleCall`). Compact mode is on by default (set `compact: false` for full signatures). Do not assign to `const tools = ...`; use a different variable name (e.g. `const discovered = ...`).";
	304	+ const discoverNote = "\n\nTip: use `tools.discover({ query, depth?, limit?, compact? })` first. It returns `{ bestPath, results, total }`; prefer `bestPath` when present, otherwise call `results[i].path` (or copy `results[i].exampleCall`). Compact mode is on by default (set `compact: false` for full signatures). Do not assign to `const tools = ...`; use a different variable name (e.g. `const discovered = ...`).";
305	305	const executionNote = "\n\nExecution tip: for migration/ETL-style tasks, discover once, then run in small batches and `return` compact summaries (counts, IDs, and top-N samples) instead of full objects.";
306	306	const graphqlNote = hasGraphqlTools
307	307	? "\n\nGraphQL tip: prefer `source.query.` / `source.mutation.` helper paths when available; GraphQL tools return `{ data, errors }`."

executor/lib/tool-discovery.test.ts+70−0

@@ -135,3 +135,73 @@ test("discover returns null bestPath when there are no matches", async () => { · 134 unmodified lines
135	135	expect(result.total).toBe(0);
136	136	expect(result.results).toHaveLength(0);
137	137	});
	138	+
	139	+test("discover bestPath prefers simpler exact intent operation", async () => {
	140	+ const tool = createDiscoverTool([
	141	+ {
	142	+ path: "linear.mutation.issuetoreleasecreate",
	143	+ description: "Create issue-to-release join",
	144	+ approval: "required",
	145	+ source: "graphql:linear",
	146	+ metadata: {
	147	+ argsType: "{ input: { issueId: string; releaseId: string } }",
	148	+ returnsType: "{ data: IssueToReleasePayload; errors: unknown[] }",
	149	+ },
	150	+ run: async () => ({ data: {}, errors: [] }),
	151	+ } satisfies ToolDefinition,
	152	+ {
	153	+ path: "linear.mutation.issuecreate",
	154	+ description: "Create issue",
	155	+ approval: "required",
	156	+ source: "graphql:linear",
	157	+ metadata: {
	158	+ argsType: "{ input: { teamId: string; title: string } }",
	159	+ returnsType: "{ data: IssuePayload; errors: unknown[] }",
	160	+ },
	161	+ run: async () => ({ data: {}, errors: [] }),
	162	+ } satisfies ToolDefinition,
	163	+ ]);
	164	+
	165	+ const result = await tool.run(
	166	+ { query: "linear issue create", depth: 2 },
	167	+ { taskId: "t", workspaceId: "w", isToolAllowed: () => true },
	168	+ ) as { bestPath: string \| null; results: Array<{ path: string }> };
	169	+
	170	+ expect(result.bestPath).toBe("linear.mutation.issuecreate");
	171	+ expect(result.results[0]?.path).toBe("linear.mutation.issuecreate");
	172	+});
	173	+
	174	+test("discover namespace hint suppresses cross-namespace bestPath", async () => {
	175	+ const tool = createDiscoverTool([
	176	+ {
	177	+ path: "github.teams.list",
	178	+ description: "List teams",
	179	+ approval: "auto",
	180	+ source: "openapi:github",
	181	+ metadata: {
	182	+ argsType: "{ org: string }",
	183	+ returnsType: "Array<Team>",
	184	+ },
	185	+ run: async () => ([]),
	186	+ } satisfies ToolDefinition,
	187	+ {
	188	+ path: "linear.query.teams",
	189	+ description: "List teams in Linear",
	190	+ approval: "auto",
	191	+ source: "graphql:linear",
	192	+ metadata: {
	193	+ argsType: "{}",
	194	+ returnsType: "{ data: TeamConnection; errors: unknown[] }",
	195	+ },
	196	+ run: async () => ({ data: {}, errors: [] }),
	197	+ } satisfies ToolDefinition,
	198	+ ]);
	199	+
	200	+ const result = await tool.run(
	201	+ { query: "linear teams list", depth: 2 },
	202	+ { taskId: "t", workspaceId: "w", isToolAllowed: () => true },
	203	+ ) as { bestPath: string \| null; results: Array<{ path: string }> };
	204	+
	205	+ expect(result.bestPath).toBe("linear.query.teams");
	206	+ expect(result.results.some((entry) => entry.path.startsWith("github."))).toBe(false);
	207	+});

executor/lib/tool_discovery.ts+111−9

@@ -13,6 +13,21 @@ interface DiscoverIndexEntry { · 12 unmodified lines
13	13	normalizedSearchText: string;
14	14	}
15	15
	16	+const DISCOVER_STOP_WORDS = new Set([
	17	+ "a",
	18	+ "an",
	19	+ "and",
	20	+ "for",
	21	+ "from",
	22	+ "in",
	23	+ "of",
	24	+ "on",
	25	+ "or",
	26	+ "the",
	27	+ "to",
	28	+ "with",
	29	+]);
	30	+
16	31	function normalizeType(type?: string): string {
17	32	return type && type.trim().length > 0 ? type : "unknown";
18	33	}
@@ -163,10 +178,79 @@ function buildIndex(tools: ToolDefinition[]): DiscoverIndexEntry[] { · 144 unmodified lines
163	178	});
164	179	}
165	180
166		−function scoreEntry(entry: DiscoverIndexEntry, terms: string[]): number {
	181	+function getTopLevelNamespace(path: string): string {
	182	+ return path.split(".")[0]?.toLowerCase() ?? "";
	183	+}
	184	+
	185	+function extractNamespaceHints(terms: string[], namespaces: Set<string>): Set<string> {
	186	+ const hints = new Set<string>();
	187	+
	188	+ for (const term of terms) {
	189	+ const direct = term.toLowerCase();
	190	+ if (namespaces.has(direct)) {
	191	+ hints.add(direct);
	192	+ continue;
	193	+ }
	194	+
	195	+ const leadingSegment = direct.split(".")[0] ?? direct;
	196	+ if (namespaces.has(leadingSegment)) {
	197	+ hints.add(leadingSegment);
	198	+ }
	199	+ }
	200	+
	201	+ return hints;
	202	+}
	203	+
	204	+function deriveIntentPhrase(terms: string[], namespaceHints: Set<string>): string {
	205	+ const important = terms
	206	+ .map((term) => term.toLowerCase())
	207	+ .filter((term) => !namespaceHints.has(term))
	208	+ .filter((term) => !DISCOVER_STOP_WORDS.has(term))
	209	+ .filter((term) => term.length > 2);
	210	+
	211	+ return normalizeSearchToken(important.join(" "));
	212	+}
	213	+
	214	+function chooseBestPath(
	215	+ ranked: Array<{ entry: DiscoverIndexEntry; score: number }>,
	216	+ termCount: number,
	217	+): string \| null {
	218	+ if (ranked.length === 0) return null;
	219	+
	220	+ const best = ranked[0];
	221	+ if (!best) return null;
	222	+
	223	+ const minScore = termCount === 0 ? 1 : Math.max(3, termCount * 2 - 1);
	224	+ if (best.score < minScore) {
	225	+ return null;
	226	+ }
	227	+
	228	+ const second = ranked[1];
	229	+ if (second && best.score - second.score < 2) {
	230	+ return null;
	231	+ }
	232	+
	233	+ return best.entry.path;
	234	+}
	235	+
	236	+function scoreEntry(
	237	+ entry: DiscoverIndexEntry,
	238	+ terms: string[],
	239	+ namespaceHints: Set<string>,
	240	+ intentPhrase: string,
	241	+): number {
167	242	let score = 0;
168	243	let matched = 0;
169	244
	245	+ if (namespaceHints.size > 0) {
	246	+ const namespace = getTopLevelNamespace(entry.path);
	247	+ if (namespaceHints.has(namespace)) {
	248	+ score += 6;
	249	+ } else {
	250	+ score -= 8;
	251	+ }
	252	+ }
	253	+
170	254	for (const term of terms) {
171	255	const normalizedTerm = normalizeSearchToken(term);
172	256	const inPath = entry.path.toLowerCase().includes(term);
@@ -179,6 +263,14 @@ function scoreEntry(entry: DiscoverIndexEntry, terms: string[]): number { · 6 unmodified lines
179	263	if (inPath \|\| inNormalizedPath) score += 2;
180	264	}
181	265
	266	+ if (intentPhrase.length >= 6) {
	267	+ if (entry.normalizedPath.includes(intentPhrase)) {
	268	+ score += 6;
	269	+ } else if (entry.normalizedSearchText.includes(intentPhrase)) {
	270	+ score += 3;
	271	+ }
	272	+ }
	273	+
182	274	if (terms.length > 0 && matched < Math.max(1, Math.ceil(terms.length / 2))) {
183	275	return -1;
184	276	}
@@ -231,13 +323,23 @@ export function createDiscoverTool(tools: ToolDefinition[]): ToolDefinition { · 46 unmodified lines
231	323	const limit = Math.max(1, Math.min(50, Number(payload.limit ?? 8)));
232	324	const compact = payload.compact === false ? false : true;
233	325	const terms = query.length > 0 ? query.split(/\s+/).filter(Boolean) : [];
234		−
235		− const ranked = index
236		− .filter((entry) => context.isToolAllowed(entry.path))
237		− .map((entry) => ({ entry, score: scoreEntry(entry, terms) }))
	326	+ const namespaces = new Set(index.map((entry) => getTopLevelNamespace(entry.path)).filter(Boolean));
	327	+ const namespaceHints = extractNamespaceHints(terms, namespaces);
	328	+ const intentPhrase = deriveIntentPhrase(terms, namespaceHints);
	329	+
	330	+ const visibleEntries = index.filter((entry) => context.isToolAllowed(entry.path));
	331	+ const namespaceScopedEntries = namespaceHints.size > 0
	332	+ ? visibleEntries.filter((entry) => namespaceHints.has(getTopLevelNamespace(entry.path)))
	333	+ : visibleEntries;
	334	+ const candidateEntries = namespaceScopedEntries.length > 0 ? namespaceScopedEntries : visibleEntries;
	335	+
	336	+ const ranked = candidateEntries
	337	+ .map((entry) => ({ entry, score: scoreEntry(entry, terms, namespaceHints, intentPhrase) }))
238	338	.filter((item) => item.score > 0 \|\| terms.length === 0)
239	339	.sort((a, b) => b.score - a.score)
240		− .slice(0, limit)
	340	+ .slice(0, limit);
	341	+
	342	+ const results = ranked
241	343	.map(({ entry }) => ({
242	344	path: entry.path,
243	345	aliases: entry.aliases,
@@ -249,9 +351,9 @@ export function createDiscoverTool(tools: ToolDefinition[]): ToolDefinition { · 5 unmodified lines
249	351	}));
250	352
251	353	return {
252		− bestPath: ranked[0]?.path ?? null,
253		− results: ranked,
254		− total: ranked.length,
	354	+ bestPath: chooseBestPath(ranked, terms.length),
	355	+ results,
	356	+ total: results.length,
255	357	};
256	358	},
257	359	};

executor/lib/typechecker.ts+2−2

@@ -225,8 +225,8 @@ export function generateToolInventory(tools: ToolDescriptor[]): string { · 224 unmodified lines
225	225	"You have access to these tool namespaces:",
226	226	` ${namespaces.join(", ")}`,
227	227	"",
228		− "Use `tools.discover({ query, depth?, limit?, compact? })` first. It returns `{ results, total }` (not an array).",
229		− "Use the exact `path` returned by discover, and prefer its `exampleCall` snippet for invocation shape.",
	228	+ "Use `tools.discover({ query, depth?, limit?, compact? })` first. It returns `{ bestPath, results, total }` (not an array).",
	229	+ "Prefer `bestPath` when present, otherwise use a `results[i].path`; use `exampleCall` for invocation shape.",
230	230	"For migration/ETL tasks: discover once, then execute in small batches and return compact summaries (counts, IDs, top-N samples).",
231	231	"Never shadow the global `tools` object (do NOT write `const tools = ...`).",
232	232	"Then call tools directly using the returned path.",