History

Add Discord API preset and enhance tool discovery logic

- Introduced a new API preset for Discord, including its specifications and authentication requirements. - Updated tool discovery logic to improve scoring and selection of best paths based on namespace hints and intent phrases. - Enhanced documentation for tool discovery to reflect changes in the output structure and scoring criteria. - Added tests to validate the new discovery behavior and ensure correct path selection based on user queries.

Rhys Sullivan committed 9 months ago

Commit b25a4b31 parent 669b5b16 files · +193 −20Browse files

Unified · Split

6 files
  1. executor/apps/web/src/components/tools-view.tsx +9 −0
  2. executor/convex/database.ts +0 −8
  3. executor/lib/mcp_server.ts +1 −1
  4. executor/lib/tool-discovery.test.ts +70 −0
  5. executor/lib/tool_discovery.ts +111 −9
  6. executor/lib/typechecker.ts +2 −2
executor/apps/web/src/components/tools-view.tsx+9−0
@@ -101,6 +101,15 @@ const API_PRESETS: ApiPreset[] = [ · 100 unmodified lines
101 baseUrl: "https://slack.com/api",101 baseUrl: "https://slack.com/api",
102 authNote: "Requires a bot token as bearer credential",102 authNote: "Requires a bot token as bearer credential",
103 },103 },
104 {
105 name: "discord",
106 label: "Discord",
107 description: "Guilds, channels, messages, interactions, webhooks",
108 type: "openapi",
109 spec: "https://raw.githubusercontent.com/discord/discord-api-spec/main/specs/openapi.json",
110 baseUrl: "https://discord.com/api/v10",
111 authNote: "Requires a bot token as bearer credential",
112 },
104 {113 {
105 name: "stripe",114 name: "stripe",
106 label: "Stripe",115 label: "Stripe",
executor/convex/database.ts+0−8
@@ -1,6 +1,5 @@
1import { v } from "convex/values";1import { v } from "convex/values";
2import type { Doc } from "./_generated/dataModel";2import type { Doc } from "./_generated/dataModel";
3import { internal } from "./_generated/api";
4import { mutation,3import { mutation,
5query,4query,
6type MutationCtx, type QueryCtx } from "./_generated/server";5type MutationCtx, type QueryCtx } from "./_generated/server";
@@ -504,13 +503,6 @@ export const createApproval = internalMutation({ · 497 unmodified lines
504 createdAt: now,503 createdAt: now,
505 });504 });
506 505
507 await ctx.scheduler.runAfter(0, internal.pushNotifications.sendApprovalRequested, {
508 workspaceId: task.workspaceId,
509 approvalId: args.id,
510 taskId: args.taskId,
511 toolPath: args.toolPath,
512 });
513
514 const created = await getApprovalDoc(ctx, args.id);506 const created = await getApprovalDoc(ctx, args.id);
515 if (!created) {507 if (!created) {
516 throw new Error(`Failed to fetch approval ${args.id}`);508 throw new Error(`Failed to fetch approval ${args.id}`);
executor/lib/mcp_server.ts+1−1
@@ -301,7 +301,7 @@ function buildRunCodeDescription(tools?: ToolDescriptor[]): string { · 300 unmodified lines
301 ? `\n\nTop-level tool keys: ${topLevelKeys.join(", ")}`301 ? `\n\nTop-level tool keys: ${topLevelKeys.join(", ")}`
302 : "";302 : "";
303 const hasGraphqlTools = toolList.some((tool) => tool.path.endsWith(".graphql"));303 const hasGraphqlTools = toolList.some((tool) => tool.path.endsWith(".graphql"));
304 const discoverNote = "\n\nTip: use `tools.discover({ query, depth?, limit?, compact? })` first. It returns `{ results, total }`; call the exact `results[i].path` (or copy `results[i].exampleCall`). Compact mode is on by default (set `compact: false` for full signatures). Do not assign to `const tools = ...`; use a different variable name (e.g. `const discovered = ...`).";304 const discoverNote = "\n\nTip: use `tools.discover({ query, depth?, limit?, compact? })` first. It returns `{ bestPath, results, total }`; prefer `bestPath` when present, otherwise call `results[i].path` (or copy `results[i].exampleCall`). Compact mode is on by default (set `compact: false` for full signatures). Do not assign to `const tools = ...`; use a different variable name (e.g. `const discovered = ...`).";
305 const executionNote = "\n\nExecution tip: for migration/ETL-style tasks, discover once, then run in small batches and `return` compact summaries (counts, IDs, and top-N samples) instead of full objects.";305 const executionNote = "\n\nExecution tip: for migration/ETL-style tasks, discover once, then run in small batches and `return` compact summaries (counts, IDs, and top-N samples) instead of full objects.";
306 const graphqlNote = hasGraphqlTools306 const graphqlNote = hasGraphqlTools
307 ? "\n\nGraphQL tip: prefer `source.query.*` / `source.mutation.*` helper paths when available; GraphQL tools return `{ data, errors }`."307 ? "\n\nGraphQL tip: prefer `source.query.*` / `source.mutation.*` helper paths when available; GraphQL tools return `{ data, errors }`."
executor/lib/tool-discovery.test.ts+70−0
@@ -135,3 +135,73 @@ test("discover returns null bestPath when there are no matches", async () => { · 134 unmodified lines
135 expect(result.total).toBe(0);135 expect(result.total).toBe(0);
136 expect(result.results).toHaveLength(0);136 expect(result.results).toHaveLength(0);
137});137});
138
139test("discover bestPath prefers simpler exact intent operation", async () => {
140 const tool = createDiscoverTool([
141 {
142 path: "linear.mutation.issuetoreleasecreate",
143 description: "Create issue-to-release join",
144 approval: "required",
145 source: "graphql:linear",
146 metadata: {
147 argsType: "{ input: { issueId: string; releaseId: string } }",
148 returnsType: "{ data: IssueToReleasePayload; errors: unknown[] }",
149 },
150 run: async () => ({ data: {}, errors: [] }),
151 } satisfies ToolDefinition,
152 {
153 path: "linear.mutation.issuecreate",
154 description: "Create issue",
155 approval: "required",
156 source: "graphql:linear",
157 metadata: {
158 argsType: "{ input: { teamId: string; title: string } }",
159 returnsType: "{ data: IssuePayload; errors: unknown[] }",
160 },
161 run: async () => ({ data: {}, errors: [] }),
162 } satisfies ToolDefinition,
163 ]);
164
165 const result = await tool.run(
166 { query: "linear issue create", depth: 2 },
167 { taskId: "t", workspaceId: "w", isToolAllowed: () => true },
168 ) as { bestPath: string | null; results: Array<{ path: string }> };
169
170 expect(result.bestPath).toBe("linear.mutation.issuecreate");
171 expect(result.results[0]?.path).toBe("linear.mutation.issuecreate");
172});
173
174test("discover namespace hint suppresses cross-namespace bestPath", async () => {
175 const tool = createDiscoverTool([
176 {
177 path: "github.teams.list",
178 description: "List teams",
179 approval: "auto",
180 source: "openapi:github",
181 metadata: {
182 argsType: "{ org: string }",
183 returnsType: "Array<Team>",
184 },
185 run: async () => ([]),
186 } satisfies ToolDefinition,
187 {
188 path: "linear.query.teams",
189 description: "List teams in Linear",
190 approval: "auto",
191 source: "graphql:linear",
192 metadata: {
193 argsType: "{}",
194 returnsType: "{ data: TeamConnection; errors: unknown[] }",
195 },
196 run: async () => ({ data: {}, errors: [] }),
197 } satisfies ToolDefinition,
198 ]);
199
200 const result = await tool.run(
201 { query: "linear teams list", depth: 2 },
202 { taskId: "t", workspaceId: "w", isToolAllowed: () => true },
203 ) as { bestPath: string | null; results: Array<{ path: string }> };
204
205 expect(result.bestPath).toBe("linear.query.teams");
206 expect(result.results.some((entry) => entry.path.startsWith("github."))).toBe(false);
207});
executor/lib/tool_discovery.ts+111−9
@@ -13,6 +13,21 @@ interface DiscoverIndexEntry { · 12 unmodified lines
13 normalizedSearchText: string;13 normalizedSearchText: string;
14}14}
15 15
16const DISCOVER_STOP_WORDS = new Set([
17 "a",
18 "an",
19 "and",
20 "for",
21 "from",
22 "in",
23 "of",
24 "on",
25 "or",
26 "the",
27 "to",
28 "with",
29]);
30
16function normalizeType(type?: string): string {31function normalizeType(type?: string): string {
17 return type && type.trim().length > 0 ? type : "unknown";32 return type && type.trim().length > 0 ? type : "unknown";
18}33}
@@ -163,10 +178,79 @@ function buildIndex(tools: ToolDefinition[]): DiscoverIndexEntry[] { · 144 unmodified lines
163 });178 });
164}179}
165 180
166function scoreEntry(entry: DiscoverIndexEntry, terms: string[]): number {181function getTopLevelNamespace(path: string): string {
182 return path.split(".")[0]?.toLowerCase() ?? "";
183}
184
185function extractNamespaceHints(terms: string[], namespaces: Set<string>): Set<string> {
186 const hints = new Set<string>();
187
188 for (const term of terms) {
189 const direct = term.toLowerCase();
190 if (namespaces.has(direct)) {
191 hints.add(direct);
192 continue;
193 }
194
195 const leadingSegment = direct.split(".")[0] ?? direct;
196 if (namespaces.has(leadingSegment)) {
197 hints.add(leadingSegment);
198 }
199 }
200
201 return hints;
202}
203
204function deriveIntentPhrase(terms: string[], namespaceHints: Set<string>): string {
205 const important = terms
206 .map((term) => term.toLowerCase())
207 .filter((term) => !namespaceHints.has(term))
208 .filter((term) => !DISCOVER_STOP_WORDS.has(term))
209 .filter((term) => term.length > 2);
210
211 return normalizeSearchToken(important.join(" "));
212}
213
214function chooseBestPath(
215 ranked: Array<{ entry: DiscoverIndexEntry; score: number }>,
216 termCount: number,
217): string | null {
218 if (ranked.length === 0) return null;
219
220 const best = ranked[0];
221 if (!best) return null;
222
223 const minScore = termCount === 0 ? 1 : Math.max(3, termCount * 2 - 1);
224 if (best.score < minScore) {
225 return null;
226 }
227
228 const second = ranked[1];
229 if (second && best.score - second.score < 2) {
230 return null;
231 }
232
233 return best.entry.path;
234}
235
236function scoreEntry(
237 entry: DiscoverIndexEntry,
238 terms: string[],
239 namespaceHints: Set<string>,
240 intentPhrase: string,
241): number {
167 let score = 0;242 let score = 0;
168 let matched = 0;243 let matched = 0;
169 244
245 if (namespaceHints.size > 0) {
246 const namespace = getTopLevelNamespace(entry.path);
247 if (namespaceHints.has(namespace)) {
248 score += 6;
249 } else {
250 score -= 8;
251 }
252 }
253
170 for (const term of terms) {254 for (const term of terms) {
171 const normalizedTerm = normalizeSearchToken(term);255 const normalizedTerm = normalizeSearchToken(term);
172 const inPath = entry.path.toLowerCase().includes(term);256 const inPath = entry.path.toLowerCase().includes(term);
@@ -179,6 +263,14 @@ function scoreEntry(entry: DiscoverIndexEntry, terms: string[]): number { · 6 unmodified lines
179 if (inPath || inNormalizedPath) score += 2;263 if (inPath || inNormalizedPath) score += 2;
180 }264 }
181 265
266 if (intentPhrase.length >= 6) {
267 if (entry.normalizedPath.includes(intentPhrase)) {
268 score += 6;
269 } else if (entry.normalizedSearchText.includes(intentPhrase)) {
270 score += 3;
271 }
272 }
273
182 if (terms.length > 0 && matched < Math.max(1, Math.ceil(terms.length / 2))) {274 if (terms.length > 0 && matched < Math.max(1, Math.ceil(terms.length / 2))) {
183 return -1;275 return -1;
184 }276 }
@@ -231,13 +323,23 @@ export function createDiscoverTool(tools: ToolDefinition[]): ToolDefinition { · 46 unmodified lines
231 const limit = Math.max(1, Math.min(50, Number(payload.limit ?? 8)));323 const limit = Math.max(1, Math.min(50, Number(payload.limit ?? 8)));
232 const compact = payload.compact === false ? false : true;324 const compact = payload.compact === false ? false : true;
233 const terms = query.length > 0 ? query.split(/\s+/).filter(Boolean) : [];325 const terms = query.length > 0 ? query.split(/\s+/).filter(Boolean) : [];
234 326 const namespaces = new Set(index.map((entry) => getTopLevelNamespace(entry.path)).filter(Boolean));
235 const ranked = index327 const namespaceHints = extractNamespaceHints(terms, namespaces);
236 .filter((entry) => context.isToolAllowed(entry.path))328 const intentPhrase = deriveIntentPhrase(terms, namespaceHints);
237 .map((entry) => ({ entry, score: scoreEntry(entry, terms) }))329
330 const visibleEntries = index.filter((entry) => context.isToolAllowed(entry.path));
331 const namespaceScopedEntries = namespaceHints.size > 0
332 ? visibleEntries.filter((entry) => namespaceHints.has(getTopLevelNamespace(entry.path)))
333 : visibleEntries;
334 const candidateEntries = namespaceScopedEntries.length > 0 ? namespaceScopedEntries : visibleEntries;
335
336 const ranked = candidateEntries
337 .map((entry) => ({ entry, score: scoreEntry(entry, terms, namespaceHints, intentPhrase) }))
238 .filter((item) => item.score > 0 || terms.length === 0)338 .filter((item) => item.score > 0 || terms.length === 0)
239 .sort((a, b) => b.score - a.score)339 .sort((a, b) => b.score - a.score)
240 .slice(0, limit)340 .slice(0, limit);
341
342 const results = ranked
241 .map(({ entry }) => ({343 .map(({ entry }) => ({
242 path: entry.path,344 path: entry.path,
243 aliases: entry.aliases,345 aliases: entry.aliases,
@@ -249,9 +351,9 @@ export function createDiscoverTool(tools: ToolDefinition[]): ToolDefinition { · 5 unmodified lines
249 }));351 }));
250 352
251 return {353 return {
252 bestPath: ranked[0]?.path ?? null,354 bestPath: chooseBestPath(ranked, terms.length),
253 results: ranked,355 results,
254 total: ranked.length,356 total: results.length,
255 };357 };
256 },358 },
257 };359 };
executor/lib/typechecker.ts+2−2
@@ -225,8 +225,8 @@ export function generateToolInventory(tools: ToolDescriptor[]): string { · 224 unmodified lines
225 "You have access to these tool namespaces:",225 "You have access to these tool namespaces:",
226 ` ${namespaces.join(", ")}`,226 ` ${namespaces.join(", ")}`,
227 "",227 "",
228 "Use `tools.discover({ query, depth?, limit?, compact? })` first. It returns `{ results, total }` (not an array).",228 "Use `tools.discover({ query, depth?, limit?, compact? })` first. It returns `{ bestPath, results, total }` (not an array).",
229 "Use the exact `path` returned by discover, and prefer its `exampleCall` snippet for invocation shape.",229 "Prefer `bestPath` when present, otherwise use a `results[i].path`; use `exampleCall` for invocation shape.",
230 "For migration/ETL tasks: discover once, then execute in small batches and return compact summaries (counts, IDs, top-N samples).",230 "For migration/ETL tasks: discover once, then execute in small batches and return compact summaries (counts, IDs, top-N samples).",
231 "Never shadow the global `tools` object (do NOT write `const tools = ...`).",231 "Never shadow the global `tools` object (do NOT write `const tools = ...`).",
232 "Then call tools directly using the returned path.",232 "Then call tools directly using the returned path.",