History

Let an empty-query namespace search enumerate the full catalog (#1387)

tools.search refused empty queries outright, and the sandbox proxy hint pointed agents at it as the way to discover tools: a namespace catalog could be counted (integrations.list reports toolCount) but never listed. Agents were left unioning keyword searches to lower-bound a catalog they could not confirm. An empty query with a namespace now enumerates: the whole catalog, path-sorted at score 0, through the existing limit/offset paging, with total an exact census that reconciles against toolCount. An empty query without a namespace still returns nothing (no scope, no signal, no arbitrary workspace dump), and ranked search is untouched. The proxy enumeration hints in all three runtimes now mention the namespace form. Fixes #1383

Rhys Sullivan committed 4 months ago

Commit 4fd94dc1 parent 0b0dab49 files · +316 −23Browse files

Unified · Split

9 files
  1. e2e/scenarios/namespace-enumeration.test.ts +167 −0
  2. packages/core/execution/src/tool-invoker.test.ts +100 −0
  3. packages/core/execution/src/tool-invoker.ts +40 −14
  4. packages/kernel/runtime-deno-subprocess/src/deno-subprocess-worker.mjs +1 −1
  5. packages/kernel/runtime-deno-subprocess/src/index.test.ts +2 −2
  6. packages/kernel/runtime-dynamic-worker/src/invocation.test.ts +2 −2
  7. packages/kernel/runtime-dynamic-worker/src/module-template.ts +1 −1
  8. packages/kernel/runtime-quickjs/src/index.test.ts +2 −2
  9. packages/kernel/runtime-quickjs/src/index.ts +1 −1
e2e/scenarios/namespace-enumeration.test.ts+167−0
@@ -0,0 +1,167 @@
1+// Cross-target: an integration's tool catalog is enumerable from the sandbox.
2+// tools.search with a namespace and an EMPTY query is enumeration: the whole
3+// catalog, path-sorted and paged, whose `total` reconciles exactly against
4+// the toolCount that executor.integrations.list reports. Before this
5+// guarantee an agent could only lower-bound a catalog by unioning keyword
6+// searches (issue #1383): the count was available and the contents were not.
7+import { randomBytes } from "node:crypto";
8+
9+import { expect } from "@effect/vitest";
10+import { Effect } from "effect";
11+import { composePluginApi } from "@executor-js/api/server";
12+import { openApiHttpPlugin } from "@executor-js/plugin-openapi/api";
13+import { AuthTemplateSlug, ConnectionName, IntegrationSlug } from "@executor-js/sdk/shared";
14+
15+import { scenario } from "../src/scenario";
16+import { Api, Target } from "../src/services";
17+
18+const api = composePluginApi([openApiHttpPlugin()] as const);
19+
20+const unique = (prefix: string) => `${prefix}_${randomBytes(4).toString("hex")}`;
21+
22+/** Three operations whose names share no verb, so no single keyword search
23+ * could ever return all of them — only enumeration can. */
24+const spec = (baseUrl: string): string =>
25+ JSON.stringify({
26+ openapi: "3.0.3",
27+ info: { title: "Enumerable API", version: "1.0.0" },
28+ servers: [{ url: baseUrl }],
29+ paths: {
30+ "/alpha": {
31+ get: {
32+ operationId: "alphaOp",
33+ summary: "First operation",
34+ responses: { "200": { description: "ok" } },
35+ },
36+ },
37+ "/bravo": {
38+ post: {
39+ operationId: "bravoOp",
40+ summary: "Second operation",
41+ responses: { "200": { description: "ok" } },
42+ },
43+ },
44+ "/charlie": {
45+ delete: {
46+ operationId: "charlieOp",
47+ summary: "Third operation",
48+ responses: { "200": { description: "ok" } },
49+ },
50+ },
51+ },
52+ });
53+
54+scenario(
55+ "Discovery · an integration's full tool catalog is enumerable and reconciles with its reported toolCount",
56+ {},
57+ Effect.gen(function* () {
58+ const target = yield* Target;
59+ const { client: makeClient } = yield* Api;
60+ const identity = yield* target.newIdentity();
61+ const client = yield* makeClient(api, identity);
62+ const slug = unique("enum");
63+
64+ yield* Effect.ensuring(
65+ Effect.gen(function* () {
66+ yield* client.openapi.addSpec({
67+ payload: {
68+ spec: { kind: "blob", value: spec("http://127.0.0.1:59999") },
69+ slug,
70+ baseUrl: "http://127.0.0.1:59999", // never contacted: discovery only
71+ authenticationTemplate: [
72+ {
73+ slug: "apiKey",
74+ type: "apiKey",
75+ headers: { "x-api-key": [{ type: "variable", name: "token" }] },
76+ },
77+ ],
78+ },
79+ });
80+ yield* client.connections.create({
81+ payload: {
82+ owner: "org",
83+ name: ConnectionName.make("main"),
84+ integration: IntegrationSlug.make(slug),
85+ template: AuthTemplateSlug.make("apiKey"),
86+ value: "tok_enum",
87+ },
88+ });
89+
90+ // Everything below runs inside the execute sandbox — the exact
91+ // surface an agent has.
92+ const executed = yield* client.executions.execute({
93+ payload: {
94+ code: `
95+const integrations = await tools.executor.integrations.list({ limit: 50 });
96+const mine = integrations.items.find((item) => item.id === ${JSON.stringify(slug)});
97+
98+const enumerated = await tools.search({ namespace: ${JSON.stringify(slug)}, query: "", limit: 50 });
99+
100+// Paged enumeration walks the same census.
101+const paged = [];
102+let offset = 0;
103+for (let i = 0; i < 10; i++) {
104+ const page = await tools.search({ namespace: ${JSON.stringify(slug)}, query: "", limit: 2, offset });
105+ paged.push(...page.items.map((item) => item.path));
106+ if (!page.hasMore) break;
107+ offset = page.nextOffset;
108+}
109+
110+// No namespace + empty query stays empty (no arbitrary workspace dump).
111+const unscoped = await tools.search({ query: "" });
112+
113+return JSON.stringify({
114+ toolCount: mine ? mine.toolCount : null,
115+ enumeratedTotal: enumerated.total,
116+ enumeratedPaths: enumerated.items.map((item) => item.path),
117+ pagedPaths: paged,
118+ unscopedTotal: unscoped.total,
119+});
120+`,
121+ autoApprove: true,
122+ },
123+ });
124+ expect(executed.status, "the sandbox execution completed").toBe("completed");
125+ const outcome = JSON.parse(executed.text) as {
126+ readonly toolCount: number | null;
127+ readonly enumeratedTotal: number;
128+ readonly enumeratedPaths: readonly string[];
129+ readonly pagedPaths: readonly string[];
130+ readonly unscopedTotal: number;
131+ };
132+
133+ // THE guarantee: enumeration returns the whole catalog, and its
134+ // total reconciles exactly against the integration's toolCount.
135+ expect(outcome.toolCount, "the integration reports its toolCount").toBe(3);
136+ expect(outcome.enumeratedTotal, "enumeration returns the full census").toBe(3);
137+ expect(
138+ outcome.enumeratedPaths.map((path) => path.split(".").at(-1)).sort(),
139+ "all three operations are enumerated, despite sharing no searchable verb",
140+ ).toEqual(["alphaOp", "bravoOp", "charlieOp"]);
141+ expect(
142+ outcome.enumeratedPaths,
143+ "enumeration is path-sorted, so paging is deterministic",
144+ ).toEqual([...outcome.enumeratedPaths].sort());
145+ expect(outcome.pagedPaths, "paged enumeration walks the same census in order").toEqual(
146+ outcome.enumeratedPaths,
147+ );
148+ expect(
149+ outcome.unscopedTotal,
150+ "an empty query without a namespace still returns nothing",
151+ ).toBe(0);
152+ }),
153+ Effect.gen(function* () {
154+ yield* client.connections
155+ .remove({
156+ params: {
157+ owner: "org",
158+ integration: IntegrationSlug.make(slug),
159+ name: ConnectionName.make("main"),
160+ },
161+ })
162+ .pipe(Effect.ignore);
163+ yield* client.openapi.removeSpec({ params: { slug } }).pipe(Effect.ignore);
164+ }),
165+ );
166+ }),
167+);
packages/core/execution/src/tool-invoker.test.ts+100−0
@@ -31,6 +31,7 @@ import { createExecutionEngine } from "./engine"; · 30 unmodified lines
3131 import { ExecutionToolError } from "./errors";
3232 import {
3333 describeTool,
34+ listExecutorIntegrations,
3435 makeExecutorToolInvoker,
3536 searchTools,
3637 type ToolDiscoveryProvider,
@@ -502,6 +503,105 @@ describe("tool discovery", () => { · 465 unmodified lines
502503 }),
503504 );
504505
506+ it.effect("enumerates a namespace's full catalog for an empty query with a namespace", () =>
507+ Effect.gen(function* () {
508+ const executor = yield* makeSearchExecutor();
509+
510+ // Enumeration, not search: every github tool, path-sorted, score 0.
511+ // `total` is the census an agent can reconcile against the integration's
512+ // reported toolCount — keyword search can only ever lower-bound it.
513+ const enumerated = yield* searchTools(executor, "", 100, { namespace: "github" });
514+ expect(enumerated.items.length).toBeGreaterThan(0);
515+ expect(enumerated.total).toBe(enumerated.items.length);
516+ expect(enumerated.items.map((item) => item.path)).toEqual(
517+ [...enumerated.items.map((item) => item.path)].sort((a, b) => a.localeCompare(b)),
518+ );
519+ expect(enumerated.items.every((item) => item.integration === "github")).toBe(true);
520+ expect(enumerated.items.every((item) => item.score === 0)).toBe(true);
521+
522+ // The census matches what the integrations list reports for the same
523+ // namespace — the reconciliation #1383 could not perform.
524+ const integrations = yield* listExecutorIntegrations(executor, { limit: 50 });
525+ const github = integrations.items.find((item) => item.id === "github");
526+ expect(github?.toolCount).toBe(enumerated.total);
527+
528+ // Enumeration pages like any other discovery result.
529+ const firstPage = yield* searchTools(executor, "", 1, { namespace: "github" });
530+ expect(firstPage.items).toEqual([enumerated.items[0]]);
531+ expect(firstPage.total).toBe(enumerated.total);
532+ expect(firstPage.hasMore).toBe(enumerated.total > 1);
533+ }),
534+ );
535+
536+ it.effect("enumeration scopes by exact slug, not the prefix matching ranked search uses", () =>
537+ Effect.gen(function* () {
538+ // Prefix-sibling integrations: "google" is a token prefix of both
539+ // "google_gmail" and "google_sheets". Ranked search deliberately treats
540+ // namespace as a fuzzy prefix filter; enumeration must NOT, or its
541+ // `total` stops being a census of one integration.
542+ const google = makeTestPlugin({
543+ pluginId: "google-test",
544+ integration: "google",
545+ tools: [
546+ {
547+ name: "ping",
548+ description: "Ping",
549+ inputJsonSchema: EmptyInputJson,
550+ validator: EmptyValidator,
551+ handler: () => Effect.succeed([]),
552+ },
553+ ],
554+ });
555+ const gmail = makeTestPlugin({
556+ pluginId: "google-gmail-test",
557+ integration: "google_gmail",
558+ tools: [
559+ {
560+ name: "listMessages",
561+ description: "List messages",
562+ inputJsonSchema: EmptyInputJson,
563+ validator: EmptyValidator,
564+ handler: () => Effect.succeed([]),
565+ },
566+ {
567+ name: "sendMessage",
568+ description: "Send a message",
569+ inputJsonSchema: EmptyInputJson,
570+ validator: EmptyValidator,
571+ handler: () => Effect.succeed([]),
572+ },
573+ ],
574+ });
575+ const executor = yield* makeExecutorWith([google, gmail] as const);
576+ yield* provision(executor as never, [
577+ { pluginId: "google-test", integration: "google" },
578+ { pluginId: "google-gmail-test", integration: "google_gmail" },
579+ ]);
580+
581+ const parent = yield* searchTools(executor, "", 100, { namespace: "google" });
582+ expect(
583+ parent.items.map((item) => item.integration),
584+ "enumerating 'google' returns only the google integration's tools",
585+ ).toEqual(["google"]);
586+ expect(parent.total).toBe(1);
587+
588+ const sibling = yield* searchTools(executor, "", 100, { namespace: "google_gmail" });
589+ expect(
590+ sibling.items.every((item) => item.integration === "google_gmail"),
591+ "enumerating the sibling returns only its own tools",
592+ ).toBe(true);
593+ expect(sibling.total).toBe(2);
594+
595+ // Ranked search keeps its fuzzy namespace semantics: a keyword search
596+ // scoped to "google" may still match tools in google_gmail.
597+ const fuzzy = yield* searchTools(executor, "message", 100, { namespace: "google" });
598+ expect(
599+ fuzzy.items.some((item) => item.integration === "google_gmail"),
600+ "ranked search still prefix-matches sibling namespaces",
601+ ).toBe(true);
602+ }),
603+ );
604+
505605 it.effect("paginates ranked matches via limit + offset with hasMore + nextOffset", () =>
506606 Effect.gen(function* () {
507607 const executor = yield* makeSearchExecutor();
packages/core/execution/src/tool-invoker.ts+40−14
@@ -94,7 +94,8 @@ const BUILTIN_TOOL_DESCRIPTIONS: ReadonlyMap<string, DescribedTool> = new Map< · 93 unmodified lines
9494 {
9595 path: "search",
9696 name: "search",
97− description: "Search available Executor tools.",
97+ description:
98+ "Search available Executor tools. An empty query with a namespace enumerates that integration's full catalog, sorted by path.",
9899 inputTypeScript: "{ query: string; namespace?: string; limit?: number; offset?: number; }",
99100 outputTypeScript:
100101 "{ items: ToolDiscoveryResult[]; total: number; hasMore: boolean; nextOffset: number | null; }",
@@ -655,15 +656,20 @@ export const searchTools = Effect.fn("executor.tools.search")(function* ( · 554 unmodified lines
655656 ...(options?.namespace ? { "executor.search.namespace": options.namespace } : {}),
656657 });
657658
658− const empty: PagedResult<ToolDiscoveryResult> = {
659− items: [],
660− total: 0,
661− hasMore: false,
662− nextOffset: null,
663− };
659+ const emptyQuery = normalizeSearchText(query).length === 0;
660+ const hasNamespace =
661+ options?.namespace !== undefined && normalizeSearchText(options.namespace).length > 0;
664662
665− if (normalizeSearchText(query).length === 0) {
666− return empty;
663+ // An empty query with no namespace stays empty: it carries neither a
664+ // ranking signal nor a scope, and listing the whole workspace "by default"
665+ // is exactly the arbitrary dump the ranked search refuses to be.
666+ if (emptyQuery && !hasNamespace) {
667+ return {
668+ items: [],
669+ total: 0,
670+ hasMore: false,
671+ nextOffset: null,
672+ } satisfies PagedResult<ToolDiscoveryResult>;
667673 }
668674
669675 const all = yield* executor.tools.list({ includeAnnotations: false }).pipe(
@@ -676,11 +682,31 @@ export const searchTools = Effect.fn("executor.tools.search")(function* ( · 6 unmodified lines
676682 ),
677683 );
678684 const searchable = all.map(toSearchableTool);
679− const ranked = searchable
680− .filter((tool: SearchableTool) => matchesNamespace(tool, options?.namespace))
681− .map((tool: SearchableTool) => scoreToolMatch(tool, query))
682− .filter(Predicate.isNotNull)
683− .sort((left, right) => right.score - left.score || left.path.localeCompare(right.path));
685+
686+ // An empty query WITH a namespace is enumeration, not search: there is no
687+ // ranking signal, so the namespace's whole catalog comes back sorted by
688+ // path (score 0) and paged. Enumeration scopes by EXACT integration slug —
689+ // the token-prefix `matchesNamespace` used for ranked search would also
690+ // sweep in prefix-sibling integrations (namespace "google" matching
691+ // google_gmail and google_sheets), which would silently break the census
692+ // guarantee: `total` here must reconcile against
693+ // `executor.integrations.list`'s per-integration toolCount.
694+ const ranked: readonly ToolDiscoveryResult[] = emptyQuery
695+ ? searchable
696+ .filter((tool) => tool.integration === options?.namespace?.trim())
697+ .sort((left, right) => left.path.localeCompare(right.path))
698+ .map((tool) => ({
699+ path: tool.path,
700+ name: tool.name,
701+ integration: tool.integration,
702+ score: 0,
703+ ...(tool.description !== undefined ? { description: tool.description } : {}),
704+ }))
705+ : searchable
706+ .filter((tool: SearchableTool) => matchesNamespace(tool, options?.namespace))
707+ .map((tool: SearchableTool) => scoreToolMatch(tool, query))
708+ .filter(Predicate.isNotNull)
709+ .sort((left, right) => right.score - left.score || left.path.localeCompare(right.path));
684710
685711 const page = paginate(ranked, offset, limit);
686712
packages/kernel/runtime-deno-subprocess/src/deno-subprocess-worker.mjs+1−1
@@ -43,7 +43,7 @@ const createToolCaller = (toolPath) => (args) => · 42 unmodified lines
4343
4444 const toolsEnumerationMessage = (path) =>
4545 (path.length === 0 ? "tools" : "tools." + path.join(".")) +
46− ' is a lazy proxy and cannot be enumerated. Use tools.search({ query: "..." }) to find tools, or tools.executor.coreTools.connections.list({}) to list saved connections.';
46+ ' is a lazy proxy and cannot be enumerated. Use tools.search({ query: "..." }) to find tools, tools.search({ namespace: "<integration>", query: "" }) to list every tool in an integration, or tools.executor.coreTools.connections.list({}) to list saved connections.';
4747
4848 const createToolsProxy = (path = []) => {
4949 const callable = () => undefined;
packages/kernel/runtime-deno-subprocess/src/index.test.ts+2−2
@@ -337,9 +337,9 @@ describe.skipIf(!isDenoAvailable())("runtime-deno-subprocess", () => { · 336 unmodified lines
337337
338338 expect(output.error).toBeUndefined();
339339 expect(output.result).toEqual({
340− keys: 'tools is a lazy proxy and cannot be enumerated. Use tools.search({ query: "..." }) to find tools, or tools.executor.coreTools.connections.list({}) to list saved connections.',
340+ keys: 'tools is a lazy proxy and cannot be enumerated. Use tools.search({ query: "..." }) to find tools, tools.search({ namespace: "<integration>", query: "" }) to list every tool in an integration, or tools.executor.coreTools.connections.list({}) to list saved connections.',
341341 spread:
342− 'tools.github is a lazy proxy and cannot be enumerated. Use tools.search({ query: "..." }) to find tools, or tools.executor.coreTools.connections.list({}) to list saved connections.',
342+ 'tools.github is a lazy proxy and cannot be enumerated. Use tools.search({ query: "..." }) to find tools, tools.search({ namespace: "<integration>", query: "" }) to list every tool in an integration, or tools.executor.coreTools.connections.list({}) to list saved connections.',
343343 });
344344 }),
345345 );
packages/kernel/runtime-dynamic-worker/src/invocation.test.ts+2−2
@@ -945,9 +945,9 @@ describe("makeDynamicWorkerExecutor", () => { · 944 unmodified lines
945945
946946 expect(result.error).toBeUndefined();
947947 expect(result.result).toEqual({
948− keys: 'tools is a lazy proxy and cannot be enumerated. Use tools.search({ query: "..." }) to find tools, or tools.executor.coreTools.connections.list({}) to list saved connections.',
948+ keys: 'tools is a lazy proxy and cannot be enumerated. Use tools.search({ query: "..." }) to find tools, tools.search({ namespace: "<integration>", query: "" }) to list every tool in an integration, or tools.executor.coreTools.connections.list({}) to list saved connections.',
949949 spread:
950− 'tools.github is a lazy proxy and cannot be enumerated. Use tools.search({ query: "..." }) to find tools, or tools.executor.coreTools.connections.list({}) to list saved connections.',
950+ 'tools.github is a lazy proxy and cannot be enumerated. Use tools.search({ query: "..." }) to find tools, tools.search({ namespace: "<integration>", query: "" }) to list every tool in an integration, or tools.executor.coreTools.connections.list({}) to list saved connections.',
951951 });
952952 }),
953953 );
packages/kernel/runtime-dynamic-worker/src/module-template.ts+1−1
@@ -178,7 +178,7 @@ export const buildExecutorModule = (body: string, timeoutMs: number): string => · 177 unmodified lines
178178 " };",
179179 " const __toolsEnumerationError = (path) => new Error(",
180180 " (path.length === 0 ? 'tools' : 'tools.' + path.join('.')) +",
181− " ' is a lazy proxy and cannot be enumerated. Use tools.search({ query: \"...\" }) to find tools, or tools.executor.coreTools.connections.list({}) to list saved connections.',",
181+ ' \' is a lazy proxy and cannot be enumerated. Use tools.search({ query: "..." }) to find tools, tools.search({ namespace: "<integration>", query: "" }) to list every tool in an integration, or tools.executor.coreTools.connections.list({}) to list saved connections.\',',
182182 " );",
183183 " const __makeToolsProxy = (path = []) => new Proxy(() => undefined, {",
184184 " get(_target, prop) {",
packages/kernel/runtime-quickjs/src/index.test.ts+2−2
@@ -363,9 +363,9 @@ describe("quickjs executor", () => { · 362 unmodified lines
363363
364364 expect(result.error).toBeUndefined();
365365 expect(result.result).toEqual({
366− keys: 'tools is a lazy proxy and cannot be enumerated. Use tools.search({ query: "..." }) to find tools, or tools.executor.coreTools.connections.list({}) to list saved connections.',
366+ keys: 'tools is a lazy proxy and cannot be enumerated. Use tools.search({ query: "..." }) to find tools, tools.search({ namespace: "<integration>", query: "" }) to list every tool in an integration, or tools.executor.coreTools.connections.list({}) to list saved connections.',
367367 spread:
368− 'tools.github is a lazy proxy and cannot be enumerated. Use tools.search({ query: "..." }) to find tools, or tools.executor.coreTools.connections.list({}) to list saved connections.',
368+ 'tools.github is a lazy proxy and cannot be enumerated. Use tools.search({ query: "..." }) to find tools, tools.search({ namespace: "<integration>", query: "" }) to list every tool in an integration, or tools.executor.coreTools.connections.list({}) to list saved connections.',
369369 });
370370 }),
371371 );
packages/kernel/runtime-quickjs/src/index.ts+1−1
@@ -170,7 +170,7 @@ const buildExecutionSource = (code: string): string => { · 169 unmodified lines
170170 "};",
171171 "const __toolsEnumerationError = (path) => new Error(",
172172 " (path.length === 0 ? 'tools' : 'tools.' + path.join('.')) +",
173− " ' is a lazy proxy and cannot be enumerated. Use tools.search({ query: \"...\" }) to find tools, or tools.executor.coreTools.connections.list({}) to list saved connections.',",
173+ ' \' is a lazy proxy and cannot be enumerated. Use tools.search({ query: "..." }) to find tools, tools.search({ namespace: "<integration>", query: "" }) to list every tool in an integration, or tools.executor.coreTools.connections.list({}) to list saved connections.\',',
174174 ");",
175175 "const __makeToolsProxy = (path = []) => new Proxy(() => undefined, {",
176176 " get(_target, prop) {",