History

Let an empty-query namespace search enumerate the full catalog (#1387)

tools.search refused empty queries outright, and the sandbox proxy hint pointed agents at it as the way to discover tools: a namespace catalog could be counted (integrations.list reports toolCount) but never listed. Agents were left unioning keyword searches to lower-bound a catalog they could not confirm. An empty query with a namespace now enumerates: the whole catalog, path-sorted at score 0, through the existing limit/offset paging, with total an exact census that reconciles against toolCount. An empty query without a namespace still returns nothing (no scope, no signal, no arbitrary workspace dump), and ranked search is untouched. The proxy enumeration hints in all three runtimes now mention the namespace form. Fixes #1383

Rhys Sullivan committed 4 months ago

Commit 4fd94dc1 parent 0b0dab49 files · +316 −23Browse files

Unified · Split

9 files
  1. e2e/scenarios/namespace-enumeration.test.ts +167 −0
  2. packages/core/execution/src/tool-invoker.test.ts +100 −0
  3. packages/core/execution/src/tool-invoker.ts +40 −14
  4. packages/kernel/runtime-deno-subprocess/src/deno-subprocess-worker.mjs +1 −1
  5. packages/kernel/runtime-deno-subprocess/src/index.test.ts +2 −2
  6. packages/kernel/runtime-dynamic-worker/src/invocation.test.ts +2 −2
  7. packages/kernel/runtime-dynamic-worker/src/module-template.ts +1 −1
  8. packages/kernel/runtime-quickjs/src/index.test.ts +2 −2
  9. packages/kernel/runtime-quickjs/src/index.ts +1 −1
e2e/scenarios/namespace-enumeration.test.ts+167−0
@@ -0,0 +1,167 @@
1// Cross-target: an integration's tool catalog is enumerable from the sandbox.
2// tools.search with a namespace and an EMPTY query is enumeration: the whole
3// catalog, path-sorted and paged, whose `total` reconciles exactly against
4// the toolCount that executor.integrations.list reports. Before this
5// guarantee an agent could only lower-bound a catalog by unioning keyword
6// searches (issue #1383): the count was available and the contents were not.
7import { randomBytes } from "node:crypto";
8
9import { expect } from "@effect/vitest";
10import { Effect } from "effect";
11import { composePluginApi } from "@executor-js/api/server";
12import { openApiHttpPlugin } from "@executor-js/plugin-openapi/api";
13import { AuthTemplateSlug, ConnectionName, IntegrationSlug } from "@executor-js/sdk/shared";
14
15import { scenario } from "../src/scenario";
16import { Api, Target } from "../src/services";
17
18const api = composePluginApi([openApiHttpPlugin()] as const);
19
20const unique = (prefix: string) => `${prefix}_${randomBytes(4).toString("hex")}`;
21
22/** Three operations whose names share no verb, so no single keyword search
23 * could ever return all of them — only enumeration can. */
24const spec = (baseUrl: string): string =>
25 JSON.stringify({
26 openapi: "3.0.3",
27 info: { title: "Enumerable API", version: "1.0.0" },
28 servers: [{ url: baseUrl }],
29 paths: {
30 "/alpha": {
31 get: {
32 operationId: "alphaOp",
33 summary: "First operation",
34 responses: { "200": { description: "ok" } },
35 },
36 },
37 "/bravo": {
38 post: {
39 operationId: "bravoOp",
40 summary: "Second operation",
41 responses: { "200": { description: "ok" } },
42 },
43 },
44 "/charlie": {
45 delete: {
46 operationId: "charlieOp",
47 summary: "Third operation",
48 responses: { "200": { description: "ok" } },
49 },
50 },
51 },
52 });
53
54scenario(
55 "Discovery · an integration's full tool catalog is enumerable and reconciles with its reported toolCount",
56 {},
57 Effect.gen(function* () {
58 const target = yield* Target;
59 const { client: makeClient } = yield* Api;
60 const identity = yield* target.newIdentity();
61 const client = yield* makeClient(api, identity);
62 const slug = unique("enum");
63
64 yield* Effect.ensuring(
65 Effect.gen(function* () {
66 yield* client.openapi.addSpec({
67 payload: {
68 spec: { kind: "blob", value: spec("http://127.0.0.1:59999") },
69 slug,
70 baseUrl: "http://127.0.0.1:59999", // never contacted: discovery only
71 authenticationTemplate: [
72 {
73 slug: "apiKey",
74 type: "apiKey",
75 headers: { "x-api-key": [{ type: "variable", name: "token" }] },
76 },
77 ],
78 },
79 });
80 yield* client.connections.create({
81 payload: {
82 owner: "org",
83 name: ConnectionName.make("main"),
84 integration: IntegrationSlug.make(slug),
85 template: AuthTemplateSlug.make("apiKey"),
86 value: "tok_enum",
87 },
88 });
89
90 // Everything below runs inside the execute sandbox — the exact
91 // surface an agent has.
92 const executed = yield* client.executions.execute({
93 payload: {
94 code: `
95const integrations = await tools.executor.integrations.list({ limit: 50 });
96const mine = integrations.items.find((item) => item.id === ${JSON.stringify(slug)});
97
98const enumerated = await tools.search({ namespace: ${JSON.stringify(slug)}, query: "", limit: 50 });
99
100// Paged enumeration walks the same census.
101const paged = [];
102let offset = 0;
103for (let i = 0; i < 10; i++) {
104 const page = await tools.search({ namespace: ${JSON.stringify(slug)}, query: "", limit: 2, offset });
105 paged.push(...page.items.map((item) => item.path));
106 if (!page.hasMore) break;
107 offset = page.nextOffset;
108}
109
110// No namespace + empty query stays empty (no arbitrary workspace dump).
111const unscoped = await tools.search({ query: "" });
112
113return JSON.stringify({
114 toolCount: mine ? mine.toolCount : null,
115 enumeratedTotal: enumerated.total,
116 enumeratedPaths: enumerated.items.map((item) => item.path),
117 pagedPaths: paged,
118 unscopedTotal: unscoped.total,
119});
120`,
121 autoApprove: true,
122 },
123 });
124 expect(executed.status, "the sandbox execution completed").toBe("completed");
125 const outcome = JSON.parse(executed.text) as {
126 readonly toolCount: number | null;
127 readonly enumeratedTotal: number;
128 readonly enumeratedPaths: readonly string[];
129 readonly pagedPaths: readonly string[];
130 readonly unscopedTotal: number;
131 };
132
133 // THE guarantee: enumeration returns the whole catalog, and its
134 // total reconciles exactly against the integration's toolCount.
135 expect(outcome.toolCount, "the integration reports its toolCount").toBe(3);
136 expect(outcome.enumeratedTotal, "enumeration returns the full census").toBe(3);
137 expect(
138 outcome.enumeratedPaths.map((path) => path.split(".").at(-1)).sort(),
139 "all three operations are enumerated, despite sharing no searchable verb",
140 ).toEqual(["alphaOp", "bravoOp", "charlieOp"]);
141 expect(
142 outcome.enumeratedPaths,
143 "enumeration is path-sorted, so paging is deterministic",
144 ).toEqual([...outcome.enumeratedPaths].sort());
145 expect(outcome.pagedPaths, "paged enumeration walks the same census in order").toEqual(
146 outcome.enumeratedPaths,
147 );
148 expect(
149 outcome.unscopedTotal,
150 "an empty query without a namespace still returns nothing",
151 ).toBe(0);
152 }),
153 Effect.gen(function* () {
154 yield* client.connections
155 .remove({
156 params: {
157 owner: "org",
158 integration: IntegrationSlug.make(slug),
159 name: ConnectionName.make("main"),
160 },
161 })
162 .pipe(Effect.ignore);
163 yield* client.openapi.removeSpec({ params: { slug } }).pipe(Effect.ignore);
164 }),
165 );
166 }),
167);
packages/core/execution/src/tool-invoker.test.ts+100−0
@@ -31,6 +31,7 @@ import { createExecutionEngine } from "./engine"; · 30 unmodified lines
31import { ExecutionToolError } from "./errors";31import { ExecutionToolError } from "./errors";
32import {32import {
33 describeTool,33 describeTool,
34 listExecutorIntegrations,
34 makeExecutorToolInvoker,35 makeExecutorToolInvoker,
35 searchTools,36 searchTools,
36 type ToolDiscoveryProvider,37 type ToolDiscoveryProvider,
@@ -502,6 +503,105 @@ describe("tool discovery", () => { · 465 unmodified lines
502 }),503 }),
503 );504 );
504 505
506 it.effect("enumerates a namespace's full catalog for an empty query with a namespace", () =>
507 Effect.gen(function* () {
508 const executor = yield* makeSearchExecutor();
509
510 // Enumeration, not search: every github tool, path-sorted, score 0.
511 // `total` is the census an agent can reconcile against the integration's
512 // reported toolCount — keyword search can only ever lower-bound it.
513 const enumerated = yield* searchTools(executor, "", 100, { namespace: "github" });
514 expect(enumerated.items.length).toBeGreaterThan(0);
515 expect(enumerated.total).toBe(enumerated.items.length);
516 expect(enumerated.items.map((item) => item.path)).toEqual(
517 [...enumerated.items.map((item) => item.path)].sort((a, b) => a.localeCompare(b)),
518 );
519 expect(enumerated.items.every((item) => item.integration === "github")).toBe(true);
520 expect(enumerated.items.every((item) => item.score === 0)).toBe(true);
521
522 // The census matches what the integrations list reports for the same
523 // namespace — the reconciliation #1383 could not perform.
524 const integrations = yield* listExecutorIntegrations(executor, { limit: 50 });
525 const github = integrations.items.find((item) => item.id === "github");
526 expect(github?.toolCount).toBe(enumerated.total);
527
528 // Enumeration pages like any other discovery result.
529 const firstPage = yield* searchTools(executor, "", 1, { namespace: "github" });
530 expect(firstPage.items).toEqual([enumerated.items[0]]);
531 expect(firstPage.total).toBe(enumerated.total);
532 expect(firstPage.hasMore).toBe(enumerated.total > 1);
533 }),
534 );
535
536 it.effect("enumeration scopes by exact slug, not the prefix matching ranked search uses", () =>
537 Effect.gen(function* () {
538 // Prefix-sibling integrations: "google" is a token prefix of both
539 // "google_gmail" and "google_sheets". Ranked search deliberately treats
540 // namespace as a fuzzy prefix filter; enumeration must NOT, or its
541 // `total` stops being a census of one integration.
542 const google = makeTestPlugin({
543 pluginId: "google-test",
544 integration: "google",
545 tools: [
546 {
547 name: "ping",
548 description: "Ping",
549 inputJsonSchema: EmptyInputJson,
550 validator: EmptyValidator,
551 handler: () => Effect.succeed([]),
552 },
553 ],
554 });
555 const gmail = makeTestPlugin({
556 pluginId: "google-gmail-test",
557 integration: "google_gmail",
558 tools: [
559 {
560 name: "listMessages",
561 description: "List messages",
562 inputJsonSchema: EmptyInputJson,
563 validator: EmptyValidator,
564 handler: () => Effect.succeed([]),
565 },
566 {
567 name: "sendMessage",
568 description: "Send a message",
569 inputJsonSchema: EmptyInputJson,
570 validator: EmptyValidator,
571 handler: () => Effect.succeed([]),
572 },
573 ],
574 });
575 const executor = yield* makeExecutorWith([google, gmail] as const);
576 yield* provision(executor as never, [
577 { pluginId: "google-test", integration: "google" },
578 { pluginId: "google-gmail-test", integration: "google_gmail" },
579 ]);
580
581 const parent = yield* searchTools(executor, "", 100, { namespace: "google" });
582 expect(
583 parent.items.map((item) => item.integration),
584 "enumerating 'google' returns only the google integration's tools",
585 ).toEqual(["google"]);
586 expect(parent.total).toBe(1);
587
588 const sibling = yield* searchTools(executor, "", 100, { namespace: "google_gmail" });
589 expect(
590 sibling.items.every((item) => item.integration === "google_gmail"),
591 "enumerating the sibling returns only its own tools",
592 ).toBe(true);
593 expect(sibling.total).toBe(2);
594
595 // Ranked search keeps its fuzzy namespace semantics: a keyword search
596 // scoped to "google" may still match tools in google_gmail.
597 const fuzzy = yield* searchTools(executor, "message", 100, { namespace: "google" });
598 expect(
599 fuzzy.items.some((item) => item.integration === "google_gmail"),
600 "ranked search still prefix-matches sibling namespaces",
601 ).toBe(true);
602 }),
603 );
604
505 it.effect("paginates ranked matches via limit + offset with hasMore + nextOffset", () =>605 it.effect("paginates ranked matches via limit + offset with hasMore + nextOffset", () =>
506 Effect.gen(function* () {606 Effect.gen(function* () {
507 const executor = yield* makeSearchExecutor();607 const executor = yield* makeSearchExecutor();
packages/core/execution/src/tool-invoker.ts+40−14
@@ -94,7 +94,8 @@ const BUILTIN_TOOL_DESCRIPTIONS: ReadonlyMap<string, DescribedTool> = new Map< · 93 unmodified lines
94 {94 {
95 path: "search",95 path: "search",
96 name: "search",96 name: "search",
97 description: "Search available Executor tools.",97 description:
98 "Search available Executor tools. An empty query with a namespace enumerates that integration's full catalog, sorted by path.",
98 inputTypeScript: "{ query: string; namespace?: string; limit?: number; offset?: number; }",99 inputTypeScript: "{ query: string; namespace?: string; limit?: number; offset?: number; }",
99 outputTypeScript:100 outputTypeScript:
100 "{ items: ToolDiscoveryResult[]; total: number; hasMore: boolean; nextOffset: number | null; }",101 "{ items: ToolDiscoveryResult[]; total: number; hasMore: boolean; nextOffset: number | null; }",
@@ -655,15 +656,20 @@ export const searchTools = Effect.fn("executor.tools.search")(function* ( · 554 unmodified lines
655 ...(options?.namespace ? { "executor.search.namespace": options.namespace } : {}),656 ...(options?.namespace ? { "executor.search.namespace": options.namespace } : {}),
656 });657 });
657 658
658 const empty: PagedResult<ToolDiscoveryResult> = {659 const emptyQuery = normalizeSearchText(query).length === 0;
659 items: [],660 const hasNamespace =
660 total: 0,661 options?.namespace !== undefined && normalizeSearchText(options.namespace).length > 0;
661 hasMore: false,
662 nextOffset: null,
663 };
664 662
665 if (normalizeSearchText(query).length === 0) {663 // An empty query with no namespace stays empty: it carries neither a
666 return empty;664 // ranking signal nor a scope, and listing the whole workspace "by default"
665 // is exactly the arbitrary dump the ranked search refuses to be.
666 if (emptyQuery && !hasNamespace) {
667 return {
668 items: [],
669 total: 0,
670 hasMore: false,
671 nextOffset: null,
672 } satisfies PagedResult<ToolDiscoveryResult>;
667 }673 }
668 674
669 const all = yield* executor.tools.list({ includeAnnotations: false }).pipe(675 const all = yield* executor.tools.list({ includeAnnotations: false }).pipe(
@@ -676,11 +682,31 @@ export const searchTools = Effect.fn("executor.tools.search")(function* ( · 6 unmodified lines
676 ),682 ),
677 );683 );
678 const searchable = all.map(toSearchableTool);684 const searchable = all.map(toSearchableTool);
679 const ranked = searchable685
680 .filter((tool: SearchableTool) => matchesNamespace(tool, options?.namespace))686 // An empty query WITH a namespace is enumeration, not search: there is no
681 .map((tool: SearchableTool) => scoreToolMatch(tool, query))687 // ranking signal, so the namespace's whole catalog comes back sorted by
682 .filter(Predicate.isNotNull)688 // path (score 0) and paged. Enumeration scopes by EXACT integration slug —
683 .sort((left, right) => right.score - left.score || left.path.localeCompare(right.path));689 // the token-prefix `matchesNamespace` used for ranked search would also
690 // sweep in prefix-sibling integrations (namespace "google" matching
691 // google_gmail and google_sheets), which would silently break the census
692 // guarantee: `total` here must reconcile against
693 // `executor.integrations.list`'s per-integration toolCount.
694 const ranked: readonly ToolDiscoveryResult[] = emptyQuery
695 ? searchable
696 .filter((tool) => tool.integration === options?.namespace?.trim())
697 .sort((left, right) => left.path.localeCompare(right.path))
698 .map((tool) => ({
699 path: tool.path,
700 name: tool.name,
701 integration: tool.integration,
702 score: 0,
703 ...(tool.description !== undefined ? { description: tool.description } : {}),
704 }))
705 : searchable
706 .filter((tool: SearchableTool) => matchesNamespace(tool, options?.namespace))
707 .map((tool: SearchableTool) => scoreToolMatch(tool, query))
708 .filter(Predicate.isNotNull)
709 .sort((left, right) => right.score - left.score || left.path.localeCompare(right.path));
684 710
685 const page = paginate(ranked, offset, limit);711 const page = paginate(ranked, offset, limit);
686 712
packages/kernel/runtime-deno-subprocess/src/deno-subprocess-worker.mjs+1−1
@@ -43,7 +43,7 @@ const createToolCaller = (toolPath) => (args) => · 42 unmodified lines
43 43
44const toolsEnumerationMessage = (path) =>44const toolsEnumerationMessage = (path) =>
45 (path.length === 0 ? "tools" : "tools." + path.join(".")) +45 (path.length === 0 ? "tools" : "tools." + path.join(".")) +
46 ' is a lazy proxy and cannot be enumerated. Use tools.search({ query: "..." }) to find tools, or tools.executor.coreTools.connections.list({}) to list saved connections.';46 ' is a lazy proxy and cannot be enumerated. Use tools.search({ query: "..." }) to find tools, tools.search({ namespace: "<integration>", query: "" }) to list every tool in an integration, or tools.executor.coreTools.connections.list({}) to list saved connections.';
47 47
48const createToolsProxy = (path = []) => {48const createToolsProxy = (path = []) => {
49 const callable = () => undefined;49 const callable = () => undefined;
packages/kernel/runtime-deno-subprocess/src/index.test.ts+2−2
@@ -337,9 +337,9 @@ describe.skipIf(!isDenoAvailable())("runtime-deno-subprocess", () => { · 336 unmodified lines
337 337
338 expect(output.error).toBeUndefined();338 expect(output.error).toBeUndefined();
339 expect(output.result).toEqual({339 expect(output.result).toEqual({
340 keys: 'tools is a lazy proxy and cannot be enumerated. Use tools.search({ query: "..." }) to find tools, or tools.executor.coreTools.connections.list({}) to list saved connections.',340 keys: 'tools is a lazy proxy and cannot be enumerated. Use tools.search({ query: "..." }) to find tools, tools.search({ namespace: "<integration>", query: "" }) to list every tool in an integration, or tools.executor.coreTools.connections.list({}) to list saved connections.',
341 spread:341 spread:
342 'tools.github is a lazy proxy and cannot be enumerated. Use tools.search({ query: "..." }) to find tools, or tools.executor.coreTools.connections.list({}) to list saved connections.',342 'tools.github is a lazy proxy and cannot be enumerated. Use tools.search({ query: "..." }) to find tools, tools.search({ namespace: "<integration>", query: "" }) to list every tool in an integration, or tools.executor.coreTools.connections.list({}) to list saved connections.',
343 });343 });
344 }),344 }),
345 );345 );
packages/kernel/runtime-dynamic-worker/src/invocation.test.ts+2−2
@@ -945,9 +945,9 @@ describe("makeDynamicWorkerExecutor", () => { · 944 unmodified lines
945 945
946 expect(result.error).toBeUndefined();946 expect(result.error).toBeUndefined();
947 expect(result.result).toEqual({947 expect(result.result).toEqual({
948 keys: 'tools is a lazy proxy and cannot be enumerated. Use tools.search({ query: "..." }) to find tools, or tools.executor.coreTools.connections.list({}) to list saved connections.',948 keys: 'tools is a lazy proxy and cannot be enumerated. Use tools.search({ query: "..." }) to find tools, tools.search({ namespace: "<integration>", query: "" }) to list every tool in an integration, or tools.executor.coreTools.connections.list({}) to list saved connections.',
949 spread:949 spread:
950 'tools.github is a lazy proxy and cannot be enumerated. Use tools.search({ query: "..." }) to find tools, or tools.executor.coreTools.connections.list({}) to list saved connections.',950 'tools.github is a lazy proxy and cannot be enumerated. Use tools.search({ query: "..." }) to find tools, tools.search({ namespace: "<integration>", query: "" }) to list every tool in an integration, or tools.executor.coreTools.connections.list({}) to list saved connections.',
951 });951 });
952 }),952 }),
953 );953 );
packages/kernel/runtime-dynamic-worker/src/module-template.ts+1−1
@@ -178,7 +178,7 @@ export const buildExecutorModule = (body: string, timeoutMs: number): string => · 177 unmodified lines
178 " };",178 " };",
179 " const __toolsEnumerationError = (path) => new Error(",179 " const __toolsEnumerationError = (path) => new Error(",
180 " (path.length === 0 ? 'tools' : 'tools.' + path.join('.')) +",180 " (path.length === 0 ? 'tools' : 'tools.' + path.join('.')) +",
181 " ' is a lazy proxy and cannot be enumerated. Use tools.search({ query: \"...\" }) to find tools, or tools.executor.coreTools.connections.list({}) to list saved connections.',",181 ' \' is a lazy proxy and cannot be enumerated. Use tools.search({ query: "..." }) to find tools, tools.search({ namespace: "<integration>", query: "" }) to list every tool in an integration, or tools.executor.coreTools.connections.list({}) to list saved connections.\',',
182 " );",182 " );",
183 " const __makeToolsProxy = (path = []) => new Proxy(() => undefined, {",183 " const __makeToolsProxy = (path = []) => new Proxy(() => undefined, {",
184 " get(_target, prop) {",184 " get(_target, prop) {",
packages/kernel/runtime-quickjs/src/index.test.ts+2−2
@@ -363,9 +363,9 @@ describe("quickjs executor", () => { · 362 unmodified lines
363 363
364 expect(result.error).toBeUndefined();364 expect(result.error).toBeUndefined();
365 expect(result.result).toEqual({365 expect(result.result).toEqual({
366 keys: 'tools is a lazy proxy and cannot be enumerated. Use tools.search({ query: "..." }) to find tools, or tools.executor.coreTools.connections.list({}) to list saved connections.',366 keys: 'tools is a lazy proxy and cannot be enumerated. Use tools.search({ query: "..." }) to find tools, tools.search({ namespace: "<integration>", query: "" }) to list every tool in an integration, or tools.executor.coreTools.connections.list({}) to list saved connections.',
367 spread:367 spread:
368 'tools.github is a lazy proxy and cannot be enumerated. Use tools.search({ query: "..." }) to find tools, or tools.executor.coreTools.connections.list({}) to list saved connections.',368 'tools.github is a lazy proxy and cannot be enumerated. Use tools.search({ query: "..." }) to find tools, tools.search({ namespace: "<integration>", query: "" }) to list every tool in an integration, or tools.executor.coreTools.connections.list({}) to list saved connections.',
369 });369 });
370 }),370 }),
371 );371 );
packages/kernel/runtime-quickjs/src/index.ts+1−1
@@ -170,7 +170,7 @@ const buildExecutionSource = (code: string): string => { · 169 unmodified lines
170 "};",170 "};",
171 "const __toolsEnumerationError = (path) => new Error(",171 "const __toolsEnumerationError = (path) => new Error(",
172 " (path.length === 0 ? 'tools' : 'tools.' + path.join('.')) +",172 " (path.length === 0 ? 'tools' : 'tools.' + path.join('.')) +",
173 " ' is a lazy proxy and cannot be enumerated. Use tools.search({ query: \"...\" }) to find tools, or tools.executor.coreTools.connections.list({}) to list saved connections.',",173 ' \' is a lazy proxy and cannot be enumerated. Use tools.search({ query: "..." }) to find tools, tools.search({ namespace: "<integration>", query: "" }) to list every tool in an integration, or tools.executor.coreTools.connections.list({}) to list saved connections.\',',
174 ");",174 ");",
175 "const __makeToolsProxy = (path = []) => new Proxy(() => undefined, {",175 "const __makeToolsProxy = (path = []) => new Proxy(() => undefined, {",
176 " get(_target, prop) {",176 " get(_target, prop) {",