import { describe, expect, it } from "@effect/vitest";import { Effect, Schema } from "effect"; import { ElicitationResponse, Source, createExecutor, inMemoryToolsPlugin, makeTestConfig, tool,} from "@executor/sdk";import { createExecutionEngine } from "./engine";import { describeTool, searchTools } from "./tool-invoker"; const EmptyInput = Schema.Struct({});const RepoInput = Schema.Struct({ owner: Schema.String, repo: Schema.String,});const ContactInput = Schema.Struct({ email: Schema.String,}); const acceptAll = () => Effect.succeed(new ElicitationResponse({ action: "accept" })); const makeSearchExecutor = () => Effect.gen(function* () { const config = makeTestConfig({ plugins: [ inMemoryToolsPlugin({ namespace: "github", tools: [ tool({ name: "listRepositoryIssues", description: "List issues for a repository", inputSchema: RepoInput, handler: () => [], }), tool({ name: "getRepositoryDetails", description: "Get repository details including the default branch", inputSchema: RepoInput, handler: () => ({ defaultBranch: "main" }), }), tool({ name: "searchDocs", description: "Search GitHub API documentation", inputSchema: EmptyInput, handler: () => [], }), ], }), inMemoryToolsPlugin({ namespace: "crm", tools: [ tool({ name: "createContact", description: "Create a CRM contact record", inputSchema: ContactInput, handler: () => ({ id: "contact_1" }), }), tool({ name: "listContacts", description: "List CRM contacts", inputSchema: EmptyInput, handler: () => [], }), ], }), ] as const, }); yield* config.sources.registerRuntime(new Source({ id: "github", name: "GitHub", kind: "in-memory", runtime: true, canRemove: false, canRefresh: false, })); yield* config.sources.registerRuntime(new Source({ id: "crm", name: "CRM", kind: "in-memory", runtime: true, canRemove: false, canRefresh: false, })); return yield* createExecutor(config); }); describe("tool discovery", () => { it.effect("ranks matches using ids, namespaces, camelCase names, and descriptions", () => Effect.gen(function* () { const executor = yield* makeSearchExecutor(); const githubMatches = yield* searchTools(executor, "github issues", 5); expect(githubMatches.map((match) => match.path)).toEqual([ "github.listRepositoryIssues", ]); expect(githubMatches[0]?.score ?? 0).toBeGreaterThan(0); const repoMatches = yield* searchTools(executor, "repo details", 5); expect(repoMatches[0]?.path).toBe("github.getRepositoryDetails"); const crmMatches = yield* searchTools(executor, "crm create contact", 5); expect(crmMatches[0]?.path).toBe("crm.createContact"); expect(crmMatches[0]?.score ?? 0).toBeGreaterThan(crmMatches[1]?.score ?? 0); }), ); it.effect("returns no matches for empty queries instead of listing arbitrary tools", () => Effect.gen(function* () { const executor = yield* makeSearchExecutor(); const matches = yield* searchTools(executor, "", 5); expect(matches).toEqual([]); }), ); it.effect("can narrow discovery to a namespace", () => Effect.gen(function* () { const executor = yield* makeSearchExecutor(); const githubOnly = yield* searchTools(executor, "list", 5, { namespace: "github", }); expect(githubOnly.map((match) => match.path)).toEqual([ "github.listRepositoryIssues", ]); const crmOnly = yield* searchTools(executor, "list", 5, { namespace: "crm", }); expect(crmOnly.map((match) => match.path)).toEqual([ "crm.listContacts", ]); const sandboxResult = yield* Effect.promise(() => createExecutionEngine({ executor }).execute( 'return await tools.search({ namespace: "crm", query: "create contact", limit: 5 });', { onElicitation: acceptAll }, ), ); expect(sandboxResult.error).toBeUndefined(); expect(sandboxResult.result).toEqual([ expect.objectContaining({ path: "crm.createContact" }), ]); }), ); it.effect("supports executor-scoped source listing and tool search", () => Effect.gen(function* () { const executor = yield* makeSearchExecutor(); const listed = yield* Effect.promise(() => createExecutionEngine({ executor }).execute( "return await tools.executor.sources.list();", { onElicitation: acceptAll }, ), ); expect(listed.error).toBeUndefined(); expect(listed.result).toEqual( expect.arrayContaining([ expect.objectContaining({ id: "github", toolCount: 3 }), expect.objectContaining({ id: "crm", toolCount: 2 }), ]), ); const searched = yield* Effect.promise(() => createExecutionEngine({ executor }).execute( 'return await tools.search({ query: "list contacts", namespace: "crm", limit: 5 });', { onElicitation: acceptAll }, ), ); expect(searched.error).toBeUndefined(); expect(searched.result).toEqual([ expect.objectContaining({ path: "crm.listContacts" }), ]); }), ); it.effect("describes tools with TypeScript previews", () => Effect.gen(function* () { const executor = yield* makeSearchExecutor(); const withoutSchemas = yield* describeTool( executor, "github.listRepositoryIssues", ); expect(withoutSchemas).toEqual({ path: "github.listRepositoryIssues", name: "listRepositoryIssues", description: "List issues for a repository", inputTypeScript: "{ owner: string; repo: string }", outputTypeScript: undefined, typeScriptDefinitions: undefined, }); const withSchemas = yield* describeTool( executor, "github.listRepositoryIssues", ); expect(withSchemas.path).toBe("github.listRepositoryIssues"); expect(withSchemas.inputTypeScript).toBe( "{ owner: string; repo: string }", ); expect(withSchemas.typeScriptDefinitions).toBeUndefined(); expect(withSchemas.outputTypeScript).toBeUndefined(); }), ); it.effect("rejects malformed discover calls inside the sandbox", () => Effect.gen(function* () { const executor = yield* makeSearchExecutor(); const engine = createExecutionEngine({ executor }); const invalid = yield* Effect.promise(() => engine.execute( [ "try {", " await tools.search(\"github issues\");", " return \"unexpected\";", "} catch (error) {", " return error instanceof Error ? error.message : String(error);", "}", ].join("\n"), { onElicitation: acceptAll }, ), ); expect(invalid.error).toBeUndefined(); expect(String(invalid.result)).toContain( "tools.search expects an object: { query?: string; namespace?: string; limit?: number }", ); const emptyQuery = yield* Effect.promise(() => engine.execute( "return await tools.search({ query: \"\", limit: 5 });", { onElicitation: acceptAll }, ), ); expect(emptyQuery.error).toBeUndefined(); expect(emptyQuery.result).toEqual([]); const invalidDescribe = yield* Effect.promise(() => engine.execute( [ "try {", " await tools.describe.tool({ path: \"github.listRepositoryIssues\", includeSchemas: true });", " return \"unexpected\";", "} catch (error) {", " return error instanceof Error ? error.message : String(error);", "}", ].join("\n"), { onElicitation: acceptAll }, ), ); expect(invalidDescribe.error).toBeUndefined(); expect(String(invalidDescribe.result)).toContain( "tools.describe.tool no longer accepts includeSchemas", ); const invalidSearch = yield* Effect.promise(() => engine.execute( 'try { return await tools.search("crm"); } catch (error) { return error instanceof Error ? error.message : String(error); }', { onElicitation: acceptAll }, ), ); expect(invalidSearch.error).toBeUndefined(); expect(String(invalidSearch.result)).toContain( "tools.search expects an object", ); }), );});
import { describe, expect, it } from "@effect/vitest";
import { Effect, Schema } from "effect";
import {
ElicitationResponse,
Source,
createExecutor,
inMemoryToolsPlugin,
makeTestConfig,
tool,
} from "@executor/sdk";
import { createExecutionEngine } from "./engine";
import { describeTool, searchTools } from "./tool-invoker";
const EmptyInput = Schema.Struct({});
const RepoInput = Schema.Struct({
owner: Schema.String,
repo: Schema.String,
});
const ContactInput = Schema.Struct({
email: Schema.String,
});
const acceptAll = () => Effect.succeed(new ElicitationResponse({ action: "accept" }));
const makeSearchExecutor = () =>
Effect.gen(function* () {
const config = makeTestConfig({
plugins: [
inMemoryToolsPlugin({
namespace: "github",
tools: [
tool({
name: "listRepositoryIssues",
description: "List issues for a repository",
inputSchema: RepoInput,
handler: () => [],
}),
tool({
name: "getRepositoryDetails",
description: "Get repository details including the default branch",
inputSchema: RepoInput,
handler: () => ({ defaultBranch: "main" }),
}),
tool({
name: "searchDocs",
description: "Search GitHub API documentation",
inputSchema: EmptyInput,
handler: () => [],
}),
],
}),
inMemoryToolsPlugin({
namespace: "crm",
tools: [
tool({
name: "createContact",
description: "Create a CRM contact record",
inputSchema: ContactInput,
handler: () => ({ id: "contact_1" }),
}),
tool({
name: "listContacts",
description: "List CRM contacts",
inputSchema: EmptyInput,
handler: () => [],
}),
],
}),
] as const,
});
yield* config.sources.registerRuntime(new Source({
id: "github",
name: "GitHub",
kind: "in-memory",
runtime: true,
canRemove: false,
canRefresh: false,
}));
yield* config.sources.registerRuntime(new Source({
id: "crm",
name: "CRM",
kind: "in-memory",
runtime: true,
canRemove: false,
canRefresh: false,
}));
return yield* createExecutor(config);
});
describe("tool discovery", () => {
it.effect("ranks matches using ids, namespaces, camelCase names, and descriptions", () =>
Effect.gen(function* () {
const executor = yield* makeSearchExecutor();
const githubMatches = yield* searchTools(executor, "github issues", 5);
expect(githubMatches.map((match) => match.path)).toEqual([
"github.listRepositoryIssues",
]);
expect(githubMatches[0]?.score ?? 0).toBeGreaterThan(0);
const repoMatches = yield* searchTools(executor, "repo details", 5);
expect(repoMatches[0]?.path).toBe("github.getRepositoryDetails");
const crmMatches = yield* searchTools(executor, "crm create contact", 5);
expect(crmMatches[0]?.path).toBe("crm.createContact");
expect(crmMatches[0]?.score ?? 0).toBeGreaterThan(crmMatches[1]?.score ?? 0);
}),
);
it.effect("returns no matches for empty queries instead of listing arbitrary tools", () =>
Effect.gen(function* () {
const executor = yield* makeSearchExecutor();
const matches = yield* searchTools(executor, "", 5);
expect(matches).toEqual([]);
}),
);
it.effect("can narrow discovery to a namespace", () =>
Effect.gen(function* () {
const executor = yield* makeSearchExecutor();
const githubOnly = yield* searchTools(executor, "list", 5, {
namespace: "github",
});
expect(githubOnly.map((match) => match.path)).toEqual([
"github.listRepositoryIssues",
]);
const crmOnly = yield* searchTools(executor, "list", 5, {
namespace: "crm",
});
expect(crmOnly.map((match) => match.path)).toEqual([
"crm.listContacts",
]);
const sandboxResult = yield* Effect.promise(() =>
createExecutionEngine({ executor }).execute(
'return await tools.search({ namespace: "crm", query: "create contact", limit: 5 });',
{ onElicitation: acceptAll },
),
);
expect(sandboxResult.error).toBeUndefined();
expect(sandboxResult.result).toEqual([
expect.objectContaining({ path: "crm.createContact" }),
]);
}),
);
it.effect("supports executor-scoped source listing and tool search", () =>
Effect.gen(function* () {
const executor = yield* makeSearchExecutor();
const listed = yield* Effect.promise(() =>
createExecutionEngine({ executor }).execute(
"return await tools.executor.sources.list();",
{ onElicitation: acceptAll },
),
);
expect(listed.error).toBeUndefined();
expect(listed.result).toEqual(
expect.arrayContaining([
expect.objectContaining({ id: "github", toolCount: 3 }),
expect.objectContaining({ id: "crm", toolCount: 2 }),
]),
);
const searched = yield* Effect.promise(() =>
createExecutionEngine({ executor }).execute(
'return await tools.search({ query: "list contacts", namespace: "crm", limit: 5 });',
{ onElicitation: acceptAll },
),
);
expect(searched.error).toBeUndefined();
expect(searched.result).toEqual([
expect.objectContaining({ path: "crm.listContacts" }),
]);
}),
);
it.effect("describes tools with TypeScript previews", () =>
Effect.gen(function* () {
const executor = yield* makeSearchExecutor();
const withoutSchemas = yield* describeTool(
executor,
"github.listRepositoryIssues",
);
expect(withoutSchemas).toEqual({
path: "github.listRepositoryIssues",
name: "listRepositoryIssues",
description: "List issues for a repository",
inputTypeScript: "{ owner: string; repo: string }",
outputTypeScript: undefined,
typeScriptDefinitions: undefined,
});
const withSchemas = yield* describeTool(
executor,
"github.listRepositoryIssues",
);
expect(withSchemas.path).toBe("github.listRepositoryIssues");
expect(withSchemas.inputTypeScript).toBe(
"{ owner: string; repo: string }",
);
expect(withSchemas.typeScriptDefinitions).toBeUndefined();
expect(withSchemas.outputTypeScript).toBeUndefined();
}),
);
it.effect("rejects malformed discover calls inside the sandbox", () =>
Effect.gen(function* () {
const executor = yield* makeSearchExecutor();
const engine = createExecutionEngine({ executor });
const invalid = yield* Effect.promise(() =>
engine.execute(
[
"try {",
" await tools.search(\"github issues\");",
" return \"unexpected\";",
"} catch (error) {",
" return error instanceof Error ? error.message : String(error);",
"}",
].join("\n"),
{ onElicitation: acceptAll },
),
);
expect(invalid.error).toBeUndefined();
expect(String(invalid.result)).toContain(
"tools.search expects an object: { query?: string; namespace?: string; limit?: number }",
);
const emptyQuery = yield* Effect.promise(() =>
engine.execute(
"return await tools.search({ query: \"\", limit: 5 });",
{ onElicitation: acceptAll },
),
);
expect(emptyQuery.error).toBeUndefined();
expect(emptyQuery.result).toEqual([]);
const invalidDescribe = yield* Effect.promise(() =>
engine.execute(
[
"try {",
" await tools.describe.tool({ path: \"github.listRepositoryIssues\", includeSchemas: true });",
" return \"unexpected\";",
"} catch (error) {",
" return error instanceof Error ? error.message : String(error);",
"}",
].join("\n"),
{ onElicitation: acceptAll },
),
);
expect(invalidDescribe.error).toBeUndefined();
expect(String(invalidDescribe.result)).toContain(
"tools.describe.tool no longer accepts includeSchemas",
);
const invalidSearch = yield* Effect.promise(() =>
engine.execute(
'try { return await tools.search("crm"); } catch (error) { return error instanceof Error ? error.message : String(error); }',
{ onElicitation: acceptAll },
),
);
expect(invalidSearch.error).toBeUndefined();
expect(String(invalidSearch.result)).toContain(
"tools.search expects an object",
);
}),
);
});