import { describe, expect, it } from "@effect/vitest"; import { Effect, Fiber, Schema } from "effect"; import { ElicitationResponse, Source, createExecutor, inMemoryToolsPlugin, makeTestConfig, tool, } from "@executor/sdk"; import { createExecutionEngine } from "./engine"; import { describeTool, searchTools } from "./tool-invoker"; const EmptyInput = Schema.Struct({}); const RepoInput = Schema.Struct({ owner: Schema.String, repo: Schema.String, }); const ContactInput = Schema.Struct({ email: Schema.String, }); import { FormElicitation } from "@executor/sdk"; const acceptAll = () => Effect.succeed(new ElicitationResponse({ action: "accept" })); const makeSearchExecutor = () => Effect.gen(function* () { const config = makeTestConfig({ plugins: [ inMemoryToolsPlugin({ namespace: "github", tools: [ tool({ name: "listRepositoryIssues", description: "List issues for a repository", inputSchema: RepoInput, handler: () => [], }), tool({ name: "getRepositoryDetails", description: "Get repository details including the default branch", inputSchema: RepoInput, handler: () => ({ defaultBranch: "main" }), }), tool({ name: "searchDocs", description: "Search GitHub API documentation", inputSchema: EmptyInput, handler: () => [], }), ], }), inMemoryToolsPlugin({ namespace: "crm", tools: [ tool({ name: "createContact", description: "Create a CRM contact record", inputSchema: ContactInput, handler: () => ({ id: "contact_1" }), }), tool({ name: "listContacts", description: "List CRM contacts", inputSchema: EmptyInput, handler: () => [], }), ], }), ] as const, }); yield* config.sources.registerRuntime( new Source({ id: "github", name: "GitHub", kind: "in-memory", runtime: true, canRemove: false, canRefresh: false, }), ); yield* config.sources.registerRuntime( new Source({ id: "crm", name: "CRM", kind: "in-memory", runtime: true, canRemove: false, canRefresh: false, }), ); return yield* createExecutor(config); }); describe("tool discovery", () => { it.effect("ranks matches using ids, namespaces, camelCase names, and descriptions", () => Effect.gen(function* () { const executor = yield* makeSearchExecutor(); const githubMatches = yield* searchTools(executor, "github issues", 5); expect(githubMatches.map((match) => match.path)).toEqual(["github.listRepositoryIssues"]); expect(githubMatches[0]?.score ?? 0).toBeGreaterThan(0); const repoMatches = yield* searchTools(executor, "repo details", 5); expect(repoMatches[0]?.path).toBe("github.getRepositoryDetails"); const crmMatches = yield* searchTools(executor, "crm create contact", 5); expect(crmMatches[0]?.path).toBe("crm.createContact"); expect(crmMatches[0]?.score ?? 0).toBeGreaterThan(crmMatches[1]?.score ?? 0); }), ); it.effect("returns no matches for empty queries instead of listing arbitrary tools", () => Effect.gen(function* () { const executor = yield* makeSearchExecutor(); const matches = yield* searchTools(executor, "", 5); expect(matches).toEqual([]); }), ); it.effect("can narrow discovery to a namespace", () => Effect.gen(function* () { const executor = yield* makeSearchExecutor(); const githubOnly = yield* searchTools(executor, "list", 5, { namespace: "github", }); expect(githubOnly.map((match) => match.path)).toEqual(["github.listRepositoryIssues"]); const crmOnly = yield* searchTools(executor, "list", 5, { namespace: "crm", }); expect(crmOnly.map((match) => match.path)).toEqual(["crm.listContacts"]); const sandboxResult = yield* Effect.promise(() => createExecutionEngine({ executor }).execute( 'return await tools.search({ namespace: "crm", query: "create contact", limit: 5 });', { onElicitation: acceptAll }, ), ); expect(sandboxResult.error).toBeUndefined(); expect(sandboxResult.result).toEqual([ expect.objectContaining({ path: "crm.createContact" }), ]); }), ); it.effect("supports executor-scoped source listing and tool search", () => Effect.gen(function* () { const executor = yield* makeSearchExecutor(); const listed = yield* Effect.promise(() => createExecutionEngine({ executor }).execute("return await tools.executor.sources.list();", { onElicitation: acceptAll, }), ); expect(listed.error).toBeUndefined(); expect(listed.result).toEqual( expect.arrayContaining([ expect.objectContaining({ id: "github", toolCount: 3 }), expect.objectContaining({ id: "crm", toolCount: 2 }), ]), ); const searched = yield* Effect.promise(() => createExecutionEngine({ executor }).execute( 'return await tools.search({ query: "list contacts", namespace: "crm", limit: 5 });', { onElicitation: acceptAll }, ), ); expect(searched.error).toBeUndefined(); expect(searched.result).toEqual([expect.objectContaining({ path: "crm.listContacts" })]); }), ); it.effect("describes tools with TypeScript previews", () => Effect.gen(function* () { const executor = yield* makeSearchExecutor(); const withoutSchemas = yield* describeTool(executor, "github.listRepositoryIssues"); expect(withoutSchemas).toEqual({ path: "github.listRepositoryIssues", name: "listRepositoryIssues", description: "List issues for a repository", inputTypeScript: "{ owner: string; repo: string }", outputTypeScript: undefined, typeScriptDefinitions: undefined, }); const withSchemas = yield* describeTool(executor, "github.listRepositoryIssues"); expect(withSchemas.path).toBe("github.listRepositoryIssues"); expect(withSchemas.inputTypeScript).toBe("{ owner: string; repo: string }"); expect(withSchemas.typeScriptDefinitions).toBeUndefined(); expect(withSchemas.outputTypeScript).toBeUndefined(); }), ); it.effect("rejects malformed discover calls inside the sandbox", () => Effect.gen(function* () { const executor = yield* makeSearchExecutor(); const engine = createExecutionEngine({ executor }); const invalid = yield* Effect.promise(() => engine.execute( [ "try {", ' await tools.search("github issues");', ' return "unexpected";', "} catch (error) {", " return error instanceof Error ? error.message : String(error);", "}", ].join("\n"), { onElicitation: acceptAll }, ), ); expect(invalid.error).toBeUndefined(); expect(String(invalid.result)).toContain( "tools.search expects an object: { query?: string; namespace?: string; limit?: number }", ); const emptyQuery = yield* Effect.promise(() => engine.execute('return await tools.search({ query: "", limit: 5 });', { onElicitation: acceptAll, }), ); expect(emptyQuery.error).toBeUndefined(); expect(emptyQuery.result).toEqual([]); const invalidDescribe = yield* Effect.promise(() => engine.execute( [ "try {", ' await tools.describe.tool({ path: "github.listRepositoryIssues", includeSchemas: true });', ' return "unexpected";', "} catch (error) {", " return error instanceof Error ? error.message : String(error);", "}", ].join("\n"), { onElicitation: acceptAll }, ), ); expect(invalidDescribe.error).toBeUndefined(); expect(String(invalidDescribe.result)).toContain( "tools.describe.tool no longer accepts includeSchemas", ); const invalidSearch = yield* Effect.promise(() => engine.execute( 'try { return await tools.search("crm"); } catch (error) { return error instanceof Error ? error.message : String(error); }', { onElicitation: acceptAll }, ), ); expect(invalidSearch.error).toBeUndefined(); expect(String(invalidSearch.result)).toContain("tools.search expects an object"); }), ); }); // --------------------------------------------------------------------------- // pause/resume — multiple elicitations in a single execution // --------------------------------------------------------------------------- describe("pause/resume with multiple elicitations", () => { const makeElicitingExecutor = () => Effect.gen(function* () { const config = makeTestConfig({ plugins: [ inMemoryToolsPlugin({ namespace: "api", tools: [ tool({ name: "multiApproval", description: "A tool that elicits twice", inputSchema: EmptyInput, handler: (_args, ctx) => Effect.gen(function* () { const r1 = yield* ctx.elicit( new FormElicitation({ message: "First approval", requestedSchema: {}, }), ); const r2 = yield* ctx.elicit( new FormElicitation({ message: "Second approval", requestedSchema: {}, }), ); return { first: r1, second: r2 }; }), }), tool({ name: "singleApproval", description: "A tool that elicits exactly once and then returns a value. Mirrors the shape of a typical `gmail.users.labels.create` style operation: one approval, one side effect, one success response.", inputSchema: EmptyInput, handler: (_args, ctx) => Effect.gen(function* () { const r = yield* ctx.elicit( new FormElicitation({ message: "Only approval", requestedSchema: {}, }), ); return { ok: true, response: r }; }), }), ], }), ] as const, }); yield* config.sources.registerRuntime( new Source({ id: "api", name: "API", kind: "in-memory", runtime: true, canRemove: false, canRefresh: false, }), ); return yield* createExecutor(config); }); it.effect( "resume does not hang when execution hits a second elicitation", () => Effect.gen(function* () { const executor = yield* makeElicitingExecutor(); const engine = createExecutionEngine({ executor }); const code = "return await tools.api.multiApproval({});"; // First executeWithPause — should pause on first elicitation const outcome1 = yield* Effect.promise(() => engine.executeWithPause(code)); expect(outcome1.status).toBe("paused"); const paused1 = outcome1 as Extract; expect(paused1.execution.elicitationContext.request.message).toBe("First approval"); // Resume first pause — execution continues to second elicitation. // resume() must not hang; it should return (either a new paused // result or the completion). const outcome2 = yield* Effect.promise(() => Promise.race([ engine.resume(paused1.execution.id, { action: "accept" }), new Promise((_, reject) => setTimeout( () => reject(new Error("resume hung — second elicitation not surfaced")), 5000, ), ), ]), ); expect(outcome2).not.toBeNull(); }), { timeout: 10000 }, ); // Regression: use separate top-level runPromise calls to match HTTP/CLI // pause/resume, and a single-elicit tool so no later pause can mask a dead // sandbox fiber. it("resume returns across separate runPromise boundaries for a single-elicit tool (HTTP-like)", async () => { const executor = await Effect.runPromise(makeElicitingExecutor()); const engine = createExecutionEngine({ executor }); const code = "return await tools.api.singleApproval({});"; const outcome1 = await engine.executeWithPause(code); expect(outcome1.status).toBe("paused"); const paused1 = outcome1 as Extract; expect(paused1.execution.elicitationContext.request.message).toBe("Only approval"); // `execution.fiber` is on `InternalPausedExecution`; the exported // `PausedExecution` type doesn't carry it. Cast to read. const sandboxFiber = ( paused1.execution as unknown as { readonly fiber: Fiber.Fiber; } ).fiber; const exitProbe = await Effect.runPromise( Effect.race( Fiber.await(sandboxFiber), Effect.map(Effect.sleep("50 millis"), () => "still-running" as const), ), ); expect(exitProbe).toBe("still-running"); const outcome2 = await Promise.race([ engine.resume(paused1.execution.id, { action: "accept" }), new Promise((_, reject) => setTimeout(() => reject(new Error("resume hung across runPromise boundaries")), 2000), ), ]); expect(outcome2).not.toBeNull(); const resumed = outcome2 as NonNullable; expect(resumed.status).toBe("completed"); if (resumed.status === "completed") { expect(resumed.result.error).toBeUndefined(); expect(resumed.result.result).toMatchObject({ ok: true }); } }, 10000); });