410 lines · 14.2 KB · TypeScript
RawDownload
import { describe, expect, it } from "@effect/vitest";import { Effect, Fiber, Schema } from "effect"; import {  ElicitationResponse,  Source,  createExecutor,  inMemoryToolsPlugin,  makeTestConfig,  tool,} from "@executor/sdk";import { createExecutionEngine } from "./engine";import { describeTool, searchTools } from "./tool-invoker"; const EmptyInput = Schema.Struct({});const RepoInput = Schema.Struct({  owner: Schema.String,  repo: Schema.String,});const ContactInput = Schema.Struct({  email: Schema.String,}); import { FormElicitation } from "@executor/sdk"; const acceptAll = () => Effect.succeed(new ElicitationResponse({ action: "accept" })); const makeSearchExecutor = () =>  Effect.gen(function* () {    const config = makeTestConfig({      plugins: [        inMemoryToolsPlugin({          namespace: "github",          tools: [            tool({              name: "listRepositoryIssues",              description: "List issues for a repository",              inputSchema: RepoInput,              handler: () => [],            }),            tool({              name: "getRepositoryDetails",              description: "Get repository details including the default branch",              inputSchema: RepoInput,              handler: () => ({ defaultBranch: "main" }),            }),            tool({              name: "searchDocs",              description: "Search GitHub API documentation",              inputSchema: EmptyInput,              handler: () => [],            }),          ],        }),        inMemoryToolsPlugin({          namespace: "crm",          tools: [            tool({              name: "createContact",              description: "Create a CRM contact record",              inputSchema: ContactInput,              handler: () => ({ id: "contact_1" }),            }),            tool({              name: "listContacts",              description: "List CRM contacts",              inputSchema: EmptyInput,              handler: () => [],            }),          ],        }),      ] as const,    });     yield* config.sources.registerRuntime(      new Source({        id: "github",        name: "GitHub",        kind: "in-memory",        runtime: true,        canRemove: false,        canRefresh: false,      }),    );    yield* config.sources.registerRuntime(      new Source({        id: "crm",        name: "CRM",        kind: "in-memory",        runtime: true,        canRemove: false,        canRefresh: false,      }),    );     return yield* createExecutor(config);  }); describe("tool discovery", () => {  it.effect("ranks matches using ids, namespaces, camelCase names, and descriptions", () =>    Effect.gen(function* () {      const executor = yield* makeSearchExecutor();       const githubMatches = yield* searchTools(executor, "github issues", 5);      expect(githubMatches.map((match) => match.path)).toEqual(["github.listRepositoryIssues"]);      expect(githubMatches[0]?.score ?? 0).toBeGreaterThan(0);       const repoMatches = yield* searchTools(executor, "repo details", 5);      expect(repoMatches[0]?.path).toBe("github.getRepositoryDetails");       const crmMatches = yield* searchTools(executor, "crm create contact", 5);      expect(crmMatches[0]?.path).toBe("crm.createContact");      expect(crmMatches[0]?.score ?? 0).toBeGreaterThan(crmMatches[1]?.score ?? 0);    }),  );   it.effect("returns no matches for empty queries instead of listing arbitrary tools", () =>    Effect.gen(function* () {      const executor = yield* makeSearchExecutor();      const matches = yield* searchTools(executor, "", 5);      expect(matches).toEqual([]);    }),  );   it.effect("can narrow discovery to a namespace", () =>    Effect.gen(function* () {      const executor = yield* makeSearchExecutor();       const githubOnly = yield* searchTools(executor, "list", 5, {        namespace: "github",      });      expect(githubOnly.map((match) => match.path)).toEqual(["github.listRepositoryIssues"]);       const crmOnly = yield* searchTools(executor, "list", 5, {        namespace: "crm",      });      expect(crmOnly.map((match) => match.path)).toEqual(["crm.listContacts"]);       const sandboxResult = yield* Effect.promise(() =>        createExecutionEngine({ executor }).execute(          'return await tools.search({ namespace: "crm", query: "create contact", limit: 5 });',          { onElicitation: acceptAll },        ),      );      expect(sandboxResult.error).toBeUndefined();      expect(sandboxResult.result).toEqual([        expect.objectContaining({ path: "crm.createContact" }),      ]);    }),  );   it.effect("supports executor-scoped source listing and tool search", () =>    Effect.gen(function* () {      const executor = yield* makeSearchExecutor();       const listed = yield* Effect.promise(() =>        createExecutionEngine({ executor }).execute("return await tools.executor.sources.list();", {          onElicitation: acceptAll,        }),      );      expect(listed.error).toBeUndefined();      expect(listed.result).toEqual(        expect.arrayContaining([          expect.objectContaining({ id: "github", toolCount: 3 }),          expect.objectContaining({ id: "crm", toolCount: 2 }),        ]),      );       const searched = yield* Effect.promise(() =>        createExecutionEngine({ executor }).execute(          'return await tools.search({ query: "list contacts", namespace: "crm", limit: 5 });',          { onElicitation: acceptAll },        ),      );      expect(searched.error).toBeUndefined();      expect(searched.result).toEqual([expect.objectContaining({ path: "crm.listContacts" })]);    }),  );   it.effect("describes tools with TypeScript previews", () =>    Effect.gen(function* () {      const executor = yield* makeSearchExecutor();       const withoutSchemas = yield* describeTool(executor, "github.listRepositoryIssues");      expect(withoutSchemas).toEqual({        path: "github.listRepositoryIssues",        name: "listRepositoryIssues",        description: "List issues for a repository",        inputTypeScript: "{ owner: string; repo: string }",        outputTypeScript: undefined,        typeScriptDefinitions: undefined,      });       const withSchemas = yield* describeTool(executor, "github.listRepositoryIssues");      expect(withSchemas.path).toBe("github.listRepositoryIssues");      expect(withSchemas.inputTypeScript).toBe("{ owner: string; repo: string }");      expect(withSchemas.typeScriptDefinitions).toBeUndefined();      expect(withSchemas.outputTypeScript).toBeUndefined();    }),  );   it.effect("rejects malformed discover calls inside the sandbox", () =>    Effect.gen(function* () {      const executor = yield* makeSearchExecutor();      const engine = createExecutionEngine({ executor });       const invalid = yield* Effect.promise(() =>        engine.execute(          [            "try {",            '  await tools.search("github issues");',            '  return "unexpected";',            "} catch (error) {",            "  return error instanceof Error ? error.message : String(error);",            "}",          ].join("\n"),          { onElicitation: acceptAll },        ),      );      expect(invalid.error).toBeUndefined();      expect(String(invalid.result)).toContain(        "tools.search expects an object: { query?: string; namespace?: string; limit?: number }",      );       const emptyQuery = yield* Effect.promise(() =>        engine.execute('return await tools.search({ query: "", limit: 5 });', {          onElicitation: acceptAll,        }),      );      expect(emptyQuery.error).toBeUndefined();      expect(emptyQuery.result).toEqual([]);       const invalidDescribe = yield* Effect.promise(() =>        engine.execute(          [            "try {",            '  await tools.describe.tool({ path: "github.listRepositoryIssues", includeSchemas: true });',            '  return "unexpected";',            "} catch (error) {",            "  return error instanceof Error ? error.message : String(error);",            "}",          ].join("\n"),          { onElicitation: acceptAll },        ),      );      expect(invalidDescribe.error).toBeUndefined();      expect(String(invalidDescribe.result)).toContain(        "tools.describe.tool no longer accepts includeSchemas",      );       const invalidSearch = yield* Effect.promise(() =>        engine.execute(          'try { return await tools.search("crm"); } catch (error) { return error instanceof Error ? error.message : String(error); }',          { onElicitation: acceptAll },        ),      );      expect(invalidSearch.error).toBeUndefined();      expect(String(invalidSearch.result)).toContain("tools.search expects an object");    }),  );}); // ---------------------------------------------------------------------------// pause/resume — multiple elicitations in a single execution// --------------------------------------------------------------------------- describe("pause/resume with multiple elicitations", () => {  const makeElicitingExecutor = () =>    Effect.gen(function* () {      const config = makeTestConfig({        plugins: [          inMemoryToolsPlugin({            namespace: "api",            tools: [              tool({                name: "multiApproval",                description: "A tool that elicits twice",                inputSchema: EmptyInput,                handler: (_args, ctx) =>                  Effect.gen(function* () {                    const r1 = yield* ctx.elicit(                      new FormElicitation({                        message: "First approval",                        requestedSchema: {},                      }),                    );                    const r2 = yield* ctx.elicit(                      new FormElicitation({                        message: "Second approval",                        requestedSchema: {},                      }),                    );                    return { first: r1, second: r2 };                  }),              }),              tool({                name: "singleApproval",                description:                  "A tool that elicits exactly once and then returns a value. Mirrors the shape of a typical `gmail.users.labels.create` style operation: one approval, one side effect, one success response.",                inputSchema: EmptyInput,                handler: (_args, ctx) =>                  Effect.gen(function* () {                    const r = yield* ctx.elicit(                      new FormElicitation({                        message: "Only approval",                        requestedSchema: {},                      }),                    );                    return { ok: true, response: r };                  }),              }),            ],          }),        ] as const,      });       yield* config.sources.registerRuntime(        new Source({          id: "api",          name: "API",          kind: "in-memory",          runtime: true,          canRemove: false,          canRefresh: false,        }),      );       return yield* createExecutor(config);    });   it.effect(    "resume does not hang when execution hits a second elicitation",    () =>      Effect.gen(function* () {        const executor = yield* makeElicitingExecutor();        const engine = createExecutionEngine({ executor });         const code = "return await tools.api.multiApproval({});";         // First executeWithPause — should pause on first elicitation        const outcome1 = yield* Effect.promise(() => engine.executeWithPause(code));        expect(outcome1.status).toBe("paused");        const paused1 = outcome1 as Extract<typeof outcome1, { status: "paused" }>;        expect(paused1.execution.elicitationContext.request.message).toBe("First approval");         // Resume first pause — execution continues to second elicitation.        // resume() must not hang; it should return (either a new paused        // result or the completion).        const outcome2 = yield* Effect.promise(() =>          Promise.race([            engine.resume(paused1.execution.id, { action: "accept" }),            new Promise<never>((_, reject) =>              setTimeout(                () => reject(new Error("resume hung — second elicitation not surfaced")),                5000,              ),            ),          ]),        );         expect(outcome2).not.toBeNull();      }),    { timeout: 10000 },  );   // Regression: use separate top-level runPromise calls to match HTTP/CLI  // pause/resume, and a single-elicit tool so no later pause can mask a dead  // sandbox fiber.  it("resume returns across separate runPromise boundaries for a single-elicit tool (HTTP-like)", async () => {    const executor = await Effect.runPromise(makeElicitingExecutor());    const engine = createExecutionEngine({ executor });     const code = "return await tools.api.singleApproval({});";     const outcome1 = await engine.executeWithPause(code);    expect(outcome1.status).toBe("paused");    const paused1 = outcome1 as Extract<typeof outcome1, { status: "paused" }>;    expect(paused1.execution.elicitationContext.request.message).toBe("Only approval");     // `execution.fiber` is on `InternalPausedExecution`; the exported    // `PausedExecution` type doesn't carry it. Cast to read.    const sandboxFiber = (      paused1.execution as unknown as {        readonly fiber: Fiber.Fiber<unknown, unknown>;      }    ).fiber;    const exitProbe = await Effect.runPromise(      Effect.race(        Fiber.await(sandboxFiber),        Effect.map(Effect.sleep("50 millis"), () => "still-running" as const),      ),    );    expect(exitProbe).toBe("still-running");     const outcome2 = await Promise.race([      engine.resume(paused1.execution.id, { action: "accept" }),      new Promise<never>((_, reject) =>        setTimeout(() => reject(new Error("resume hung across runPromise boundaries")), 2000),      ),    ]);     expect(outcome2).not.toBeNull();    const resumed = outcome2 as NonNullable<typeof outcome2>;    expect(resumed.status).toBe("completed");    if (resumed.status === "completed") {      expect(resumed.result.error).toBeUndefined();      expect(resumed.result.result).toMatchObject({ ok: true });    }  }, 10000);});

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t