import { randomBytes } from "node:crypto"; import { expect } from "@effect/vitest"; import { Effect } from "effect"; import { composePluginApi } from "@executor-js/api/server"; import { openApiHttpPlugin } from "@executor-js/plugin-openapi/api"; import { toolkitsPlugin } from "@executor-js/plugin-toolkits/server"; import { AuthTemplateSlug, ConnectionName, IntegrationSlug } from "@executor-js/sdk/shared"; import { scenario } from "../src/scenario"; import { Api, Browser, Target } from "../src/services"; const api = composePluginApi([toolkitsPlugin(), openApiHttpPlugin()] as const); const hiddenPersonalSpec = (baseUrl: string): string => JSON.stringify({ openapi: "3.0.3", info: { title: "Personal Hidden API", version: "1.0.0" }, servers: [{ url: baseUrl }], paths: { "/personal-only": { get: { operationId: "personalOnly", responses: { "200": { description: "Personal-only response" } }, }, }, }, }); scenario( "Toolkits ยท self-host UI creates a toolkit and configures tools", { timeout: 180_000 }, Effect.gen(function* () { const target = yield* Target; const browser = yield* Browser; const { client: makeApiClient } = yield* Api; const identity = yield* target.newIdentity(); const client = yield* makeApiClient(api, identity); const suffix = randomBytes(4).toString("hex"); const prefix = `toolkits-ui-${suffix}`; const name = `${prefix}-created`; const slug = name; const hiddenPersonalIntegration = `${prefix}-personal-api`; const hiddenPersonalConnection = "mine"; const seededToolkits = [ { owner: "org" as const, name: `${prefix}-workspace-a` }, { owner: "org" as const, name: `${prefix}-workspace-b` }, { owner: "user" as const, name: `${prefix}-personal-a` }, { owner: "user" as const, name: `${prefix}-personal-b` }, { owner: "user" as const, name: `${prefix}-personal-c` }, ]; let addedConnectionPattern = ""; const blockPattern = "executor.coreTools.policies.list"; const cleanup = Effect.gen(function* () { yield* client.connections .remove({ params: { owner: "user", integration: IntegrationSlug.make(hiddenPersonalIntegration), name: ConnectionName.make(hiddenPersonalConnection), }, }) .pipe(Effect.ignore); yield* client.openapi .removeSpec({ params: { slug: hiddenPersonalIntegration } }) .pipe(Effect.ignore); const listed = yield* client.toolkits.list(); yield* Effect.forEach( listed.toolkits.filter((row) => row.slug.startsWith(prefix)), (toolkit) => client.toolkits.remove({ params: { toolkitId: toolkit.id } }), { discard: true }, ); }).pipe(Effect.ignore); yield* Effect.gen(function* () { yield* Effect.forEach( seededToolkits, (toolkit) => client.toolkits.create({ payload: toolkit }), { discard: true }, ); yield* client.openapi.addSpec({ payload: { spec: { kind: "blob", value: hiddenPersonalSpec("http://127.0.0.1:59999") }, slug: IntegrationSlug.make(hiddenPersonalIntegration), baseUrl: "http://127.0.0.1:59999", authenticationTemplate: [ { slug: "apiKey", type: "apiKey", headers: { "x-api-key": [{ type: "variable", name: "token" }] }, }, ], }, }); yield* client.connections.create({ payload: { owner: "user", name: ConnectionName.make(hiddenPersonalConnection), integration: IntegrationSlug.make(hiddenPersonalIntegration), template: AuthTemplateSlug.make("apiKey"), value: "unused-token", }, }); yield* browser.session(identity, async ({ page, step }) => { await step("Open the Toolkits plugin page", async () => { await page.goto("/default/toolkits/", { waitUntil: "domcontentloaded" }); await page.getByRole("heading", { name: "Toolkits" }).waitFor(); await page.getByRole("heading", { name: "Workspace" }).waitFor(); await page.getByRole("heading", { name: "Personal" }).waitFor(); await page.getByRole("button", { name: "Add workspace toolkit" }).waitFor(); await page.getByRole("button", { name: "Add personal toolkit" }).waitFor(); await page.locator('main [data-slot="skeleton"]').first().waitFor({ state: "detached" }); expect(await page.locator('main [data-slot="skeleton"]').count()).toBe(0); const seededCard = page.getByRole("link", { name: `Open toolkit ${prefix}-workspace-a`, }); await seededCard.waitFor(); expect(await seededCard.getByText("/mcp/toolkits").count()).toBe(0); expect(await seededCard.getByText("Workspace tools").count()).toBe(0); expect(await page.getByLabel("New toolkit").count()).toBe(0); }); await step("Create a workspace toolkit from the add card", async () => { const workspaceSection = page.locator("section").filter({ has: page.getByRole("heading", { name: "Workspace" }), }); await workspaceSection.getByRole("button", { name: "Add workspace toolkit" }).click(); await page.getByRole("dialog", { name: "New workspace toolkit" }).waitFor(); await page.getByLabel("Toolkit name").fill(name); await page.getByRole("button", { name: "Create toolkit" }).click(); await page.getByRole("link", { name: `Open toolkit ${name}` }).waitFor(); }); await step("Validate owner sections render as three-column grids", async () => { const workspaceSection = page.locator("section").filter({ has: page.getByRole("heading", { name: "Workspace" }), }); const personalSection = page.locator("section").filter({ has: page.getByRole("heading", { name: "Personal" }), }); const workspaceColumns = await workspaceSection .getByRole("link", { name: /^Open toolkit/ }) .evaluateAll((nodes) => nodes.slice(0, 3).map((node) => Math.round(node.getBoundingClientRect().left)), ); const personalColumns = await personalSection .getByRole("link", { name: /^Open toolkit/ }) .evaluateAll((nodes) => nodes.slice(0, 3).map((node) => Math.round(node.getBoundingClientRect().left)), ); expect(new Set(workspaceColumns).size).toBe(3); expect(new Set(personalColumns).size).toBe(3); }); await step("Open the created toolkit from the grid", async () => { await page.getByRole("link", { name: `Open toolkit ${name}` }).click(); await page.waitForURL(new RegExp(`/toolkits/${slug}$`)); expect(page.url()).toMatch(new RegExp(`/toolkits/${slug}$`)); await page .locator("code") .filter({ hasText: `/mcp/toolkits/${slug}` }) .waitFor(); await page.getByText("No connections added").waitFor(); expect(await page.getByLabel("New toolkit").count()).toBe(0); }); await step("Return to the toolkit grid with browser-visible routing", async () => { await page.getByRole("button", { name: "Toolkits" }).click(); await page.waitForURL(/\/toolkits\/?$/); expect(page.url()).toMatch(/\/toolkits\/?$/); await page.getByRole("heading", { name: "Workspace" }).waitFor(); await page.getByRole("link", { name: `Open toolkit ${name}` }).waitFor(); }); await step("Open the created toolkit from a direct URL", async () => { await page.goto(`/default/toolkits/${slug}`, { waitUntil: "domcontentloaded" }); expect(page.url()).toMatch(new RegExp(`/toolkits/${slug}$`)); await page .locator("code") .filter({ hasText: `/mcp/toolkits/${slug}` }) .waitFor(); await page.getByText("No connections added").waitFor(); expect(await page.getByLabel("New toolkit").count()).toBe(0); }); await step("Cancel toolkit deletion from the confirmation modal", async () => { await page.getByRole("button", { name: "Delete toolkit" }).click(); const dialog = page.getByRole("alertdialog", { name: `Delete ${name}?` }); await dialog.waitFor(); await dialog.getByRole("button", { name: "Cancel" }).click(); await dialog.waitFor({ state: "detached" }); await page.getByRole("heading", { name }).waitFor(); }); await step("The connection picker explains hidden personal connections", async () => { await page.getByRole("button", { name: "Manage toolkit connections" }).click(); const dialog = page.getByRole("dialog", { name: "Manage connections" }); await dialog.waitFor(); // The contract is the explanation, not the count: this suite's own // hidden connection guarantees at least one, but the workspace is // shared, so another scenario's personal connection may legitimately // raise the number (asserting a global count is what e2e/AGENTS.md // forbids). await dialog .getByText( /You have \d+ personal connections? that (?:is|are) not shown because this is a shared toolkit\./, ) .waitFor(); }); await step("Add a connection to the toolkit", async () => { const dialog = page.getByRole("dialog", { name: "Manage connections" }); await dialog.waitFor(); await dialog.getByLabel("Search connections and tools").fill("policies.list"); expect(await dialog.getByRole("button", { name: /^Add tool/ }).count()).toBe(0); addedConnectionPattern = "executor.*"; expect(await dialog.getByText(addedConnectionPattern, { exact: true }).count()).toBe(0); await dialog .getByRole("button", { name: /^Add connection / }) .first() .click(); await dialog.getByRole("button", { name: /^Remove connection / }).waitFor(); await page.keyboard.press("Escape"); await dialog.waitFor({ state: "hidden" }); const toolkitTools = page.getByRole("region", { name: "Toolkit tools" }); await toolkitTools.waitFor(); await toolkitTools.getByLabel("Filter tools").fill("policies.list"); await toolkitTools.getByRole("button").filter({ hasText: "list" }).last().waitFor(); await toolkitTools.getByLabel("Filter tools").clear(); }); await step("The add connection list reflects the saved toolkit connection", async () => { await page.getByRole("button", { name: "Manage toolkit connections" }).click(); const dialog = page.getByRole("dialog", { name: "Manage connections" }); await dialog.waitFor(); await dialog.getByLabel("Search connections and tools").fill("policies.list"); await dialog.getByRole("button", { name: /^Remove connection / }).waitFor(); expect(await dialog.getByRole("button", { name: /^Add connection / }).count()).toBe(0); await page.keyboard.press("Escape"); await dialog.waitFor({ state: "hidden" }); }); await step("Remove the connection from the manage modal", async () => { await page.getByRole("button", { name: "Manage toolkit connections" }).click(); const removeDialog = page.getByRole("dialog", { name: "Manage connections" }); await removeDialog.waitFor(); await removeDialog.getByLabel("Search connections and tools").fill("policies.list"); await removeDialog .getByRole("button", { name: /^Remove connection / }) .first() .click(); await removeDialog.getByRole("button", { name: /^Add connection / }).waitFor(); await page.keyboard.press("Escape"); await removeDialog.waitFor({ state: "hidden" }); await page.getByText("No connections added").waitFor(); await page.getByRole("button", { name: "Manage toolkit connections" }).click(); const dialog = page.getByRole("dialog", { name: "Manage connections" }); await dialog.waitFor(); await dialog.getByLabel("Search connections and tools").fill("policies.list"); await dialog .getByRole("button", { name: /^Add connection / }) .first() .click(); await dialog.getByRole("button", { name: /^Remove connection / }).waitFor(); await page.keyboard.press("Escape"); await dialog.waitFor({ state: "hidden" }); const toolkitTools = page.getByRole("region", { name: "Toolkit tools" }); await toolkitTools.getByLabel("Filter tools").fill("policies.list"); await toolkitTools.getByRole("button").filter({ hasText: "list" }).last().waitFor(); await toolkitTools.getByLabel("Filter tools").clear(); }); await step("Block one tool from the toolkit tools list", async () => { const toolkitTools = page.getByRole("region", { name: "Toolkit tools" }); await toolkitTools.getByLabel("Filter tools").fill("policies.list"); await toolkitTools.getByRole("button").filter({ hasText: "list" }).last().click(); await page.getByRole("button", { name: "Set policy", exact: true }).click(); await page.getByText(blockPattern, { exact: true }).waitFor(); await page.getByRole("menuitem", { name: "Block" }).click(); await toolkitTools.getByRole("button").filter({ hasText: "list" }).last().waitFor(); await page.getByText("This tool is not available through the current toolkit.").waitFor(); }); }); const listed = yield* client.toolkits.list(); const toolkit = listed.toolkits.find((row) => row.slug === slug); expect(toolkit, "the UI-created toolkit persisted").toBeDefined(); if (!toolkit) return; expect(toolkit.owner).toBe("org"); const { policies } = yield* client.toolkits.listPolicies({ params: { toolkitId: toolkit.id }, }); const { connections } = yield* client.toolkits.listConnections({ params: { toolkitId: toolkit.id }, }); expect(addedConnectionPattern.length, "the UI selected a connection").toBeGreaterThan(0); expect( connections.map((connection) => connection.pattern), "the UI-authored toolkit connection persisted", ).toContain(addedConnectionPattern); expect( policies.map((policy) => `${policy.pattern} ${policy.action}`).sort(), "the UI-authored toolkit access persisted with its action", ).toEqual([`${blockPattern} block`]); }).pipe(Effect.ensuring(cleanup)); }), );