| @@ -14,9 +14,18 @@ produce a Playwright trace, video, and step screenshots for debugging. · 13 unmodified lines |
| 14 | | 14 | |
| 15 | ## File placement | 15 | ## File placement |
| 16 | | 16 | |
| 17 | - `scenarios/*.test.ts` — runs on every target (cloud + selfhost) | 17 | Scenario directories map to vitest projects (`vitest.config.ts` is the |
| 18 | - `cloud/*.test.ts` — cloud-only (e.g. billing, WorkOS-session UI) | 18 | authoritative list of targets and what each one includes): |
| 19 | - `selfhost/*.test.ts` — selfhost-only | 19 | |
| | 20 | - `scenarios/*.test.ts` — cross-target; runs on cloud + selfhost by default, |
| | 21 | and selected files also run on selfhost-docker and cloudflare |
| | 22 | - `cloud/*.test.ts` — cloud-only (e.g. billing, WorkOS-session UI, telemetry) |
| | 23 | - `selfhost/*.test.ts` — selfhost-only (also runs on selfhost-docker) |
| | 24 | - `cloudflare/*.test.ts` — the Cloudflare self-host worker |
| | 25 | - `local/*.test.ts` — the single-user local app; each scenario boots its own |
| | 26 | `executor web` |
| | 27 | - `cli/*.test.ts` — the supervised CLI daemon inside guest VMs |
| | 28 | - `desktop/`, `desktop-packaged/`, `desktop-vm/` — see Desktop targets below |
| 20 | | 29 | |
| 21 | ## Anatomy | 30 | ## Anatomy |
| 22 | | 31 | |
| @@ -25,32 +34,47 @@ import { expect } from "@effect/vitest"; · 2 unmodified lines |
| 25 | import { Effect } from "effect"; | 34 | import { Effect } from "effect"; |
| 26 | import { composePluginApi } from "@executor-js/api/server"; | 35 | import { composePluginApi } from "@executor-js/api/server"; |
| 27 | import { scenario } from "../src/scenario"; | 36 | import { scenario } from "../src/scenario"; |
| | 37 | import { Api, Target } from "../src/services"; |
| 28 | | 38 | |
| 29 | const coreApi = composePluginApi([] as const); // tools/integrations/connections/providers/executions/oauth/policies | 39 | const coreApi = composePluginApi([] as const); // tools/integrations/connections/providers/executions/oauth/policies |
| 30 | | 40 | |
| 31 | scenario("Tools · a fresh workspace advertises the built-in tools", { needs: ["api"] }, (ctx) => | 41 | scenario( |
| | 42 | "Tools · a fresh workspace advertises the built-in tools", |
| | 43 | {}, // options: { timeout?: number } |
| 32 | Effect.gen(function* () { | 44 | Effect.gen(function* () { |
| 33 | const identity = yield* ctx.target.newIdentity(); // fresh isolated user+org | 45 | const target = yield* Target; |
| 34 | const client = yield* ctx.api.client(coreApi, identity); // typed HttpApiClient | 46 | const { client } = yield* Api; |
| 35 | const tools = yield* client.tools.list(); | 47 | const identity = yield* target.newIdentity(); // fresh isolated user+org |
| | 48 | const api = yield* client(coreApi, identity); // typed HttpApiClient |
| | 49 | const tools = yield* api.tools.list({ query: {} }); |
| 36 | expect(tools.length, "at least one tool is exposed").toBeGreaterThan(0); | 50 | expect(tools.length, "at least one tool is exposed").toBeGreaterThan(0); |
| 37 | }), | 51 | }), |
| 38 | ); | 52 | ); |
| 39 | ``` | 53 | ``` |
| 40 | | 54 | |
| 41 | - Capabilities (`needs`): `api`, `browser` (cloud only today), `mcp-oauth` | 55 | - A scenario declares what it needs by **yielding services** from |
| 42 | (selfhost only today), `billing` (cloud only). | 56 | `src/services.ts` (`Target`, `Api`, `Browser`, `Mcp`, `Billing`, `Cli`, |
| | 57 | `Telemetry`, …). There is no `needs` list: yielding a service the current |
| | 58 | target can't provide skips the test and records why in `skipped.json`. |
| | 59 | - Which target provides what (from `targets/*.ts`): `api` — everything except |
| | 60 | local and the desktop targets; `browser` — cloud, selfhost, selfhost-docker, |
| | 61 | cloudflare, local; `mcp-oauth` — cloud, selfhost, selfhost-docker, |
| | 62 | cloudflare (dev-auth on cloudflare, so no real consent hop); `billing` — |
| | 63 | cloud only. `Telemetry` and `Autumn` appear when the suite booted motel / |
| | 64 | the Autumn emulator (cloud). |
| 43 | - Resources created in a test must be cleaned up with `Effect.ensuring` (a | 65 | - Resources created in a test must be cleaned up with `Effect.ensuring` (a |
| 44 | finalizer), not trailing statements — a mid-test failure must not leak state | 66 | finalizer), not trailing statements — a mid-test failure must not leak state |
| 45 | into the shared instance. | 67 | into the shared instance. |
| 46 | | 68 | |
| 47 | ## Browser scenarios (cloud) | 69 | ## Browser scenarios |
| 48 | | 70 | |
| 49 | ```ts | 71 | ```ts |
| 50 | const identity = yield * ctx.target.newIdentity(); // logged in, has an org | 72 | const target = yield * Target; |
| | 73 | const browser = yield * Browser; |
| | 74 | const identity = yield * target.newIdentity(); // logged in, has an org |
| 51 | // or newIdentity({ org: false }) for the onboarding flow | 75 | // or newIdentity({ org: false }) for the onboarding flow |
| 52 | yield * | 76 | yield * |
| 53 | ctx.browser.session(identity, async ({ page, step }) => { | 77 | browser.session(identity, async ({ page, step }) => { |
| 54 | await step("A fresh user lands on the integrations page", async () => { | 78 | await step("A fresh user lands on the integrations page", async () => { |
| 55 | await page.goto("/", { waitUntil: "networkidle" }); | 79 | await page.goto("/", { waitUntil: "networkidle" }); |
| 56 | await page.getByText("Integrations").first().waitFor(); | 80 | await page.getByText("Integrations").first().waitFor(); |
| @@ -68,10 +92,11 @@ yield * · 11 unmodified lines |
| 68 | opening menus: `await page.waitForLoadState("networkidle")`. | 92 | opening menus: `await page.waitForLoadState("networkidle")`. |
| 69 | - The stub user renders as "Test User" / `test@example.com`. | 93 | - The stub user renders as "Test User" / `test@example.com`. |
| 70 | | 94 | |
| 71 | ## MCP scenarios (selfhost) | 95 | ## MCP scenarios |
| 72 | | 96 | |
| 73 | ```ts | 97 | ```ts |
| 74 | const session = ctx.mcp.session(identity); | 98 | const mcp = yield * Mcp; |
| | 99 | const session = mcp.session(identity); |
| 75 | const tools = yield * session.listTools(); // OAuth happens headlessly here | 100 | const tools = yield * session.listTools(); // OAuth happens headlessly here |
| 76 | const r = yield * session.call("execute", { code: "return 1 + 1;" }); | 101 | const r = yield * session.call("execute", { code: "return 1 + 1;" }); |
| 77 | // human-in-the-loop: session.approvePaused(r.text) resumes a paused execution | 102 | // human-in-the-loop: session.approvePaused(r.text) resumes a paused execution |
| @@ -97,6 +122,8 @@ expect(span.span.tags["executor.tool.outcome"]).toBe("fail"); · 19 unmodified lines |
| 97 | | 122 | |
| 98 | - `expectSpan` polls (~20s): exporters batch, so arrival is | 123 | - `expectSpan` polls (~20s): exporters batch, so arrival is |
| 99 | eventually-consistent — "the span reaches the store, soon" IS the contract. | 124 | eventually-consistent — "the span reaches the store, soon" IS the contract. |
| | 125 | - The cloud globalsetup boots motel automatically; `bun run motel` runs the |
| | 126 | same store standalone (browse it, or point a dev server's exporter at it). |
| 100 | - Spec gotcha for fixtures: give operations explicit `tags` — tool addresses | 127 | - Spec gotcha for fixtures: give operations explicit `tags` — tool addresses |
| 101 | are `group.leaf`, and an untagged op derives its group from the URL path, | 128 | are `group.leaf`, and an untagged op derives its group from the URL path, |
| 102 | so `/fail` does NOT produce a `.fail`-suffixed address. | 129 | so `/fail` does NOT produce a `.fail`-suffixed address. |
| @@ -106,14 +133,20 @@ expect(span.span.tags["executor.tool.outcome"]).toBe("fail"); · 3 unmodified lines |
| 106 | | 133 | |
| 107 | ```sh | 134 | ```sh |
| 108 | cd e2e | 135 | cd e2e |
| 109 | bun run test # boots both dev servers, runs everything | 136 | bun run test # boots both dev servers, runs cloud + selfhost |
| 110 | bun run test:cloud # one target | 137 | bun run test:cloud # one target (also: test:selfhost, test:selfhost-docker, |
| | 138 | # test:cloudflare, test:local, test:desktop, test:watch) |
| 111 | bun run ports # print THIS checkout's derived ports | 139 | bun run ports # print THIS checkout's derived ports |
| | 140 | bun run summary # pass/fail digest per target from runs/ |
| 112 | # attach to an already-running server while iterating (use `bun run ports` URLs): | 141 | # attach to an already-running server while iterating (use `bun run ports` URLs): |
| 113 | E2E_CLOUD_URL=http://127.0.0.1:<port> ../node_modules/.bin/vitest run --project cloud <file> | 142 | E2E_CLOUD_URL=http://127.0.0.1:<port> ../node_modules/.bin/vitest run --project cloud <file> |
| 114 | E2E_SELFHOST_URL=http://localhost:<port> ../node_modules/.bin/vitest run --project selfhost <file> | 143 | E2E_SELFHOST_URL=http://localhost:<port> ../node_modules/.bin/vitest run --project selfhost <file> |
| 115 | ``` | 144 | ``` |
| 116 | | 145 | |
| | 146 | For interactive work against a live instance (boot, mint identities, typed API |
| | 147 | calls, MCP calls, emulator ledger) use the dev CLI: `bun run cli` — full |
| | 148 | command list in [RUNNING.md](../RUNNING.md). |
| | 149 | |
| 117 | Ports are claimed at boot (see `src/ports.ts`): each checkout hashes its repo | 150 | Ports are claimed at boot (see `src/ports.ts`): each checkout hashes its repo |
| 118 | root to a preferred block, atomically locks it (a held lock port makes races | 151 | root to a preferred block, atomically locks it (a held lock port makes races |
| 119 | impossible), and walks to the next free block if it's locked or squatted — so | 152 | impossible), and walks to the next free block if it's locked or squatted — so |
| @@ -124,7 +157,10 @@ if a suite moved. `E2E_*_PORT` env vars pin ports explicitly (no probing) and · 4 unmodified lines |
| 124 | | 157 | |
| 125 | Each run writes `runs/<target>/<slug>/result.json` plus any browser artifacts | 158 | Each run writes `runs/<target>/<slug>/result.json` plus any browser artifacts |
| 126 | (trace.zip / session.mp4 / screenshots). `bun run serve` hosts the scenario × | 159 | (trace.zip / session.mp4 / screenshots). `bun run serve` hosts the scenario × |
| 127 | target matrix; a run page links the trace into Playwright's trace viewer. | 160 | target matrix; a run page links the trace into Playwright's trace viewer. The |
| | 161 | viewer itself is a Vite/React SPA in `viewer/` (rebuilt into `runs/` by |
| | 162 | `bun run viewer:build`); `bun e2e/scripts/pr-media.ts runs/<target>/<slug>` |
| | 163 | turns a run's recording into PR-ready markdown. |
| 128 | | 164 | |
| 129 | When handing results to the user, follow the evidence contract in the root | 165 | When handing results to the user, follow the evidence contract in the root |
| 130 | [AGENTS.md](../AGENTS.md) (direct run links + a live instance + what to try); | 166 | [AGENTS.md](../AGENTS.md) (direct run links + a live instance + what to try); |
| @@ -149,6 +185,18 @@ project + globalsetup per guest OS. · 18 unmodified lines |
| 149 | ```sh | 185 | ```sh |
| 150 | vitest run --project desktop-macos # or desktop-linux | 186 | vitest run --project desktop-macos # or desktop-linux |
| 151 | ``` | 187 | ``` |
| | 188 | - **`desktop-windows`** — same scenario, but ATTACHES to a long-lived dockur |
| | 189 | Windows guest over an SSH jump instead of provisioning one (no bundle build). |
| | 190 | |
| | 191 | There are also **`cli-macos` / `cli-linux` / `cli-windows`** projects (the |
| | 192 | supervised CLI daemon inside a guest VM, `cli/*.test.ts` + |
| | 193 | `scenarios/restart-persistence.test.ts`): the globalsetup provisions the VM |
| | 194 | and `executor service install`s the daemon; `restart()` reboots the guest for |
| | 195 | real, proving the boot-time auto-start path. tart for macOS/Linux, EC2 for |
| | 196 | Windows. |
| | 197 | |
| | 198 | macOS-guest gotchas (VNC login first, single-instance lock vs a host |
| | 199 | Executor.app, guest log paths): see [notes/testing-on-mac.md](notes/testing-on-mac.md). |
| 152 | | 200 | |
| 153 | The guests run tart `--no-graphics` (no host window, never steals focus) but | 201 | The guests run tart `--no-graphics` (no host window, never steals focus) but |
| 154 | still have a usable display: | 202 | still have a usable display: |