The browse -> promote pipeline (drive a flow, record the steps, codegen a test) was more than intended. Developing against the UI is just running Playwright, which is already in-repo and is the same thing a scenario() runs. Remove src/journey/ (steps/codegen/run), the browse/promote commands in scripts/cli.ts, and the "Authoring from a live browser" doc section. The cross-OS desktop targets are unaffected.
Unified · Split
| @@ -130,38 +130,6 @@ When handing results to the user, follow the evidence contract in the root · 129 unmodified lines | |||
| 130 | [AGENTS.md](../AGENTS.md) (direct run links + a live instance + what to try); | 130 | [AGENTS.md](../AGENTS.md) (direct run links + a live instance + what to try); |
| 131 | [RUNNING.md](../RUNNING.md) has the current sharing/demo mechanics. | 131 | [RUNNING.md](../RUNNING.md) has the current sharing/demo mechanics. |
| 132 | 132 | ||
| 133 | ## Authoring from a live browser (`browse` → `promote`) | ||
| 134 | |||
| 135 | You don't have to hand-write a browser scenario. Drive a running instance's web | ||
| 136 | UI one step at a time, then turn the recorded journey into a committed scenario. | ||
| 137 | The generated test drives the same Browser surface the exploration drove, so it | ||
| 138 | is the real test, not a transcript of one — develop the flow, then crystallize | ||
| 139 | it. | ||
| 140 | |||
| 141 | ```sh | ||
| 142 | cd e2e | ||
| 143 | bun run cli up cloud # a live instance to develop against | ||
| 144 | bun run cli browse cloud goto / # each step REPLAYS the whole flow from a | ||
| 145 | bun run cli browse cloud click link Policies # clean browser and prints the page's controls | ||
| 146 | bun run cli browse cloud at-url /policies # (role · name) + a screenshot, so the next | ||
| 147 | bun run cli browse cloud see "No policies yet" # step is written against what's actually there | ||
| 148 | bun run cli promote cloud "Policies · a fresh workspace has none" | ||
| 149 | ``` | ||
| 150 | |||
| 151 | Each `browse` replays every step so far, so what you are building is, at every | ||
| 152 | moment, exactly what `promote` emits — a step that doesn't reproduce fails here, | ||
| 153 | not in CI. Steps: `goto <path>`, `click <role> <name>`, `click-text <text>`, | ||
| 154 | `fill <field> <value>`, `press <key>`, and the assertions `see <text>` / | ||
| 155 | `at-url <substring>`. `--label "…"` names a step (it becomes the `step(...)` | ||
| 156 | group); `browse <target> show | undo | reset` manages the journey. | ||
| 157 | |||
| 158 | `promote` writes `<target>/<slug>.gen.test.ts` and runs it against the live | ||
| 159 | instance, producing the usual run artifacts (session.mp4, step screenshots, | ||
| 160 | trace). A journey with no assertion is refused — a scenario must prove | ||
| 161 | something. From then on the file is an ordinary scenario: edit it, add API/MCP | ||
| 162 | checks, drop the `.gen` once it's yours. The journey itself lives in | ||
| 163 | `.dev/<target>.journey.json` (gitignored), not the repo. | ||
| 164 | |||
| 165 | ## Desktop targets (the app on real OSes, filmed) | 133 | ## Desktop targets (the app on real OSes, filmed) |
| 166 | 134 | ||
| 167 | The packaged desktop app runs as its own targets, each landing in its own | 135 | The packaged desktop app runs as its own targets, each landing in its own |
| @@ -7,8 +7,6 @@ · 6 unmodified lines | |||
| 7 | // bun scripts/cli.ts api <target> <group.endpoint> [json] | 7 | // bun scripts/cli.ts api <target> <group.endpoint> [json] |
| 8 | // bun scripts/cli.ts mcp <target> tools | call <tool> [json] | 8 | // bun scripts/cli.ts mcp <target> tools | call <tool> [json] |
| 9 | // bun scripts/cli.ts ledger <target> [workos|autumn] | 9 | // bun scripts/cli.ts ledger <target> [workos|autumn] |
| 10 | // bun scripts/cli.ts browse <target> <step…> (goto/click/see/…; show|undo|reset) | ||
| 11 | // bun scripts/cli.ts promote <target> "<scenario name>" | ||
| 12 | // bun scripts/cli.ts logs <target> | 10 | // bun scripts/cli.ts logs <target> |
| 13 | // bun scripts/cli.ts down <target> | 11 | // bun scripts/cli.ts down <target> |
| 14 | // | 12 | // |
| @@ -27,9 +25,6 @@ import { networkInterfaces } from "node:os"; · 12 unmodified lines | |||
| 27 | import { join } from "node:path"; | 25 | import { join } from "node:path"; |
| 28 | import { fileURLToPath } from "node:url"; | 26 | import { fileURLToPath } from "node:url"; |
| 29 | 27 | ||
| 30 | import type { JourneyFile } from "../src/journey/codegen"; | ||
| 31 | import type { Role, Step } from "../src/journey/steps"; | ||
| 32 | |||
| 33 | const e2eDir = fileURLToPath(new URL("..", import.meta.url)); | 28 | const e2eDir = fileURLToPath(new URL("..", import.meta.url)); |
| 34 | const devDir = join(e2eDir, ".dev"); | 29 | const devDir = join(e2eDir, ".dev"); |
| 35 | const cliPath = fileURLToPath(import.meta.url); | 30 | const cliPath = fileURLToPath(import.meta.url); |
| @@ -448,308 +443,6 @@ const ledger = async (targetName: string, service = "workos") => { · 412 unmodified lines | |||
| 448 | console.log(JSON.stringify(entries, null, 2)); | 443 | console.log(JSON.stringify(entries, null, 2)); |
| 449 | }; | 444 | }; |
| 450 | 445 | ||
| 451 | // --- browser journeys ------------------------------------------------------ | ||
| 452 | // Drive a live instance's web UI step by step. Each step appends to a journey | ||
| 453 | // file and REPLAYS the whole flow from a clean browser, so what you build is, at | ||
| 454 | // every moment, exactly what `promote` emits as a scenario — develop the flow, | ||
| 455 | // then crystallize it (e2e/AGENTS.md) with no translation gap. The agent can't | ||
| 456 | // see the screen, so every step returns the page's interactive controls (the | ||
| 457 | // vocabulary the next step is written against) plus a screenshot for a human. | ||
| 458 | |||
| 459 | const journeyPaths = (target: string) => ({ | ||
| 460 | file: join(devDir, `${target}.journey.json`), | ||
| 461 | shotsDir: join(devDir, `${target}.journey`), | ||
| 462 | }); | ||
| 463 | |||
| 464 | const readJourney = (target: string): JourneyFile => { | ||
| 465 | try { | ||
| 466 | return JSON.parse(readFileSync(journeyPaths(target).file, "utf8")) as JourneyFile; | ||
| 467 | } catch { | ||
| 468 | return { target, org: true, steps: [] }; | ||
| 469 | } | ||
| 470 | }; | ||
| 471 | |||
| 472 | const writeJourney = (journey: JourneyFile) => { | ||
| 473 | mkdirSync(devDir, { recursive: true }); | ||
| 474 | writeFileSync(journeyPaths(journey.target).file, JSON.stringify(journey, null, 1)); | ||
| 475 | }; | ||
| 476 | |||
| 477 | const STEP_ROLES = new Set<Role>([ | ||
| 478 | "link", | ||
| 479 | "button", | ||
| 480 | "heading", | ||
| 481 | "textbox", | ||
| 482 | "tab", | ||
| 483 | "menuitem", | ||
| 484 | "checkbox", | ||
| 485 | ]); | ||
| 486 | |||
| 487 | /** One CLI step verb → a Step. The verbs are deliberately plain English so the | ||
| 488 | * journey reads like instructions: goto / click / click-text / fill / press / | ||
| 489 | * see / at-url / run / request. */ | ||
| 490 | const parseStep = ( | ||
| 491 | verb: string, | ||
| 492 | args: ReadonlyArray<string>, | ||
| 493 | opts: { readonly label?: string; readonly contains?: string }, | ||
| 494 | ): Step => { | ||
| 495 | const withLabel = opts.label ? { label: opts.label } : {}; | ||
| 496 | const withContains = opts.contains !== undefined ? { contains: opts.contains } : {}; | ||
| 497 | switch (verb) { | ||
| 498 | case "goto": { | ||
| 499 | const path = args[0]; | ||
| 500 | if (!path) throw new Error("usage: browse <target> goto <path> (e.g. goto /)"); | ||
| 501 | return { kind: "goto", path, ...withLabel }; | ||
| 502 | } | ||
| 503 | case "click": { | ||
| 504 | const role = args[0]; | ||
| 505 | const name = args.slice(1).join(" "); | ||
| 506 | if (!role || !name) { | ||
| 507 | throw new Error( | ||
| 508 | "usage: browse <target> click <role> <name> (role: " + [...STEP_ROLES].join("|") + ")", | ||
| 509 | ); | ||
| 510 | } | ||
| 511 | if (!STEP_ROLES.has(role as Role)) throw new Error(`unknown role ${JSON.stringify(role)}`); | ||
| 512 | return { kind: "clickRole", role: role as Role, name, ...withLabel }; | ||
| 513 | } | ||
| 514 | case "click-text": { | ||
| 515 | const text = args.join(" "); | ||
| 516 | if (!text) throw new Error("usage: browse <target> click-text <text>"); | ||
| 517 | return { kind: "clickText", text, ...withLabel }; | ||
| 518 | } | ||
| 519 | case "fill": { | ||
| 520 | const field = args[0]; | ||
| 521 | const value = args.slice(1).join(" "); | ||
| 522 | if (!field || args.length < 2) throw new Error("usage: browse <target> fill <field> <value>"); | ||
| 523 | return { kind: "fill", field, value, ...withLabel }; | ||
| 524 | } | ||
| 525 | case "press": { | ||
| 526 | const key = args[0]; | ||
| 527 | if (!key) throw new Error("usage: browse <target> press <key> (e.g. press Enter)"); | ||
| 528 | return { kind: "press", key, ...withLabel }; | ||
| 529 | } | ||
| 530 | case "see": { | ||
| 531 | const text = args.join(" "); | ||
| 532 | if (!text) | ||
| 533 | throw new Error("usage: browse <target> see <text> (asserts the text is visible)"); | ||
| 534 | return { kind: "expectText", text, ...withLabel }; | ||
| 535 | } | ||
| 536 | case "at-url": { | ||
| 537 | const contains = args[0]; | ||
| 538 | if (!contains) throw new Error("usage: browse <target> at-url <substring>"); | ||
| 539 | return { kind: "expectUrl", contains, ...withLabel }; | ||
| 540 | } | ||
| 541 | case "run": { | ||
| 542 | const command = args.join(" "); | ||
| 543 | if (!command) { | ||
| 544 | throw new Error( | ||
| 545 | 'usage: browse <target> run "<command>" [--contains <text>] ({base} = the instance URL)', | ||
| 546 | ); | ||
| 547 | } | ||
| 548 | return { kind: "run", command, ...withContains, ...withLabel }; | ||
| 549 | } | ||
| 550 | case "request": { | ||
| 551 | const method = (args[0] ?? "").toUpperCase(); | ||
| 552 | const path = args[1]; | ||
| 553 | if (!method || !path) { | ||
| 554 | throw new Error("usage: browse <target> request <METHOD> <path> [--contains <text>]"); | ||
| 555 | } | ||
| 556 | return { kind: "request", method, path, ...withContains, ...withLabel }; | ||
| 557 | } | ||
| 558 | default: | ||
| 559 | throw new Error( | ||
| 560 | `unknown step ${JSON.stringify(verb)} — goto | click | click-text | fill | press | see | at-url | run | request`, | ||
| 561 | ); | ||
| 562 | } | ||
| 563 | }; | ||
| 564 | |||
| 565 | const printJourney = (journey: JourneyFile) => { | ||
| 566 | if (journey.steps.length === 0) return console.log(`${journey.target}: empty journey`); | ||
| 567 | console.log(`${journey.target} journey (${journey.steps.length} steps, org=${journey.org}):`); | ||
| 568 | journey.steps.forEach((step, index) => { | ||
| 569 | console.log(` ${String(index + 1).padStart(2)}. ${stepLabelOf(step)}`); | ||
| 570 | }); | ||
| 571 | }; | ||
| 572 | |||
| 573 | // A local copy of the label default so printing doesn't pull in the browser | ||
| 574 | // module (which imports playwright). Kept trivially in sync with steps.ts. | ||
| 575 | const stepLabelOf = (step: Step): string => { | ||
| 576 | if (step.label) return step.label; | ||
| 577 | if (step.kind === "goto") return `goto ${step.path}`; | ||
| 578 | if (step.kind === "clickRole") return `click ${step.role} ${JSON.stringify(step.name)}`; | ||
| 579 | if (step.kind === "clickText") return `click-text ${JSON.stringify(step.text)}`; | ||
| 580 | if (step.kind === "fill") return `fill ${JSON.stringify(step.field)}`; | ||
| 581 | if (step.kind === "press") return `press ${step.key}`; | ||
| 582 | if (step.kind === "expectText") return `see ${JSON.stringify(step.text)}`; | ||
| 583 | if (step.kind === "expectUrl") return `at-url ${JSON.stringify(step.contains)}`; | ||
| 584 | if (step.kind === "run") { | ||
| 585 | return `run ${JSON.stringify(step.command)}${step.contains ? ` → ${JSON.stringify(step.contains)}` : ""}`; | ||
| 586 | } | ||
| 587 | return `request ${step.method} ${step.path}${step.contains ? ` → ${JSON.stringify(step.contains)}` : ""}`; | ||
| 588 | }; | ||
| 589 | |||
| 590 | const browse = async (raw: ReadonlyArray<string>) => { | ||
| 591 | const target = raw[0]; | ||
| 592 | if (!target) throw new Error("usage: browse <target> <step…> (or show | undo | reset)"); | ||
| 593 | |||
| 594 | // Re-parse tokens here: --label takes a value, which the top-level flag/arg | ||
| 595 | // split would mangle. | ||
| 596 | let label: string | undefined; | ||
| 597 | let contains: string | undefined; | ||
| 598 | const positional: string[] = []; | ||
| 599 | const bools = new Set<string>(); | ||
| 600 | const tokens = raw.slice(1); | ||
| 601 | for (let i = 0; i < tokens.length; i++) { | ||
| 602 | const token = tokens[i]!; | ||
| 603 | if (token === "--label") { | ||
| 604 | label = tokens[++i]; | ||
| 605 | continue; | ||
| 606 | } | ||
| 607 | if (token.startsWith("--label=")) { | ||
| 608 | label = token.slice("--label=".length); | ||
| 609 | continue; | ||
| 610 | } | ||
| 611 | if (token === "--contains") { | ||
| 612 | contains = tokens[++i]; | ||
| 613 | continue; | ||
| 614 | } | ||
| 615 | if (token.startsWith("--contains=")) { | ||
| 616 | contains = token.slice("--contains=".length); | ||
| 617 | continue; | ||
| 618 | } | ||
| 619 | if (token.startsWith("--")) { | ||
| 620 | bools.add(token); | ||
| 621 | continue; | ||
| 622 | } | ||
| 623 | positional.push(token); | ||
| 624 | } | ||
| 625 | |||
| 626 | const verb = positional[0]; | ||
| 627 | const { file, shotsDir } = journeyPaths(target); | ||
| 628 | let journey = readJourney(target); | ||
| 629 | |||
| 630 | if (verb === "show") return printJourney(journey); | ||
| 631 | if (verb === "reset") { | ||
| 632 | rmSync(file, { force: true }); | ||
| 633 | rmSync(shotsDir, { recursive: true, force: true }); | ||
| 634 | return console.log(`${target}: journey reset`); | ||
| 635 | } | ||
| 636 | if (!verb) throw new Error("usage: browse <target> <step…> (or show | undo | reset)"); | ||
| 637 | |||
| 638 | // The flow runs as ONE minted identity per replay — same as the generated | ||
| 639 | // test mints one per run, so a stateful journey behaves identically. | ||
| 640 | const { target: resolved } = await loadTarget(target); | ||
| 641 | const identity = await runEffect<import("../src/target").Identity>( | ||
| 642 | resolved.newIdentity(journey.org ? undefined : { org: false }), | ||
| 643 | ); | ||
| 644 | const { replayJourney } = await import("../src/journey/run"); | ||
| 645 | |||
| 646 | let steps = [...journey.steps]; | ||
| 647 | if (verb === "undo") { | ||
| 648 | if (steps.length === 0) throw new Error("nothing to undo"); | ||
| 649 | steps = steps.slice(0, -1); | ||
| 650 | } else { | ||
| 651 | if (steps.length === 0 && bools.has("--no-org")) journey = { ...journey, org: false }; | ||
| 652 | steps = [...steps, parseStep(verb, positional.slice(1), { label, contains })]; | ||
| 653 | } | ||
| 654 | |||
| 655 | mkdirSync(shotsDir, { recursive: true }); | ||
| 656 | const shot = join(shotsDir, `${String(steps.length).padStart(2, "0")}.png`); | ||
| 657 | const observation = await replayJourney(resolved, identity, steps, { screenshotPath: shot }); | ||
| 658 | |||
| 659 | if (observation.failedStep) { | ||
| 660 | const failed = observation.failedStep; | ||
| 661 | const isNew = verb !== "undo" && failed.index === steps.length - 1; | ||
| 662 | console.log( | ||
| 663 | `✗ ${isNew ? "this step" : `step ${failed.index + 1} (${stepLabelOf(steps[failed.index]!)})`} failed — journey unchanged.`, | ||
| 664 | ); | ||
| 665 | console.log(` ${failed.error.split("\n")[0]}`); | ||
| 666 | console.log(` screenshot: ${observation.screenshotPath}`); | ||
| 667 | return; | ||
| 668 | } | ||
| 669 | |||
| 670 | journey = { ...journey, steps }; | ||
| 671 | writeJourney(journey); | ||
| 672 | |||
| 673 | console.log(`→ ${observation.url} ${JSON.stringify(observation.title)}`); | ||
| 674 | console.log(` screenshot: ${observation.screenshotPath}`); | ||
| 675 | if (observation.lastOutput) { | ||
| 676 | const lines = observation.lastOutput.split("\n").slice(0, 12); | ||
| 677 | console.log("output:"); | ||
| 678 | for (const line of lines) console.log(` ${line}`); | ||
| 679 | } | ||
| 680 | printJourney(journey); | ||
| 681 | if (observation.controls.length > 0) { | ||
| 682 | console.log("controls on the page (role · name):"); | ||
| 683 | for (const control of observation.controls.slice(0, 40)) { | ||
| 684 | console.log(` ${control.role.padEnd(9)} ${control.name}`); | ||
| 685 | } | ||
| 686 | if (observation.controls.length > 40) { | ||
| 687 | console.log(` … ${observation.controls.length - 40} more`); | ||
| 688 | } | ||
| 689 | } | ||
| 690 | }; | ||
| 691 | |||
| 692 | const promote = async (raw: ReadonlyArray<string>) => { | ||
| 693 | const target = raw[0]; | ||
| 694 | const positional = raw.slice(1).filter((arg) => !arg.startsWith("--")); | ||
| 695 | const name = positional[0]; | ||
| 696 | const noRun = raw.includes("--no-run"); | ||
| 697 | if (!target || !name) { | ||
| 698 | throw new Error('usage: promote <target> "<scenario name>" [--no-run]'); | ||
| 699 | } | ||
| 700 | const journey = readJourney(target); | ||
| 701 | if (journey.steps.length === 0) { | ||
| 702 | throw new Error(`no journey for ${target} — build one with \`browse ${target} …\` first`); | ||
| 703 | } | ||
| 704 | const { codegenScenario, journeyHasAssertion, journeyHasBrowserStep } = | ||
| 705 | await import("../src/journey/codegen"); | ||
| 706 | if (!journeyHasBrowserStep(journey)) { | ||
| 707 | throw new Error( | ||
| 708 | "this journey is all terminal/HTTP steps — `promote` generates browser-anchored scenarios. Add a browser step, or write a CLI/API test directly.", | ||
| 709 | ); | ||
| 710 | } | ||
| 711 | if (!journeyHasAssertion(journey)) { | ||
| 712 | throw new Error( | ||
| 713 | "this journey has no assertion (a `see`, `at-url`, `request`, or `run --contains` step), so the scenario would prove nothing. Add one, then promote.", | ||
| 714 | ); | ||
| 715 | } | ||
| 716 | |||
| 717 | const source = codegenScenario(name, journey); | ||
| 718 | const slug = name | ||
| 719 | .toLowerCase() | ||
| 720 | .replace(/[^a-z0-9]+/g, "-") | ||
| 721 | .replace(/^-+|-+$/g, "") | ||
| 722 | .slice(0, 80); | ||
| 723 | // Scope the file to the target's own dir (cloud/, selfhost/) so it runs only | ||
| 724 | // where its UI selectors apply, not on every cross-target host. | ||
| 725 | const relFile = join(target, `${slug}.gen.test.ts`); | ||
| 726 | writeFileSync(join(e2eDir, relFile), source); | ||
| 727 | console.log(`wrote ${relFile} (${journey.steps.length} steps from the ${target} journey)`); | ||
| 728 | |||
| 729 | const urlKey = `E2E_${target.toUpperCase()}_URL`; | ||
| 730 | const url = readState(target)?.env?.[urlKey]; | ||
| 731 | if (noRun || !url) { | ||
| 732 | console.log( | ||
| 733 | `run it against the live instance:\n ${urlKey}=<url> bunx vitest run --project ${target} ${relFile}`, | ||
| 734 | ); | ||
| 735 | return; | ||
| 736 | } | ||
| 737 | |||
| 738 | console.log(`running it against the live instance (${url}) …`); | ||
| 739 | const proc = spawn("bunx", ["vitest", "run", "--project", target, relFile], { | ||
| 740 | cwd: e2eDir, | ||
| 741 | stdio: "inherit", | ||
| 742 | env: { ...process.env, [urlKey]: url }, | ||
| 743 | }); | ||
| 744 | const code: number = await new Promise((resolve) => proc.on("exit", (c) => resolve(c ?? 1))); | ||
| 745 | const slugDir = join("runs", target, slug); | ||
| 746 | console.log( | ||
| 747 | code === 0 | ||
| 748 | ? `\n✓ passed. Artifacts (test.ts, result.json, session.mp4/film.mp4): e2e/${slugDir}\n view: cd e2e && bun run serve → #/${target}/${slug}` | ||
| 749 | : `\n✗ the generated test failed (exit ${code}). The flow that passed live did not pass as a committed test — inspect ${relFile} and e2e/${slugDir}.`, | ||
| 750 | ); | ||
| 751 | }; | ||
| 752 | |||
| 753 | // --- lifecycle commands ---------------------------------------------------- | 446 | // --- lifecycle commands ---------------------------------------------------- |
| 754 | 447 | ||
| 755 | const status = () => { | 448 | const status = () => { |
| @@ -815,24 +508,9 @@ const HELP = `e2e dev CLI — the scenario primitives, interactive (see e2e/AGEN · 59 unmodified lines | |||
| 815 | api <target> <group.endpoint> [json] typed API call as a fresh identity | 508 | api <target> <group.endpoint> [json] typed API call as a fresh identity |
| 816 | mcp <target> tools | call <tool> [json] MCP session call | 509 | mcp <target> tools | call <tool> [json] MCP session call |
| 817 | ledger <target> [workos|autumn] the emulator's request ledger (cloud) | 510 | ledger <target> [workos|autumn] the emulator's request ledger (cloud) |
| 818 | browse <target> <step> drive the live web UI, one step at a time; each step | ||
| 819 | replays the whole flow and prints the page's controls | ||
| 820 | steps: goto <path> | click <role> <name> | click-text <text> | ||
| 821 | | fill <field> <value> | press <key> | ||
| 822 | | see <text> | at-url <substring> | ||
| 823 | | run "<cmd>" | request <METHOD> <path> | ||
| 824 | flags: --label "…", --no-org, --contains <text> | ||
| 825 | ({base} in a run command = the instance URL) | ||
| 826 | browse <target> show | undo | reset | ||
| 827 | promote <target> "<name>" turn the recorded journey into a committed scenario | ||
| 828 | (<target>/<slug>.gen.test.ts) and run it (--no-run) | ||
| 829 | logs <target> dump the instance's dev-server log | 511 | logs <target> dump the instance's dev-server log |
| 830 | down <target> tear down (kills servers, removes tailscale serves) | 512 | down <target> tear down (kills servers, removes tailscale serves) |
| 831 | 513 | ||
| 832 | A browser journey IS the scenario: develop the flow with \`browse\`, then | ||
| 833 | \`promote\` it. The generated test drives the same surface, so a reviewer judges | ||
| 834 | it by reading the test and watching its video. | ||
| 835 | |||
| 836 | Instances live in e2e/.dev/<target>.json — a state file marks a DELIBERATE | 514 | Instances live in e2e/.dev/<target>.json — a state file marks a DELIBERATE |
| 837 | long-lived instance. Use the booted instance for e2e too: | 515 | long-lived instance. Use the booted instance for e2e too: |
| 838 | E2E_SELFHOST_URL=<app url> vitest run --project selfhost <file>`; | 516 | E2E_SELFHOST_URL=<app url> vitest run --project selfhost <file>`; |
| @@ -856,10 +534,6 @@ const main = async () => { · 17 unmodified lines | |||
| 856 | return mcpCall(args[0] ?? "", args[1], args.slice(2)); | 534 | return mcpCall(args[0] ?? "", args[1], args.slice(2)); |
| 857 | case "ledger": | 535 | case "ledger": |
| 858 | return ledger(args[0] ?? "", args[1]); | 536 | return ledger(args[0] ?? "", args[1]); |
| 859 | case "browse": | ||
| 860 | return browse(rest); | ||
| 861 | case "promote": | ||
| 862 | return promote(rest); | ||
| 863 | case "logs": | 537 | case "logs": |
| 864 | return logs(args[0] ?? ""); | 538 | return logs(args[0] ?? ""); |
| 865 | case "down": | 539 | case "down": |
| @@ -1,78 +0,0 @@ | |||
| 1 | // Generate a committed scenario from a recorded journey. The output is a normal | ||
| 2 | // scenario file (the same shape a human writes, see e2e/AGENTS.md): an Effect | ||
| 3 | // body that yields Target + Browser, mints a fresh identity, and drives one | ||
| 4 | // browser session of labelled steps. Terminal (`run`) and HTTP (`request`) steps | ||
| 5 | // run INSIDE that session, interleaved with the UI, so page state is never lost | ||
| 6 | // between them. It is meant to be read and edited after generation — promotion | ||
| 7 | // is the START of a scenario's life, not a frozen artifact. | ||
| 8 | import { codegenStep, isAssertion, isBrowserStep, stepLabel, type Step } from "./steps"; | ||
| 9 | |||
| 10 | export interface JourneyFile { | ||
| 11 | readonly target: string; | ||
| 12 | readonly org: boolean; | ||
| 13 | readonly steps: ReadonlyArray<Step>; | ||
| 14 | } | ||
| 15 | |||
| 16 | const INDENT = " "; | ||
| 17 | |||
| 18 | const stepBlock = (step: Step): string => { | ||
| 19 | const label = JSON.stringify(stepLabel(step)); | ||
| 20 | const body = codegenStep(step) | ||
| 21 | .split("\n") | ||
| 22 | .map((line) => `${INDENT} ${line}`) | ||
| 23 | .join("\n"); | ||
| 24 | return `${INDENT}await step(${label}, async () => {\n${body}\n${INDENT}});`; | ||
| 25 | }; | ||
| 26 | |||
| 27 | /** A journey with no assertion proves nothing; one with no browser step isn't | ||
| 28 | * what this tool generates (write a CLI/API test directly). `promote` checks | ||
| 29 | * both. */ | ||
| 30 | export const journeyHasAssertion = (journey: JourneyFile): boolean => | ||
| 31 | journey.steps.some(isAssertion); | ||
| 32 | |||
| 33 | export const journeyHasBrowserStep = (journey: JourneyFile): boolean => | ||
| 34 | journey.steps.some(isBrowserStep); | ||
| 35 | |||
| 36 | export const codegenScenario = (name: string, journey: JourneyFile): string => { | ||
| 37 | const identityArg = journey.org ? "" : "{ org: false }"; | ||
| 38 | const body = journey.steps.map(stepBlock).join("\n"); | ||
| 39 | |||
| 40 | const needsExec = journey.steps.some((step) => step.kind === "run"); | ||
| 41 | const needsExpect = journey.steps.some( | ||
| 42 | (step) => step.kind === "request" || (step.kind === "run" && step.contains !== undefined), | ||
| 43 | ); | ||
| 44 | |||
| 45 | const imports: string[] = []; | ||
| 46 | if (needsExec) { | ||
| 47 | imports.push(`import { execFile } from "node:child_process";`); | ||
| 48 | imports.push(`import { promisify } from "node:util";`); | ||
| 49 | imports.push(""); | ||
| 50 | } | ||
| 51 | if (needsExpect) imports.push(`import { expect } from "@effect/vitest";`); | ||
| 52 | imports.push(`import { Effect } from "effect";`); | ||
| 53 | imports.push(""); | ||
| 54 | imports.push(`import { scenario } from "../src/scenario";`); | ||
| 55 | imports.push(`import { Browser, Target } from "../src/services";`); | ||
| 56 | |||
| 57 | const execHelper = needsExec ? "\nconst execFileAsync = promisify(execFile);\n" : ""; | ||
| 58 | |||
| 59 | return `// Generated from an interactive browser journey: \`bun scripts/cli.ts promote ${journey.target} "<name>"\`. | ||
| 60 | // This is now an ordinary scenario — edit it freely. It drives the same Browser | ||
| 61 | // surface the exploration used, so a reviewer can judge the guarantee by | ||
| 62 | // reading it. Re-run with: E2E_${journey.target.toUpperCase()}_URL=<url> vitest run --project ${journey.target} <this file> | ||
| 63 | ${imports.join("\n")} | ||
| 64 | ${execHelper} | ||
| 65 | scenario( | ||
| 66 | ${JSON.stringify(name)}, | ||
| 67 | { timeout: 120_000 }, | ||
| 68 | Effect.gen(function* () { | ||
| 69 | const target = yield* Target; | ||
| 70 | const browser = yield* Browser; | ||
| 71 | const identity = yield* target.newIdentity(${identityArg}); | ||
| 72 | yield* browser.session(identity, async ({ page, step }) => { | ||
| 73 | ${body} | ||
| 74 | }); | ||
| 75 | }), | ||
| 76 | ); | ||
| 77 | `; | ||
| 78 | }; | ||
| @@ -1,104 +0,0 @@ | |||
| 1 | // Replay a journey live, from a clean browser, and report what the page looks | ||
| 2 | // like afterward. This is the development loop: each `browse` command appends a | ||
| 3 | // step and replays the WHOLE journey from scratch, so the flow the agent is | ||
| 4 | // building is, at every moment, exactly what the generated test will run — a | ||
| 5 | // step that doesn't reproduce fails here, not later. The returned observation | ||
| 6 | // (url, title, the page's interactive controls) is how the agent, which can't | ||
| 7 | // see the screen, decides the next step; the screenshot is for a human. | ||
| 8 | import { chromium, type Page } from "playwright"; | ||
| 9 | |||
| 10 | import type { Identity, Target } from "../target"; | ||
| 11 | import { executeStep, type Step } from "./steps"; | ||
| 12 | |||
| 13 | export interface Control { | ||
| 14 | readonly role: string; | ||
| 15 | readonly name: string; | ||
| 16 | } | ||
| 17 | |||
| 18 | export interface Observation { | ||
| 19 | readonly url: string; | ||
| 20 | readonly title: string; | ||
| 21 | /** The interactive elements on the page, as (role, accessible name) — the | ||
| 22 | * vocabulary the next clickRole/fill step is written against. */ | ||
| 23 | readonly controls: ReadonlyArray<Control>; | ||
| 24 | readonly screenshotPath: string; | ||
| 25 | /** Textual output of the last terminal/HTTP step, if any — so the agent sees | ||
| 26 | * what a `run`/`request` produced, not just the page. */ | ||
| 27 | readonly lastOutput?: string; | ||
| 28 | /** Index of the step that threw, with its message — undefined on success. */ | ||
| 29 | readonly failedStep?: { readonly index: number; readonly error: string }; | ||
| 30 | } | ||
| 31 | |||
| 32 | const OBSERVED_ROLES = ["link", "button", "textbox", "tab", "menuitem", "checkbox"] as const; | ||
| 33 | const PER_ROLE_CAP = 30; | ||
| 34 | |||
| 35 | /** A compact, deduped list of the page's interactive controls. Names come from | ||
| 36 | * the accessible name (text / aria-label / placeholder), trimmed. */ | ||
| 37 | const snapshotControls = async (page: Page): Promise<Control[]> => { | ||
| 38 | const seen = new Set<string>(); | ||
| 39 | const controls: Control[] = []; | ||
| 40 | for (const role of OBSERVED_ROLES) { | ||
| 41 | const elements = await page.getByRole(role).all(); | ||
| 42 | for (const element of elements.slice(0, PER_ROLE_CAP)) { | ||
| 43 | const raw = | ||
| 44 | (await element.textContent().catch(() => null))?.trim() || | ||
| 45 | (await element.getAttribute("aria-label").catch(() => null)) || | ||
| 46 | (await element.getAttribute("placeholder").catch(() => null)) || | ||
| 47 | ""; | ||
| 48 | const name = raw.replace(/\s+/g, " ").trim().slice(0, 70); | ||
| 49 | if (!name) continue; | ||
| 50 | const key = `${role}:${name}`; | ||
| 51 | if (seen.has(key)) continue; | ||
| 52 | seen.add(key); | ||
| 53 | controls.push({ role, name }); | ||
| 54 | } | ||
| 55 | } | ||
| 56 | return controls; | ||
| 57 | }; | ||
| 58 | |||
| 59 | export const replayJourney = async ( | ||
| 60 | target: Target, | ||
| 61 | identity: Identity, | ||
| 62 | steps: ReadonlyArray<Step>, | ||
| 63 | options: { readonly screenshotPath: string }, | ||
| 64 | ): Promise<Observation> => { | ||
| 65 | const browser = await chromium.launch(); | ||
| 66 | let failedStep: Observation["failedStep"]; | ||
| 67 | try { | ||
| 68 | const context = await browser.newContext({ | ||
| 69 | colorScheme: "dark", | ||
| 70 | viewport: { width: 1280, height: 800 }, | ||
| 71 | baseURL: target.baseUrl, | ||
| 72 | }); | ||
| 73 | // Same identity injection the Browser surface does, so the live page is the | ||
| 74 | // logged-in page the generated test will drive. | ||
| 75 | if (identity.cookies?.length) { | ||
| 76 | await context.addCookies( | ||
| 77 | identity.cookies.map((cookie) => ({ ...cookie, url: target.baseUrl })), | ||
| 78 | ); | ||
| 79 | } | ||
| 80 | const page = await context.newPage(); | ||
| 81 | let lastOutput: string | undefined; | ||
| 82 | for (let index = 0; index < steps.length; index++) { | ||
| 83 | try { | ||
| 84 | const output = await executeStep({ page, baseUrl: target.baseUrl }, steps[index]!); | ||
| 85 | if (output !== undefined) lastOutput = output; | ||
| 86 | } catch (error) { | ||
| 87 | failedStep = { index, error: error instanceof Error ? error.message : String(error) }; | ||
| 88 | break; | ||
| 89 | } | ||
| 90 | } | ||
| 91 | await page.waitForLoadState("networkidle").catch(() => {}); | ||
| 92 | await page.screenshot({ path: options.screenshotPath }).catch(() => {}); | ||
| 93 | return { | ||
| 94 | url: page.url(), | ||
| 95 | title: await page.title().catch(() => ""), | ||
| 96 | controls: await snapshotControls(page).catch(() => []), | ||
| 97 | screenshotPath: options.screenshotPath, | ||
| 98 | lastOutput, | ||
| 99 | failedStep, | ||
| 100 | }; | ||
| 101 | } finally { | ||
| 102 | await browser.close(); | ||
| 103 | } | ||
| 104 | }; | ||
| @@ -1,236 +0,0 @@ | |||
| 1 | // The journey Step DSL: the single source of truth shared by interactive | ||
| 2 | // exploration and the generated scenario. ONE step description is both | ||
| 3 | // (a) executed live against the real product while the agent develops a flow | ||
| 4 | // (`executeStep`), and (b) emitted as the matching Playwright line inside a | ||
| 5 | // committed scenario (`codegenStep`). Because both sides read the same record, | ||
| 6 | // "turn what I just did into a test" is a translation, not a reimplementation: | ||
| 7 | // the generated test drives the exact surface the exploration drove. | ||
| 8 | // | ||
| 9 | // Steps are plain JSON (they persist to .dev/<target>.journey.json between CLI | ||
| 10 | // invocations), so they carry no closures — every action is a named primitive. | ||
| 11 | import { execFile } from "node:child_process"; | ||
| 12 | import { promisify } from "node:util"; | ||
| 13 | |||
| 14 | import type { Page } from "playwright"; | ||
| 15 | |||
| 16 | const execFileAsync = promisify(execFile); | ||
| 17 | |||
| 18 | /** ARIA roles the journey can target. A closed set keeps codegen honest and | ||
| 19 | * the live `getByRole` calls type-safe (Playwright's role arg is a union). */ | ||
| 20 | export type Role = "link" | "button" | "heading" | "textbox" | "tab" | "menuitem" | "checkbox"; | ||
| 21 | |||
| 22 | export type Step = | ||
| 23 | | { readonly kind: "goto"; readonly path: string; readonly label?: string } | ||
| 24 | | { | ||
| 25 | readonly kind: "clickRole"; | ||
| 26 | readonly role: Role; | ||
| 27 | readonly name: string; | ||
| 28 | readonly label?: string; | ||
| 29 | } | ||
| 30 | | { readonly kind: "clickText"; readonly text: string; readonly label?: string } | ||
| 31 | | { | ||
| 32 | readonly kind: "fill"; | ||
| 33 | readonly field: string; | ||
| 34 | readonly value: string; | ||
| 35 | readonly label?: string; | ||
| 36 | } | ||
| 37 | | { readonly kind: "press"; readonly key: string; readonly label?: string } | ||
| 38 | | { readonly kind: "expectText"; readonly text: string; readonly label?: string } | ||
| 39 | | { readonly kind: "expectUrl"; readonly contains: string; readonly label?: string } | ||
| 40 | // A terminal command. `{base}` expands to the target's base URL, so a journey | ||
| 41 | // can hit the same instance the UI is driving (curl, npx add-mcp, executor …). | ||
| 42 | // `contains` (when set) asserts on the combined stdout+stderr. | ||
| 43 | | { | ||
| 44 | readonly kind: "run"; | ||
| 45 | readonly command: string; | ||
| 46 | readonly contains?: string; | ||
| 47 | readonly label?: string; | ||
| 48 | } | ||
| 49 | // An HTTP call through the page's own authenticated session (relative paths | ||
| 50 | // resolve against the base URL). `contains` asserts on the response body; | ||
| 51 | // without it, the assertion is a 2xx. | ||
| 52 | | { | ||
| 53 | readonly kind: "request"; | ||
| 54 | readonly method: string; | ||
| 55 | readonly path: string; | ||
| 56 | readonly contains?: string; | ||
| 57 | readonly label?: string; | ||
| 58 | }; | ||
| 59 | |||
| 60 | /** Assertions are the steps a reviewer reads as the guarantee — a journey with | ||
| 61 | * none asserts nothing, so `promote` refuses it. A `run`/`request` is an | ||
| 62 | * assertion when it carries an expectation (`contains`, or `request`'s 2xx). */ | ||
| 63 | export const isAssertion = (step: Step): boolean => | ||
| 64 | step.kind === "expectText" || | ||
| 65 | step.kind === "expectUrl" || | ||
| 66 | step.kind === "request" || | ||
| 67 | (step.kind === "run" && step.contains !== undefined); | ||
| 68 | |||
| 69 | export const isBrowserStep = (step: Step): boolean => | ||
| 70 | step.kind !== "run" && step.kind !== "request"; | ||
| 71 | |||
| 72 | /** The human-readable step name (the `step(label, …)` group + screenshot | ||
| 73 | * caption). The agent can override per step; this is the sensible default so a | ||
| 74 | * generated test reads as a journey even when labels were left implicit. */ | ||
| 75 | export const stepLabel = (step: Step): string => { | ||
| 76 | if (step.label) return step.label; | ||
| 77 | switch (step.kind) { | ||
| 78 | case "goto": | ||
| 79 | return `Open ${step.path}`; | ||
| 80 | case "clickRole": | ||
| 81 | return `Click the ${JSON.stringify(step.name)} ${step.role}`; | ||
| 82 | case "clickText": | ||
| 83 | return `Click ${JSON.stringify(step.text)}`; | ||
| 84 | case "fill": | ||
| 85 | return `Fill ${JSON.stringify(step.field)}`; | ||
| 86 | case "press": | ||
| 87 | return `Press ${step.key}`; | ||
| 88 | case "expectText": | ||
| 89 | return `See ${JSON.stringify(step.text)}`; | ||
| 90 | case "expectUrl": | ||
| 91 | return `Land on a URL containing ${JSON.stringify(step.contains)}`; | ||
| 92 | case "run": | ||
| 93 | return step.contains | ||
| 94 | ? `Run ${JSON.stringify(step.command)} and see ${JSON.stringify(step.contains)}` | ||
| 95 | : `Run ${JSON.stringify(step.command)}`; | ||
| 96 | case "request": | ||
| 97 | return step.contains | ||
| 98 | ? `${step.method} ${step.path} returns ${JSON.stringify(step.contains)}` | ||
| 99 | : `${step.method} ${step.path} succeeds`; | ||
| 100 | } | ||
| 101 | }; | ||
| 102 | |||
| 103 | const ASSERT_TIMEOUT = 15_000; | ||
| 104 | |||
| 105 | export interface StepContext { | ||
| 106 | readonly page: Page; | ||
| 107 | /** The target's base URL — `{base}` in a `run` command expands to this. */ | ||
| 108 | readonly baseUrl: string; | ||
| 109 | } | ||
| 110 | |||
| 111 | /** Expand `{base}` so a terminal command can reach the instance under test. */ | ||
| 112 | const withBase = (command: string, baseUrl: string): string => | ||
| 113 | command.replaceAll("{base}", baseUrl); | ||
| 114 | |||
| 115 | /** Drive one step against a live page. Assertions throw on failure (a missing | ||
| 116 | * text, a wrong URL, a non-matching command output) so it surfaces immediately | ||
| 117 | * while exploring, the same way it would fail the generated test. Returns any | ||
| 118 | * textual output (terminal / HTTP) so the caller can show it. */ | ||
| 119 | export const executeStep = async (ctx: StepContext, step: Step): Promise<string | undefined> => { | ||
| 120 | const { page } = ctx; | ||
| 121 | switch (step.kind) { | ||
| 122 | case "goto": | ||
| 123 | await page.goto(step.path, { waitUntil: "networkidle" }); | ||
| 124 | return; | ||
| 125 | case "clickRole": | ||
| 126 | await page.getByRole(step.role, { name: step.name }).first().click(); | ||
| 127 | return; | ||
| 128 | case "clickText": | ||
| 129 | await page.getByText(step.text).first().click(); | ||
| 130 | return; | ||
| 131 | case "fill": | ||
| 132 | await page.getByLabel(step.field).first().fill(step.value); | ||
| 133 | return; | ||
| 134 | case "press": | ||
| 135 | await page.keyboard.press(step.key); | ||
| 136 | return; | ||
| 137 | case "expectText": | ||
| 138 | await page | ||
| 139 | .getByText(step.text) | ||
| 140 | .first() | ||
| 141 | .waitFor({ state: "visible", timeout: ASSERT_TIMEOUT }); | ||
| 142 | return; | ||
| 143 | case "expectUrl": | ||
| 144 | await page.waitForURL((url) => url.toString().includes(step.contains), { | ||
| 145 | timeout: ASSERT_TIMEOUT, | ||
| 146 | }); | ||
| 147 | return; | ||
| 148 | case "run": { | ||
| 149 | const result = await execFileAsync("sh", ["-c", withBase(step.command, ctx.baseUrl)]).catch( | ||
| 150 | (error: { stdout?: string; stderr?: string }) => ({ | ||
| 151 | stdout: error.stdout ?? "", | ||
| 152 | stderr: error.stderr ?? String(error), | ||
| 153 | }), | ||
| 154 | ); | ||
| 155 | const output = `${result.stdout}${result.stderr}`; | ||
| 156 | if (step.contains !== undefined && !output.includes(step.contains)) { | ||
| 157 | throw new Error( | ||
| 158 | `\`run\` output did not contain ${JSON.stringify(step.contains)}\n${output.slice(0, 1000)}`, | ||
| 159 | ); | ||
| 160 | } | ||
| 161 | return output.trim().slice(0, 2000); | ||
| 162 | } | ||
| 163 | case "request": { | ||
| 164 | const response = await page.request.fetch(step.path, { method: step.method }); | ||
| 165 | const body = await response.text(); | ||
| 166 | if (step.contains !== undefined) { | ||
| 167 | if (!body.includes(step.contains)) { | ||
| 168 | throw new Error( | ||
| 169 | `${step.method} ${step.path} body did not contain ${JSON.stringify(step.contains)} (status ${response.status()})`, | ||
| 170 | ); | ||
| 171 | } | ||
| 172 | } else if (!response.ok()) { | ||
| 173 | throw new Error(`${step.method} ${step.path} returned ${response.status()}`); | ||
| 174 | } | ||
| 175 | return `${response.status()} ${body.slice(0, 800)}`; | ||
| 176 | } | ||
| 177 | } | ||
| 178 | }; | ||
| 179 | |||
| 180 | /** The Playwright line(s) for this step, as they appear inside the generated | ||
| 181 | * scenario's `step(label, async () => { … })` body. Mirrors `executeStep` | ||
| 182 | * exactly — same locator, same call — so live behavior and the test match. */ | ||
| 183 | export const codegenStep = (step: Step): string => { | ||
| 184 | const s = (value: string): string => JSON.stringify(value); | ||
| 185 | switch (step.kind) { | ||
| 186 | case "goto": | ||
| 187 | return `await page.goto(${s(step.path)}, { waitUntil: "networkidle" });`; | ||
| 188 | case "clickRole": | ||
| 189 | return `await page.getByRole(${s(step.role)}, { name: ${s(step.name)} }).first().click();`; | ||
| 190 | case "clickText": | ||
| 191 | return `await page.getByText(${s(step.text)}).first().click();`; | ||
| 192 | case "fill": | ||
| 193 | return `await page.getByLabel(${s(step.field)}).first().fill(${s(step.value)});`; | ||
| 194 | case "press": | ||
| 195 | return `await page.keyboard.press(${s(step.key)});`; | ||
| 196 | case "expectText": | ||
| 197 | // The repo's browser-assertion idiom: waiting for the element IS the | ||
| 198 | // assertion (a timeout fails the step with the locator in the message). | ||
| 199 | return `await page.getByText(${s(step.text)}).first().waitFor();`; | ||
| 200 | case "expectUrl": | ||
| 201 | return `await page.waitForURL((url) => url.toString().includes(${s(step.contains)}));`; | ||
| 202 | case "run": { | ||
| 203 | const lines = [ | ||
| 204 | `const { stdout } = await execFileAsync("sh", ["-c", ${backtick(step.command)}]);`, | ||
| 205 | ]; | ||
| 206 | if (step.contains !== undefined) { | ||
| 207 | lines.push( | ||
| 208 | `expect(stdout, "the command output is as expected").toContain(${s(step.contains)});`, | ||
| 209 | ); | ||
| 210 | } | ||
| 211 | return lines.join("\n"); | ||
| 212 | } | ||
| 213 | case "request": { | ||
| 214 | const lines = [ | ||
| 215 | `const response = await page.request.fetch(${s(step.path)}, { method: ${s(step.method)} });`, | ||
| 216 | ]; | ||
| 217 | lines.push( | ||
| 218 | step.contains !== undefined | ||
| 219 | ? `expect(await response.text(), "the response is as expected").toContain(${s(step.contains)});` | ||
| 220 | : `expect(response.ok(), "the request succeeded").toBe(true);`, | ||
| 221 | ); | ||
| 222 | return lines.join("\n"); | ||
| 223 | } | ||
| 224 | } | ||
| 225 | }; | ||
| 226 | |||
| 227 | /** A terminal command as a template literal so `{base}` becomes `target.baseUrl` | ||
| 228 | * (which is in scope in the generated body). Backticks in the command are | ||
| 229 | * escaped so the literal stays valid. */ | ||
| 230 | const backtick = (command: string): string => | ||
| 231 | "`" + | ||
| 232 | command | ||
| 233 | .replaceAll("\\", "\\\\") | ||
| 234 | .replaceAll("`", "\\`") | ||
| 235 | .replaceAll("{base}", "${target.baseUrl}") + | ||
| 236 | "`"; | ||