History

e2e: remove the browse/promote authoring loop

The browse -> promote pipeline (drive a flow, record the steps, codegen a test) was more than intended. Developing against the UI is just running Playwright, which is already in-repo and is the same thing a scenario() runs. Remove src/journey/ (steps/codegen/run), the browse/promote commands in scripts/cli.ts, and the "Authoring from a live browser" doc section. The cross-OS desktop targets are unaffected.

Rhys Sullivan committed 4 months ago

Commit bb751b91 parent 9586e685 files · +0 −776Browse files

Unified · Split

5 files
  1. e2e/AGENTS.md +0 −32
  2. e2e/scripts/cli.ts +0 −326
  3. e2e/src/journey/codegen.ts +0 −78
  4. e2e/src/journey/run.ts +0 −104
  5. e2e/src/journey/steps.ts +0 −236
e2e/AGENTS.md+0−32
@@ -130,38 +130,6 @@ When handing results to the user, follow the evidence contract in the root · 129 unmodified lines
130130 [AGENTS.md](../AGENTS.md) (direct run links + a live instance + what to try);
131131 [RUNNING.md](../RUNNING.md) has the current sharing/demo mechanics.
132132
133−## Authoring from a live browser (`browse` → `promote`)
134−
135−You don't have to hand-write a browser scenario. Drive a running instance's web
136−UI one step at a time, then turn the recorded journey into a committed scenario.
137−The generated test drives the same Browser surface the exploration drove, so it
138−is the real test, not a transcript of one — develop the flow, then crystallize
139−it.
140−
141−```sh
142−cd e2e
143−bun run cli up cloud # a live instance to develop against
144−bun run cli browse cloud goto / # each step REPLAYS the whole flow from a
145−bun run cli browse cloud click link Policies # clean browser and prints the page's controls
146−bun run cli browse cloud at-url /policies # (role · name) + a screenshot, so the next
147−bun run cli browse cloud see "No policies yet" # step is written against what's actually there
148−bun run cli promote cloud "Policies · a fresh workspace has none"
149−```
150−
151−Each `browse` replays every step so far, so what you are building is, at every
152−moment, exactly what `promote` emits — a step that doesn't reproduce fails here,
153−not in CI. Steps: `goto <path>`, `click <role> <name>`, `click-text <text>`,
154−`fill <field> <value>`, `press <key>`, and the assertions `see <text>` /
155−`at-url <substring>`. `--label "…"` names a step (it becomes the `step(...)`
156−group); `browse <target> show | undo | reset` manages the journey.
157−
158−`promote` writes `<target>/<slug>.gen.test.ts` and runs it against the live
159−instance, producing the usual run artifacts (session.mp4, step screenshots,
160−trace). A journey with no assertion is refused — a scenario must prove
161−something. From then on the file is an ordinary scenario: edit it, add API/MCP
162−checks, drop the `.gen` once it's yours. The journey itself lives in
163−`.dev/<target>.journey.json` (gitignored), not the repo.
164−
165133 ## Desktop targets (the app on real OSes, filmed)
166134
167135 The packaged desktop app runs as its own targets, each landing in its own
e2e/scripts/cli.ts+0−326
@@ -7,8 +7,6 @@ · 6 unmodified lines
77 // bun scripts/cli.ts api <target> <group.endpoint> [json]
88 // bun scripts/cli.ts mcp <target> tools | call <tool> [json]
99 // bun scripts/cli.ts ledger <target> [workos|autumn]
10−// bun scripts/cli.ts browse <target> <step…> (goto/click/see/…; show|undo|reset)
11−// bun scripts/cli.ts promote <target> "<scenario name>"
1210 // bun scripts/cli.ts logs <target>
1311 // bun scripts/cli.ts down <target>
1412 //
@@ -27,9 +25,6 @@ import { networkInterfaces } from "node:os"; · 12 unmodified lines
2725 import { join } from "node:path";
2826 import { fileURLToPath } from "node:url";
2927
30−import type { JourneyFile } from "../src/journey/codegen";
31−import type { Role, Step } from "../src/journey/steps";
32−
3328 const e2eDir = fileURLToPath(new URL("..", import.meta.url));
3429 const devDir = join(e2eDir, ".dev");
3530 const cliPath = fileURLToPath(import.meta.url);
@@ -448,308 +443,6 @@ const ledger = async (targetName: string, service = "workos") => { · 412 unmodified lines
448443 console.log(JSON.stringify(entries, null, 2));
449444 };
450445
451−// --- browser journeys ------------------------------------------------------
452−// Drive a live instance's web UI step by step. Each step appends to a journey
453−// file and REPLAYS the whole flow from a clean browser, so what you build is, at
454−// every moment, exactly what `promote` emits as a scenario — develop the flow,
455−// then crystallize it (e2e/AGENTS.md) with no translation gap. The agent can't
456−// see the screen, so every step returns the page's interactive controls (the
457−// vocabulary the next step is written against) plus a screenshot for a human.
458−
459−const journeyPaths = (target: string) => ({
460− file: join(devDir, `${target}.journey.json`),
461− shotsDir: join(devDir, `${target}.journey`),
462−});
463−
464−const readJourney = (target: string): JourneyFile => {
465− try {
466− return JSON.parse(readFileSync(journeyPaths(target).file, "utf8")) as JourneyFile;
467− } catch {
468− return { target, org: true, steps: [] };
469− }
470−};
471−
472−const writeJourney = (journey: JourneyFile) => {
473− mkdirSync(devDir, { recursive: true });
474− writeFileSync(journeyPaths(journey.target).file, JSON.stringify(journey, null, 1));
475−};
476−
477−const STEP_ROLES = new Set<Role>([
478− "link",
479− "button",
480− "heading",
481− "textbox",
482− "tab",
483− "menuitem",
484− "checkbox",
485−]);
486−
487−/** One CLI step verb → a Step. The verbs are deliberately plain English so the
488− * journey reads like instructions: goto / click / click-text / fill / press /
489− * see / at-url / run / request. */
490−const parseStep = (
491− verb: string,
492− args: ReadonlyArray<string>,
493− opts: { readonly label?: string; readonly contains?: string },
494−): Step => {
495− const withLabel = opts.label ? { label: opts.label } : {};
496− const withContains = opts.contains !== undefined ? { contains: opts.contains } : {};
497− switch (verb) {
498− case "goto": {
499− const path = args[0];
500− if (!path) throw new Error("usage: browse <target> goto <path> (e.g. goto /)");
501− return { kind: "goto", path, ...withLabel };
502− }
503− case "click": {
504− const role = args[0];
505− const name = args.slice(1).join(" ");
506− if (!role || !name) {
507− throw new Error(
508− "usage: browse <target> click <role> <name> (role: " + [...STEP_ROLES].join("|") + ")",
509− );
510− }
511− if (!STEP_ROLES.has(role as Role)) throw new Error(`unknown role ${JSON.stringify(role)}`);
512− return { kind: "clickRole", role: role as Role, name, ...withLabel };
513− }
514− case "click-text": {
515− const text = args.join(" ");
516− if (!text) throw new Error("usage: browse <target> click-text <text>");
517− return { kind: "clickText", text, ...withLabel };
518− }
519− case "fill": {
520− const field = args[0];
521− const value = args.slice(1).join(" ");
522− if (!field || args.length < 2) throw new Error("usage: browse <target> fill <field> <value>");
523− return { kind: "fill", field, value, ...withLabel };
524− }
525− case "press": {
526− const key = args[0];
527− if (!key) throw new Error("usage: browse <target> press <key> (e.g. press Enter)");
528− return { kind: "press", key, ...withLabel };
529− }
530− case "see": {
531− const text = args.join(" ");
532− if (!text)
533− throw new Error("usage: browse <target> see <text> (asserts the text is visible)");
534− return { kind: "expectText", text, ...withLabel };
535− }
536− case "at-url": {
537− const contains = args[0];
538− if (!contains) throw new Error("usage: browse <target> at-url <substring>");
539− return { kind: "expectUrl", contains, ...withLabel };
540− }
541− case "run": {
542− const command = args.join(" ");
543− if (!command) {
544− throw new Error(
545− 'usage: browse <target> run "<command>" [--contains <text>] ({base} = the instance URL)',
546− );
547− }
548− return { kind: "run", command, ...withContains, ...withLabel };
549− }
550− case "request": {
551− const method = (args[0] ?? "").toUpperCase();
552− const path = args[1];
553− if (!method || !path) {
554− throw new Error("usage: browse <target> request <METHOD> <path> [--contains <text>]");
555− }
556− return { kind: "request", method, path, ...withContains, ...withLabel };
557− }
558− default:
559− throw new Error(
560− `unknown step ${JSON.stringify(verb)} — goto | click | click-text | fill | press | see | at-url | run | request`,
561− );
562− }
563−};
564−
565−const printJourney = (journey: JourneyFile) => {
566− if (journey.steps.length === 0) return console.log(`${journey.target}: empty journey`);
567− console.log(`${journey.target} journey (${journey.steps.length} steps, org=${journey.org}):`);
568− journey.steps.forEach((step, index) => {
569− console.log(` ${String(index + 1).padStart(2)}. ${stepLabelOf(step)}`);
570− });
571−};
572−
573−// A local copy of the label default so printing doesn't pull in the browser
574−// module (which imports playwright). Kept trivially in sync with steps.ts.
575−const stepLabelOf = (step: Step): string => {
576− if (step.label) return step.label;
577− if (step.kind === "goto") return `goto ${step.path}`;
578− if (step.kind === "clickRole") return `click ${step.role} ${JSON.stringify(step.name)}`;
579− if (step.kind === "clickText") return `click-text ${JSON.stringify(step.text)}`;
580− if (step.kind === "fill") return `fill ${JSON.stringify(step.field)}`;
581− if (step.kind === "press") return `press ${step.key}`;
582− if (step.kind === "expectText") return `see ${JSON.stringify(step.text)}`;
583− if (step.kind === "expectUrl") return `at-url ${JSON.stringify(step.contains)}`;
584− if (step.kind === "run") {
585− return `run ${JSON.stringify(step.command)}${step.contains ? ` → ${JSON.stringify(step.contains)}` : ""}`;
586− }
587− return `request ${step.method} ${step.path}${step.contains ? ` → ${JSON.stringify(step.contains)}` : ""}`;
588−};
589−
590−const browse = async (raw: ReadonlyArray<string>) => {
591− const target = raw[0];
592− if (!target) throw new Error("usage: browse <target> <step…> (or show | undo | reset)");
593−
594− // Re-parse tokens here: --label takes a value, which the top-level flag/arg
595− // split would mangle.
596− let label: string | undefined;
597− let contains: string | undefined;
598− const positional: string[] = [];
599− const bools = new Set<string>();
600− const tokens = raw.slice(1);
601− for (let i = 0; i < tokens.length; i++) {
602− const token = tokens[i]!;
603− if (token === "--label") {
604− label = tokens[++i];
605− continue;
606− }
607− if (token.startsWith("--label=")) {
608− label = token.slice("--label=".length);
609− continue;
610− }
611− if (token === "--contains") {
612− contains = tokens[++i];
613− continue;
614− }
615− if (token.startsWith("--contains=")) {
616− contains = token.slice("--contains=".length);
617− continue;
618− }
619− if (token.startsWith("--")) {
620− bools.add(token);
621− continue;
622− }
623− positional.push(token);
624− }
625−
626− const verb = positional[0];
627− const { file, shotsDir } = journeyPaths(target);
628− let journey = readJourney(target);
629−
630− if (verb === "show") return printJourney(journey);
631− if (verb === "reset") {
632− rmSync(file, { force: true });
633− rmSync(shotsDir, { recursive: true, force: true });
634− return console.log(`${target}: journey reset`);
635− }
636− if (!verb) throw new Error("usage: browse <target> <step…> (or show | undo | reset)");
637−
638− // The flow runs as ONE minted identity per replay — same as the generated
639− // test mints one per run, so a stateful journey behaves identically.
640− const { target: resolved } = await loadTarget(target);
641− const identity = await runEffect<import("../src/target").Identity>(
642− resolved.newIdentity(journey.org ? undefined : { org: false }),
643− );
644− const { replayJourney } = await import("../src/journey/run");
645−
646− let steps = [...journey.steps];
647− if (verb === "undo") {
648− if (steps.length === 0) throw new Error("nothing to undo");
649− steps = steps.slice(0, -1);
650− } else {
651− if (steps.length === 0 && bools.has("--no-org")) journey = { ...journey, org: false };
652− steps = [...steps, parseStep(verb, positional.slice(1), { label, contains })];
653− }
654−
655− mkdirSync(shotsDir, { recursive: true });
656− const shot = join(shotsDir, `${String(steps.length).padStart(2, "0")}.png`);
657− const observation = await replayJourney(resolved, identity, steps, { screenshotPath: shot });
658−
659− if (observation.failedStep) {
660− const failed = observation.failedStep;
661− const isNew = verb !== "undo" && failed.index === steps.length - 1;
662− console.log(
663− `✗ ${isNew ? "this step" : `step ${failed.index + 1} (${stepLabelOf(steps[failed.index]!)})`} failed — journey unchanged.`,
664− );
665− console.log(` ${failed.error.split("\n")[0]}`);
666− console.log(` screenshot: ${observation.screenshotPath}`);
667− return;
668− }
669−
670− journey = { ...journey, steps };
671− writeJourney(journey);
672−
673− console.log(`→ ${observation.url} ${JSON.stringify(observation.title)}`);
674− console.log(` screenshot: ${observation.screenshotPath}`);
675− if (observation.lastOutput) {
676− const lines = observation.lastOutput.split("\n").slice(0, 12);
677− console.log("output:");
678− for (const line of lines) console.log(` ${line}`);
679− }
680− printJourney(journey);
681− if (observation.controls.length > 0) {
682− console.log("controls on the page (role · name):");
683− for (const control of observation.controls.slice(0, 40)) {
684− console.log(` ${control.role.padEnd(9)} ${control.name}`);
685− }
686− if (observation.controls.length > 40) {
687− console.log(` … ${observation.controls.length - 40} more`);
688− }
689− }
690−};
691−
692−const promote = async (raw: ReadonlyArray<string>) => {
693− const target = raw[0];
694− const positional = raw.slice(1).filter((arg) => !arg.startsWith("--"));
695− const name = positional[0];
696− const noRun = raw.includes("--no-run");
697− if (!target || !name) {
698− throw new Error('usage: promote <target> "<scenario name>" [--no-run]');
699− }
700− const journey = readJourney(target);
701− if (journey.steps.length === 0) {
702− throw new Error(`no journey for ${target} — build one with \`browse ${target} …\` first`);
703− }
704− const { codegenScenario, journeyHasAssertion, journeyHasBrowserStep } =
705− await import("../src/journey/codegen");
706− if (!journeyHasBrowserStep(journey)) {
707− throw new Error(
708− "this journey is all terminal/HTTP steps — `promote` generates browser-anchored scenarios. Add a browser step, or write a CLI/API test directly.",
709− );
710− }
711− if (!journeyHasAssertion(journey)) {
712− throw new Error(
713− "this journey has no assertion (a `see`, `at-url`, `request`, or `run --contains` step), so the scenario would prove nothing. Add one, then promote.",
714− );
715− }
716−
717− const source = codegenScenario(name, journey);
718− const slug = name
719− .toLowerCase()
720− .replace(/[^a-z0-9]+/g, "-")
721− .replace(/^-+|-+$/g, "")
722− .slice(0, 80);
723− // Scope the file to the target's own dir (cloud/, selfhost/) so it runs only
724− // where its UI selectors apply, not on every cross-target host.
725− const relFile = join(target, `${slug}.gen.test.ts`);
726− writeFileSync(join(e2eDir, relFile), source);
727− console.log(`wrote ${relFile} (${journey.steps.length} steps from the ${target} journey)`);
728−
729− const urlKey = `E2E_${target.toUpperCase()}_URL`;
730− const url = readState(target)?.env?.[urlKey];
731− if (noRun || !url) {
732− console.log(
733− `run it against the live instance:\n ${urlKey}=<url> bunx vitest run --project ${target} ${relFile}`,
734− );
735− return;
736− }
737−
738− console.log(`running it against the live instance (${url}) …`);
739− const proc = spawn("bunx", ["vitest", "run", "--project", target, relFile], {
740− cwd: e2eDir,
741− stdio: "inherit",
742− env: { ...process.env, [urlKey]: url },
743− });
744− const code: number = await new Promise((resolve) => proc.on("exit", (c) => resolve(c ?? 1)));
745− const slugDir = join("runs", target, slug);
746− console.log(
747− code === 0
748− ? `\n✓ passed. Artifacts (test.ts, result.json, session.mp4/film.mp4): e2e/${slugDir}\n view: cd e2e && bun run serve → #/${target}/${slug}`
749− : `\n✗ the generated test failed (exit ${code}). The flow that passed live did not pass as a committed test — inspect ${relFile} and e2e/${slugDir}.`,
750− );
751−};
752−
753446 // --- lifecycle commands ----------------------------------------------------
754447
755448 const status = () => {
@@ -815,24 +508,9 @@ const HELP = `e2e dev CLI — the scenario primitives, interactive (see e2e/AGEN · 59 unmodified lines
815508 api <target> <group.endpoint> [json] typed API call as a fresh identity
816509 mcp <target> tools | call <tool> [json] MCP session call
817510 ledger <target> [workos|autumn] the emulator's request ledger (cloud)
818− browse <target> <step> drive the live web UI, one step at a time; each step
819− replays the whole flow and prints the page's controls
820− steps: goto <path> | click <role> <name> | click-text <text>
821− | fill <field> <value> | press <key>
822− | see <text> | at-url <substring>
823− | run "<cmd>" | request <METHOD> <path>
824− flags: --label "…", --no-org, --contains <text>
825− ({base} in a run command = the instance URL)
826− browse <target> show | undo | reset
827− promote <target> "<name>" turn the recorded journey into a committed scenario
828− (<target>/<slug>.gen.test.ts) and run it (--no-run)
829511 logs <target> dump the instance's dev-server log
830512 down <target> tear down (kills servers, removes tailscale serves)
831513
832−A browser journey IS the scenario: develop the flow with \`browse\`, then
833−\`promote\` it. The generated test drives the same surface, so a reviewer judges
834−it by reading the test and watching its video.
835−
836514 Instances live in e2e/.dev/<target>.json — a state file marks a DELIBERATE
837515 long-lived instance. Use the booted instance for e2e too:
838516 E2E_SELFHOST_URL=<app url> vitest run --project selfhost <file>`;
@@ -856,10 +534,6 @@ const main = async () => { · 17 unmodified lines
856534 return mcpCall(args[0] ?? "", args[1], args.slice(2));
857535 case "ledger":
858536 return ledger(args[0] ?? "", args[1]);
859− case "browse":
860− return browse(rest);
861− case "promote":
862− return promote(rest);
863537 case "logs":
864538 return logs(args[0] ?? "");
865539 case "down":
e2e/src/journey/codegen.ts+0−78
@@ -1,78 +0,0 @@
1−// Generate a committed scenario from a recorded journey. The output is a normal
2−// scenario file (the same shape a human writes, see e2e/AGENTS.md): an Effect
3−// body that yields Target + Browser, mints a fresh identity, and drives one
4−// browser session of labelled steps. Terminal (`run`) and HTTP (`request`) steps
5−// run INSIDE that session, interleaved with the UI, so page state is never lost
6−// between them. It is meant to be read and edited after generation — promotion
7−// is the START of a scenario's life, not a frozen artifact.
8−import { codegenStep, isAssertion, isBrowserStep, stepLabel, type Step } from "./steps";
9−
10−export interface JourneyFile {
11− readonly target: string;
12− readonly org: boolean;
13− readonly steps: ReadonlyArray<Step>;
14−}
15−
16−const INDENT = " ";
17−
18−const stepBlock = (step: Step): string => {
19− const label = JSON.stringify(stepLabel(step));
20− const body = codegenStep(step)
21− .split("\n")
22− .map((line) => `${INDENT} ${line}`)
23− .join("\n");
24− return `${INDENT}await step(${label}, async () => {\n${body}\n${INDENT}});`;
25−};
26−
27−/** A journey with no assertion proves nothing; one with no browser step isn't
28− * what this tool generates (write a CLI/API test directly). `promote` checks
29− * both. */
30−export const journeyHasAssertion = (journey: JourneyFile): boolean =>
31− journey.steps.some(isAssertion);
32−
33−export const journeyHasBrowserStep = (journey: JourneyFile): boolean =>
34− journey.steps.some(isBrowserStep);
35−
36−export const codegenScenario = (name: string, journey: JourneyFile): string => {
37− const identityArg = journey.org ? "" : "{ org: false }";
38− const body = journey.steps.map(stepBlock).join("\n");
39−
40− const needsExec = journey.steps.some((step) => step.kind === "run");
41− const needsExpect = journey.steps.some(
42− (step) => step.kind === "request" || (step.kind === "run" && step.contains !== undefined),
43− );
44−
45− const imports: string[] = [];
46− if (needsExec) {
47− imports.push(`import { execFile } from "node:child_process";`);
48− imports.push(`import { promisify } from "node:util";`);
49− imports.push("");
50− }
51− if (needsExpect) imports.push(`import { expect } from "@effect/vitest";`);
52− imports.push(`import { Effect } from "effect";`);
53− imports.push("");
54− imports.push(`import { scenario } from "../src/scenario";`);
55− imports.push(`import { Browser, Target } from "../src/services";`);
56−
57− const execHelper = needsExec ? "\nconst execFileAsync = promisify(execFile);\n" : "";
58−
59− return `// Generated from an interactive browser journey: \`bun scripts/cli.ts promote ${journey.target} "<name>"\`.
60−// This is now an ordinary scenario — edit it freely. It drives the same Browser
61−// surface the exploration used, so a reviewer can judge the guarantee by
62−// reading it. Re-run with: E2E_${journey.target.toUpperCase()}_URL=<url> vitest run --project ${journey.target} <this file>
63−${imports.join("\n")}
64−${execHelper}
65−scenario(
66− ${JSON.stringify(name)},
67− { timeout: 120_000 },
68− Effect.gen(function* () {
69− const target = yield* Target;
70− const browser = yield* Browser;
71− const identity = yield* target.newIdentity(${identityArg});
72− yield* browser.session(identity, async ({ page, step }) => {
73−${body}
74− });
75− }),
76−);
77−`;
78−};
e2e/src/journey/run.ts+0−104
@@ -1,104 +0,0 @@
1−// Replay a journey live, from a clean browser, and report what the page looks
2−// like afterward. This is the development loop: each `browse` command appends a
3−// step and replays the WHOLE journey from scratch, so the flow the agent is
4−// building is, at every moment, exactly what the generated test will run — a
5−// step that doesn't reproduce fails here, not later. The returned observation
6−// (url, title, the page's interactive controls) is how the agent, which can't
7−// see the screen, decides the next step; the screenshot is for a human.
8−import { chromium, type Page } from "playwright";
9−
10−import type { Identity, Target } from "../target";
11−import { executeStep, type Step } from "./steps";
12−
13−export interface Control {
14− readonly role: string;
15− readonly name: string;
16−}
17−
18−export interface Observation {
19− readonly url: string;
20− readonly title: string;
21− /** The interactive elements on the page, as (role, accessible name) — the
22− * vocabulary the next clickRole/fill step is written against. */
23− readonly controls: ReadonlyArray<Control>;
24− readonly screenshotPath: string;
25− /** Textual output of the last terminal/HTTP step, if any — so the agent sees
26− * what a `run`/`request` produced, not just the page. */
27− readonly lastOutput?: string;
28− /** Index of the step that threw, with its message — undefined on success. */
29− readonly failedStep?: { readonly index: number; readonly error: string };
30−}
31−
32−const OBSERVED_ROLES = ["link", "button", "textbox", "tab", "menuitem", "checkbox"] as const;
33−const PER_ROLE_CAP = 30;
34−
35−/** A compact, deduped list of the page's interactive controls. Names come from
36− * the accessible name (text / aria-label / placeholder), trimmed. */
37−const snapshotControls = async (page: Page): Promise<Control[]> => {
38− const seen = new Set<string>();
39− const controls: Control[] = [];
40− for (const role of OBSERVED_ROLES) {
41− const elements = await page.getByRole(role).all();
42− for (const element of elements.slice(0, PER_ROLE_CAP)) {
43− const raw =
44− (await element.textContent().catch(() => null))?.trim() ||
45− (await element.getAttribute("aria-label").catch(() => null)) ||
46− (await element.getAttribute("placeholder").catch(() => null)) ||
47− "";
48− const name = raw.replace(/\s+/g, " ").trim().slice(0, 70);
49− if (!name) continue;
50− const key = `${role}:${name}`;
51− if (seen.has(key)) continue;
52− seen.add(key);
53− controls.push({ role, name });
54− }
55− }
56− return controls;
57−};
58−
59−export const replayJourney = async (
60− target: Target,
61− identity: Identity,
62− steps: ReadonlyArray<Step>,
63− options: { readonly screenshotPath: string },
64−): Promise<Observation> => {
65− const browser = await chromium.launch();
66− let failedStep: Observation["failedStep"];
67− try {
68− const context = await browser.newContext({
69− colorScheme: "dark",
70− viewport: { width: 1280, height: 800 },
71− baseURL: target.baseUrl,
72− });
73− // Same identity injection the Browser surface does, so the live page is the
74− // logged-in page the generated test will drive.
75− if (identity.cookies?.length) {
76− await context.addCookies(
77− identity.cookies.map((cookie) => ({ ...cookie, url: target.baseUrl })),
78− );
79− }
80− const page = await context.newPage();
81− let lastOutput: string | undefined;
82− for (let index = 0; index < steps.length; index++) {
83− try {
84− const output = await executeStep({ page, baseUrl: target.baseUrl }, steps[index]!);
85− if (output !== undefined) lastOutput = output;
86− } catch (error) {
87− failedStep = { index, error: error instanceof Error ? error.message : String(error) };
88− break;
89− }
90− }
91− await page.waitForLoadState("networkidle").catch(() => {});
92− await page.screenshot({ path: options.screenshotPath }).catch(() => {});
93− return {
94− url: page.url(),
95− title: await page.title().catch(() => ""),
96− controls: await snapshotControls(page).catch(() => []),
97− screenshotPath: options.screenshotPath,
98− lastOutput,
99− failedStep,
100− };
101− } finally {
102− await browser.close();
103− }
104−};
e2e/src/journey/steps.ts+0−236
@@ -1,236 +0,0 @@
1−// The journey Step DSL: the single source of truth shared by interactive
2−// exploration and the generated scenario. ONE step description is both
3−// (a) executed live against the real product while the agent develops a flow
4−// (`executeStep`), and (b) emitted as the matching Playwright line inside a
5−// committed scenario (`codegenStep`). Because both sides read the same record,
6−// "turn what I just did into a test" is a translation, not a reimplementation:
7−// the generated test drives the exact surface the exploration drove.
8−//
9−// Steps are plain JSON (they persist to .dev/<target>.journey.json between CLI
10−// invocations), so they carry no closures — every action is a named primitive.
11−import { execFile } from "node:child_process";
12−import { promisify } from "node:util";
13−
14−import type { Page } from "playwright";
15−
16−const execFileAsync = promisify(execFile);
17−
18−/** ARIA roles the journey can target. A closed set keeps codegen honest and
19− * the live `getByRole` calls type-safe (Playwright's role arg is a union). */
20−export type Role = "link" | "button" | "heading" | "textbox" | "tab" | "menuitem" | "checkbox";
21−
22−export type Step =
23− | { readonly kind: "goto"; readonly path: string; readonly label?: string }
24− | {
25− readonly kind: "clickRole";
26− readonly role: Role;
27− readonly name: string;
28− readonly label?: string;
29− }
30− | { readonly kind: "clickText"; readonly text: string; readonly label?: string }
31− | {
32− readonly kind: "fill";
33− readonly field: string;
34− readonly value: string;
35− readonly label?: string;
36− }
37− | { readonly kind: "press"; readonly key: string; readonly label?: string }
38− | { readonly kind: "expectText"; readonly text: string; readonly label?: string }
39− | { readonly kind: "expectUrl"; readonly contains: string; readonly label?: string }
40− // A terminal command. `{base}` expands to the target's base URL, so a journey
41− // can hit the same instance the UI is driving (curl, npx add-mcp, executor …).
42− // `contains` (when set) asserts on the combined stdout+stderr.
43− | {
44− readonly kind: "run";
45− readonly command: string;
46− readonly contains?: string;
47− readonly label?: string;
48− }
49− // An HTTP call through the page's own authenticated session (relative paths
50− // resolve against the base URL). `contains` asserts on the response body;
51− // without it, the assertion is a 2xx.
52− | {
53− readonly kind: "request";
54− readonly method: string;
55− readonly path: string;
56− readonly contains?: string;
57− readonly label?: string;
58− };
59−
60−/** Assertions are the steps a reviewer reads as the guarantee — a journey with
61− * none asserts nothing, so `promote` refuses it. A `run`/`request` is an
62− * assertion when it carries an expectation (`contains`, or `request`'s 2xx). */
63−export const isAssertion = (step: Step): boolean =>
64− step.kind === "expectText" ||
65− step.kind === "expectUrl" ||
66− step.kind === "request" ||
67− (step.kind === "run" && step.contains !== undefined);
68−
69−export const isBrowserStep = (step: Step): boolean =>
70− step.kind !== "run" && step.kind !== "request";
71−
72−/** The human-readable step name (the `step(label, …)` group + screenshot
73− * caption). The agent can override per step; this is the sensible default so a
74− * generated test reads as a journey even when labels were left implicit. */
75−export const stepLabel = (step: Step): string => {
76− if (step.label) return step.label;
77− switch (step.kind) {
78− case "goto":
79− return `Open ${step.path}`;
80− case "clickRole":
81− return `Click the ${JSON.stringify(step.name)} ${step.role}`;
82− case "clickText":
83− return `Click ${JSON.stringify(step.text)}`;
84− case "fill":
85− return `Fill ${JSON.stringify(step.field)}`;
86− case "press":
87− return `Press ${step.key}`;
88− case "expectText":
89− return `See ${JSON.stringify(step.text)}`;
90− case "expectUrl":
91− return `Land on a URL containing ${JSON.stringify(step.contains)}`;
92− case "run":
93− return step.contains
94− ? `Run ${JSON.stringify(step.command)} and see ${JSON.stringify(step.contains)}`
95− : `Run ${JSON.stringify(step.command)}`;
96− case "request":
97− return step.contains
98− ? `${step.method} ${step.path} returns ${JSON.stringify(step.contains)}`
99− : `${step.method} ${step.path} succeeds`;
100− }
101−};
102−
103−const ASSERT_TIMEOUT = 15_000;
104−
105−export interface StepContext {
106− readonly page: Page;
107− /** The target's base URL — `{base}` in a `run` command expands to this. */
108− readonly baseUrl: string;
109−}
110−
111−/** Expand `{base}` so a terminal command can reach the instance under test. */
112−const withBase = (command: string, baseUrl: string): string =>
113− command.replaceAll("{base}", baseUrl);
114−
115−/** Drive one step against a live page. Assertions throw on failure (a missing
116− * text, a wrong URL, a non-matching command output) so it surfaces immediately
117− * while exploring, the same way it would fail the generated test. Returns any
118− * textual output (terminal / HTTP) so the caller can show it. */
119−export const executeStep = async (ctx: StepContext, step: Step): Promise<string | undefined> => {
120− const { page } = ctx;
121− switch (step.kind) {
122− case "goto":
123− await page.goto(step.path, { waitUntil: "networkidle" });
124− return;
125− case "clickRole":
126− await page.getByRole(step.role, { name: step.name }).first().click();
127− return;
128− case "clickText":
129− await page.getByText(step.text).first().click();
130− return;
131− case "fill":
132− await page.getByLabel(step.field).first().fill(step.value);
133− return;
134− case "press":
135− await page.keyboard.press(step.key);
136− return;
137− case "expectText":
138− await page
139− .getByText(step.text)
140− .first()
141− .waitFor({ state: "visible", timeout: ASSERT_TIMEOUT });
142− return;
143− case "expectUrl":
144− await page.waitForURL((url) => url.toString().includes(step.contains), {
145− timeout: ASSERT_TIMEOUT,
146− });
147− return;
148− case "run": {
149− const result = await execFileAsync("sh", ["-c", withBase(step.command, ctx.baseUrl)]).catch(
150− (error: { stdout?: string; stderr?: string }) => ({
151− stdout: error.stdout ?? "",
152− stderr: error.stderr ?? String(error),
153− }),
154− );
155− const output = `${result.stdout}${result.stderr}`;
156− if (step.contains !== undefined && !output.includes(step.contains)) {
157− throw new Error(
158− `\`run\` output did not contain ${JSON.stringify(step.contains)}\n${output.slice(0, 1000)}`,
159− );
160− }
161− return output.trim().slice(0, 2000);
162− }
163− case "request": {
164− const response = await page.request.fetch(step.path, { method: step.method });
165− const body = await response.text();
166− if (step.contains !== undefined) {
167− if (!body.includes(step.contains)) {
168− throw new Error(
169− `${step.method} ${step.path} body did not contain ${JSON.stringify(step.contains)} (status ${response.status()})`,
170− );
171− }
172− } else if (!response.ok()) {
173− throw new Error(`${step.method} ${step.path} returned ${response.status()}`);
174− }
175− return `${response.status()} ${body.slice(0, 800)}`;
176− }
177− }
178−};
179−
180−/** The Playwright line(s) for this step, as they appear inside the generated
181− * scenario's `step(label, async () => { … })` body. Mirrors `executeStep`
182− * exactly — same locator, same call — so live behavior and the test match. */
183−export const codegenStep = (step: Step): string => {
184− const s = (value: string): string => JSON.stringify(value);
185− switch (step.kind) {
186− case "goto":
187− return `await page.goto(${s(step.path)}, { waitUntil: "networkidle" });`;
188− case "clickRole":
189− return `await page.getByRole(${s(step.role)}, { name: ${s(step.name)} }).first().click();`;
190− case "clickText":
191− return `await page.getByText(${s(step.text)}).first().click();`;
192− case "fill":
193− return `await page.getByLabel(${s(step.field)}).first().fill(${s(step.value)});`;
194− case "press":
195− return `await page.keyboard.press(${s(step.key)});`;
196− case "expectText":
197− // The repo's browser-assertion idiom: waiting for the element IS the
198− // assertion (a timeout fails the step with the locator in the message).
199− return `await page.getByText(${s(step.text)}).first().waitFor();`;
200− case "expectUrl":
201− return `await page.waitForURL((url) => url.toString().includes(${s(step.contains)}));`;
202− case "run": {
203− const lines = [
204− `const { stdout } = await execFileAsync("sh", ["-c", ${backtick(step.command)}]);`,
205− ];
206− if (step.contains !== undefined) {
207− lines.push(
208− `expect(stdout, "the command output is as expected").toContain(${s(step.contains)});`,
209− );
210− }
211− return lines.join("\n");
212− }
213− case "request": {
214− const lines = [
215− `const response = await page.request.fetch(${s(step.path)}, { method: ${s(step.method)} });`,
216− ];
217− lines.push(
218− step.contains !== undefined
219− ? `expect(await response.text(), "the response is as expected").toContain(${s(step.contains)});`
220− : `expect(response.ok(), "the request succeeded").toBe(true);`,
221− );
222− return lines.join("\n");
223− }
224− }
225−};
226−
227−/** A terminal command as a template literal so `{base}` becomes `target.baseUrl`
228− * (which is in scope in the generated body). Backticks in the command are
229− * escaped so the literal stays valid. */
230−const backtick = (command: string): string =>
231− "`" +
232− command
233− .replaceAll("\\", "\\\\")
234− .replaceAll("`", "\\`")
235− .replaceAll("{base}", "${target.baseUrl}") +
236− "`";