The Desk: film a scenario on one virtual desktop, one recording

e2e/desk/run.sh builds a Linux image (Xvfb + openbox + xterm + headed Chromium + ffmpeg) and runs a scenario inside it with E2E_DESK=1: the chat renderer opens in a visible UTF-8 xterm (events over a FIFO instead of a PTY), the browser hop is a real headed window raised over it, and a single x11grab films the whole screen for the whole run — no splicing. The film replaces session.mp4 in the scenario's run dir; the scenario file itself is unchanged (chat-theater switches transports on E2E_DESK). First run builds the image and fills a node_modules volume; subsequent films of the connect-handoff session take ~2 minutes end to end.

Rhys Sullivan committed 5 months ago

Commit 75632d01 parent 82895ca7 files · +249 −39Browse files

Unified · Split

7 files

e2e/AGENTS.md +29 −0
e2e/desk/Dockerfile +27 −0
e2e/desk/entry.sh +43 −0
e2e/desk/run.sh +17 −0
e2e/src/clients/agent-chat-tui.ts +16 −2
e2e/src/clients/chat-theater.ts +106 −36
e2e/src/surfaces/browser.ts +11 −1

e2e/AGENTS.md+29−0

@@ -133,3 +133,32 @@ When handing results to the user, follow the evidence contract in the root · 132 unmodified lines
133	133	values, not booleans — `expect(list).toContain(x)`, never
134	134	`expect(list.includes(x)).toBe(true)` — so failures show the data.
135	135	- Keep it deterministic: no sleeps; wait on conditions.
	136	+
	137	+## Developer-session recordings (chat theater + desk)
	138	+
	139	+Some scenarios are meant to be WATCHED — they show the product the way a
	140	+developer actually uses it. Three tiers, pick deliberately:
	141	+
	142	+1. Chat theater (`src/clients/chat-theater.ts`): the default for
	143	+ product-flow recordings. The "agent" is a chat renderer in a recorded
	144	+ PTY; every tool spinner brackets a REAL mcporter MCP call (OAuth,
	145	+ execute, approval resume). No inference, no third-party binary.
	146	+ Exemplar: `scenarios/connect-handoff-session.test.ts`. Artifacts:
	147	+ `terminal.cast` (the chat) + `session.mp4` (browser hops); the viewer
	148	+ plays them in story order.
	149	+2. Replay brain + real client (`src/clients/replay-brain.ts`): when the
	150	+ third-party CLIENT's behavior is under test (OpenCode/Claude Code
	151	+ protocol handling). A scripted OpenAI-wire server plays the LLM; the
	152	+ real client does everything else. Script by transcript inspection, never
	153	+ turn counting.
	154	+3. Real-inference evals: a different axis (performance distributions,
	155	+ not pass/fail). Not in this suite.
	156	+
	157	+The Desk (`desk/`): films a scenario on one virtual Linux desktop — the
	158	+chat renderer in a visible xterm, the browser as a real headed window, one
	159	+ffmpeg x11grab. The film replaces session.mp4 in the run dir; the scenario
	160	+file is unchanged (chat-theater switches transports on `E2E_DESK=1`).
	161	+
	162	+```
	163	+e2e/desk/run.sh [scenario] [project] # docker; first run builds + installs
	164	+```

e2e/desk/Dockerfile+27−0

@@ -0,0 +1,27 @@
	1	+# The Desk: a virtual Linux desktop that e2e scenarios are FILMED on — one
	2	+# Xvfb display showing a real terminal window (the chat) and a real headed
	3	+# Chromium (the browser hop), captured by a single ffmpeg x11grab into one
	4	+# session.mp4. No splicing: the film is the screen.
	5	+#
	6	+# The repo is bind-mounted at /repo at runtime; node_modules and caches live
	7	+# in named volumes (host artifacts are macOS-native and can't be reused).
	8	+FROM node:22-bookworm
	9	+
	10	+RUN apt-get update && apt-get install -y --no-install-recommends \
	11	+ xvfb openbox xterm ffmpeg xdotool x11-utils x11-xserver-utils \
	12	+ fonts-dejavu-core fonts-noto-color-emoji \
	13	+ curl unzip ca-certificates git \
	14	+ && rm -rf /var/lib/apt/lists/*
	15	+
	16	+# Bun — the repo's package manager and script runner.
	17	+RUN curl -fsSL https://bun.sh/install \| bash -s "bun-v1.3.13" \
	18	+ && ln -s /root/.bun/bin/bun /usr/local/bin/bun \
	19	+ && ln -s /root/.bun/bin/bun /usr/local/bin/bunx
	20	+
	21	+# Chromium + system deps baked into the image, pinned to the repo's
	22	+# playwright version so the runtime install check is a no-op.
	23	+ENV PLAYWRIGHT_BROWSERS_PATH=/ms-playwright
	24	+RUN npx -y playwright@1.60.0 install --with-deps chromium && rm -rf /root/.npm
	25	+
	26	+WORKDIR /repo
	27	+ENTRYPOINT ["bash", "e2e/desk/entry.sh"]

e2e/desk/entry.sh+43−0

@@ -0,0 +1,43 @@
	1	+#!/usr/bin/env bash
	2	+# Desk entrypoint (runs inside the desk image): boot a display, roll one
	3	+# camera on it, run the scenario with E2E_DESK=1 so its windows land on the
	4	+# display, then drop the film into the scenario's run dir as session.mp4.
	5	+set -euo pipefail
	6	+
	7	+SCENARIO="${DESK_SCENARIO:-scenarios/connect-handoff-session.test.ts}"
	8	+PROJECT="${DESK_PROJECT:-cloud}"
	9	+SIZE="${DESK_SIZE:-1440x900}"
	10	+
	11	+echo "[desk] installing dependencies (cached in a volume after first run)…"
	12	+bun install --frozen-lockfile
	13	+
	14	+echo "[desk] display :99 at ${SIZE}"
	15	+Xvfb :99 -screen 0 "${SIZE}x24" -nolisten tcp &
	16	+export DISPLAY=:99
	17	+sleep 0.5
	18	+openbox &
	19	+xsetroot -solid "#0b0b10" \|\| true
	20	+
	21	+mkdir -p /desk-out
	22	+echo "[desk] camera rolling"
	23	+ffmpeg -loglevel error -f x11grab -framerate 24 -video_size "$SIZE" -i :99 \
	24	+ -c:v libx264 -preset veryfast -crf 24 -pix_fmt yuv420p -y /desk-out/desk.mp4 &
	25	+FFMPEG_PID=$!
	26	+
	27	+set +e
	28	+(cd e2e && E2E_DESK=1 npx vitest run --project "$PROJECT" "$SCENARIO")
	29	+STATUS=$?
	30	+set -e
	31	+
	32	+sleep 1
	33	+kill -INT "$FFMPEG_PID" 2>/dev/null \|\| true
	34	+wait "$FFMPEG_PID" 2>/dev/null \|\| true
	35	+
	36	+# The desk film IS the session recording — it replaces the browser-only
	37	+# video the surface recorded inside the run.
	38	+RUN_DIR=$(ls -dt e2e/runs/"$PROJECT"/*/ 2>/dev/null \| head -1)
	39	+if [ -n "$RUN_DIR" ] && [ -f /desk-out/desk.mp4 ]; then
	40	+ cp /desk-out/desk.mp4 "${RUN_DIR}session.mp4"
	41	+ echo "[desk] film → ${RUN_DIR}session.mp4"
	42	+fi
	43	+exit "$STATUS"

e2e/desk/run.sh+17−0

@@ -0,0 +1,17 @@
	1	+#!/usr/bin/env bash
	2	+# Film a scenario on the Desk (one virtual desktop, one recording).
	3	+# e2e/desk/run.sh [scenario-path] [project]
	4	+# First run builds the image and fills the node_modules volume; later runs
	5	+# reuse both.
	6	+set -euo pipefail
	7	+cd "$(dirname "$0")/../.."
	8	+
	9	+docker build -t executor-e2e-desk e2e/desk
	10	+
	11	+docker run --rm \
	12	+ -v "$PWD":/repo \
	13	+ -v executor-desk-node-modules:/repo/node_modules \
	14	+ -v executor-desk-bun-cache:/root/.bun/install/cache \
	15	+ -e DESK_SCENARIO="${1:-scenarios/connect-handoff-session.test.ts}" \
	16	+ -e DESK_PROJECT="${2:-cloud}" \
	17	+ executor-e2e-desk

e2e/src/clients/agent-chat-tui.ts+16−2

@@ -5,7 +5,13 @@ · 4 unmodified lines
5	5	// pacing of a chat session (typed user input, streamed agent text, live
6	6	// tool spinners that run exactly as long as the real call did).
7	7	//
8		−// Run with: bun agent-chat-tui.ts "<session title>"
	8	+// Run with: bun agent-chat-tui.ts "<session title>" [events-fifo]
	9	+//
	10	+// Two transports: events arrive base64-encoded one-per-line either on stdin
	11	+// (PTY mode — the chat theater types them into the recorded terminal) or on
	12	+// a FIFO path (desk mode — the renderer runs inside a visible xterm on the
	13	+// virtual desktop, where stdin belongs to the terminal emulator).
	14	+import { createReadStream, writeFileSync } from "node:fs";
9	15	import { createInterface } from "node:readline";
10	16
11	17	type TheaterEvent =
@@ -35,6 +41,7 @@ const SPINNER = ["⠋", "⠙", "⠹", "⠸", "⠼", "⠴", "⠦", "⠧", "⠇", · 23 unmodified lines
35	41	const sleep = (ms: number) => new Promise<void>((tick) => setTimeout(tick, ms));
36	42
37	43	const title = process.argv[2] ?? "executor session";
	44	+const eventsFifo = process.argv[3];
38	45	out(`${BOLD}${MAGENTA}●${RESET} ${BOLD}${title}${RESET}\n`);
39	46	out(`${DIM}${"─".repeat(Math.min(96, title.length + 24))}${RESET}\n`);
40	47
@@ -92,6 +99,12 @@ const handle = async (event: TheaterEvent): Promise<boolean> => { · 51 unmodified lines
92	99	}
93	100	case "done": {
94	101	out(`\n${DIM}${"─".repeat(40)}${RESET}\n${GREEN}✦ session complete${RESET}\n`);
	102	+ if (eventsFifo) {
	103	+ // Desk mode: ack completion to the driver, then linger so the
	104	+ // closing frame stays on camera before the xterm vanishes.
	105	+ writeFileSync(`${eventsFifo}.done`, "");
	106	+ await sleep(4_000);
	107	+ }
95	108	return false;
96	109	}
97	110	}
@@ -115,7 +128,8 @@ const drain = async () => { · 17 unmodified lines
115	128	draining = false;
116	129	};
117	130
118		−createInterface({ input: process.stdin, terminal: false }).on("line", (line) => {
	131	+const input = eventsFifo ? createReadStream(eventsFifo) : process.stdin;
	132	+createInterface({ input, terminal: false }).on("line", (line) => {
119	133	const trimmed = line.trim();
120	134	if (!trimmed) return;
121	135	// oxlint-disable-next-line executor/no-try-catch-or-throw -- boundary: PTY stdin may echo stray keystrokes that aren't base64 JSON; ignore them

e2e/src/clients/chat-theater.ts+106−36

@@ -1,9 +1,14 @@
1	1	// Chat theater: agent-chat presentation over REAL mcporter MCP calls — no
2	2	// inference, no third-party agent binary. The scenario stays in Effect land
3		−// making real calls; a forked PTY runs the chat renderer
4		−// (agent-chat-tui.ts) and this driver feeds it events, so the terminal.cast
5		−// reads like a developer chatting with an agent while every tool spinner
6		−// brackets the genuine call it narrates.
	3	+// making real calls; a chat renderer (agent-chat-tui.ts) paints them, so
	4	+// the recording reads like a developer chatting with an agent while every
	5	+// tool spinner brackets the genuine call it narrates.
	6	+//
	7	+// Two stages for the same play:
	8	+// - PTY mode (default): the renderer runs in a recorded PTY → terminal.cast
	9	+// - Desk mode (E2E_DESK=1): the renderer runs in a visible xterm on the
	10	+// virtual desktop (events over a FIFO) and the desk's single x11grab
	11	+// films it together with the headed browser — one screen, one mp4
7	12	//
8	13	// Division of labor in the e2e stack:
9	14	// - chat theater (this): deterministic PRODUCT-flow recordings
@@ -11,6 +16,10 @@ · 1 unmodified line
11	16	// (OpenCode/Claude Code protocol handling), still no inference
12	17	// - real-inference evals: a separate axis entirely — performance
13	18	// distributions, not pass/fail scenarios
	19	+import { spawn, spawnSync } from "node:child_process";
	20	+import { createWriteStream, existsSync, mkdtempSync } from "node:fs";
	21	+import { tmpdir } from "node:os";
	22	+import { join } from "node:path";
14	23	import { fileURLToPath } from "node:url";
15	24
16	25	import { Effect, Exit, Fiber } from "effect";
@@ -40,31 +49,63 @@ export interface ChatTheater { · 23 unmodified lines
40	49	readonly status: (text: string) => Effect.Effect<void>;
41	50	}
42	51
	52	+const sleep = (ms: number) => new Promise<void>((tick) => setTimeout(tick, ms));
	53	+
	54	+const encode = (event: TheaterEvent) =>
	55	+ `${Buffer.from(JSON.stringify(event)).toString("base64")}\n`;
	56	+
	57	+const makeTheater = (push: (event: TheaterEvent) => Effect.Effect<void>): ChatTheater => ({
	58	+ user: (text) => push({ type: "user", text }),
	59	+ assistant: (text) => push({ type: "assistant", text }),
	60	+ status: (text) => push({ type: "status", text }),
	61	+ tool: (label, work, note) =>
	62	+ Effect.gen(function* () {
	63	+ yield* push({ type: "tool-start", label });
	64	+ const exit = yield* Effect.exit(work);
	65	+ if (Exit.isSuccess(exit)) {
	66	+ yield* push({ type: "tool-end", ok: true, note: note?.(exit.value) });
	67	+ return exit.value;
	68	+ }
	69	+ yield* push({ type: "tool-end", ok: false, note: "failed" });
	70	+ return yield* exit;
	71	+ }),
	72	+});
	73	+
	74	+export interface TheaterOptions {
	75	+ readonly title: string;
	76	+ readonly record: string;
	77	+ readonly viewport?: { readonly cols: number; readonly rows: number };
	78	+}
	79	+
43	80	/**
44		− * Open a recorded PTY running the chat renderer, hand the scenario a
45		− * ChatTheater handle, and close the session (footer + exit) when the body
46		− * finishes — success or failure, so the cast always ends cleanly.
	81	+ * Open the chat renderer, hand the scenario a ChatTheater handle, and close
	82	+ * the session (footer + exit) when the body finishes — success or failure,
	83	+ * so the recording always ends cleanly.
47	84	*/
48	85	export const withChatTheater = <A, E, R>(
49	86	cli: CliSurface,
50		− options: {
51		− readonly title: string;
52		− readonly record: string;
53		− readonly viewport?: { readonly cols: number; readonly rows: number };
54		− },
	87	+ options: TheaterOptions,
	88	+ body: (theater: ChatTheater) => Effect.Effect<A, E, R>,
	89	+): Effect.Effect<A, E, R> =>
	90	+ process.env.E2E_DESK === "1" ? deskTheater(options, body) : ptyTheater(cli, options, body);
	91	+
	92	+// ---------------------------------------------------------------------------
	93	+// PTY mode: renderer in a recorded PTY; events typed in as base64 lines.
	94	+// ---------------------------------------------------------------------------
	95	+
	96	+const ptyTheater = <A, E, R>(
	97	+ cli: CliSurface,
	98	+ options: TheaterOptions,
55	99	body: (theater: ChatTheater) => Effect.Effect<A, E, R>,
56	100	): Effect.Effect<A, E, R> =>
57	101	Effect.gen(function* () {
58	102	const queue: TheaterEvent[] = [];
59	103	let closed = false;
60		−
61	104	const push = (event: TheaterEvent) =>
62	105	Effect.sync(() => {
63	106	if (!closed) queue.push(event);
64	107	});
65	108
66		− // The PTY pump: forward queued events as base64 lines (keystroke-safe),
67		− // and return once the renderer paints its completion footer.
68	109	const pump = cli.session(
69	110	["bun", RENDERER, options.title],
70	111	async (term) => {
@@ -76,13 +117,12 @@ export const withChatTheater = <A, E, R>( · 5 unmodified lines
76	117	for (;;) {
77	118	const event = queue.shift();
78	119	if (event) {
79		− const line = Buffer.from(JSON.stringify(event)).toString("base64");
80		− await term.keyboard.type(`${line}\n`);
	120	+ await term.keyboard.type(encode(event));
81	121	if (event.type === "done") break;
82	122	continue;
83	123	}
84	124	if (Date.now() > deadline) break;
85		− await new Promise<void>((tick) => setTimeout(tick, 40));
	125	+ await sleep(40);
86	126	}
87	127	await term.screen.waitForText("session complete", { timeoutMs: 60_000 });
88	128	},
@@ -93,26 +133,56 @@ export const withChatTheater = <A, E, R>( · 4 unmodified lines
93	133	);
94	134	const pumpFiber = yield* Effect.forkChild(Effect.exit(pump));
95	135
96		− const theater: ChatTheater = {
97		− user: (text) => push({ type: "user", text }),
98		− assistant: (text) => push({ type: "assistant", text }),
99		− status: (text) => push({ type: "status", text }),
100		− tool: (label, work, note) =>
101		− Effect.gen(function* () {
102		− yield* push({ type: "tool-start", label });
103		− const exit = yield* Effect.exit(work);
104		− if (Exit.isSuccess(exit)) {
105		− yield* push({ type: "tool-end", ok: true, note: note?.(exit.value) });
106		− return exit.value;
107		− }
108		− yield* push({ type: "tool-end", ok: false, note: "failed" });
109		− return yield* exit;
110		− }),
111		− };
112		−
113		− const result = yield* Effect.exit(body(theater));
	136	+ const result = yield* Effect.exit(body(makeTheater(push)));
114	137	yield* push({ type: "done" });
115	138	closed = true;
116	139	yield* Fiber.join(pumpFiber);
117	140	return yield* result;
118	141	});
	142	+
	143	+// ---------------------------------------------------------------------------
	144	+// Desk mode: renderer in a visible xterm on the virtual desktop; events over
	145	+// a FIFO. No cast — the desk's x11grab films the screen.
	146	+// ---------------------------------------------------------------------------
	147	+
	148	+const deskTheater = <A, E, R>(
	149	+ options: TheaterOptions,
	150	+ body: (theater: ChatTheater) => Effect.Effect<A, E, R>,
	151	+): Effect.Effect<A, E, R> =>
	152	+ Effect.gen(function* () {
	153	+ const root = mkdtempSync(join(tmpdir(), "desk-theater-"));
	154	+ const fifo = join(root, "events.fifo");
	155	+ spawnSync("mkfifo", [fifo]);
	156	+
	157	+ const xterm = spawn(
	158	+ "xterm",
	159	+ [
	160	+ // -u8 + a UTF-8 locale: the renderer speaks box-drawing and bullets.
	161	+ ...["-u8", "-fa", "DejaVu Sans Mono", "-fs", "13"],
	162	+ ...["-bg", "#0b0b10", "-fg", "#e8e8ea"],
	163	+ ...["-geometry", "112x34+48+40", "-T", options.title],
	164	+ ...["-e", "bun", RENDERER, options.title, fifo],
	165	+ ],
	166	+ { stdio: "ignore", env: { ...process.env, LANG: "C.UTF-8", LC_ALL: "C.UTF-8" } },
	167	+ );
	168	+ // Opening the write end parks until the renderer opens the read end.
	169	+ const writer = createWriteStream(fifo);
	170	+ const push = (event: TheaterEvent) =>
	171	+ Effect.sync(() => {
	172	+ writer.write(encode(event));
	173	+ });
	174	+
	175	+ const result = yield* Effect.exit(body(makeTheater(push)));
	176	+ yield* push({ type: "done" });
	177	+ yield* Effect.promise(async () => {
	178	+ // Renderer acks the footer, lingers for the camera, then exits with
	179	+ // its xterm. Don't hang the run on a wedged terminal.
	180	+ const ackDeadline = Date.now() + 30_000;
	181	+ while (!existsSync(`${fifo}.done`) && Date.now() < ackDeadline) await sleep(200);
	182	+ const exitDeadline = Date.now() + 10_000;
	183	+ while (xterm.exitCode === null && Date.now() < exitDeadline) await sleep(200);
	184	+ if (xterm.exitCode === null) xterm.kill();
	185	+ writer.end();
	186	+ });
	187	+ return yield* result;
	188	+ });

e2e/src/surfaces/browser.ts+11−1

@@ -42,7 +42,17 @@ export const makeBrowserSurface = (dir: string, target: Target): BrowserSurface · 41 unmodified lines
42	42	const videoTmp = join(dir, ".video-tmp");
43	43	mkdirSync(videoTmp, { recursive: true });
44	44
45		− const browser = await chromium.launch();
	45	+ // On the desk (E2E_DESK), the browser is a real headed window on the
	46	+ // virtual display — the desk's single screen recording films it next
	47	+ // to the chat terminal, exactly like a developer tabbing over.
	48	+ const browser = await chromium.launch(
	49	+ process.env.E2E_DESK === "1"
	50	+ ? {
	51	+ headless: false,
	52	+ args: ["--window-position=300,40", "--window-size=1100,830"],
	53	+ }
	54	+ : {},
	55	+ );
46	56	const context = await browser.newContext({
47	57	colorScheme: "dark",
48	58	viewport: { width: 1280, height: 800 },