History

Focus is derived from operations: surfaces emit the run's focus timeline

Acting on a window IS focusing it: a Playwright step marks the browser focused, a chat-theater event marks the terminal focused. Surfaces record these transitions (plus each recording's start anchor) into the run's timeline.json as a side effect of normal driving — no scenario changes, no declared focus anywhere. film.ts now builds its cut list from that timeline (any number of hops, each act mapped into its recording's own clock via the anchors), keeping the narrator-line/largest-gap heuristic only as a fallback for runs without one. The film is the operations, edited by where they happened.

Rhys Sullivan committed 5 months ago

Commit 60bb6a01 parent 18d66d54 files · +160 −54Browse files

Unified · Split

4 files
  1. e2e/scripts/film.ts +96 −52
  2. e2e/src/clients/chat-theater.ts +11 −2
  3. e2e/src/surfaces/browser.ts +6 −0
  4. e2e/src/timeline.ts +47 −0
e2e/scripts/film.ts+96−52
@@ -1,23 +1,25 @@
1// Film a run: turn a chat-theater run's two real recordings into ONE mp41// Film a run: turn a chat-theater run's real recordings into ONE mp4 that
2// that plays like a screen recording of a developer tabbing between2// plays like a screen recording of a developer tabbing between full-screen
3// full-screen windows — terminal, browser, terminal. No virtual desktop,3// windows — terminal, browser, terminal. Both segments are the genuine
4// no splice of anything synthetic: both segments are the real recordings,4// recordings; the only editorial act is the cut, exactly like real tabbing.
5// cut at the moment the session actually moved between them.
6//5//
7// bun scripts/film.ts runs/<target>/<slug>6// bun scripts/film.ts runs/<target>/<slug>
8//7//
9// Mechanics: the terminal.cast contains dead air while the "user" was in8// The cut list comes from the run's focus timeline (src/timeline.ts):
10// the browser (the chat was genuinely idle). We render the cast to video9// surfaces mark focus as a side effect of acting — a Playwright step
11// with NO idle compression (so cast time == video time), cut it at the10// focuses the browser, a chat event focuses the terminal — so the
12// browser hop, and put the browser session.mp4 in the gap. The hop is11// operations themselves decide where the film cuts, for any number of
13// located by the scenario's narrator line ("…in the browser…"), falling12// hops. Runs without a timeline fall back to the narrator-line / largest-
14// back to the largest output gap. Output: film.mp4 next to the inputs —13// gap heuristic. The cast renders with NO idle compression so cast time
15// the viewer plays it as the session when present.14// equals video time. Output: film.mp4, registered in result.json — the
15// viewer plays it as the session when present.
16import { execFileSync } from "node:child_process";16import { execFileSync } from "node:child_process";
17import { existsSync, mkdtempSync, readFileSync, rmSync } from "node:fs";17import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
18import { tmpdir } from "node:os";18import { tmpdir } from "node:os";
19import { join, resolve } from "node:path";19import { join, resolve } from "node:path";
20 20
21import { readTimeline } from "../src/timeline";
22
21const runDir = resolve(process.argv[2] ?? "");23const runDir = resolve(process.argv[2] ?? "");
22const castPath = join(runDir, "terminal.cast");24const castPath = join(runDir, "terminal.cast");
23const browserPath = join(runDir, "session.mp4");25const browserPath = join(runDir, "session.mp4");
@@ -38,7 +40,7 @@ const probeSeconds = (file: string): number => · 14 unmodified lines
38 );40 );
39 41
40// ---------------------------------------------------------------------------42// ---------------------------------------------------------------------------
41// Locate the browser hop in cast time.43// Build the cut list: acts of { source, from, to } in each source's clock.
42// ---------------------------------------------------------------------------44// ---------------------------------------------------------------------------
43 45
44interface CastEvent {46interface CastEvent {
@@ -56,51 +58,87 @@ const events: CastEvent[] = readFileSync(castPath, "utf8") · 11 unmodified lines
56 58
57const castEnd = events.at(-1)?.at ?? 0;59const castEnd = events.at(-1)?.at ?? 0;
58 60
59/** Hop start: the narrator line the scenario prints as it moves to the61interface Act {
60 * browser; hop end: the next visible output after it. Falls back to the62 readonly source: "terminal" | "browser";
61 * largest inter-event gap (the hop is the only long silence). */63 readonly from: number;
62const findHop = (): { start: number; end: number } => {64 readonly to: number;
63 const markerIndex = events.findIndex((event) => event.text.includes("in the browser"));65}
64 if (markerIndex !== -1) {66
65 const start = events[markerIndex];67/** Acts from the focus timeline: each contiguous focus run becomes a cut of
66 const after = events68 * that window's recording, mapped via the recording-start anchors. */
67 .slice(markerIndex + 1)69const actsFromTimeline = (): Act[] | null => {
68 .find((event) => event.at > (start?.at ?? 0) + 1 && event.text.trim().length > 0);70 const timeline = readTimeline(runDir);
69 if (start && after) return { start: start.at + 0.8, end: after.at };71 if (!timeline || timeline.focus.length < 2) return null;
72 const terminalAnchor = timeline.anchors.terminal;
73 const browserAnchor = timeline.anchors.browser;
74 if (terminalAnchor === undefined || browserAnchor === undefined) return null;
75
76 const browserEnd = probeSeconds(browserPath);
77 const toMedia = (window: Act["source"], wallMs: number) =>
78 Math.max(0, (wallMs - (window === "terminal" ? terminalAnchor : browserAnchor)) / 1000);
79
80 const acts: Act[] = [];
81 for (let i = 0; i < timeline.focus.length; i += 1) {
82 const current = timeline.focus[i];
83 if (!current) continue;
84 const nextAt = timeline.focus[i + 1]?.at;
85 const from = i === 0 && current.window === "terminal" ? 0 : toMedia(current.window, current.at);
86 const mediaEnd = current.window === "terminal" ? castEnd : browserEnd;
87 const to =
88 nextAt === undefined ? mediaEnd : Math.min(toMedia(current.window, nextAt), mediaEnd);
89 if (to - from > 0.5) acts.push({ source: current.window, from, to });
70 }90 }
71 let best = { start: 0, end: 0 };91 return acts.length >= 2 ? acts : null;
72 for (let i = 1; i < events.length; i += 1) {92};
73 const previous = events[i - 1];93
74 const current = events[i];94/** Fallback for runs without a timeline: one browser hop located by the
75 if (previous && current && current.at - previous.at > best.end - best.start) {95 * narrator line, or the largest output gap in the cast. */
76 best = { start: previous.at, end: current.at };96const actsFromHeuristic = (): Act[] => {
97 const findHop = (): { start: number; end: number } => {
98 const markerIndex = events.findIndex((event) => event.text.includes("in the browser"));
99 if (markerIndex !== -1) {
100 const start = events[markerIndex];
101 const after = events
102 .slice(markerIndex + 1)
103 .find((event) => event.at > (start?.at ?? 0) + 1 && event.text.trim().length > 0);
104 if (start && after) return { start: start.at + 0.8, end: after.at };
105 }
106 let best = { start: 0, end: 0 };
107 for (let i = 1; i < events.length; i += 1) {
108 const previous = events[i - 1];
109 const current = events[i];
110 if (previous && current && current.at - previous.at > best.end - best.start) {
111 best = { start: previous.at, end: current.at };
112 }
77 }113 }
114 return best;
115 };
116 const hop = findHop();
117 if (hop.end - hop.start < 2) {
118 console.error(`film: no browser hop found in the cast (gap ${hop.end - hop.start}s)`);
119 process.exit(1);
78 }120 }
79 return best;121 return [
122 { source: "terminal", from: 0, to: hop.start },
123 { source: "browser", from: 0, to: probeSeconds(browserPath) },
124 { source: "terminal", from: hop.end, to: castEnd },
125 ];
80};126};
81 127
82const hop = findHop();128const acts = actsFromTimeline() ?? actsFromHeuristic();
83if (hop.end - hop.start < 2) {
84 console.error(`film: no browser hop found in the cast (gap ${hop.end - hop.start}s)`);
85 process.exit(1);
86}
87 129
88// ---------------------------------------------------------------------------130// ---------------------------------------------------------------------------
89// Render + cut + concatenate. One canvas (the browser video's size), every131// Render + cut + concatenate onto one canvas — full-screen cuts, like
90// segment scaled/padded onto it — full-screen cuts, like tabbing.132// tabbing.
91// ---------------------------------------------------------------------------133// ---------------------------------------------------------------------------
92 134
93const work = mkdtempSync(join(tmpdir(), "e2e-film-"));135const work = mkdtempSync(join(tmpdir(), "e2e-film-"));
94const castGif = join(work, "cast.gif");136const castGif = join(work, "cast.gif");
95const castVideo = join(work, "cast.mp4");137const castVideo = join(work, "cast.mp4");
96 138
97// No idle compression: cast video time must equal cast event time so the
98// cut points land exactly.
99run("agg", [139run("agg", [
100 "--idle-time-limit",140 ...["--idle-time-limit", String(Math.ceil(castEnd) + 60)],
101 String(Math.ceil(castEnd) + 60),141 ...["--font-size", "16"],
102 "--font-size",
103 "16",
104 castPath,142 castPath,
105 castGif,143 castGif,
106]);144]);
@@ -111,21 +149,23 @@ run("ffmpeg", [ · 4 unmodified lines
111 ...["-pix_fmt", "yuv420p", "-r", "24", castVideo],149 ...["-pix_fmt", "yuv420p", "-r", "24", castVideo],
112]);150]);
113 151
114const browserSeconds = probeSeconds(browserPath);
115const FIT =152const FIT =
116 "scale=1280:800:force_original_aspect_ratio=decrease,pad=1280:800:(ow-iw)/2:(oh-ih)/2:color=0x0b0b10,setsar=1,fps=24,format=yuv420p";153 "scale=1280:800:force_original_aspect_ratio=decrease,pad=1280:800:(ow-iw)/2:(oh-ih)/2:color=0x0b0b10,setsar=1,fps=24,format=yuv420p";
117 154
118const filmPath = join(runDir, "film.mp4");155const filmPath = join(runDir, "film.mp4");
156const inputIndex = { terminal: 0, browser: 1 } as const;
157const filters = acts.map(
158 (act, index) =>
159 `[${inputIndex[act.source]}:v]trim=${act.from.toFixed(2)}:${act.to.toFixed(2)},setpts=PTS-STARTPTS,${FIT}[act${index}]`,
160);
119run("ffmpeg", [161run("ffmpeg", [
120 "-y",162 "-y",
121 ...["-i", castVideo],163 ...["-i", castVideo],
122 ...["-i", browserPath],164 ...["-i", browserPath],
123 "-filter_complex",165 "-filter_complex",
124 [166 [
125 `[0:v]trim=0:${hop.start.toFixed(2)},setpts=PTS-STARTPTS,${FIT}[act1]`,167 ...filters,
126 `[1:v]${FIT}[act2]`,168 `${acts.map((_, index) => `[act${index}]`).join("")}concat=n=${acts.length}:v=1:a=0[out]`,
127 `[0:v]trim=${hop.end.toFixed(2)},setpts=PTS-STARTPTS,${FIT}[act3]`,
128 `[act1][act2][act3]concat=n=3:v=1:a=0[out]`,
129 ].join(";"),169 ].join(";"),
130 ...["-map", "[out]"],170 ...["-map", "[out]"],
131 ...["-c:v", "libx264", "-preset", "veryfast", "-crf", "24", "-movflags", "+faststart"],171 ...["-c:v", "libx264", "-preset", "veryfast", "-crf", "24", "-movflags", "+faststart"],
@@ -139,11 +179,15 @@ if (existsSync(resultPath)) { · 7 unmodified lines
139 const result = JSON.parse(readFileSync(resultPath, "utf8")) as { artifacts?: string[] };179 const result = JSON.parse(readFileSync(resultPath, "utf8")) as { artifacts?: string[] };
140 if (Array.isArray(result.artifacts) && !result.artifacts.includes("film.mp4")) {180 if (Array.isArray(result.artifacts) && !result.artifacts.includes("film.mp4")) {
141 result.artifacts.push("film.mp4");181 result.artifacts.push("film.mp4");
142 const { writeFileSync } = await import("node:fs");
143 writeFileSync(resultPath, JSON.stringify(result, null, 1));182 writeFileSync(resultPath, JSON.stringify(result, null, 1));
144 }183 }
145}184}
146 185
147console.log(186console.log(
148 `film: ${filmPath}\n act 1 terminal 0–${hop.start.toFixed(1)}s · act 2 browser ${browserSeconds.toFixed(1)}s · act 3 terminal ${(castEnd - hop.end).toFixed(1)}s`,187 `film: ${filmPath}\n${acts
188 .map(
189 (act, index) =>
190 ` act ${index + 1} ${act.source} ${act.from.toFixed(1)}–${act.to.toFixed(1)}s`,
191 )
192 .join("\n")}`,
149);193);
e2e/src/clients/chat-theater.ts+11−2
@@ -19,11 +19,12 @@ · 18 unmodified lines
19import { spawn, spawnSync } from "node:child_process";19import { spawn, spawnSync } from "node:child_process";
20import { createWriteStream, existsSync, mkdtempSync } from "node:fs";20import { createWriteStream, existsSync, mkdtempSync } from "node:fs";
21import { tmpdir } from "node:os";21import { tmpdir } from "node:os";
22import { join } from "node:path";22import { dirname, join } from "node:path";
23import { fileURLToPath } from "node:url";23import { fileURLToPath } from "node:url";
24 24
25import { Effect, Exit, Fiber } from "effect";25import { Effect, Exit, Fiber } from "effect";
26 26
27import { markFocus, markRecordingStart } from "../timeline";
27import type { CliSurface } from "../surfaces/cli";28import type { CliSurface } from "../surfaces/cli";
28 29
29const RENDERER = fileURLToPath(new URL("./agent-chat-tui.ts", import.meta.url));30const RENDERER = fileURLToPath(new URL("./agent-chat-tui.ts", import.meta.url));
@@ -99,11 +100,16 @@ const ptyTheater = <A, E, R>( · 69 unmodified lines
99 body: (theater: ChatTheater) => Effect.Effect<A, E, R>,100 body: (theater: ChatTheater) => Effect.Effect<A, E, R>,
100): Effect.Effect<A, E, R> =>101): Effect.Effect<A, E, R> =>
101 Effect.gen(function* () {102 Effect.gen(function* () {
103 const runDir = dirname(options.record);
102 const queue: TheaterEvent[] = [];104 const queue: TheaterEvent[] = [];
103 let closed = false;105 let closed = false;
104 const push = (event: TheaterEvent) =>106 const push = (event: TheaterEvent) =>
105 Effect.sync(() => {107 Effect.sync(() => {
106 if (!closed) queue.push(event);108 if (!closed) {
109 // Chatting IS focusing the terminal window.
110 if (event.type !== "done") markFocus(runDir, "terminal");
111 queue.push(event);
112 }
107 });113 });
108 114
109 const pump = cli.session(115 const pump = cli.session(
@@ -113,6 +119,9 @@ const ptyTheater = <A, E, R>( · 3 unmodified lines
113 // it hasn't set raw mode yet, and the PTY would echo the event line119 // it hasn't set raw mode yet, and the PTY would echo the event line
114 // into the recording.120 // into the recording.
115 await term.screen.waitForText(options.title, { timeoutMs: 30_000 });121 await term.screen.waitForText(options.title, { timeoutMs: 30_000 });
122 // The cast's clock started with the PTY moments ago; anchor it for
123 // the run's focus timeline (scripts/film.ts cuts on these).
124 markRecordingStart(runDir, "terminal");
116 const deadline = Date.now() + 30 * 60 * 1000;125 const deadline = Date.now() + 30 * 60 * 1000;
117 for (;;) {126 for (;;) {
118 const event = queue.shift();127 const event = queue.shift();
e2e/src/surfaces/browser.ts+6−0
@@ -11,6 +11,7 @@ import { promisify } from "node:util"; · 10 unmodified lines
11import { Effect } from "effect";11import { Effect } from "effect";
12import { chromium, type Page } from "playwright";12import { chromium, type Page } from "playwright";
13 13
14import { markFocus, markRecordingStart } from "../timeline";
14import type { Identity, Target } from "../target";15import type { Identity, Target } from "../target";
15 16
16export interface BrowserSession {17export interface BrowserSession {
@@ -66,11 +67,16 @@ export const makeBrowserSurface = (dir: string, target: Target): BrowserSurface · 49 unmodified lines
66 );67 );
67 }68 }
68 const page = await context.newPage();69 const page = await context.newPage();
70 // The session video's clock starts with the page; anchor it for the
71 // run's focus timeline (scripts/film.ts cuts on these).
72 markRecordingStart(dir, "browser");
69 return { browser, context, page, videoTmp, shots: { count: 0 } };73 return { browser, context, page, videoTmp, shots: { count: 0 } };
70 }),74 }),
71 ({ page, context, shots }) =>75 ({ page, context, shots }) =>
72 Effect.promise(async () => {76 Effect.promise(async () => {
73 const step = async (label: string, action: (page: Page) => Promise<void>) => {77 const step = async (label: string, action: (page: Page) => Promise<void>) => {
78 // Acting on the page IS focusing the browser window.
79 markFocus(dir, "browser");
74 await context.tracing.group(label);80 await context.tracing.group(label);
75 try {81 try {
76 await action(page);82 await action(page);
e2e/src/timeline.ts+47−0
@@ -0,0 +1,47 @@
1// The run's focus timeline: which window the scenario was acting on, when.
2//
3// Focus is DERIVED, never declared — driving a Playwright page focuses the
4// browser window; pushing a chat/terminal event focuses the terminal. The
5// surfaces call markFocus as a side effect of normal operations, so any
6// scenario gets a faithful "where was the developer looking" track for
7// free, and scripts/film.ts can cut the session recordings exactly where
8// the action moved. Anchors map wall-clock to each recording's own clock.
9import { existsSync, readFileSync, writeFileSync } from "node:fs";
10import { join } from "node:path";
11
12export type TimelineWindow = "terminal" | "browser";
13
14export interface Timeline {
15 /** Wall-clock ms when each recording's clock started. */
16 readonly anchors: { terminal?: number; browser?: number };
17 /** Focus transitions (first event per contiguous run of a window). */
18 readonly focus: Array<{ at: number; window: TimelineWindow }>;
19}
20
21const fileFor = (runDir: string) => join(runDir, "timeline.json");
22
23const read = (runDir: string): Timeline => {
24 const file = fileFor(runDir);
25 if (!existsSync(file)) return { anchors: {}, focus: [] };
26 return JSON.parse(readFileSync(file, "utf8")) as Timeline;
27};
28
29const write = (runDir: string, timeline: Timeline) =>
30 writeFileSync(fileFor(runDir), JSON.stringify(timeline, null, 1));
31
32/** Record that `window`'s recording clock starts now. */
33export const markRecordingStart = (runDir: string, window: TimelineWindow): void => {
34 const timeline = read(runDir);
35 write(runDir, { ...timeline, anchors: { ...timeline.anchors, [window]: Date.now() } });
36};
37
38/** Record that the scenario is acting on `window` (deduped per run). */
39export const markFocus = (runDir: string, window: TimelineWindow): void => {
40 const timeline = read(runDir);
41 if (timeline.focus.at(-1)?.window === window) return;
42 timeline.focus.push({ at: Date.now(), window });
43 write(runDir, timeline);
44};
45
46export const readTimeline = (runDir: string): Timeline | null =>
47 existsSync(fileFor(runDir)) ? read(runDir) : null;