History

Focus is derived from operations: surfaces emit the run's focus timeline

Acting on a window IS focusing it: a Playwright step marks the browser focused, a chat-theater event marks the terminal focused. Surfaces record these transitions (plus each recording's start anchor) into the run's timeline.json as a side effect of normal driving — no scenario changes, no declared focus anywhere. film.ts now builds its cut list from that timeline (any number of hops, each act mapped into its recording's own clock via the anchors), keeping the narrator-line/largest-gap heuristic only as a fallback for runs without one. The film is the operations, edited by where they happened.

Rhys Sullivan committed 5 months ago

Commit 60bb6a01 parent 18d66d54 files · +160 −54Browse files

Unified · Split

4 files
  1. e2e/scripts/film.ts +96 −52
  2. e2e/src/clients/chat-theater.ts +11 −2
  3. e2e/src/surfaces/browser.ts +6 −0
  4. e2e/src/timeline.ts +47 −0
e2e/scripts/film.ts+96−52
@@ -1,23 +1,25 @@
1−// Film a run: turn a chat-theater run's two real recordings into ONE mp4
2−// that plays like a screen recording of a developer tabbing between
3−// full-screen windows — terminal, browser, terminal. No virtual desktop,
4−// no splice of anything synthetic: both segments are the real recordings,
5−// cut at the moment the session actually moved between them.
1+// Film a run: turn a chat-theater run's real recordings into ONE mp4 that
2+// plays like a screen recording of a developer tabbing between full-screen
3+// windows — terminal, browser, terminal. Both segments are the genuine
4+// recordings; the only editorial act is the cut, exactly like real tabbing.
65 //
76 // bun scripts/film.ts runs/<target>/<slug>
87 //
9−// Mechanics: the terminal.cast contains dead air while the "user" was in
10−// the browser (the chat was genuinely idle). We render the cast to video
11−// with NO idle compression (so cast time == video time), cut it at the
12−// browser hop, and put the browser session.mp4 in the gap. The hop is
13−// located by the scenario's narrator line ("…in the browser…"), falling
14−// back to the largest output gap. Output: film.mp4 next to the inputs —
15−// the viewer plays it as the session when present.
8+// The cut list comes from the run's focus timeline (src/timeline.ts):
9+// surfaces mark focus as a side effect of acting — a Playwright step
10+// focuses the browser, a chat event focuses the terminal — so the
11+// operations themselves decide where the film cuts, for any number of
12+// hops. Runs without a timeline fall back to the narrator-line / largest-
13+// gap heuristic. The cast renders with NO idle compression so cast time
14+// equals video time. Output: film.mp4, registered in result.json — the
15+// viewer plays it as the session when present.
1616 import { execFileSync } from "node:child_process";
17−import { existsSync, mkdtempSync, readFileSync, rmSync } from "node:fs";
17+import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
1818 import { tmpdir } from "node:os";
1919 import { join, resolve } from "node:path";
2020
21+import { readTimeline } from "../src/timeline";
22+
2123 const runDir = resolve(process.argv[2] ?? "");
2224 const castPath = join(runDir, "terminal.cast");
2325 const browserPath = join(runDir, "session.mp4");
@@ -38,7 +40,7 @@ const probeSeconds = (file: string): number => · 14 unmodified lines
3840 );
3941
4042 // ---------------------------------------------------------------------------
41−// Locate the browser hop in cast time.
43+// Build the cut list: acts of { source, from, to } in each source's clock.
4244 // ---------------------------------------------------------------------------
4345
4446 interface CastEvent {
@@ -56,51 +58,87 @@ const events: CastEvent[] = readFileSync(castPath, "utf8") · 11 unmodified lines
5658
5759 const castEnd = events.at(-1)?.at ?? 0;
5860
59−/** Hop start: the narrator line the scenario prints as it moves to the
60− * browser; hop end: the next visible output after it. Falls back to the
61− * largest inter-event gap (the hop is the only long silence). */
62−const findHop = (): { start: number; end: number } => {
63− const markerIndex = events.findIndex((event) => event.text.includes("in the browser"));
64− if (markerIndex !== -1) {
65− const start = events[markerIndex];
66− const after = events
67− .slice(markerIndex + 1)
68− .find((event) => event.at > (start?.at ?? 0) + 1 && event.text.trim().length > 0);
69− if (start && after) return { start: start.at + 0.8, end: after.at };
61+interface Act {
62+ readonly source: "terminal" | "browser";
63+ readonly from: number;
64+ readonly to: number;
65+}
66+
67+/** Acts from the focus timeline: each contiguous focus run becomes a cut of
68+ * that window's recording, mapped via the recording-start anchors. */
69+const actsFromTimeline = (): Act[] | null => {
70+ const timeline = readTimeline(runDir);
71+ if (!timeline || timeline.focus.length < 2) return null;
72+ const terminalAnchor = timeline.anchors.terminal;
73+ const browserAnchor = timeline.anchors.browser;
74+ if (terminalAnchor === undefined || browserAnchor === undefined) return null;
75+
76+ const browserEnd = probeSeconds(browserPath);
77+ const toMedia = (window: Act["source"], wallMs: number) =>
78+ Math.max(0, (wallMs - (window === "terminal" ? terminalAnchor : browserAnchor)) / 1000);
79+
80+ const acts: Act[] = [];
81+ for (let i = 0; i < timeline.focus.length; i += 1) {
82+ const current = timeline.focus[i];
83+ if (!current) continue;
84+ const nextAt = timeline.focus[i + 1]?.at;
85+ const from = i === 0 && current.window === "terminal" ? 0 : toMedia(current.window, current.at);
86+ const mediaEnd = current.window === "terminal" ? castEnd : browserEnd;
87+ const to =
88+ nextAt === undefined ? mediaEnd : Math.min(toMedia(current.window, nextAt), mediaEnd);
89+ if (to - from > 0.5) acts.push({ source: current.window, from, to });
7090 }
71− let best = { start: 0, end: 0 };
72− for (let i = 1; i < events.length; i += 1) {
73− const previous = events[i - 1];
74− const current = events[i];
75− if (previous && current && current.at - previous.at > best.end - best.start) {
76− best = { start: previous.at, end: current.at };
91+ return acts.length >= 2 ? acts : null;
92+};
93+
94+/** Fallback for runs without a timeline: one browser hop located by the
95+ * narrator line, or the largest output gap in the cast. */
96+const actsFromHeuristic = (): Act[] => {
97+ const findHop = (): { start: number; end: number } => {
98+ const markerIndex = events.findIndex((event) => event.text.includes("in the browser"));
99+ if (markerIndex !== -1) {
100+ const start = events[markerIndex];
101+ const after = events
102+ .slice(markerIndex + 1)
103+ .find((event) => event.at > (start?.at ?? 0) + 1 && event.text.trim().length > 0);
104+ if (start && after) return { start: start.at + 0.8, end: after.at };
105+ }
106+ let best = { start: 0, end: 0 };
107+ for (let i = 1; i < events.length; i += 1) {
108+ const previous = events[i - 1];
109+ const current = events[i];
110+ if (previous && current && current.at - previous.at > best.end - best.start) {
111+ best = { start: previous.at, end: current.at };
112+ }
77113 }
114+ return best;
115+ };
116+ const hop = findHop();
117+ if (hop.end - hop.start < 2) {
118+ console.error(`film: no browser hop found in the cast (gap ${hop.end - hop.start}s)`);
119+ process.exit(1);
78120 }
79− return best;
121+ return [
122+ { source: "terminal", from: 0, to: hop.start },
123+ { source: "browser", from: 0, to: probeSeconds(browserPath) },
124+ { source: "terminal", from: hop.end, to: castEnd },
125+ ];
80126 };
81127
82−const hop = findHop();
83−if (hop.end - hop.start < 2) {
84− console.error(`film: no browser hop found in the cast (gap ${hop.end - hop.start}s)`);
85− process.exit(1);
86−}
128+const acts = actsFromTimeline() ?? actsFromHeuristic();
87129
88130 // ---------------------------------------------------------------------------
89−// Render + cut + concatenate. One canvas (the browser video's size), every
90−// segment scaled/padded onto it — full-screen cuts, like tabbing.
131+// Render + cut + concatenate onto one canvas — full-screen cuts, like
132+// tabbing.
91133 // ---------------------------------------------------------------------------
92134
93135 const work = mkdtempSync(join(tmpdir(), "e2e-film-"));
94136 const castGif = join(work, "cast.gif");
95137 const castVideo = join(work, "cast.mp4");
96138
97−// No idle compression: cast video time must equal cast event time so the
98−// cut points land exactly.
99139 run("agg", [
100− "--idle-time-limit",
101− String(Math.ceil(castEnd) + 60),
102− "--font-size",
103− "16",
140+ ...["--idle-time-limit", String(Math.ceil(castEnd) + 60)],
141+ ...["--font-size", "16"],
104142 castPath,
105143 castGif,
106144 ]);
@@ -111,21 +149,23 @@ run("ffmpeg", [ · 4 unmodified lines
111149 ...["-pix_fmt", "yuv420p", "-r", "24", castVideo],
112150 ]);
113151
114−const browserSeconds = probeSeconds(browserPath);
115152 const FIT =
116153 "scale=1280:800:force_original_aspect_ratio=decrease,pad=1280:800:(ow-iw)/2:(oh-ih)/2:color=0x0b0b10,setsar=1,fps=24,format=yuv420p";
117154
118155 const filmPath = join(runDir, "film.mp4");
156+const inputIndex = { terminal: 0, browser: 1 } as const;
157+const filters = acts.map(
158+ (act, index) =>
159+ `[${inputIndex[act.source]}:v]trim=${act.from.toFixed(2)}:${act.to.toFixed(2)},setpts=PTS-STARTPTS,${FIT}[act${index}]`,
160+);
119161 run("ffmpeg", [
120162 "-y",
121163 ...["-i", castVideo],
122164 ...["-i", browserPath],
123165 "-filter_complex",
124166 [
125− `[0:v]trim=0:${hop.start.toFixed(2)},setpts=PTS-STARTPTS,${FIT}[act1]`,
126− `[1:v]${FIT}[act2]`,
127− `[0:v]trim=${hop.end.toFixed(2)},setpts=PTS-STARTPTS,${FIT}[act3]`,
128− `[act1][act2][act3]concat=n=3:v=1:a=0[out]`,
167+ ...filters,
168+ `${acts.map((_, index) => `[act${index}]`).join("")}concat=n=${acts.length}:v=1:a=0[out]`,
129169 ].join(";"),
130170 ...["-map", "[out]"],
131171 ...["-c:v", "libx264", "-preset", "veryfast", "-crf", "24", "-movflags", "+faststart"],
@@ -139,11 +179,15 @@ if (existsSync(resultPath)) { · 7 unmodified lines
139179 const result = JSON.parse(readFileSync(resultPath, "utf8")) as { artifacts?: string[] };
140180 if (Array.isArray(result.artifacts) && !result.artifacts.includes("film.mp4")) {
141181 result.artifacts.push("film.mp4");
142− const { writeFileSync } = await import("node:fs");
143182 writeFileSync(resultPath, JSON.stringify(result, null, 1));
144183 }
145184 }
146185
147186 console.log(
148− `film: ${filmPath}\n act 1 terminal 0–${hop.start.toFixed(1)}s · act 2 browser ${browserSeconds.toFixed(1)}s · act 3 terminal ${(castEnd - hop.end).toFixed(1)}s`,
187+ `film: ${filmPath}\n${acts
188+ .map(
189+ (act, index) =>
190+ ` act ${index + 1} ${act.source} ${act.from.toFixed(1)}–${act.to.toFixed(1)}s`,
191+ )
192+ .join("\n")}`,
149193 );
e2e/src/clients/chat-theater.ts+11−2
@@ -19,11 +19,12 @@ · 18 unmodified lines
1919 import { spawn, spawnSync } from "node:child_process";
2020 import { createWriteStream, existsSync, mkdtempSync } from "node:fs";
2121 import { tmpdir } from "node:os";
22−import { join } from "node:path";
22+import { dirname, join } from "node:path";
2323 import { fileURLToPath } from "node:url";
2424
2525 import { Effect, Exit, Fiber } from "effect";
2626
27+import { markFocus, markRecordingStart } from "../timeline";
2728 import type { CliSurface } from "../surfaces/cli";
2829
2930 const RENDERER = fileURLToPath(new URL("./agent-chat-tui.ts", import.meta.url));
@@ -99,11 +100,16 @@ const ptyTheater = <A, E, R>( · 69 unmodified lines
99100 body: (theater: ChatTheater) => Effect.Effect<A, E, R>,
100101 ): Effect.Effect<A, E, R> =>
101102 Effect.gen(function* () {
103+ const runDir = dirname(options.record);
102104 const queue: TheaterEvent[] = [];
103105 let closed = false;
104106 const push = (event: TheaterEvent) =>
105107 Effect.sync(() => {
106− if (!closed) queue.push(event);
108+ if (!closed) {
109+ // Chatting IS focusing the terminal window.
110+ if (event.type !== "done") markFocus(runDir, "terminal");
111+ queue.push(event);
112+ }
107113 });
108114
109115 const pump = cli.session(
@@ -113,6 +119,9 @@ const ptyTheater = <A, E, R>( · 3 unmodified lines
113119 // it hasn't set raw mode yet, and the PTY would echo the event line
114120 // into the recording.
115121 await term.screen.waitForText(options.title, { timeoutMs: 30_000 });
122+ // The cast's clock started with the PTY moments ago; anchor it for
123+ // the run's focus timeline (scripts/film.ts cuts on these).
124+ markRecordingStart(runDir, "terminal");
116125 const deadline = Date.now() + 30 * 60 * 1000;
117126 for (;;) {
118127 const event = queue.shift();
e2e/src/surfaces/browser.ts+6−0
@@ -11,6 +11,7 @@ import { promisify } from "node:util"; · 10 unmodified lines
1111 import { Effect } from "effect";
1212 import { chromium, type Page } from "playwright";
1313
14+import { markFocus, markRecordingStart } from "../timeline";
1415 import type { Identity, Target } from "../target";
1516
1617 export interface BrowserSession {
@@ -66,11 +67,16 @@ export const makeBrowserSurface = (dir: string, target: Target): BrowserSurface · 49 unmodified lines
6667 );
6768 }
6869 const page = await context.newPage();
70+ // The session video's clock starts with the page; anchor it for the
71+ // run's focus timeline (scripts/film.ts cuts on these).
72+ markRecordingStart(dir, "browser");
6973 return { browser, context, page, videoTmp, shots: { count: 0 } };
7074 }),
7175 ({ page, context, shots }) =>
7276 Effect.promise(async () => {
7377 const step = async (label: string, action: (page: Page) => Promise<void>) => {
78+ // Acting on the page IS focusing the browser window.
79+ markFocus(dir, "browser");
7480 await context.tracing.group(label);
7581 try {
7682 await action(page);
e2e/src/timeline.ts+47−0
@@ -0,0 +1,47 @@
1+// The run's focus timeline: which window the scenario was acting on, when.
2+//
3+// Focus is DERIVED, never declared — driving a Playwright page focuses the
4+// browser window; pushing a chat/terminal event focuses the terminal. The
5+// surfaces call markFocus as a side effect of normal operations, so any
6+// scenario gets a faithful "where was the developer looking" track for
7+// free, and scripts/film.ts can cut the session recordings exactly where
8+// the action moved. Anchors map wall-clock to each recording's own clock.
9+import { existsSync, readFileSync, writeFileSync } from "node:fs";
10+import { join } from "node:path";
11+
12+export type TimelineWindow = "terminal" | "browser";
13+
14+export interface Timeline {
15+ /** Wall-clock ms when each recording's clock started. */
16+ readonly anchors: { terminal?: number; browser?: number };
17+ /** Focus transitions (first event per contiguous run of a window). */
18+ readonly focus: Array<{ at: number; window: TimelineWindow }>;
19+}
20+
21+const fileFor = (runDir: string) => join(runDir, "timeline.json");
22+
23+const read = (runDir: string): Timeline => {
24+ const file = fileFor(runDir);
25+ if (!existsSync(file)) return { anchors: {}, focus: [] };
26+ return JSON.parse(readFileSync(file, "utf8")) as Timeline;
27+};
28+
29+const write = (runDir: string, timeline: Timeline) =>
30+ writeFileSync(fileFor(runDir), JSON.stringify(timeline, null, 1));
31+
32+/** Record that `window`'s recording clock starts now. */
33+export const markRecordingStart = (runDir: string, window: TimelineWindow): void => {
34+ const timeline = read(runDir);
35+ write(runDir, { ...timeline, anchors: { ...timeline.anchors, [window]: Date.now() } });
36+};
37+
38+/** Record that the scenario is acting on `window` (deduped per run). */
39+export const markFocus = (runDir: string, window: TimelineWindow): void => {
40+ const timeline = read(runDir);
41+ if (timeline.focus.at(-1)?.window === window) return;
42+ timeline.focus.push({ at: Date.now(), window });
43+ write(runDir, timeline);
44+};
45+
46+export const readTimeline = (runDir: string): Timeline | null =>
47+ existsSync(fileFor(runDir)) ? read(runDir) : null;