History

fix(mcp): keep preview snapshots usable by the agent and let it save them (#10501)

Theo Browne committed 2 months ago

Commit 061543e1 parent b5d89036 files · +594 −66Browse files

Unified · Split

6 files
  1. apps/desktop/src/preview/Manager.ts +7 −1
  2. apps/server/src/config.ts +3 −0
  3. apps/server/src/mcp/McpHttpServer.test.ts +316 −34
  4. apps/server/src/mcp/McpHttpServer.ts +243 −25
  5. apps/server/src/mcp/toolkits/preview/handlers.ts +5 −3
  6. apps/server/src/mcp/toolkits/preview/tools.ts +20 −3
apps/desktop/src/preview/Manager.ts+7−1
@@ -117,6 +117,12 @@ const ZOOM_EPSILON = 0.001; · 116 unmodified lines
117117 const MAX_EVALUATION_BYTES = 64_000;
118118 const MAX_VISIBLE_TEXT_LENGTH = 20_000;
119119 const MAX_INTERACTIVE_ELEMENTS = 200;
120+/**
121+ * A `[role]` container's innerText is its whole subtree, which turned one
122+ * snapshot's element list into 60 KB of repeated page text. Names are labels,
123+ * not content, so cap them where they are read.
124+ */
125+const MAX_INTERACTIVE_ELEMENT_NAME_LENGTH = 200;
120126 const MAX_SCREENSHOT_WIDTH = 1280;
121127 /** How long an armed tab keeps the exclusive display-media slot before another tab may take it. */
122128 const RECORDING_ARM_GRACE_MS = 10_000;
@@ -3582,7 +3588,7 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function · 3459 unmodified lines
35823588 return {
35833589 tag: element.tagName.toLowerCase(),
35843590 role: element.getAttribute("role"),
3585− name: element.getAttribute("aria-label") || element.innerText || element.getAttribute("name") || "",
3591+ name: (element.getAttribute("aria-label") || element.innerText || element.getAttribute("name") || "").slice(0, ${MAX_INTERACTIVE_ELEMENT_NAME_LENGTH}),
35863592 selector: selectorFor(element),
35873593 x: rect.x,
35883594 y: rect.y,
apps/server/src/config.ts+3−0
@@ -38,6 +38,8 @@ export interface ServerDerivedPaths { · 37 unmodified lines
3838 readonly providerStatusCacheDir: string;
3939 readonly worktreesDir: string;
4040 readonly attachmentsDir: string;
41+ /** Screenshots the agent asks the collaborative browser to keep for the user. */
42+ readonly browserArtifactsDir: string;
4143 readonly logsDir: string;
4244 readonly serverLogPath: string;
4345 readonly serverTracePath: string;
@@ -125,6 +127,7 @@ export const deriveServerPaths = Effect.fn(function* ( · 81 unmodified lines
125127 providerStatusCacheDir,
126128 worktreesDir: join(baseDir, "worktrees"),
127129 attachmentsDir,
130+ browserArtifactsDir: join(stateDir, "browser-artifacts"),
128131 logsDir,
129132 serverLogPath: join(logsDir, "server.log"),
130133 serverTracePath: join(logsDir, "server.trace.ndjson"),
apps/server/src/mcp/McpHttpServer.test.ts+316−34
@@ -4,12 +4,15 @@ import * as NodeServices from "@effect/platform-node/NodeServices"; · 3 unmodified lines
44 import { EnvironmentId, PreviewTabId, ProviderInstanceId, ThreadId } from "@t3tools/contracts";
55 import * as Deferred from "effect/Deferred";
66 import * as Effect from "effect/Effect";
7+import * as FileSystem from "effect/FileSystem";
78 import * as Layer from "effect/Layer";
9+import * as Path from "effect/Path";
810 import * as Schema from "effect/Schema";
911 import * as Stream from "effect/Stream";
1012 import { McpProtocol, McpSchema, McpServer } from "effect/unstable/ai";
1113 import { HttpBody, HttpClient, HttpRouter, HttpServerResponse } from "effect/unstable/http";
1214
15+import * as ServerConfig from "../config.ts";
1316 import * as McpHttpServer from "./McpHttpServer.ts";
1417 import * as McpInvocationContext from "./McpInvocationContext.ts";
1518 import * as PreviewAutomationBroker from "./PreviewAutomationBroker.ts";
@@ -18,7 +21,7 @@ const environmentId = EnvironmentId.make("environment-mcp-test"); · 2 unmodified lines
1821 const threadId = ThreadId.make("thread-mcp-test");
1922 const tabId = PreviewTabId.make("tab-mcp-test");
2023 const alternateTabId = PreviewTabId.make("tab-mcp-alternate");
21−const encodeJsonText = Schema.encodeSync(Schema.fromJsonString(Schema.Unknown));
24+const decodeJsonText = Schema.decodeUnknownSync(Schema.fromJsonString(Schema.Unknown));
2225 const invocation = {
2326 environmentId,
2427 threadId,
@@ -41,9 +44,62 @@ const client = McpSchema.McpServerClient.of({ · 16 unmodified lines
4144 });
4245 const TestLayer = McpHttpServer.PreviewToolkitRegistrationLive.pipe(
4346 Layer.provideMerge(McpServer.McpServer.layer),
44− Layer.provideMerge(PreviewAutomationBroker.layer.pipe(Layer.provide(NodeServices.layer))),
47+ Layer.provideMerge(PreviewAutomationBroker.layer),
48+ Layer.provideMerge(ServerConfig.layerTest(process.cwd(), { prefix: "t3-mcp-http-server-test-" })),
49+ Layer.provideMerge(NodeServices.layer),
4550 );
4651
52+const snapshotResult = {
53+ url: "http://example.test/",
54+ title: "Example",
55+ loading: false,
56+ visibleText: "Example",
57+ interactiveElements: [],
58+ accessibilityTree: {},
59+ consoleEntries: [],
60+ networkEntries: [],
61+ actionTimeline: [],
62+ screenshot: {
63+ mimeType: "image/png",
64+ data: Buffer.from("png").toString("base64"),
65+ width: 10,
66+ height: 5,
67+ },
68+};
69+
70+/** Answers every snapshot request on a fresh broker host with the given result. */
71+const serveSnapshots = (clientId: string, result: unknown) =>
72+ Effect.gen(function* () {
73+ const broker = yield* PreviewAutomationBroker.PreviewAutomationBroker;
74+ const connected = yield* Deferred.make<void>();
75+ const inputs: Array<unknown> = [];
76+ const events = yield* broker.connect({ clientId, environmentId });
77+ yield* Stream.runForEach(events, (event) => {
78+ if (event.type === "connected") return Deferred.succeed(connected, undefined);
79+ inputs.push(event.request.input);
80+ return broker.respond({
81+ clientId,
82+ connectionId: event.connectionId,
83+ requestId: event.request.requestId,
84+ ok: true,
85+ result,
86+ });
87+ }).pipe(Effect.forkScoped);
88+ yield* Deferred.await(connected);
89+ return inputs;
90+ });
91+
92+const callSnapshot = (args: Record<string, unknown>) =>
93+ Effect.gen(function* () {
94+ const server = yield* McpServer.McpServer;
95+ return yield* server
96+ .callTool({ name: "preview_snapshot", arguments: args })
97+ .pipe(
98+ Effect.provideService(McpInvocationContext.McpInvocationContext, invocation),
99+ Effect.provideService(McpSchema.McpServerClient, client),
100+ );
101+ });
102+
47103 it("normalizes empty successful notification responses to accepted", () => {
48104 const notificationResponse = McpHttpServer.normalizeMcpHttpResponse(
49105 HttpServerResponse.text("", { status: 200, contentType: "application/json" }),
@@ -92,7 +148,9 @@ it.effect.each([{}, { includeImage: false }])( · 42 unmodified lines
92148 );
93149
94150 expect(snapshot.isError).toBe(true);
95− expect(snapshot.content).toEqual([{ type: "text", text: "Preview snapshot failed." }]);
151+ expect(snapshot.content).toEqual([
152+ { type: "text", text: "Preview snapshot failed: PreviewAutomationExecutionError." },
153+ ]);
96154 expect(snapshot.structuredContent).toEqual({
97155 error: {
98156 _tag: "PreviewAutomationExecutionError",
@@ -174,10 +232,16 @@ it.effect.each([ · 75 unmodified lines
174232 Effect.provideService(McpSchema.McpServerClient, client),
175233 );
176234 const metadata = { ...page, title: `Snapshot ${call}`, screenshot };
235+ const { accessibilityTree: _tree, ...boundedMetadata } = metadata;
177236 expect(snapshot.isError).toBe(false);
178237 expect(snapshot.structuredContent).toEqual(metadata);
179− expect(snapshot.content).toEqual([
180− { type: "text", text: encodeJsonText(snapshot.structuredContent) },
238+ const [text, ...rest] = snapshot.content;
239+ expect(text?.type === "text" ? decodeJsonText(text.text) : null).toEqual(boundedMetadata);
240+ expect(rest).toEqual([
241+ {
242+ type: "text",
243+ text: "Snapshot text was bounded. Omitted: accessibilityTree (use interactiveElements locators or preview_evaluate).",
244+ },
181245 ...(images
182246 ? [
183247 {
@@ -200,7 +264,7 @@ it.effect.each([ · 16 unmodified lines
200264 Effect.provideService(McpInvocationContext.McpInvocationContext, invocation),
201265 Effect.provideService(McpSchema.McpServerClient, client),
202266 );
203− expect(nextDefault.content.map((content) => content.type)).toEqual(["text", "image"]);
267+ expect(nextDefault.content.map((content) => content.type)).toEqual(["text", "text", "image"]);
204268 expect(nextDefault.structuredContent).toEqual({ ...page, title: "Snapshot 7", screenshot });
205269 expect(requests).toBe(7);
206270 }),
@@ -221,7 +285,7 @@ it.effect("rejects non-boolean snapshot image options before selecting a browser · 14 unmodified lines
221285 Effect.provideService(McpSchema.McpServerClient, client),
222286 );
223287 expect(result.isError).toBe(true);
224− expect(result.content).toEqual([{ type: "text", text: "Preview snapshot failed." }]);
288+ expect(result.content).toEqual([{ type: "text", text: "Preview snapshot failed: AiError." }]);
225289 expect(result.structuredContent).toEqual({
226290 error: { _tag: "AiError", operation: "snapshot", failureCount: 1 },
227291 });
@@ -229,6 +293,222 @@ it.effect("rejects non-boolean snapshot image options before selecting a browser · 1 unmodified line
229293 }).pipe(Effect.provide(TestLayer)),
230294 );
231295
296+it.effect("saves the snapshot PNG on request and reports its path", () =>
297+ Effect.scoped(
298+ Effect.gen(function* () {
299+ const config = yield* ServerConfig.ServerConfig;
300+ const fileSystem = yield* FileSystem.FileSystem;
301+ const path = yield* Path.Path;
302+ const inputs = yield* serveSnapshots("mcp-save-client", snapshotResult);
303+
304+ const snapshot = yield* callSnapshot({ save: true });
305+
306+ expect(snapshot.isError).toBe(false);
307+ // The browser never receives the server-only `save` flag.
308+ expect(inputs).toEqual([{}]);
309+ const structured = snapshot.structuredContent as { readonly screenshotPath?: string };
310+ const screenshotPath = structured.screenshotPath;
311+ expect(typeof screenshotPath).toBe("string");
312+ expect(path.dirname(screenshotPath!)).toBe(config.browserArtifactsDir);
313+ expect(path.basename(screenshotPath!)).toMatch(
314+ /^browser-screenshot-example-test-[0-9a-z]+-[0-9a-f]{8}\.png$/,
315+ );
316+ expect(Buffer.from(yield* fileSystem.readFile(screenshotPath!)).toString()).toBe("png");
317+ const text = snapshot.content.find((content) => content.type === "text");
318+ expect(text?.type === "text" ? text.text : "").toContain(screenshotPath);
319+
320+ const unsaved = yield* callSnapshot({});
321+ expect(unsaved.structuredContent).not.toHaveProperty("screenshotPath");
322+ }),
323+ ).pipe(Effect.provide(TestLayer)),
324+);
325+
326+it.effect("reports a tagged error when the screenshot cannot be saved", () =>
327+ Effect.scoped(
328+ Effect.gen(function* () {
329+ const config = yield* ServerConfig.ServerConfig;
330+ const fileSystem = yield* FileSystem.FileSystem;
331+ // A regular file where the artifacts directory should be makes every write fail.
332+ yield* fileSystem.writeFileString(config.browserArtifactsDir, "");
333+ yield* serveSnapshots("mcp-save-failure-client", snapshotResult);
334+
335+ const snapshot = yield* callSnapshot({ save: true });
336+
337+ expect(snapshot.isError).toBe(true);
338+ expect(snapshot.content).toEqual([
339+ { type: "text", text: "Preview snapshot failed: PreviewScreenshotSaveError." },
340+ ]);
341+ expect(snapshot.structuredContent).toEqual({
342+ error: { _tag: "PreviewScreenshotSaveError", operation: "snapshot", failureCount: 1 },
343+ });
344+ }),
345+ ).pipe(Effect.provide(TestLayer)),
346+);
347+
348+it.effect("keeps the snapshot text under the agent's output ceiling", () =>
349+ Effect.scoped(
350+ Effect.gen(function* () {
351+ // Mirrors the real failure: a [role] container whose innerText is the whole
352+ // project list, repeated for several elements, plus a big AX tree.
353+ const pageText = "/Users/theo/Code/project\nClaude, Codex · 79 threads\n".repeat(600);
354+ const element = (name: string, index: number) => ({
355+ tag: "div",
356+ role: "presentation",
357+ name,
358+ selector: `div:nth-of-type(${index})`,
359+ x: 0,
360+ y: 0,
361+ width: 10,
362+ height: 10,
363+ });
364+ const oversized = {
365+ ...snapshotResult,
366+ visibleText: pageText,
367+ interactiveElements: [
368+ element(pageText, 1),
369+ element(pageText, 2),
370+ element(pageText, 3),
371+ element("Continue", 4),
372+ ],
373+ accessibilityTree: { nodes: Array.from({ length: 2_000 }, (_, i) => ({ nodeId: `${i}` })) },
374+ consoleEntries: Array.from({ length: 100 }, (_, i) => ({
375+ level: "log",
376+ text: `entry ${i}`,
377+ timestamp: "t",
378+ })),
379+ };
380+ yield* serveSnapshots("mcp-bounded-client", oversized);
381+
382+ const snapshot = yield* callSnapshot({ includeImage: false });
383+
384+ expect(snapshot.isError).toBe(false);
385+ const [text, notice] = snapshot.content;
386+ expect(text?.type).toBe("text");
387+ const body = text?.type === "text" ? text.text : "";
388+ expect(Buffer.byteLength(body, "utf8")).toBeLessThanOrEqual(
389+ McpHttpServer.MAX_SNAPSHOT_TEXT_BYTES,
390+ );
391+ const parsed = decodeJsonText(body) as {
392+ readonly accessibilityTree?: unknown;
393+ readonly visibleText: string;
394+ readonly interactiveElements: ReadonlyArray<{ readonly name: string }>;
395+ readonly consoleEntries: ReadonlyArray<{ readonly text: string }>;
396+ };
397+ expect(parsed.accessibilityTree).toBeUndefined();
398+ expect(parsed.visibleText.length).toBeLessThanOrEqual(8_001);
399+ expect(parsed.interactiveElements).toHaveLength(4);
400+ expect(parsed.interactiveElements[0]?.name.length).toBeLessThanOrEqual(201);
401+ expect(parsed.interactiveElements[3]?.name).toBe("Continue");
402+ expect(parsed.consoleEntries).toHaveLength(40);
403+ expect(parsed.consoleEntries[0]?.text).toBe("entry 60");
404+ expect(notice?.type === "text" ? notice.text : "").toContain("accessibilityTree");
405+ expect(notice?.type === "text" ? notice.text : "").toContain("60 older console entries");
406+ // The structured result is untouched; only the text the agent reads is bounded.
407+ expect(snapshot.structuredContent).toMatchObject({
408+ accessibilityTree: oversized.accessibilityTree,
409+ });
410+ }),
411+ ).pipe(Effect.provide(TestLayer)),
412+);
413+
414+it.effect("bounds the snapshot text even when nothing but logs and the title are large", () =>
415+ Effect.scoped(
416+ Effect.gen(function* () {
417+ const oversized = {
418+ ...snapshotResult,
419+ title: "t".repeat(70_000),
420+ interactiveElements: [],
421+ consoleEntries: [{ level: "log", text: "x".repeat(70_000), timestamp: "t" }],
422+ };
423+ yield* serveSnapshots("mcp-bounded-logs-client", oversized);
424+
425+ const snapshot = yield* callSnapshot({ includeImage: false });
426+
427+ const [text] = snapshot.content;
428+ const body = text?.type === "text" ? text.text : "";
429+ expect(Buffer.byteLength(body, "utf8")).toBeLessThanOrEqual(
430+ McpHttpServer.MAX_SNAPSHOT_TEXT_BYTES,
431+ );
432+ const parsed = decodeJsonText(body) as {
433+ readonly title: string;
434+ readonly consoleEntries: ReadonlyArray<{ readonly text: string }>;
435+ };
436+ expect(parsed.title.length).toBe(2_049);
437+ expect(parsed.consoleEntries[0]?.text.length).toBe(501);
438+ const notice = snapshot.content[1];
439+ const noticeText = notice?.type === "text" ? notice.text : "";
440+ expect(noticeText).toContain("url or title after 2048 characters");
441+ expect(noticeText).toContain("console entries text after 500 characters");
442+ }),
443+ ).pipe(Effect.provide(TestLayer)),
444+);
445+
446+it.effect("sheds log entries before locators when every list is full", () =>
447+ Effect.scoped(
448+ Effect.gen(function* () {
449+ const long = "x".repeat(2_000);
450+ const oversized = {
451+ ...snapshotResult,
452+ interactiveElements: Array.from({ length: 20 }, (_, i) => ({
453+ tag: "button",
454+ role: "button",
455+ name: `Button ${i}`,
456+ selector: `#button-${i}`,
457+ x: 0,
458+ y: 0,
459+ width: 10,
460+ height: 10,
461+ })),
462+ consoleEntries: Array.from({ length: 200 }, () => ({
463+ level: long,
464+ text: long,
465+ timestamp: long,
466+ source: long,
467+ })),
468+ networkEntries: Array.from({ length: 200 }, () => ({
469+ url: long,
470+ method: long,
471+ status: 200,
472+ failed: false,
473+ errorText: long,
474+ timestamp: long,
475+ })),
476+ actionTimeline: Array.from({ length: 200 }, () => ({
477+ id: long,
478+ action: long,
479+ status: "succeeded",
480+ startedAt: long,
481+ completedAt: long,
482+ error: long,
483+ })),
484+ };
485+ yield* serveSnapshots("mcp-full-logs-client", oversized);
486+
487+ const snapshot = yield* callSnapshot({ includeImage: false });
488+
489+ const [text, notice] = snapshot.content;
490+ const body = text?.type === "text" ? text.text : "";
491+ expect(Buffer.byteLength(body, "utf8")).toBeLessThanOrEqual(
492+ McpHttpServer.MAX_SNAPSHOT_TEXT_BYTES,
493+ );
494+ const parsed = decodeJsonText(body) as {
495+ readonly interactiveElements: ReadonlyArray<unknown>;
496+ readonly consoleEntries: ReadonlyArray<unknown>;
497+ readonly networkEntries: ReadonlyArray<unknown>;
498+ readonly actionTimeline: ReadonlyArray<unknown>;
499+ };
500+ // Locators survive; the log lists take the cut.
501+ expect(parsed.interactiveElements).toHaveLength(20);
502+ expect(
503+ parsed.consoleEntries.length + parsed.networkEntries.length + parsed.actionTimeline.length,
504+ ).toBeLessThan(120);
505+ const noticeText = notice?.type === "text" ? notice.text : "";
506+ expect(noticeText).toContain("40 of 40 actionTimeline");
507+ expect(noticeText).not.toMatch(/\d+ of \d+ interactiveElements/);
508+ }),
509+ ).pipe(Effect.provide(TestLayer)),
510+);
511+
232512 it.effect("terminates HTTP MCP sessions with DELETE", () =>
233513 Effect.scoped(
234514 Effect.gen(function* () {
@@ -306,33 +586,19 @@ it.effect("registers annotated tools and preserves authenticated request context · 71 unmodified lines
306586 ok: true,
307587 result:
308588 event.request.operation === "snapshot"
309− ? {
310− url: "http://example.test/",
311− title: "Example",
312− loading: false,
313− visibleText: "Example",
314− interactiveElements: [],
315− accessibilityTree: {},
316− consoleEntries: [],
317− networkEntries: [],
318− actionTimeline: [],
319− screenshot: {
320− mimeType: "image/png",
321− data: Buffer.from("png").toString("base64"),
322− width: 10,
323− height: 5,
324− },
325− }
326− : event.request.operation === "press"
327− ? undefined
328− : {
329− available: true,
330− visible: true,
331− tabId,
332− url: "http://example.test/",
333− title: "Example",
334− loading: false,
335− },
589+ ? snapshotResult
590+ : event.request.operation === "evaluate"
591+ ? ["Connect", "Continue"]
592+ : event.request.operation === "press"
593+ ? undefined
594+ : {
595+ available: true,
596+ visible: true,
597+ tabId,
598+ url: "http://example.test/",
599+ title: "Example",
600+ loading: false,
601+ },
336602 });
337603 }).pipe(Effect.forkScoped);
338604 yield* Effect.yieldNow;
@@ -397,6 +663,22 @@ it.effect("registers annotated tools and preserves authenticated request context · 58 unmodified lines
397663 alternateTabId,
398664 );
399665
666+ // Arrays and primitives are wrapped so structuredContent stays a JSON object.
667+ // Claude Code rejects the whole result otherwise.
668+ const evaluateTool = server.tools.find(({ tool }) => tool.name === "preview_evaluate");
669+ expect(evaluateTool?.tool.outputSchema).toMatchObject({ type: "object" });
670+ const evaluated = yield* server
671+ .callTool({ name: "preview_evaluate", arguments: { expression: "buttons()" } })
672+ .pipe(
673+ Effect.provideService(McpInvocationContext.McpInvocationContext, invocation),
674+ Effect.provideService(McpSchema.McpServerClient, client),
675+ );
676+ expect(evaluated.isError).toBe(false);
677+ expect(evaluated.structuredContent).toEqual({ value: ["Connect", "Continue"] });
678+ expect(evaluated.content).toEqual([
679+ { type: "text", text: '{"value":["Connect","Continue"]}' },
680+ ]);
681+
400682 const actionRequests = [
401683 { name: "preview_click", arguments: { x: 10, y: 10 } },
402684 { name: "preview_type", arguments: { text: "Hello" } },
apps/server/src/mcp/McpHttpServer.ts+243−25
@@ -1,8 +1,13 @@
1+import * as NodeCrypto from "node:crypto";
12 import * as Cause from "effect/Cause";
3+import * as Clock from "effect/Clock";
24 import * as Context from "effect/Context";
35 import * as Effect from "effect/Effect";
6+import * as FileSystem from "effect/FileSystem";
47 import * as Layer from "effect/Layer";
58 import * as Option from "effect/Option";
9+import * as Path from "effect/Path";
10+import * as Schema from "effect/Schema";
611 import * as Sink from "effect/Sink";
712 import * as Stream from "effect/Stream";
813 import type * as Types from "effect/Types";
@@ -10,6 +15,7 @@ import { McpProtocol, McpSchema, McpServer, Tool } from "effect/unstable/ai"; · 1 unmodified line
1015 import { HttpRouter, HttpServerRequest, HttpServerResponse } from "effect/unstable/http";
1116
1217 import packageJson from "../../package.json" with { type: "json" };
18+import * as ServerConfig from "../config.ts";
1319 import * as McpInvocationContext from "./McpInvocationContext.ts";
1420 import * as McpSessionRegistry from "./McpSessionRegistry.ts";
1521 import * as PreviewAutomationBroker from "./PreviewAutomationBroker.ts";
@@ -95,6 +101,204 @@ const McpAuthMiddlewareLive = HttpRouter.middleware<{ · 79 unmodified lines
95101 provides: McpInvocationContext.McpInvocationContext;
96102 }>()(makeMcpAuthMiddleware).layer;
97103
104+/**
105+ * Claude Code drops every MCP result above 25k tokens (~100 KB of text) and
106+ * hands the agent a truncation notice instead, so a snapshot that carries the
107+ * full accessibility tree and 20 KB of page text loses its locators too. Keep
108+ * the text under that ceiling and tell the agent what was cut.
109+ */
110+export const MAX_SNAPSHOT_TEXT_BYTES = 60_000;
111+const MAX_SNAPSHOT_VISIBLE_TEXT_CHARS = 8_000;
112+const MAX_SNAPSHOT_ELEMENT_NAME_CHARS = 200;
113+const MAX_SNAPSHOT_LOG_ENTRIES = 40;
114+const MAX_SNAPSHOT_LOG_TEXT_CHARS = 500;
115+const MAX_SNAPSHOT_IDENTIFIER_CHARS = 2_048;
116+
117+const encodeJsonText = Schema.encodeSync(Schema.fromJsonString(Schema.Unknown));
118+const utf8Length = (text: string) => Buffer.byteLength(text, "utf8");
119+const cutText = (text: string, max: number) =>
120+ text.length > max ? `${text.slice(0, max)}…` : text;
121+
122+/** Shortens every string field of a log entry; other fields pass through. */
123+const cutEntryStrings = <A>(entry: A): A =>
124+ typeof entry === "object" && entry !== null
125+ ? (Object.fromEntries(
126+ Object.entries(entry).map(([key, value]) => [
127+ key,
128+ typeof value === "string" ? cutText(value, MAX_SNAPSHOT_LOG_TEXT_CHARS) : value,
129+ ]),
130+ ) as A)
131+ : entry;
132+
133+const hasLongString = (entry: unknown, max: number) =>
134+ typeof entry === "object" &&
135+ entry !== null &&
136+ Object.values(entry).some((value) => typeof value === "string" && value.length > max);
137+
138+type SnapshotMetadata = {
139+ readonly url: string;
140+ readonly title: string;
141+ readonly visibleText: string;
142+ readonly interactiveElements: ReadonlyArray<{
143+ readonly name: string;
144+ readonly [key: string]: unknown;
145+ }>;
146+ readonly consoleEntries: ReadonlyArray<unknown>;
147+ readonly networkEntries: ReadonlyArray<unknown>;
148+ readonly actionTimeline: ReadonlyArray<unknown>;
149+ readonly [key: string]: unknown;
150+};
151+
152+/**
153+ * Drops the accessibility tree, shortens page text, element names, identifiers,
154+ * and log strings, keeps only the newest log entries, and finally sheds
155+ * interactive elements until the JSON fits. Returns the text plus notes on
156+ * what is missing so the agent can reach for preview_evaluate.
157+ */
158+const boundSnapshotMetadata = (
159+ metadata: SnapshotMetadata,
160+): { readonly text: string; readonly omitted: ReadonlyArray<string> } => {
161+ const omitted: Array<string> = [];
162+ const { accessibilityTree, ...withoutTree } = metadata;
163+ if (accessibilityTree !== undefined) {
164+ omitted.push("accessibilityTree (use interactiveElements locators or preview_evaluate)");
165+ }
166+ const tail = <A>(entries: ReadonlyArray<A>, label: string) => {
167+ if (entries.length > MAX_SNAPSHOT_LOG_ENTRIES) {
168+ omitted.push(`${entries.length - MAX_SNAPSHOT_LOG_ENTRIES} older ${label}`);
169+ }
170+ const kept = entries.slice(-MAX_SNAPSHOT_LOG_ENTRIES);
171+ if (kept.some((entry) => hasLongString(entry, MAX_SNAPSHOT_LOG_TEXT_CHARS))) {
172+ omitted.push(`${label} text after ${MAX_SNAPSHOT_LOG_TEXT_CHARS} characters`);
173+ }
174+ return kept.map(cutEntryStrings);
175+ };
176+ if (
177+ metadata.url.length > MAX_SNAPSHOT_IDENTIFIER_CHARS ||
178+ metadata.title.length > MAX_SNAPSHOT_IDENTIFIER_CHARS
179+ ) {
180+ omitted.push(`url or title after ${MAX_SNAPSHOT_IDENTIFIER_CHARS} characters`);
181+ }
182+ if (
183+ metadata.interactiveElements.some(
184+ (element) => element.name.length > MAX_SNAPSHOT_ELEMENT_NAME_CHARS,
185+ )
186+ ) {
187+ omitted.push(`element names longer than ${MAX_SNAPSHOT_ELEMENT_NAME_CHARS} characters`);
188+ }
189+ if (metadata.visibleText.length > MAX_SNAPSHOT_VISIBLE_TEXT_CHARS) {
190+ omitted.push(
191+ `visibleText after ${MAX_SNAPSHOT_VISIBLE_TEXT_CHARS} characters (use preview_evaluate for more)`,
192+ );
193+ }
194+ const bounded = {
195+ ...withoutTree,
196+ url: cutText(metadata.url, MAX_SNAPSHOT_IDENTIFIER_CHARS),
197+ title: cutText(metadata.title, MAX_SNAPSHOT_IDENTIFIER_CHARS),
198+ visibleText: cutText(metadata.visibleText, MAX_SNAPSHOT_VISIBLE_TEXT_CHARS),
199+ interactiveElements: metadata.interactiveElements.map((element) => ({
200+ ...element,
201+ name: cutText(element.name, MAX_SNAPSHOT_ELEMENT_NAME_CHARS),
202+ })),
203+ consoleEntries: tail(metadata.consoleEntries, "console entries"),
204+ networkEntries: tail(metadata.networkEntries, "network entries"),
205+ actionTimeline: tail(metadata.actionTimeline, "action timeline entries"),
206+ };
207+
208+ // Per-field caps do not sum below the ceiling: three log arrays of 40 capped
209+ // entries alone can pass 60 KB. Shed the least useful lists first, halving
210+ // one list per round, until the JSON fits. With every list empty the rest
211+ // is bounded by the identifier and visibleText caps, so this terminates.
212+ const shedOrder = [
213+ "actionTimeline",
214+ "networkEntries",
215+ "consoleEntries",
216+ "interactiveElements",
217+ ] as const;
218+ const lists: Record<(typeof shedOrder)[number], ReadonlyArray<unknown>> = {
219+ interactiveElements: bounded.interactiveElements,
220+ consoleEntries: bounded.consoleEntries,
221+ networkEntries: bounded.networkEntries,
222+ actionTimeline: bounded.actionTimeline,
223+ };
224+ const dropped: Record<(typeof shedOrder)[number], number> = {
225+ interactiveElements: 0,
226+ consoleEntries: 0,
227+ networkEntries: 0,
228+ actionTimeline: 0,
229+ };
230+ let text = encodeJsonText({ ...bounded, ...lists });
231+ while (utf8Length(text) > MAX_SNAPSHOT_TEXT_BYTES) {
232+ // Elements carry the locators, so they go last; logs shed newest-last.
233+ const key =
234+ shedOrder.find(
235+ (candidate) => candidate !== "interactiveElements" && lists[candidate].length > 0,
236+ ) ?? (lists.interactiveElements.length > 0 ? "interactiveElements" : undefined);
237+ if (key === undefined) break;
238+ const keep = Math.floor(lists[key].length / 2);
239+ dropped[key] += lists[key].length - keep;
240+ // slice(-0) keeps everything, so spell out the empty case.
241+ lists[key] =
242+ keep === 0
243+ ? []
244+ : key === "interactiveElements"
245+ ? lists[key].slice(0, keep)
246+ : lists[key].slice(-keep);
247+ text = encodeJsonText({ ...bounded, ...lists });
248+ }
249+ for (const key of shedOrder) {
250+ if (dropped[key] > 0) {
251+ omitted.push(`${dropped[key]} of ${bounded[key].length} ${key}`);
252+ }
253+ }
254+ return { text, omitted };
255+};
256+
257+export class PreviewScreenshotSaveError extends Schema.TaggedError<PreviewScreenshotSaveError>()(
258+ "PreviewScreenshotSaveError",
259+ { screenshotPath: Schema.String, cause: Schema.Defect() },
260+) {
261+ override get message(): string {
262+ return `Could not save preview screenshot to ${this.screenshotPath}.`;
263+ }
264+}
265+
266+const MAX_SCREENSHOT_SITE_SLUG_LENGTH = 40;
267+
268+/** Hostname reduced to a filename-safe slug, matching the desktop's own screenshot names. */
269+const screenshotSiteSlug = (rawUrl: string): string => {
270+ try {
271+ const slug = new URL(rawUrl).hostname
272+ .toLowerCase()
273+ .replace(/[^a-z0-9]+/g, "-")
274+ .replace(/^-+|-+$/g, "")
275+ .slice(0, MAX_SCREENSHOT_SITE_SLUG_LENGTH)
276+ .replace(/-+$/g, "");
277+ return slug || "site";
278+ } catch {
279+ return "site";
280+ }
281+};
282+
283+/** Writes the snapshot PNG under the browser artifacts directory and returns its path. */
284+const saveScreenshot = Effect.fn("McpHttpServer.saveScreenshot")(function* (
285+ pageUrl: string,
286+ data: Uint8Array,
287+) {
288+ const config = yield* ServerConfig.ServerConfig;
289+ const fileSystem = yield* FileSystem.FileSystem;
290+ const path = yield* Path.Path;
291+ const millis = yield* Clock.currentTimeMillis;
292+ // Two saves in the same millisecond must not overwrite each other.
293+ const fileName = `browser-screenshot-${screenshotSiteSlug(pageUrl)}-${millis.toString(36)}-${NodeCrypto.randomUUID().slice(0, 8)}.png`;
294+ const screenshotPath = path.join(config.browserArtifactsDir, fileName);
295+ yield* fileSystem.makeDirectory(config.browserArtifactsDir, { recursive: true }).pipe(
296+ Effect.andThen(fileSystem.writeFile(screenshotPath, data)),
297+ Effect.mapError((cause) => new PreviewScreenshotSaveError({ screenshotPath, cause })),
298+ );
299+ return screenshotPath;
300+});
301+
98302 const previewSnapshotFailure = <E>(cause: Cause.Cause<E>) => {
99303 if (Cause.hasInterrupts(cause) || cause.reasons.some(Cause.isDieReason)) {
100304 return Effect.failCause(cause).pipe(Effect.orDie);
@@ -117,7 +321,8 @@ const previewSnapshotFailure = <E>(cause: Cause.Cause<E>) => { · 16 unmodified lines
117321 failureCount: failures.length,
118322 },
119323 },
120− content: [{ type: "text", text: "Preview snapshot failed." }],
324+ // Agents usually see only the text content, so name the tag there too.
325+ content: [{ type: "text", text: `Preview snapshot failed: ${errorTag}.` }],
121326 });
122327 return Effect.logWarning("preview snapshot failed", {
123328 operation: "snapshot",
@@ -129,6 +334,10 @@ const previewSnapshotFailure = <E>(cause: Cause.Cause<E>) => { · 5 unmodified lines
129334 const registerPreviewSnapshot = Effect.fn("McpHttpServer.registerPreviewSnapshot")(function* () {
130335 const server = yield* McpServer.McpServer;
131336 const broker = yield* PreviewAutomationBroker.PreviewAutomationBroker;
337+ // The MCP tool runner only supplies the client, so hand the save path its services here.
338+ const saveServices = yield* Effect.context<
339+ ServerConfig.ServerConfig | FileSystem.FileSystem | Path.Path
340+ >();
132341 const built = yield* PreviewSnapshotToolkit;
133342 const tool = PreviewSnapshotTool;
134343 yield* server.addTool({
@@ -160,19 +369,21 @@ const registerPreviewSnapshot = Effect.fn("McpHttpServer.registerPreviewSnapshot · 25 unmodified lines
160369 Effect.flatMap(Effect.fromOption),
161370 Effect.provideService(PreviewAutomationBroker.PreviewAutomationBroker, broker),
162371 Effect.provideService(McpInvocationContext.McpInvocationContext, invocation),
163− Effect.matchCauseEffect({
164− onFailure: previewSnapshotFailure,
165− onSuccess: ({ encodedResult }) => {
166− const snapshot = encodedResult as {
372+ Effect.flatMap(({ encodedResult }) =>
373+ Effect.gen(function* () {
374+ const snapshot = encodedResult as SnapshotMetadata & {
375+ readonly url: string;
167376 readonly screenshot: {
168377 readonly mimeType: "image/png";
169378 readonly data: string;
170379 readonly width: number;
171380 readonly height: number;
172381 };
173− readonly [key: string]: unknown;
174382 };
175383 const { screenshot, ...page } = snapshot;
384+ const png = new Uint8Array(Buffer.from(screenshot.data, "base64"));
385+ const screenshotPath =
386+ payload?.save === true ? yield* saveScreenshot(snapshot.url, png) : undefined;
176387 const metadata = {
177388 ...page,
178389 screenshot: {
@@ -180,26 +391,33 @@ const registerPreviewSnapshot = Effect.fn("McpHttpServer.registerPreviewSnapshot · 1 unmodified line
180391 width: screenshot.width,
181392 height: screenshot.height,
182393 },
394+ ...(screenshotPath === undefined ? {} : { screenshotPath }),
183395 };
184− return Effect.succeed(
185− new McpSchema.CallToolResult({
186− isError: false,
187− structuredContent: metadata,
188− content: [
189− { type: "text", text: JSON.stringify(metadata) },
190− ...(payload?.includeImage === false
191− ? []
192− : [
193− {
194− type: "image" as const,
195− data: new Uint8Array(Buffer.from(screenshot.data, "base64")),
196− mimeType: screenshot.mimeType,
197− },
198− ]),
199− ],
200− }),
201− );
202− },
396+ const bounded = boundSnapshotMetadata(metadata);
397+ return new McpSchema.CallToolResult({
398+ isError: false,
399+ structuredContent: metadata,
400+ content: [
401+ { type: "text", text: bounded.text },
402+ ...(bounded.omitted.length === 0
403+ ? []
404+ : [
405+ {
406+ type: "text" as const,
407+ text: `Snapshot text was bounded. Omitted: ${bounded.omitted.join("; ")}.`,
408+ },
409+ ]),
410+ ...(payload?.includeImage === false
411+ ? []
412+ : [{ type: "image" as const, data: png, mimeType: screenshot.mimeType }]),
413+ ],
414+ });
415+ }),
416+ ),
417+ Effect.provide(saveServices),
418+ Effect.matchCauseEffect({
419+ onFailure: previewSnapshotFailure,
420+ onSuccess: Effect.succeed,
203421 }),
204422 );
205423 }),
apps/server/src/mcp/toolkits/preview/handlers.ts+5−3
@@ -166,8 +166,8 @@ const handlers = { · 165 unmodified lines
166166 preview_set_appearance: (input) =>
167167 invokeTargeted<PreviewAutomationSetColorSchemeResult>("setColorScheme", input),
168168 preview_snapshot: (input) => {
169− // Output selection is MCP-only; the browser still produces a complete snapshot.
170− const { includeImage: _includeImage, ...operationInput } = input ?? {};
169+ // Output selection and saving are MCP-only; the browser still produces a complete snapshot.
170+ const { includeImage: _includeImage, save: _save, ...operationInput } = input ?? {};
171171 return invokeTargeted<PreviewAutomationSnapshot>("snapshot", operationInput);
172172 },
173173 preview_click: (input) =>
@@ -176,7 +176,9 @@ const handlers = { · 2 unmodified lines
176176 preview_press: (input) => invokeTargeted<void>("press", input).pipe(Effect.as({})),
177177 preview_scroll: (input) => invokeTargeted<void>("scroll", input).pipe(Effect.as({})),
178178 preview_evaluate: (input) =>
179− invokeTargeted<unknown>("evaluate", input).pipe(Effect.map((result) => result ?? null)),
179+ invokeTargeted<unknown>("evaluate", input).pipe(
180+ Effect.map((result) => ({ value: result ?? null })),
181+ ),
180182 preview_wait_for: (input) =>
181183 invokeTargeted<void>("waitFor", input, input.timeoutMs).pipe(Effect.as({})),
182184 preview_recording_start: (input) =>
apps/server/src/mcp/toolkits/preview/tools.ts+20−3
@@ -113,7 +113,7 @@ const PreviewSetAppearanceTool = safeBrowserTool( · 112 unmodified lines
113113 export const PreviewSnapshotTool = readonlyBrowserTool(
114114 Tool.make("preview_snapshot", {
115115 description:
116− "Inspect a page before interacting. Pass tabId to inspect a specific tab; omit it to use this agent session's current tab. Returns page state, semantic elements, diagnostics, action history, and a PNG screenshot. Set includeImage=false for text-only output with the same page metadata.",
116+ "Inspect a page before interacting. Pass tabId to inspect a specific tab; omit it to use this agent session's current tab. Returns page state, semantic elements, diagnostics, action history, and a PNG screenshot. Set includeImage=false for text-only output with the same page metadata. Set save=true to also write the PNG to disk and get screenshotPath back; embed that path in your reply as ![alt](screenshotPath) so the user sees it. This is the only way to show the user a screenshot; the image in the tool result is not saved anywhere.",
117117 parameters: Schema.Struct({
118118 ...PreviewAutomationTabTargetInput.fields,
119119 includeImage: Schema.optional(
@@ -122,6 +122,12 @@ export const PreviewSnapshotTool = readonlyBrowserTool( · 2 unmodified lines
122122 "Include the PNG image in the tool response. Defaults to true. Set false for text-only output.",
123123 }),
124124 ),
125+ save: Schema.optional(
126+ Schema.Boolean.annotate({
127+ description:
128+ "Write the screenshot PNG to disk and return its absolute path as screenshotPath. Defaults to false.",
129+ }),
130+ ),
125131 }),
126132 success: PreviewAutomationSnapshot,
127133 failure: PreviewAutomationError,
@@ -173,12 +179,23 @@ const PreviewScrollTool = safeBrowserTool( · 45 unmodified lines
173179 }).annotate(Tool.Title, "Scroll preview page"),
174180 );
175181
182+/**
183+ * MCP `structuredContent` must be a JSON object, and Claude Code rejects the
184+ * whole result when it is not. Wrapping keeps arrays, strings, numbers, and
185+ * null valid instead of failing only for non-object expressions.
186+ */
187+export const PreviewEvaluateResult = Schema.Struct({
188+ value: Schema.Unknown.annotate({
189+ description: "The JSON-serializable value the expression produced, or null.",
190+ }),
191+}).annotate({ description: "The evaluated expression result." });
192+
176193 const PreviewEvaluateTool = browserTool(
177194 Tool.make("preview_evaluate", {
178195 description:
179− "Evaluate JavaScript in the tab selected by tabId, or this agent session's current tab when omitted. Returns a serializable result up to 64 KB; the expression may mutate page state.",
196+ "Evaluate JavaScript in the tab selected by tabId, or this agent session's current tab when omitted. Returns {value} with a serializable result up to 64 KB; the expression may mutate page state.",
180197 parameters: PreviewAutomationEvaluateInput,
181− success: Schema.Unknown,
198+ success: PreviewEvaluateResult,
182199 failure: PreviewAutomationError,
183200 dependencies,
184201 }).annotate(Tool.Title, "Evaluate JavaScript in preview"),