diff --git a/node_modules/agents/.bun-tag-37e67b70862be2b2 b/.bun-tag-37e67b70862be2b2new file mode 100644index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391diff --git a/node_modules/agents/.bun-tag-61b2f1517ab5ced4 b/.bun-tag-61b2f1517ab5ced4new file mode 100644index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391diff --git a/node_modules/agents/.bun-tag-61d29b56a6079f1d b/.bun-tag-61d29b56a6079f1dnew file mode 100644index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391diff --git a/node_modules/agents/.bun-tag-a6a47855632a2623 b/.bun-tag-a6a47855632a2623new file mode 100644index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391diff --git a/node_modules/agents/.bun-tag-c0c639aa2299e502 b/.bun-tag-c0c639aa2299e502new file mode 100644index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391diff --git a/dist/agent-tool-types-CNyE1iz_.d.ts b/dist/agent-tool-types-CNyE1iz_.d.tsindex 571eececebd5a1eaf7f2fbf5278801c4d34728ba..010811ae74ef2d27f12f7fb2506848f1939d91d6 100644--- a/dist/agent-tool-types-CNyE1iz_.d.ts+++ b/dist/agent-tool-types-CNyE1iz_.d.ts@@ -480,7 +480,10 @@ declare class DurableObjectEventStore implements EventStore { private readonly seqByStream; private readonly seqInit; constructor(storage: DurableObjectStorage);- storeEvent(streamId: StreamId, message: JSONRPCMessage): Promise<EventId>;+ /** Resolves `undefined` for a message too large for DO storage's 128 KiB+ * per-value cap: the event is delivered live without a replay id rather+ * than failing the send. */+ storeEvent(streamId: StreamId, message: JSONRPCMessage): Promise<EventId | undefined>; getStreamIdForEventId(eventId: EventId): Promise<StreamId | undefined>; replayEventsAfter( lastEventId: EventId,@@ -538,6 +541,42 @@ declare abstract class McpAgent< * @internal */ private static readonly STREAM_REQS_KEY_PREFIX;+ /**+ * Monotonic id for THIS Durable Object incarnation, bumped once per live+ * instance. Ledger rows are stamped with it so a row that outlived the+ * isolate which was going to answer it is recognisable as stale.+ *+ * @internal+ */+ currentSessionEpoch(): Promise<number>;+ /**+ * Ledger rows stamped by an earlier incarnation: requests that nothing will+ * ever answer. Read-only — the caller writes the client's error first.+ *+ * @internal+ */+ getStaleEpochStreamRequestIds(): Promise<+ ReadonlyArray<{+ readonly streamId: string;+ readonly requestIds: RequestId[];+ readonly epoch: number;+ readonly currentEpoch: number;+ }>+ >;+ /**+ * Every POST stream still awaiting a response, with the incarnation that+ * accepted it. Callers that mean "work still running" must drop rows whose+ * `epoch` is below {@link currentSessionEpoch}.+ *+ * @internal+ */+ getOpenStreamRequestIds(): Promise<+ ReadonlyArray<{+ readonly streamId: string;+ readonly requestIds: RequestId[];+ readonly epoch: number;+ }>+ >; /** Persist the `requestIds` for a POST stream. @internal */ setStreamRequestIds(streamId: string, requestIds: RequestId[]): Promise<void>; /** Read the persisted `requestIds` for a POST stream. @internal */diff --git a/dist/mcp/index.d.ts b/dist/mcp/index.d.tsindex c8fad448e8797b89690a99d93490d1363851b225..d80f66f1532c29dda0a0477dc1ec9ef5fd1fca47 100644--- a/dist/mcp/index.d.ts+++ b/dist/mcp/index.d.ts@@ -29,6 +29,8 @@ import { xt as MCPClientOAuthCallbackConfig, zt as ElicitResult } from "../agent-tool-types-CNyE1iz_.js";+declare const MAX_SSE_AGE_MS = 1800000;+declare const MCP_POST_RESPONSE_DEADLINE_MS = 660000; export { type ClearableEventStore, type CreateMcpHandlerOptions,@@ -41,6 +43,8 @@ export { type MCPConnectionResult, type MCPDiscoverResult, type MCPServerOptions,+ MAX_SSE_AGE_MS,+ MCP_POST_RESPONSE_DEADLINE_MS, MCP_SERVER_ID_MAX_LENGTH, McpAgent, type McpAuthContext,diff --git a/dist/mcp/index.js b/dist/mcp/index.jsindex 1edcf0c8c9e67aa211ae515e7672cdf79912101e..231220c26b995a48e40dba993773c28786cca392 100644--- a/dist/mcp/index.js+++ b/dist/mcp/index.js@@ -28,13 +28,60 @@ import { WebStandardStreamableHTTPServerTransport } from "@modelcontextprotocol/ const KEEPALIVE_INTERVAL_MS = 25e3; /** SSE comment frame the parser drops before any event dispatch. */ const KEEPALIVE_FRAME = ": keepalive\n\n";+// Max age is a stalled-client memory backstop, not session idleness. Keep it+// well above Executor's 5 minute session idle timeout so active clients rarely+// rotate.+const MAX_SSE_AGE_MS = 30 * 60 * 1000;+/**+* JSON-RPC error code for "the isolate that was running this call is gone".+* In the implementation-defined server range (-32000..-32099) and deliberately+* NOT the -32001 the worker already uses for session/transport failures, so a+* client can tell "the call you are waiting on died mid-flight, re-run it" from+* "this session id is dead, reconnect".+*/+const SESSION_RESET_ERROR_CODE = -32010;+const SESSION_RESET_ERROR_MESSAGE = "Execution lost: the session was reset before it produced a result. Re-run the call.";+/**+* How long the POST bridge waits for the Durable Object to answer a JSON-RPC+* request before answering for it.+*+* This file is a vendored dist and cannot import the two constants that+* actually bound how long an answer may legitimately take, so this MUST be kept+* at or above their sum:+*+* DEFAULT_TIMEOUT_MS packages/kernel/runtime-dynamic-worker/src/executor.ts+* (5 min sandbox execution ceiling)+* BROWSER_APPROVAL_WAIT_TIMEOUT_MS packages/hosts/mcp/src/tool-server.ts+* (PAUSED_APPROVAL_TIMEOUT_MS + 1s = 4 min 1 s)+*+* plus ~60s of margin for DO restore, storage and edge latency. 11 minutes+* clears 10 min 1 s. Drift is caught by+* packages/hosts/cloudflare/src/mcp/agents-post-stream-loss.test.ts,+* which recomputes the sum from those two source files.+*/+const MCP_POST_RESPONSE_DEADLINE_MS = 11 * 60 * 1000;+/**+* The JSON-RPC error response sent for a request that will never get a real+* answer. `reason` distinguishes the two ways we find out: `session_reset`+* (the DO end of the bridge went away, or a ledger row outlived its+* incarnation) and `response_deadline` (nothing came back in time).+*/+const sessionResetErrorResponse = (id, reason) => ({+ error: {+ code: SESSION_RESET_ERROR_CODE,+ data: { reason },+ message: SESSION_RESET_ERROR_MESSAGE+ },+ id,+ jsonrpc: "2.0"+}); /** * Start an SSE keepalive on `writer`. Returns a `clearInterval` handle * that the stream cleanup must invoke when the stream closes. */-function startKeepalive(writer, encoder) {+function startKeepalive(writeFrame, encoder) { const handle = setInterval(() => {- writer.write(encoder.encode(KEEPALIVE_FRAME)).catch(() => clearInterval(handle));+ writeFrame(encoder.encode(KEEPALIVE_FRAME)); }, KEEPALIVE_INTERVAL_MS); return handle; }@@ -180,10 +227,30 @@ const createStreamingHttpHandler = (basePath, namespace, options = {}) => { }); return new Response(body, { status: 404 }); }- const { readable, writable } = new TransformStream();- const writer = writable.getWriter();- const encoder = new TextEncoder();- const existingHeaders = {};+ const { readable, writable } = new TransformStream();+ const writer = writable.getWriter();+ const encoder = new TextEncoder();+ const MAX_PENDING_SSE_BYTES = 8 * 1024 * 1024;+ let __pendingBytes = 0;+ let __writeChain = Promise.resolve();+ let __sseClosed = false;+ let keepAlive;+ let __responseDeadline;+ /**+ * JSON-RPC ids this POST still owes the client a response for.+ * Entries are removed as the DO reports responses (the+ * `respondedIds` field writeSSEEvent puts on the envelope).+ *+ * A non-empty set when the bridge finishes abnormally is exactly+ * the "lost execution" case: the client already holds a 200 with+ * an open SSE body, so a silently ended stream leaves it waiting+ * forever. Keyed by type+value because JSON-RPC ids may be+ * strings or numbers and 1 must not match "1".+ */+ const __outstanding = new Map();+ const __idKey = (id) => `${typeof id}:${String(id)}`;+ let __finished = false;+ const existingHeaders = {}; request.headers.forEach((value, key) => { existingHeaders[key] = value; });@@ -206,47 +273,159 @@ const createStreamingHttpHandler = (basePath, namespace, options = {}) => { jsonrpc: "2.0" }); return new Response(body, { status: 500 });- }- ws.accept();- if (messages.every((msg) => isJSONRPCNotification(msg) || isJSONRPCResultResponse(msg))) {- ws.close();- return new Response(null, {+ }+ ws.accept();+ const __closeSse = () => {+ if (__sseClosed) return;+ __sseClosed = true;+ __finished = true;+ try {+ clearInterval(keepAlive);+ clearTimeout(__responseDeadline);+ } catch {}+ try {+ ws.close(1013, "SSE client not draining");+ } catch {}+ writer.abort(new Error("SSE client not draining")).catch(() => {});+ };+ const __markSseClientClosed = () => {+ if (__sseClosed) return;+ __sseClosed = true;+ __finished = true;+ try {+ clearInterval(keepAlive);+ clearTimeout(__responseDeadline);+ } catch {}+ writer.abort(new Error("SSE client disconnected")).catch(() => {});+ };+ writer.closed?.catch(() => {+ __markSseClientClosed();+ });+ request.signal.addEventListener("abort", __closeSse, { once: true });+ const __forwardSse = (frame) => {+ if (__sseClosed) return __writeChain;+ if (__pendingBytes + frame.byteLength > MAX_PENDING_SSE_BYTES) {+ __closeSse();+ return Promise.resolve();+ }+ __pendingBytes += frame.byteLength;+ __writeChain = __writeChain.then(() => writer.write(frame)).catch(() => {+ __closeSse();+ }).finally(() => {+ __pendingBytes -= frame.byteLength;+ });+ return __writeChain;+ };+ keepAlive = startKeepalive(__forwardSse, encoder);+ if (messages.every((msg) => isJSONRPCNotification(msg) || isJSONRPCResultResponse(msg))) {+ clearInterval(keepAlive);+ ws.close();+ return new Response(null, { headers: corsHeaders(request, options.corsOptions), status: 202- });- }- const keepAlive = startKeepalive(writer, encoder);- ws.addEventListener("message", (event) => {+ });+ }+ for (const msg of messages) if (isJSONRPCRequest(msg)) __outstanding.set(__idKey(msg.id), msg.id);+ /**+ * End the response stream when it will not end on its own, and+ * answer whatever is still outstanding first.+ *+ * The Durable Object can be reset out from under an in-flight+ * execute (isolate memory/CPU limit, storage timeout, deploy). The+ * front worker has already returned HTTP 200 with an SSE body, so+ * before this the WS close simply closed the writer and the client+ * saw a cleanly terminated stream carrying no JSON-RPC response —+ * indistinguishable from "still working" until its own timeout.+ *+ * The happy path is byte-identical: every id has been reported+ * responded by then, so the set is empty and nothing is written.+ */+ const __finishAbnormally = async (reason) => {+ if (__finished) return;+ __finished = true;+ clearTimeout(__responseDeadline);+ clearInterval(keepAlive);+ if (__outstanding.size > 0) {+ console.warn(JSON.stringify({+ event: "mcp_post_stream_lost",+ outstandingCount: __outstanding.size,+ reason,+ sessionId+ }));+ for (const id of __outstanding.values()) __forwardSse(encoder.encode(`event: message\ndata: ${JSON.stringify(sessionResetErrorResponse(id, reason))}\n\n`));+ __outstanding.clear();+ }+ const pending = __writeChain;+ __sseClosed = true;+ await pending.catch(() => {});+ await writer.close().catch(() => {});+ };+ __responseDeadline = setTimeout(() => {+ __finishAbnormally("response_deadline").catch(console.error);+ }, MCP_POST_RESPONSE_DEADLINE_MS);+ ws.addEventListener("message", (event) => { async function onMessage(event) { try {- const data = typeof event.data === "string" ? event.data : new TextDecoder().decode(event.data);- const message = JSON.parse(data);- if (message.type !== "cf_mcp_agent_event") return;- await writer.write(encoder.encode(message.event));- if (message.close) {- clearInterval(keepAlive);- ws?.close();- await writer.close().catch(() => {});- }+ const data = typeof event.data === "string" ? event.data : new TextDecoder().decode(event.data);+ const message = JSON.parse(data);+ if (message.type !== "cf_mcp_agent_event") return;+ if (Array.isArray(message.respondedIds)) for (const respondedId of message.respondedIds) __outstanding.delete(__idKey(respondedId));+ const writePromise = __forwardSse(encoder.encode(message.event));+ if (message.close) {+ __finished = true;+ clearTimeout(__responseDeadline);+ clearInterval(keepAlive);+ await writePromise;+ await writer.close();+ if (!__sseClosed && !request.signal.aborted) {+ // workerd resolves writer.close() even when the client+ // canceled the POST response body, and request.signal does+ // not reliably fire for that cancellation, so a successful+ // close is NOT proof of delivery. Never ack POST-stream+ // deliveries: the DO keeps each response persisted for its+ // own Last-Event-ID recovery or an explicitly cursorless+ // fresh GET. A client that DID receive the result closes the+ // POST body reader without recovery, so the worst case of+ // this at-least-once choice is a duplicate on that fresh-GET+ // fallback, never a response injected into another cursor's+ // recovery stream.+ ws?.close(1000, "SSE response delivered");+ }+ } } catch (error) { console.error("Error forwarding message to SSE:", error); } } onMessage(event).catch(console.error); });- ws.addEventListener("error", (error) => {- async function onError(_error) {- clearInterval(keepAlive);- await writer.close().catch(() => {});+ ws.addEventListener("error", (error) => {+ async function onError(_error) {+ // The DO end of the bridge failed. Nothing still+ // outstanding will ever be answered.+ await __finishAbnormally("session_reset"); } onError(error).catch(console.error); });- ws.addEventListener("close", () => {- async function onClose() {- clearInterval(keepAlive);- await writer.close().catch(() => {});+ ws.addEventListener("close", (event) => {+ async function onClose(closeEvent) {+ // 1000/"SSE response delivered" is this bridge's OWN+ // happy-path close coming back round. Every other code —+ // including no code at all, which is what a reset Durable+ // Object produces — means the far end went away, so+ // anything unanswered gets a session_reset error. The+ // outstanding set makes this safe either way: after a+ // normal delivery it is empty and nothing is written.+ if (closeEvent?.code === 1000 && closeEvent?.reason === "SSE response delivered") {+ __finished = true;+ clearTimeout(__responseDeadline);+ __sseClosed = true;+ clearInterval(keepAlive);+ await writer.close().catch(() => {});+ return;+ }+ await __finishAbnormally("session_reset"); }- onClose().catch(console.error);+ onClose(event).catch(console.error); }); return new Response(readable, { headers: {@@ -279,10 +458,16 @@ const createStreamingHttpHandler = (basePath, namespace, options = {}) => { id: null, jsonrpc: "2.0" }), { status: 400 });- const { readable, writable } = new TransformStream();- const writer = writable.getWriter();- const encoder = new TextEncoder();- const agent = await getAgentByName(namespace, `streamable-http:${sessionId}`, {+ const { readable, writable } = new TransformStream();+ const writer = writable.getWriter();+ const encoder = new TextEncoder();+ const __openedAt = Date.now();+ const MAX_PENDING_SSE_BYTES = 8 * 1024 * 1024;+ let __pendingBytes = 0;+ let __writeChain = Promise.resolve();+ let __sseClosed = false;+ let keepAlive;+ const agent = await getAgentByName(namespace, `streamable-http:${sessionId}`, { props: ctx.props, jurisdiction: options.jurisdiction });@@ -306,27 +491,116 @@ const createStreamingHttpHandler = (basePath, namespace, options = {}) => { if (!ws) { await writer.close(); return new Response("Failed to establish WS to DO", { status: 500 });- }- ws.accept();- ws.addEventListener("message", (event) => {+ }+ ws.accept();+ const __abortSse = () => {+ if (__sseClosed) return;+ __sseClosed = true;+ try {+ clearInterval(keepAlive);+ } catch {}+ try {+ ws.close(1013, "SSE client not draining");+ } catch {}+ writer.abort(new Error("SSE client not draining")).catch(() => {});+ };+ writer.closed?.catch(() => {+ __abortSse();+ });+ request.signal.addEventListener("abort", __abortSse, { once: true });+ const __closeSseGracefully = () => {+ if (__sseClosed) return __writeChain;+ __sseClosed = true;+ try {+ clearInterval(keepAlive);+ } catch {}+ try {+ const finalFrame = encoder.encode(": max-age rotation, reconnect\n\n");+ __writeChain = __writeChain.then(() => writer.write(finalFrame)).catch(() => {}).then(() => writer.close()).catch(() => {+ writer.abort().catch(() => {});+ });+ } catch {+ __writeChain = writer.close().catch(() => {+ writer.abort().catch(() => {});+ });+ }+ try {+ ws.close(1000, "sse_max_age_rotation");+ } catch {}+ return __writeChain;+ };+ const __forwardSse = (frame) => {+ if (__sseClosed) return __writeChain;+ const ageMs = Date.now() - __openedAt;+ if (ageMs > MAX_SSE_AGE_MS) {+ console.log(JSON.stringify({+ event: "sse_max_age_close",+ sessionId,+ variant: "streamable-get",+ ageMs,+ pendingBytes: __pendingBytes+ }));+ __closeSseGracefully();+ return Promise.resolve();+ }+ if (__pendingBytes + frame.byteLength > MAX_PENDING_SSE_BYTES) {+ __abortSse();+ return Promise.resolve();+ }+ __pendingBytes += frame.byteLength;+ __writeChain = __writeChain.then(() => writer.write(frame)).catch(() => {+ __abortSse();+ }).finally(() => {+ __pendingBytes -= frame.byteLength;+ });+ return __writeChain;+ };+ keepAlive = startKeepalive(__forwardSse, encoder);+ ws.addEventListener("message", (event) => { try { async function onMessage(ev) {- const data = typeof ev.data === "string" ? ev.data : new TextDecoder().decode(ev.data);- const message = JSON.parse(data);- if (message.type !== "cf_mcp_agent_event") return;- await writer.write(encoder.encode(message.event));- }+ const data = typeof ev.data === "string" ? ev.data : new TextDecoder().decode(ev.data);+ const message = JSON.parse(data);+ if (message.type !== "cf_mcp_agent_event") return;+ const writePromise = __forwardSse(encoder.encode(message.event));+ if (message.close) {+ clearInterval(keepAlive);+ await writePromise;+ await writer.close();+ if (!__sseClosed && !request.signal.aborted) {+ // Storage is cleared only on this ack, which fires only+ // after writer.close() resolved with the client still+ // attached; a replayed response enqueued into a dead GET+ // never acks and stays replayable. `ackStreamIds` lets a+ // replay-complete frame confirm several replayed streamsLines 1–500 of 1436. Load more
diff --git a/node_modules/agents/.bun-tag-37e67b70862be2b2 b/.bun-tag-37e67b70862be2b2 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/node_modules/agents/.bun-tag-61b2f1517ab5ced4 b/.bun-tag-61b2f1517ab5ced4 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/node_modules/agents/.bun-tag-61d29b56a6079f1d b/.bun-tag-61d29b56a6079f1d new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/node_modules/agents/.bun-tag-a6a47855632a2623 b/.bun-tag-a6a47855632a2623 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/node_modules/agents/.bun-tag-c0c639aa2299e502 b/.bun-tag-c0c639aa2299e502 new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/dist/agent-tool-types-CNyE1iz_.d.ts b/dist/agent-tool-types-CNyE1iz_.d.ts index 571eececebd5a1eaf7f2fbf5278801c4d34728ba..010811ae74ef2d27f12f7fb2506848f1939d91d6 100644 --- a/dist/agent-tool-types-CNyE1iz_.d.ts +++ b/dist/agent-tool-types-CNyE1iz_.d.ts @@ -480,7 +480,10 @@ declare class DurableObjectEventStore implements EventStore { private readonly seqByStream; private readonly seqInit; constructor(storage: DurableObjectStorage); - storeEvent(streamId: StreamId, message: JSONRPCMessage): Promise<EventId>; + /** Resolves `undefined` for a message too large for DO storage's 128 KiB + * per-value cap: the event is delivered live without a replay id rather + * than failing the send. */ + storeEvent(streamId: StreamId, message: JSONRPCMessage): Promise<EventId | undefined>; getStreamIdForEventId(eventId: EventId): Promise<StreamId | undefined>; replayEventsAfter( lastEventId: EventId, @@ -538,6 +541,42 @@ declare abstract class McpAgent< * @internal */ private static readonly STREAM_REQS_KEY_PREFIX; + /** + * Monotonic id for THIS Durable Object incarnation, bumped once per live + * instance. Ledger rows are stamped with it so a row that outlived the + * isolate which was going to answer it is recognisable as stale. + * + * @internal + */ + currentSessionEpoch(): Promise<number>; + /** + * Ledger rows stamped by an earlier incarnation: requests that nothing will + * ever answer. Read-only — the caller writes the client's error first. + * + * @internal + */ + getStaleEpochStreamRequestIds(): Promise< + ReadonlyArray<{ + readonly streamId: string; + readonly requestIds: RequestId[]; + readonly epoch: number; + readonly currentEpoch: number; + }> + >; + /** + * Every POST stream still awaiting a response, with the incarnation that + * accepted it. Callers that mean "work still running" must drop rows whose + * `epoch` is below {@link currentSessionEpoch}. + * + * @internal + */ + getOpenStreamRequestIds(): Promise< + ReadonlyArray<{ + readonly streamId: string; + readonly requestIds: RequestId[]; + readonly epoch: number; + }> + >; /** Persist the `requestIds` for a POST stream. @internal */ setStreamRequestIds(streamId: string, requestIds: RequestId[]): Promise<void>; /** Read the persisted `requestIds` for a POST stream. @internal */ diff --git a/dist/mcp/index.d.ts b/dist/mcp/index.d.ts index c8fad448e8797b89690a99d93490d1363851b225..d80f66f1532c29dda0a0477dc1ec9ef5fd1fca47 100644 --- a/dist/mcp/index.d.ts +++ b/dist/mcp/index.d.ts @@ -29,6 +29,8 @@ import { xt as MCPClientOAuthCallbackConfig, zt as ElicitResult } from "../agent-tool-types-CNyE1iz_.js"; +declare const MAX_SSE_AGE_MS = 1800000; +declare const MCP_POST_RESPONSE_DEADLINE_MS = 660000; export { type ClearableEventStore, type CreateMcpHandlerOptions, @@ -41,6 +43,8 @@ export { type MCPConnectionResult, type MCPDiscoverResult, type MCPServerOptions, + MAX_SSE_AGE_MS, + MCP_POST_RESPONSE_DEADLINE_MS, MCP_SERVER_ID_MAX_LENGTH, McpAgent, type McpAuthContext, diff --git a/dist/mcp/index.js b/dist/mcp/index.js index 1edcf0c8c9e67aa211ae515e7672cdf79912101e..231220c26b995a48e40dba993773c28786cca392 100644 --- a/dist/mcp/index.js +++ b/dist/mcp/index.js @@ -28,13 +28,60 @@ import { WebStandardStreamableHTTPServerTransport } from "@modelcontextprotocol/ const KEEPALIVE_INTERVAL_MS = 25e3; /** SSE comment frame the parser drops before any event dispatch. */ const KEEPALIVE_FRAME = ": keepalive\n\n"; +// Max age is a stalled-client memory backstop, not session idleness. Keep it +// well above Executor's 5 minute session idle timeout so active clients rarely +// rotate. +const MAX_SSE_AGE_MS = 30 * 60 * 1000; +/** +* JSON-RPC error code for "the isolate that was running this call is gone". +* In the implementation-defined server range (-32000..-32099) and deliberately +* NOT the -32001 the worker already uses for session/transport failures, so a +* client can tell "the call you are waiting on died mid-flight, re-run it" from +* "this session id is dead, reconnect". +*/ +const SESSION_RESET_ERROR_CODE = -32010; +const SESSION_RESET_ERROR_MESSAGE = "Execution lost: the session was reset before it produced a result. Re-run the call."; +/** +* How long the POST bridge waits for the Durable Object to answer a JSON-RPC +* request before answering for it. +* +* This file is a vendored dist and cannot import the two constants that +* actually bound how long an answer may legitimately take, so this MUST be kept +* at or above their sum: +* +* DEFAULT_TIMEOUT_MS packages/kernel/runtime-dynamic-worker/src/executor.ts +* (5 min sandbox execution ceiling) +* BROWSER_APPROVAL_WAIT_TIMEOUT_MS packages/hosts/mcp/src/tool-server.ts +* (PAUSED_APPROVAL_TIMEOUT_MS + 1s = 4 min 1 s) +* +* plus ~60s of margin for DO restore, storage and edge latency. 11 minutes +* clears 10 min 1 s. Drift is caught by +* packages/hosts/cloudflare/src/mcp/agents-post-stream-loss.test.ts, +* which recomputes the sum from those two source files. +*/ +const MCP_POST_RESPONSE_DEADLINE_MS = 11 * 60 * 1000; +/** +* The JSON-RPC error response sent for a request that will never get a real +* answer. `reason` distinguishes the two ways we find out: `session_reset` +* (the DO end of the bridge went away, or a ledger row outlived its +* incarnation) and `response_deadline` (nothing came back in time). +*/ +const sessionResetErrorResponse = (id, reason) => ({ + error: { + code: SESSION_RESET_ERROR_CODE, + data: { reason }, + message: SESSION_RESET_ERROR_MESSAGE + }, + id, + jsonrpc: "2.0" +}); /** * Start an SSE keepalive on `writer`. Returns a `clearInterval` handle * that the stream cleanup must invoke when the stream closes. */ -function startKeepalive(writer, encoder) { +function startKeepalive(writeFrame, encoder) { const handle = setInterval(() => { - writer.write(encoder.encode(KEEPALIVE_FRAME)).catch(() => clearInterval(handle)); + writeFrame(encoder.encode(KEEPALIVE_FRAME)); }, KEEPALIVE_INTERVAL_MS); return handle; } @@ -180,10 +227,30 @@ const createStreamingHttpHandler = (basePath, namespace, options = {}) => { }); return new Response(body, { status: 404 }); } - const { readable, writable } = new TransformStream(); - const writer = writable.getWriter(); - const encoder = new TextEncoder(); - const existingHeaders = {}; + const { readable, writable } = new TransformStream(); + const writer = writable.getWriter(); + const encoder = new TextEncoder(); + const MAX_PENDING_SSE_BYTES = 8 * 1024 * 1024; + let __pendingBytes = 0; + let __writeChain = Promise.resolve(); + let __sseClosed = false; + let keepAlive; + let __responseDeadline; + /** + * JSON-RPC ids this POST still owes the client a response for. + * Entries are removed as the DO reports responses (the + * `respondedIds` field writeSSEEvent puts on the envelope). + * + * A non-empty set when the bridge finishes abnormally is exactly + * the "lost execution" case: the client already holds a 200 with + * an open SSE body, so a silently ended stream leaves it waiting + * forever. Keyed by type+value because JSON-RPC ids may be + * strings or numbers and 1 must not match "1". + */ + const __outstanding = new Map(); + const __idKey = (id) => `${typeof id}:${String(id)}`; + let __finished = false; + const existingHeaders = {}; request.headers.forEach((value, key) => { existingHeaders[key] = value; }); @@ -206,47 +273,159 @@ const createStreamingHttpHandler = (basePath, namespace, options = {}) => { jsonrpc: "2.0" }); return new Response(body, { status: 500 }); - } - ws.accept(); - if (messages.every((msg) => isJSONRPCNotification(msg) || isJSONRPCResultResponse(msg))) { - ws.close(); - return new Response(null, { + } + ws.accept(); + const __closeSse = () => { + if (__sseClosed) return; + __sseClosed = true; + __finished = true; + try { + clearInterval(keepAlive); + clearTimeout(__responseDeadline); + } catch {} + try { + ws.close(1013, "SSE client not draining"); + } catch {} + writer.abort(new Error("SSE client not draining")).catch(() => {}); + }; + const __markSseClientClosed = () => { + if (__sseClosed) return; + __sseClosed = true; + __finished = true; + try { + clearInterval(keepAlive); + clearTimeout(__responseDeadline); + } catch {} + writer.abort(new Error("SSE client disconnected")).catch(() => {}); + }; + writer.closed?.catch(() => { + __markSseClientClosed(); + }); + request.signal.addEventListener("abort", __closeSse, { once: true }); + const __forwardSse = (frame) => { + if (__sseClosed) return __writeChain; + if (__pendingBytes + frame.byteLength > MAX_PENDING_SSE_BYTES) { + __closeSse(); + return Promise.resolve(); + } + __pendingBytes += frame.byteLength; + __writeChain = __writeChain.then(() => writer.write(frame)).catch(() => { + __closeSse(); + }).finally(() => { + __pendingBytes -= frame.byteLength; + }); + return __writeChain; + }; + keepAlive = startKeepalive(__forwardSse, encoder); + if (messages.every((msg) => isJSONRPCNotification(msg) || isJSONRPCResultResponse(msg))) { + clearInterval(keepAlive); + ws.close(); + return new Response(null, { headers: corsHeaders(request, options.corsOptions), status: 202 - }); - } - const keepAlive = startKeepalive(writer, encoder); - ws.addEventListener("message", (event) => { + }); + } + for (const msg of messages) if (isJSONRPCRequest(msg)) __outstanding.set(__idKey(msg.id), msg.id); + /** + * End the response stream when it will not end on its own, and + * answer whatever is still outstanding first. + * + * The Durable Object can be reset out from under an in-flight + * execute (isolate memory/CPU limit, storage timeout, deploy). The + * front worker has already returned HTTP 200 with an SSE body, so + * before this the WS close simply closed the writer and the client + * saw a cleanly terminated stream carrying no JSON-RPC response — + * indistinguishable from "still working" until its own timeout. + * + * The happy path is byte-identical: every id has been reported + * responded by then, so the set is empty and nothing is written. + */ + const __finishAbnormally = async (reason) => { + if (__finished) return; + __finished = true; + clearTimeout(__responseDeadline); + clearInterval(keepAlive); + if (__outstanding.size > 0) { + console.warn(JSON.stringify({ + event: "mcp_post_stream_lost", + outstandingCount: __outstanding.size, + reason, + sessionId + })); + for (const id of __outstanding.values()) __forwardSse(encoder.encode(`event: message\ndata: ${JSON.stringify(sessionResetErrorResponse(id, reason))}\n\n`)); + __outstanding.clear(); + } + const pending = __writeChain; + __sseClosed = true; + await pending.catch(() => {}); + await writer.close().catch(() => {}); + }; + __responseDeadline = setTimeout(() => { + __finishAbnormally("response_deadline").catch(console.error); + }, MCP_POST_RESPONSE_DEADLINE_MS); + ws.addEventListener("message", (event) => { async function onMessage(event) { try { - const data = typeof event.data === "string" ? event.data : new TextDecoder().decode(event.data); - const message = JSON.parse(data); - if (message.type !== "cf_mcp_agent_event") return; - await writer.write(encoder.encode(message.event)); - if (message.close) { - clearInterval(keepAlive); - ws?.close(); - await writer.close().catch(() => {}); - } + const data = typeof event.data === "string" ? event.data : new TextDecoder().decode(event.data); + const message = JSON.parse(data); + if (message.type !== "cf_mcp_agent_event") return; + if (Array.isArray(message.respondedIds)) for (const respondedId of message.respondedIds) __outstanding.delete(__idKey(respondedId)); + const writePromise = __forwardSse(encoder.encode(message.event)); + if (message.close) { + __finished = true; + clearTimeout(__responseDeadline); + clearInterval(keepAlive); + await writePromise; + await writer.close(); + if (!__sseClosed && !request.signal.aborted) { + // workerd resolves writer.close() even when the client + // canceled the POST response body, and request.signal does + // not reliably fire for that cancellation, so a successful + // close is NOT proof of delivery. Never ack POST-stream + // deliveries: the DO keeps each response persisted for its + // own Last-Event-ID recovery or an explicitly cursorless + // fresh GET. A client that DID receive the result closes the + // POST body reader without recovery, so the worst case of + // this at-least-once choice is a duplicate on that fresh-GET + // fallback, never a response injected into another cursor's + // recovery stream. + ws?.close(1000, "SSE response delivered"); + } + } } catch (error) { console.error("Error forwarding message to SSE:", error); } } onMessage(event).catch(console.error); }); - ws.addEventListener("error", (error) => { - async function onError(_error) { - clearInterval(keepAlive); - await writer.close().catch(() => {}); + ws.addEventListener("error", (error) => { + async function onError(_error) { + // The DO end of the bridge failed. Nothing still + // outstanding will ever be answered. + await __finishAbnormally("session_reset"); } onError(error).catch(console.error); }); - ws.addEventListener("close", () => { - async function onClose() { - clearInterval(keepAlive); - await writer.close().catch(() => {}); + ws.addEventListener("close", (event) => { + async function onClose(closeEvent) { + // 1000/"SSE response delivered" is this bridge's OWN + // happy-path close coming back round. Every other code — + // including no code at all, which is what a reset Durable + // Object produces — means the far end went away, so + // anything unanswered gets a session_reset error. The + // outstanding set makes this safe either way: after a + // normal delivery it is empty and nothing is written. + if (closeEvent?.code === 1000 && closeEvent?.reason === "SSE response delivered") { + __finished = true; + clearTimeout(__responseDeadline); + __sseClosed = true; + clearInterval(keepAlive); + await writer.close().catch(() => {}); + return; + } + await __finishAbnormally("session_reset"); } - onClose().catch(console.error); + onClose(event).catch(console.error); }); return new Response(readable, { headers: { @@ -279,10 +458,16 @@ const createStreamingHttpHandler = (basePath, namespace, options = {}) => { id: null, jsonrpc: "2.0" }), { status: 400 }); - const { readable, writable } = new TransformStream(); - const writer = writable.getWriter(); - const encoder = new TextEncoder(); - const agent = await getAgentByName(namespace, `streamable-http:${sessionId}`, { + const { readable, writable } = new TransformStream(); + const writer = writable.getWriter(); + const encoder = new TextEncoder(); + const __openedAt = Date.now(); + const MAX_PENDING_SSE_BYTES = 8 * 1024 * 1024; + let __pendingBytes = 0; + let __writeChain = Promise.resolve(); + let __sseClosed = false; + let keepAlive; + const agent = await getAgentByName(namespace, `streamable-http:${sessionId}`, { props: ctx.props, jurisdiction: options.jurisdiction }); @@ -306,27 +491,116 @@ const createStreamingHttpHandler = (basePath, namespace, options = {}) => { if (!ws) { await writer.close(); return new Response("Failed to establish WS to DO", { status: 500 }); - } - ws.accept(); - ws.addEventListener("message", (event) => { + } + ws.accept(); + const __abortSse = () => { + if (__sseClosed) return; + __sseClosed = true; + try { + clearInterval(keepAlive); + } catch {} + try { + ws.close(1013, "SSE client not draining"); + } catch {} + writer.abort(new Error("SSE client not draining")).catch(() => {}); + }; + writer.closed?.catch(() => { + __abortSse(); + }); + request.signal.addEventListener("abort", __abortSse, { once: true }); + const __closeSseGracefully = () => { + if (__sseClosed) return __writeChain; + __sseClosed = true; + try { + clearInterval(keepAlive); + } catch {} + try { + const finalFrame = encoder.encode(": max-age rotation, reconnect\n\n"); + __writeChain = __writeChain.then(() => writer.write(finalFrame)).catch(() => {}).then(() => writer.close()).catch(() => { + writer.abort().catch(() => {}); + }); + } catch { + __writeChain = writer.close().catch(() => { + writer.abort().catch(() => {}); + }); + } + try { + ws.close(1000, "sse_max_age_rotation"); + } catch {} + return __writeChain; + }; + const __forwardSse = (frame) => { + if (__sseClosed) return __writeChain; + const ageMs = Date.now() - __openedAt; + if (ageMs > MAX_SSE_AGE_MS) { + console.log(JSON.stringify({ + event: "sse_max_age_close", + sessionId, + variant: "streamable-get", + ageMs, + pendingBytes: __pendingBytes + })); + __closeSseGracefully(); + return Promise.resolve(); + } + if (__pendingBytes + frame.byteLength > MAX_PENDING_SSE_BYTES) { + __abortSse(); + return Promise.resolve(); + } + __pendingBytes += frame.byteLength; + __writeChain = __writeChain.then(() => writer.write(frame)).catch(() => { + __abortSse(); + }).finally(() => { + __pendingBytes -= frame.byteLength; + }); + return __writeChain; + }; + keepAlive = startKeepalive(__forwardSse, encoder); + ws.addEventListener("message", (event) => { try { async function onMessage(ev) { - const data = typeof ev.data === "string" ? ev.data : new TextDecoder().decode(ev.data); - const message = JSON.parse(data); - if (message.type !== "cf_mcp_agent_event") return; - await writer.write(encoder.encode(message.event)); - } + const data = typeof ev.data === "string" ? ev.data : new TextDecoder().decode(ev.data); + const message = JSON.parse(data); + if (message.type !== "cf_mcp_agent_event") return; + const writePromise = __forwardSse(encoder.encode(message.event)); + if (message.close) { + clearInterval(keepAlive); + await writePromise; + await writer.close(); + if (!__sseClosed && !request.signal.aborted) { + // Storage is cleared only on this ack, which fires only + // after writer.close() resolved with the client still + // attached; a replayed response enqueued into a dead GET + // never acks and stays replayable. `ackStreamIds` lets a + // replay-complete frame confirm several replayed streams