/** * Classification of Cloudflare *platform* Durable Object failures. * * A Durable Object can be torn out from under its own code by the runtime, and * when that happens the only thing the caller receives is a plain `Error` whose * message is the entire signal. Those are not application defects: the object * was healthy, the request was valid, and the correct answer to the client is * "this id is dead, reconnect" or "come back in a moment" — never an unhandled * 500 and never an error report. * * The codebase already classifies transient-vs-definitive failures carefully * for the auth provider; this is the same discipline for the one other * dependency that fails without a typed error: the DO platform itself. * * Everything not listed here stays UNCLASSIFIED on purpose. An unrecognized * failure keeps its current behaviour — rethrow, report, page — because the * cost of silently swallowing a real defect is much higher than the cost of one * more retryable envelope. */import { Cause } from "effect"; import { jsonRpcErrorBody, UNAVAILABLE_RETRY_AFTER_SECONDS } from "@executor-js/host-mcp"; /** * The specific platform condition, kept granular so it can be recorded on a * span (`mcp.do.reset_kind`) and the aggregate bucket split by cause in * production instead of being one opaque pile. */export type DurableObjectFailureKind = /** The object called `ctx.abort("destroyed")` on itself — session teardown. */ | "destroyed" /** A deploy replaced the script and the runtime reset every live object. */ | "code_update" /** A storage op ran past the platform's ceiling; the object was reset. */ | "storage_timeout" /** The storage backend failed internally and reset the object. */ | "storage_internal" /** The storage backend failed internally *while bringing the object up*. */ | "startup_internal_error" /** `blockConcurrencyWhile()` ran past its cap and was cancelled. */ | "concurrency_reset" /** An invocation ran past the per-invocation CPU ceiling; the object was reset. */ | "cpu_limit" /** A generic platform blip: `internal error; reference = <id>`. */ | "internal_error" /** The runtime itself flagged the error as retryable. */ | "retryable"; /** * What the caller should do about it. * * - `session_dead` — the object is gone for good; the id will never work again, * so the client must mint a new session. * - `transient` — the object was reset but the id is still valid; the very next * attempt is likely to succeed. */export type DurableObjectFailureDisposition = "session_dead" | "transient"; export type DurableObjectFailure = { readonly kind: DurableObjectFailureKind; readonly disposition: DurableObjectFailureDisposition;}; /** * Message fragments workerd puts on the plain `Error` it throws. Matched * case-insensitively on a substring because the runtime appends ids and * punctuation ("… reference = 0123abcd") and has reworded these before. */const MESSAGE_PATTERNS: ReadonlyArray<{ readonly fragment: string; readonly failure: DurableObjectFailure;}> = [ { fragment: "durable object reset because its code was updated", failure: { kind: "code_update", disposition: "transient" }, }, { fragment: "storage operation exceeded timeout", failure: { kind: "storage_timeout", disposition: "transient" }, }, { fragment: "internal error in durable object storage", failure: { kind: "storage_internal", disposition: "transient" }, }, { // The startup sibling of the entry above: the storage backend faults while // the object is being brought up rather than while it is serving, and the // runtime says so in a different clause ("… while starting up Durable // Object storage caused object to be reset; reference = <id>"). // // Deliberately stops before "storage": the two known members of this family // agree on "Internal error … Durable Object" and disagree on everything // that follows the verb, so the qualifier is the part most likely to move // and is not what identifies the failure. Equally deliberately NOT // shortened to the shared tail "caused object to be reset" — that tail is // common to several unrelated storage faults and would stop this bucket // from meaning anything on a span. // // Transient for the same reason as its sibling: nothing about the request // caused it, the id still routes, and the object gets a fresh start on the // next attempt. fragment: "internal error while starting up durable object", failure: { kind: "startup_internal_error", disposition: "transient" }, }, { // Deliberately the whole phrase, not the bare method name: an application // defect thrown from inside a `blockConcurrencyWhile` callback also resets // the object, and a message that merely names the method must not be read // as a platform reset and quietly turned into a retry. fragment: "blockconcurrencywhile() in a durable object waited for too long", failure: { kind: "concurrency_reset", disposition: "transient" }, }, { // Deliberately starts at the verb, not at "Durable Object": the runtime's // resource-limit messages disagree about the subject noun (the memory // variant says "Durable Object's isolate exceeded its memory limit"), and // pinning a subject here would let a rewording defeat the match. From // "exceeded its CPU time limit" onward the phrase is the runtime's alone. // // Transient, not a defect: the invocation was cut off but the object's // durable storage is untouched and the session id still routes, so the next // attempt — a smaller unit of work, or the same one under a warm isolate — // can succeed. Retrying the same id is strictly better than the unhandled // 500 this produced before. fragment: "exceeded its cpu time limit and was reset", failure: { kind: "cpu_limit", disposition: "transient" }, }, { // Only the bare blip. A reference id at the end of the message is NOT the // marker: the runtime also appends one to described faults such as the // startup failure above, and because this fragment includes the semicolon // that immediately follows "internal error", any interposed description // defeats it. Those variants each need their own entry rather than a // loosened version of this one, which would turn every referenced error // into an opaque "internal_error" bucket. fragment: "internal error; reference =", failure: { kind: "internal_error", disposition: "transient" }, }, // Not listed, on purpose: the sibling memory-limit reset ("Durable Object's // isolate exceeded its memory limit due to overflowing the storage cache … // All objects in the isolate were reset."). The runtime tags that one as a // user error, and it names its own cause — too many un-awaited writes, or one // oversized read. Retrying reproduces it, so calling it transient would bury // an application defect behind a 503 instead of surfacing it.]; /** * `ctx.abort("destroyed")` surfaces as an `Error` whose message is exactly the * abort reason, so this one is matched whole rather than as a substring — a * message that merely mentions the word must not condemn a live session. */const DESTROYED_MESSAGE = "destroyed"; /** How deep to follow `error.cause` before giving up. */const MAX_UNWRAP_DEPTH = 5; const messageOf = (error: unknown): string | null => { if (typeof error === "string") return error; if (typeof error !== "object" || error === null) return null; // oxlint-disable-next-line executor/no-unknown-error-message -- platform boundary: workerd rejects with an untyped Error whose message IS the only signal; reading it here is the entire purpose of this module, and it exists so no other file has to const message = (error as { readonly message?: unknown }).message; return typeof message === "string" ? message : null;}; const isRuntimeRetryable = (error: unknown): boolean => typeof error === "object" && error !== null && (error as { readonly retryable?: unknown }).retryable === true; const classifyOne = (error: unknown): DurableObjectFailure | null => { const message = messageOf(error); if (message !== null) { const normalized = message.trim().toLowerCase(); if (normalized === DESTROYED_MESSAGE) { return { kind: "destroyed", disposition: "session_dead" }; } for (const pattern of MESSAGE_PATTERNS) { if (normalized.includes(pattern.fragment)) return pattern.failure; } } // Checked last: the message is the more specific signal, and the runtime sets // `retryable` on some of the same errors. if (isRuntimeRetryable(error)) return { kind: "retryable", disposition: "transient" }; return null;}; /** * Recognize a Cloudflare platform Durable Object failure. * * Accepts whatever the caller happens to be holding: the raw `Error` a stub RPC * rejected with, an `Error` that wrapped it as its `cause`, or an Effect * `Cause` — the DO seam only ever sees the last of those, and forcing every * call site to unwrap first is how a classifier ends up with three subtly * different copies. * * Returns `null` for anything unrecognized — the caller must then treat the * error exactly as it did before this module existed. */export const classifyDurableObjectError = (error: unknown): DurableObjectFailure | null => { if (Cause.isCause(error)) { for (const inner of Cause.prettyErrors(error)) { const failure = classifyDurableObjectError(inner); if (failure) return failure; } return null; } let current: unknown = error; for (let depth = 0; depth < MAX_UNWRAP_DEPTH && current !== null && current !== undefined; ) { const failure = classifyOne(current); if (failure) return failure; depth += 1; if (typeof current !== "object") break; current = (current as { readonly cause?: unknown }).cause; } return null;}; /** * Render a classified platform failure as the MCP protocol error the client * should act on. * * Both branches reuse envelopes the MCP host already speaks, because the client * side of this is not new: a dead session id has always been "reconnect", and a * transient dependency failure has always been a 503 carrying `Retry-After`. * The only thing that was missing is that a platform reset never reached * either, and fell out of the worker as an unhandled 500 instead. * * The JSON-RPC code is -32001 for both; the HTTP STATUS is the discriminator * clients act on — 404 = the id is dead, mint a new session; 503 = retry the * SAME id after the advertised delay. */export const durableObjectFailureResponse = (failure: DurableObjectFailure): Response => failure.disposition === "session_dead" ? jsonRpcErrorBody(404, -32001, "Session timed out, please reconnect") : jsonRpcErrorBody(503, -32001, "MCP session is restarting, please retry", { retryAfterSeconds: UNAVAILABLE_RETRY_AFTER_SECONDS, });
/**
* Classification of Cloudflare *platform* Durable Object failures.
*
* A Durable Object can be torn out from under its own code by the runtime, and
* when that happens the only thing the caller receives is a plain `Error` whose
* message is the entire signal. Those are not application defects: the object
* was healthy, the request was valid, and the correct answer to the client is
* "this id is dead, reconnect" or "come back in a moment" — never an unhandled
* 500 and never an error report.
*
* The codebase already classifies transient-vs-definitive failures carefully
* for the auth provider; this is the same discipline for the one other
* dependency that fails without a typed error: the DO platform itself.
*
* Everything not listed here stays UNCLASSIFIED on purpose. An unrecognized
* failure keeps its current behaviour — rethrow, report, page — because the
* cost of silently swallowing a real defect is much higher than the cost of one
* more retryable envelope.
*/
import { Cause } from "effect";
import { jsonRpcErrorBody, UNAVAILABLE_RETRY_AFTER_SECONDS } from "@executor-js/host-mcp";
/**
* The specific platform condition, kept granular so it can be recorded on a
* span (`mcp.do.reset_kind`) and the aggregate bucket split by cause in
* production instead of being one opaque pile.
*/
export type DurableObjectFailureKind =
/** The object called `ctx.abort("destroyed")` on itself — session teardown. */
| "destroyed"
/** A deploy replaced the script and the runtime reset every live object. */
| "code_update"
/** A storage op ran past the platform's ceiling; the object was reset. */
| "storage_timeout"
/** The storage backend failed internally and reset the object. */
| "storage_internal"
/** The storage backend failed internally *while bringing the object up*. */
| "startup_internal_error"
/** `blockConcurrencyWhile()` ran past its cap and was cancelled. */
| "concurrency_reset"
/** An invocation ran past the per-invocation CPU ceiling; the object was reset. */
| "cpu_limit"
/** A generic platform blip: `internal error; reference = <id>`. */
| "internal_error"
/** The runtime itself flagged the error as retryable. */
| "retryable";
/**
* What the caller should do about it.
*
* - `session_dead` — the object is gone for good; the id will never work again,
* so the client must mint a new session.
* - `transient` — the object was reset but the id is still valid; the very next
* attempt is likely to succeed.
*/
export type DurableObjectFailureDisposition = "session_dead" | "transient";
export type DurableObjectFailure = {
readonly kind: DurableObjectFailureKind;
readonly disposition: DurableObjectFailureDisposition;
};
/**
* Message fragments workerd puts on the plain `Error` it throws. Matched
* case-insensitively on a substring because the runtime appends ids and
* punctuation ("… reference = 0123abcd") and has reworded these before.
*/
const MESSAGE_PATTERNS: ReadonlyArray<{
readonly fragment: string;
readonly failure: DurableObjectFailure;
}> = [
{
fragment: "durable object reset because its code was updated",
failure: { kind: "code_update", disposition: "transient" },
},
{
fragment: "storage operation exceeded timeout",
failure: { kind: "storage_timeout", disposition: "transient" },
},
{
fragment: "internal error in durable object storage",
failure: { kind: "storage_internal", disposition: "transient" },
},
{
// The startup sibling of the entry above: the storage backend faults while
// the object is being brought up rather than while it is serving, and the
// runtime says so in a different clause ("… while starting up Durable
// Object storage caused object to be reset; reference = <id>").
//
// Deliberately stops before "storage": the two known members of this family
// agree on "Internal error … Durable Object" and disagree on everything
// that follows the verb, so the qualifier is the part most likely to move
// and is not what identifies the failure. Equally deliberately NOT
// shortened to the shared tail "caused object to be reset" — that tail is
// common to several unrelated storage faults and would stop this bucket
// from meaning anything on a span.
//
// Transient for the same reason as its sibling: nothing about the request
// caused it, the id still routes, and the object gets a fresh start on the
// next attempt.
fragment: "internal error while starting up durable object",
failure: { kind: "startup_internal_error", disposition: "transient" },
},
{
// Deliberately the whole phrase, not the bare method name: an application
// defect thrown from inside a `blockConcurrencyWhile` callback also resets
// the object, and a message that merely names the method must not be read
// as a platform reset and quietly turned into a retry.
fragment: "blockconcurrencywhile() in a durable object waited for too long",
failure: { kind: "concurrency_reset", disposition: "transient" },
},
{
// Deliberately starts at the verb, not at "Durable Object": the runtime's
// resource-limit messages disagree about the subject noun (the memory
// variant says "Durable Object's isolate exceeded its memory limit"), and
// pinning a subject here would let a rewording defeat the match. From
// "exceeded its CPU time limit" onward the phrase is the runtime's alone.
//
// Transient, not a defect: the invocation was cut off but the object's
// durable storage is untouched and the session id still routes, so the next
// attempt — a smaller unit of work, or the same one under a warm isolate —
// can succeed. Retrying the same id is strictly better than the unhandled
// 500 this produced before.
fragment: "exceeded its cpu time limit and was reset",
failure: { kind: "cpu_limit", disposition: "transient" },
},
{
// Only the bare blip. A reference id at the end of the message is NOT the
// marker: the runtime also appends one to described faults such as the
// startup failure above, and because this fragment includes the semicolon
// that immediately follows "internal error", any interposed description
// defeats it. Those variants each need their own entry rather than a
// loosened version of this one, which would turn every referenced error
// into an opaque "internal_error" bucket.
fragment: "internal error; reference =",
failure: { kind: "internal_error", disposition: "transient" },
},
// Not listed, on purpose: the sibling memory-limit reset ("Durable Object's
// isolate exceeded its memory limit due to overflowing the storage cache …
// All objects in the isolate were reset."). The runtime tags that one as a
// user error, and it names its own cause — too many un-awaited writes, or one
// oversized read. Retrying reproduces it, so calling it transient would bury
// an application defect behind a 503 instead of surfacing it.
];
/**
* `ctx.abort("destroyed")` surfaces as an `Error` whose message is exactly the
* abort reason, so this one is matched whole rather than as a substring — a
* message that merely mentions the word must not condemn a live session.
*/
const DESTROYED_MESSAGE = "destroyed";
/** How deep to follow `error.cause` before giving up. */
const MAX_UNWRAP_DEPTH = 5;
const messageOf = (error: unknown): string | null => {
if (typeof error === "string") return error;
if (typeof error !== "object" || error === null) return null;
// oxlint-disable-next-line executor/no-unknown-error-message -- platform boundary: workerd rejects with an untyped Error whose message IS the only signal; reading it here is the entire purpose of this module, and it exists so no other file has to
const message = (error as { readonly message?: unknown }).message;
return typeof message === "string" ? message : null;
};
const isRuntimeRetryable = (error: unknown): boolean =>
typeof error === "object" &&
error !== null &&
(error as { readonly retryable?: unknown }).retryable === true;
const classifyOne = (error: unknown): DurableObjectFailure | null => {
const message = messageOf(error);
if (message !== null) {
const normalized = message.trim().toLowerCase();
if (normalized === DESTROYED_MESSAGE) {
return { kind: "destroyed", disposition: "session_dead" };
}
for (const pattern of MESSAGE_PATTERNS) {
if (normalized.includes(pattern.fragment)) return pattern.failure;
}
}
// Checked last: the message is the more specific signal, and the runtime sets
// `retryable` on some of the same errors.
if (isRuntimeRetryable(error)) return { kind: "retryable", disposition: "transient" };
return null;
};
/**
* Recognize a Cloudflare platform Durable Object failure.
*
* Accepts whatever the caller happens to be holding: the raw `Error` a stub RPC
* rejected with, an `Error` that wrapped it as its `cause`, or an Effect
* `Cause` — the DO seam only ever sees the last of those, and forcing every
* call site to unwrap first is how a classifier ends up with three subtly
* different copies.
*
* Returns `null` for anything unrecognized — the caller must then treat the
* error exactly as it did before this module existed.
*/
export const classifyDurableObjectError = (error: unknown): DurableObjectFailure | null => {
if (Cause.isCause(error)) {
for (const inner of Cause.prettyErrors(error)) {
const failure = classifyDurableObjectError(inner);
if (failure) return failure;
}
return null;
}
let current: unknown = error;
for (let depth = 0; depth < MAX_UNWRAP_DEPTH && current !== null && current !== undefined; ) {
const failure = classifyOne(current);
if (failure) return failure;
depth += 1;
if (typeof current !== "object") break;
current = (current as { readonly cause?: unknown }).cause;
}
return null;
};
/**
* Render a classified platform failure as the MCP protocol error the client
* should act on.
*
* Both branches reuse envelopes the MCP host already speaks, because the client
* side of this is not new: a dead session id has always been "reconnect", and a
* transient dependency failure has always been a 503 carrying `Retry-After`.
* The only thing that was missing is that a platform reset never reached
* either, and fell out of the worker as an unhandled 500 instead.
*
* The JSON-RPC code is -32001 for both; the HTTP STATUS is the discriminator
* clients act on — 404 = the id is dead, mint a new session; 503 = retry the
* SAME id after the advertised delay.
*/
export const durableObjectFailureResponse = (failure: DurableObjectFailure): Response =>
failure.disposition === "session_dead"
? jsonRpcErrorBody(404, -32001, "Session timed out, please reconnect")
: jsonRpcErrorBody(503, -32001, "MCP session is restarting, please retry", {
retryAfterSeconds: UNAVAILABLE_RETRY_AFTER_SECONDS,
});