// --------------------------------------------------------------------------- // Cloud-app implementation of the shared `ErrorCapture` service. This is the // only file in the cloud-app that imports `@sentry/cloudflare` for error // capture — handlers, plugin SDKs, and storage code all stay // Sentry-agnostic and request the `ErrorCapture` tag instead. // // `withObservability` (in @executor-js/api) wraps every handler effect; when // it sees an unmapped cause it asks `ErrorCapture.captureException` for a // trace id and fails with `InternalError({ traceId })`. The client gets // the opaque id, we get the full cause + stack in Sentry. // --------------------------------------------------------------------------- import * as Sentry from "@sentry/cloudflare"; import type { ErrorEvent, Scope } from "@sentry/cloudflare"; import { Cause, Effect, Layer, Predicate } from "effect"; import type * as Tracer from "effect/Tracer"; import { ErrorCapture } from "@executor-js/api"; import { classifyDurableObjectError } from "@executor-js/cloudflare/mcp/durable-object-errors"; import { withStableGroupingFingerprint } from "@executor-js/sdk/sentry-grouping"; // Drizzle/postgres-js include the failing SQL (params + bound values) in // their error message. For OpenAPI source inserts that's 1MB+ of spec // text which blows past terminal scrollback and hides the actual pg // error. Sentry still receives the full, untruncated cause via // `setExtra`; only the dev-console mirror is capped. const MAX_CONSOLE_CAUSE_CHARS = 4_000; const OTEL_TRACE_ID_PATTERN = /^[0-9a-f]{32}$/; const OTEL_SPAN_ID_PATTERN = /^[0-9a-f]{16}$/; export const OTEL_TRACE_ID_TAG = "otel_trace_id"; export const OTEL_SPAN_ID_TAG = "otel_span_id"; export const SENTRY_EVENT_ID_ATTRIBUTE = "sentry.event_id"; /** * Set by the MCP session Durable Object when it has finished deciding what to * do about a cause — reported it, or classified it as an expected Cloudflare * platform reset and deliberately not reported it. * * `instrumentDurableObjectWithSentry` wraps the DO's entry points and captures * the same rejection again as it escapes, which is why one platform reset * opened two issues for the same event. The DO is the better owner — it has * the session, the org, the OTEL correlation and the classification — so its * claim wins and the auto-instrumentation's echo is dropped in `beforeSend`. * Nothing the DO does not claim is affected. */ export const DO_CAUSE_OWNER_TAG = "mcp.do.cause_owner"; export const DO_CAUSE_OWNER_VALUE = "durable_object"; export type OtelCorrelationContext = { readonly traceId: string; readonly spanId: string; }; const truncate = (s: string): string => s.length <= MAX_CONSOLE_CAUSE_CHARS ? s : `${s.slice(0, MAX_CONSOLE_CAUSE_CHARS)}\n…[truncated ${s.length - MAX_CONSOLE_CAUSE_CHARS} chars]`; const validOtelContext = (context: OtelCorrelationContext): boolean => OTEL_TRACE_ID_PATTERN.test(context.traceId) && OTEL_SPAN_ID_PATTERN.test(context.spanId); export const otelCorrelationContextFromEffectSpan = ( span: Tracer.Span, ): OtelCorrelationContext | null => { const context = { traceId: span.traceId, spanId: span.spanId }; return validOtelContext(context) ? context : null; }; export const otelCorrelationContextFromOpenTelemetrySpan = (span: { readonly spanContext: () => { readonly traceId: string; readonly spanId: string }; }): OtelCorrelationContext | null => { const { traceId, spanId } = span.spanContext(); const context = { traceId, spanId }; return validOtelContext(context) ? context : null; }; export const addOtelCorrelationTags = }>( event: T, context: OtelCorrelationContext | null, ): T => { if (!context) return event; return { ...event, tags: { ...event.tags, [OTEL_TRACE_ID_TAG]: context.traceId, [OTEL_SPAN_ID_TAG]: context.spanId, }, }; }; export const tagSentryScopeWithOtelContext = ( scope: Scope, context: OtelCorrelationContext | null, ): void => { if (!context) return; scope.setTag(OTEL_TRACE_ID_TAG, context.traceId); scope.setTag(OTEL_SPAN_ID_TAG, context.spanId); }; export const tagCurrentSentryScopeWithOtelContext = ( context: OtelCorrelationContext | null, ): void => { tagSentryScopeWithOtelContext(Sentry.getCurrentScope(), context); }; const currentOtelContext = Effect.map( Effect.currentSpan, otelCorrelationContextFromEffectSpan, ).pipe(Effect.orElseSucceed(() => null)); export const tagCurrentSentryScopeWithCurrentOtelSpan: Effect.Effect = Effect.map(currentOtelContext, (context) => { tagCurrentSentryScopeWithOtelContext(context); return context; }); /** * True when this event is the auto-instrumentation's copy of a cause the * Durable Object already claimed. * * Both conditions matter. The tag alone would drop the DO's own report; the * mechanism alone would drop genuinely unhandled DO failures (an alarm crash, * say) that nothing else reports. Together they identify exactly the echo. */ const isClaimedDurableObjectEcho = (event: ErrorEvent): boolean => { if (event.tags?.[DO_CAUSE_OWNER_TAG] !== DO_CAUSE_OWNER_VALUE) return false; const mechanism = event.exception?.values?.[0]?.mechanism?.type; return typeof mechanism === "string" && mechanism.startsWith("auto."); }; /** * The mechanism `instrumentDurableObjectWithSentry` stamps on a rejection it * catches escaping a Durable Object entry point. */ const DO_FAAS_MECHANISM = "auto.faas.cloudflare.durable_object"; /** * True when this event is the Durable Object instrumentation reporting the * platform tearing its own object down. * * The capture-owner scheme above covers every reset a worker seam observed: the * seam classifies the failure, answers the client with a retryable envelope and * claims the cause, so the instrumentation's echo is dropped. A deploy has no * such seam. The runtime resets every live object as the new script rolls out, * and the instrumentation captures that from INSIDE the dying object, where * there is no request to own it — so a handful of these file on every deploy * with nothing behind them. The client-facing side is already handled: the * in-flight request fails at the worker seam and is rendered as a 503 there. * * Both conditions are required, and the classifier is the one the seams use so * this can never recognize a different set of messages than they do: * * - the mechanism must be the DO instrumentation, so a reset message arriving * through any other path (an ordinary worker capture, the DO's own * `captureCause` seam) is untouched; * - the message must classify as a TRANSIENT platform failure, so a real defect * thrown inside a DO — an alarm crash, a broken handler — still reports. The * classifier returns null for everything it does not recognize, and this * drops nothing on a null. * * `session_dead` is deliberately not dropped: that disposition means our own * code called `ctx.abort`, which is an act of the application rather than the * platform, and how often it happens is worth seeing. */ const isDurableObjectPlatformResetNoise = (event: ErrorEvent): boolean => { const exception = event.exception?.values?.[0]; if (exception?.mechanism?.type !== DO_FAAS_MECHANISM) return false; // The exception's `value` is the workerd message verbatim, and the classifier // already accepts a bare string as its input. return classifyDurableObjectError(exception.value)?.disposition === "transient"; }; export const beforeSendWithOtelCorrelation = ( event: ErrorEvent, options?: { readonly logPayload?: boolean }, ): ErrorEvent | null => { if (isClaimedDurableObjectEcho(event)) return null; if (isDurableObjectPlatformResetNoise(event)) return null; if (options?.logPayload) { console.info( JSON.stringify({ event: "sentry_before_send_otel_correlation", sentry_event_id: event.event_id ?? "", otel_trace_id: String(event.tags?.[OTEL_TRACE_ID_TAG] ?? ""), otel_span_id: String(event.tags?.[OTEL_SPAN_ID_TAG] ?? ""), }), ); } return event; }; /** * The single `beforeSend` the worker and its Durable Objects install. * * The worker ships as content-hashed chunks and its frames are not resolved * back to source, so Sentry's default grouping keys on names like * `execution-rate-limit-` and re-opens every issue on the next deploy. * `withStableGroupingFingerprint` pins a key with the hash normalized out; * events with no hashed grouping input are left on the default algorithm. * * The two stages are independent and compose in this order: the reporting pass * decides WHETHER the event is reported at all (a cause the Durable Object * already claimed, or a platform reset the DO instrumentation caught from * inside a dying object, is dropped, and a dropped event is never * fingerprinted), and the grouping pass then decides HOW whatever survives is * grouped. */ export const beforeSendCloudEvent = ( event: ErrorEvent, options?: { readonly logPayload?: boolean }, ): ErrorEvent | null => { const reported = beforeSendWithOtelCorrelation(event, options); return reported === null ? null : withStableGroupingFingerprint(reported); }; /** * The Sentry options the worker and every Durable Object install. It lives * beside the `beforeSend` it wires so the composition is covered by * observability.test.ts; `server.ts` only passes this through. * * NOTE: do NOT enable `instrumentPrototypeMethods`. It walks the DO prototype * and reads every property — including accessors — to find methods to wrap, * which invokes the `sessionId` getter with `this` bound to the prototype * (where `ctx` is undefined) and throws during construction, 500ing every * session create / cold restore. The DO captures its own errors via the * `captureCause` seam (→ Sentry) instead. */ export const cloudSentryOptions = (env: Env) => ({ dsn: env.SENTRY_DSN, tracesSampleRate: 0, enableLogs: true, sendDefaultPii: true, skipOpenTelemetrySetup: true, beforeSend: (event: ErrorEvent) => beforeSendCloudEvent(event, { logPayload: !env.SENTRY_DSN || env.SENTRY_OTEL_LOG_PAYLOAD === "true", }), }); export const addCurrentOtelCorrelationTags = < T extends { readonly tags?: Record }, >( event: T, ): Effect.Effect => Effect.map(currentOtelContext, (context) => addOtelCorrelationTags(event, context)); // Sentry's `captureException` can't serialize Effect's `CauseImpl` (it logs // `'CauseImpl' captured as exception with keys: reasons, ~effect/Cause` and // drops the real failure). `Cause.squash` isn't enough on its own: when an // inner `runPromise` rejects with a CauseImpl from its own `causeSquash` // (Effect v4's behaviour), `Effect.promise` re-wraps it as `Die(causeImpl)`, // and `Cause.squash(outer)` then hands the CauseImpl straight back. Use // `Cause.prettyErrors` instead — it always produces real `Error` instances, // even for non-Error defects (including a CauseImpl defect, which gets // wrapped via `causePrettyMessage`). export const sentryPayloadForCause = ( input: unknown, ): { primary: unknown; pretty: string | null } => { if (Cause.isCause(input)) { const pretty = Cause.pretty(input); const errors = Cause.prettyErrors(input); // oxlint-disable-next-line executor/no-error-constructor -- boundary: Sentry captureException needs an Error-like primary payload for pretty Effect causes return { primary: errors[0] ?? new Error(pretty), pretty }; } return { primary: input, pretty: null }; }; // Safe classification fields our tagged errors carry (`UserStoreError.operation` // / `.reason`, `WorkOSError.status`). They are promoted to Sentry TAGS because // the pretty cause is only an `extra`, and Sentry's server-side scrubber // replaces that extra with "[Filtered]" — leaving an issue with no failing // operation and no reason in it at all. Tags survive, group, and are // searchable. Values are failure modes and operation names; never a query, a // value, or anything customer-derived. const CLASSIFICATION_TAG_FIELDS = ["operation", "reason", "status"] as const; /** The errors those fields are read from. An allowlist, because the fields are * only known to be safe on the errors this app defines. */ const CLASSIFIED_ERROR_TAGS = [ "UserStoreError", "WorkOSError", "McpSessionMetaUnavailableError", ] as const; const MAX_CLASSIFICATION_TAG_CHARS = 120; const MAX_CAUSE_NESTING = 3; /** Every error value a cause carries, failures and defects alike. A defect can * itself be a `Cause` (an inner `runPromise` rejecting with its own squashed * cause), so the walk unwraps a few levels. */ const errorValuesOf = (input: unknown, depth = 0): readonly unknown[] => { if (depth >= MAX_CAUSE_NESTING) return []; if (!Cause.isCause(input)) return [input]; const values: unknown[] = []; for (const reason of input.reasons) { if (Cause.isFailReason(reason)) values.push(reason.error); else if (Cause.isDieReason(reason)) values.push(...errorValuesOf(reason.defect, depth + 1)); } return values; }; /** Read the classification fields off the tagged errors inside a cause. First * writer wins, so the innermost reported error names the issue. */ const classificationTagsOf = (input: unknown): Readonly> => { const tags: Record = {}; for (const candidate of errorValuesOf(input)) { if (typeof candidate !== "object" || candidate === null) continue; // Only the errors whose fields we know are safe classifications. Without // this, any foreign object in the cause that happens to carry an // `operation` / `reason` / `status` field would have that value promoted // onto a tag, and a foreign field is not known to be free of a query, a // value, or anything customer-derived. if (!CLASSIFIED_ERROR_TAGS.some((tag) => Predicate.isTagged(candidate, tag))) continue; const tagged = candidate as Record; for (const field of CLASSIFICATION_TAG_FIELDS) { const value = tagged[field]; if (tags[field] !== undefined) continue; if (typeof value === "string" || typeof value === "number") { tags[field] = String(value).slice(0, MAX_CLASSIFICATION_TAG_CHARS); } } } return tags; }; export const captureCause = ( input: unknown, context: OtelCorrelationContext | null = null, ): string | undefined => { const { primary, pretty } = sentryPayloadForCause(input); const classification = classificationTagsOf(input); tagCurrentSentryScopeWithOtelContext(context); return Sentry.captureException(primary, (scope) => { tagSentryScopeWithOtelContext(scope, context); for (const [key, value] of Object.entries(classification)) scope.setTag(key, value); if (pretty !== null) scope.setExtra("cause", pretty); return scope; }); }; export const captureCauseEffect = (input: unknown): Effect.Effect => Effect.gen(function* () { const context = yield* tagCurrentSentryScopeWithCurrentOtelSpan; const eventId = yield* Effect.sync(() => captureCause(input, context)); if (eventId && context) { yield* Effect.annotateCurrentSpan(SENTRY_EVENT_ID_ATTRIBUTE, eventId); } return eventId; }); /** * Mark the current Sentry scope as "the Durable Object has already dealt with * this cause". Call it AFTER any `captureCauseEffect`, so the DO's own event — * captured against the scope as it was — is not itself mistaken for the echo. */ export const claimCauseHandledByDurableObject: Effect.Effect = Effect.sync(() => { Sentry.getCurrentScope().setTag(DO_CAUSE_OWNER_TAG, DO_CAUSE_OWNER_VALUE); }); export const ErrorCaptureLive: Layer.Layer = Layer.succeed( ErrorCapture, ErrorCapture.of({ captureException: (cause) => Effect.gen(function* () { console.error("[api] unhandled cause:", truncate(Cause.pretty(cause))); return (yield* captureCauseEffect(cause)) ?? ""; }), }), );