Correlate Sentry errors with OTel traces (#1320)

* Correlate Sentry errors with OTel traces * Default the Sentry OTel verification flags on in the e2e cloud stack

Rhys Sullivan committed 4 months ago

Commit 4a84b0d1 parent dfbc10413 files · +382 −40Browse files

Unified · Split

13 files

apps/cloud/src/api/error-response.ts +21 −8
apps/cloud/src/engine/execution-gate.ts +7 −5
apps/cloud/src/engine/execution-rate-limit.ts +7 −5
apps/cloud/src/env-augment.d.ts +2 −0
apps/cloud/src/extensions/billing/route.ts +6 −2
apps/cloud/src/extensions/billing/service.ts +4 −3
apps/cloud/src/mcp/session-durable-object.ts +15 −1
apps/cloud/src/observability/index.ts +114 −3
apps/cloud/src/observability/observability.test.ts +54 −2
apps/cloud/src/server.ts +30 −0
e2e/cloud/sentry-otel-correlation.test.ts +70 −0
e2e/setup/cloud.globalsetup.ts +16 −1
packages/hosts/cloudflare/src/mcp/agent-session-durable-object.ts +36 −10

apps/cloud/src/api/error-response.ts+21−8

@@ -1,7 +1,7 @@
1	1	import { Cause, Data, Effect, Predicate, Result } from "effect";
2	2	import { HttpServerRespondable, HttpServerResponse } from "effect/unstable/http";
3	3
4		−import { captureCause } from "../observability";
	4	+import { captureCause, captureCauseEffect } from "../observability";
5	5
6	6	// Implements `Respondable` so the framework's default cause→response
7	7	// pipeline (`HttpServerRespondable.toResponseOrElse`) renders this as the
@@ -13,13 +13,14 @@ export class HttpResponseError extends Data.TaggedError("HttpResponseError")<{ · 5 unmodified lines
13	13	readonly message: string;
14	14	}> {
15	15	[HttpServerRespondable.symbol](): Effect.Effect<HttpServerResponse.HttpServerResponse> {
16		− if (this.status >= 500) captureCause(this);
17		− return Effect.succeed(
18		− HttpServerResponse.jsonUnsafe(
19		− { error: this.message, code: this.code },
20		− { status: this.status },
21		− ),
22		− );
	16	+ const self = this;
	17	+ return Effect.gen(function* () {
	18	+ if (self.status >= 500) yield* captureCauseEffect(self);
	19	+ return HttpServerResponse.jsonUnsafe(
	20	+ { error: self.message, code: self.code },
	21	+ { status: self.status },
	22	+ );
	23	+ });
23	24	}
24	25	}
25	26
@@ -65,3 +66,15 @@ export const toErrorServerResponse = (error: unknown): HttpServerResponse.HttpSe · 39 unmodified lines
65	66	{ status: mapped.status },
66	67	);
67	68	};
	69	+
	70	+export const toErrorServerResponseEffect = (
	71	+ error: unknown,
	72	+): Effect.Effect<HttpServerResponse.HttpServerResponse> =>
	73	+ Effect.gen(function* () {
	74	+ const mapped = toHttpResponseError(error);
	75	+ if (mapped.status >= 500) yield* captureCauseEffect(mapped);
	76	+ return HttpServerResponse.jsonUnsafe(
	77	+ { error: mapped.message, code: mapped.code },
	78	+ { status: mapped.status },
	79	+ );
	80	+ });

apps/cloud/src/engine/execution-gate.ts+7−5

@@ -24,12 +24,12 @@ · 23 unmodified lines
24	24	// `CodeCompilationError` / `SandboxRuntimeError` into `ExecuteResult.error`.
25	25	// ---------------------------------------------------------------------------
26	26
27		−import * as Sentry from "@sentry/cloudflare";
28	27	import { Data, Effect } from "effect";
29	28	import type * as Cause from "effect/Cause";
30	29
31	30	import type { ExecutionEngine, ExecutionResult } from "@executor-js/execution";
32	31
	32	+import { captureCauseEffect } from "../observability";
33	33	import { EXECUTION_LIMIT_BLOCKED_MESSAGE } from "./execution-limit-messages";
34	34
35	35	// The engine's completed-result payload (`ExecuteResult` in codemode-core),
@@ -166,10 +166,12 @@ export const makeExecutionLimitGate = (checkBalance: ExecutionBalanceCheck) => { · 130 unmodified lines
166	166	// must never block executions. Reported like `trackExecution` so a
167	167	// billing outage still pages; the error outcome is never cached.
168	168	Effect.catch((error: unknown) =>
169		− Effect.sync((): GateDecision => {
170		− console.warn("[billing] execution balance check failed open:", error);
171		− Sentry.captureException(error);
172		− return { blocked: false };
	169	+ Effect.gen(function* () {
	170	+ yield* Effect.sync(() => {
	171	+ console.warn("[billing] execution balance check failed open:", error);
	172	+ });
	173	+ yield* captureCauseEffect(error);
	174	+ return { blocked: false } as const satisfies GateDecision;
173	175	}),
174	176	),
175	177	);

apps/cloud/src/engine/execution-rate-limit.ts+7−5

@@ -14,12 +14,12 @@ · 13 unmodified lines
14	14	// ---------------------------------------------------------------------------
15	15
16	16	import { DurableObject, env } from "cloudflare:workers";
17		−import * as Sentry from "@sentry/cloudflare";
18	17	import { Data, Effect } from "effect";
19	18	import type * as Cause from "effect/Cause";
20	19
21	20	import type { ExecutionEngine } from "@executor-js/execution";
22	21
	22	+import { captureCauseEffect } from "../observability";
23	23	import { withPreExecutionGate, type GateDecision } from "./execution-gate";
24	24	import { RATE_LIMIT_BLOCKED_MESSAGE } from "./execution-limit-messages";
25	25
@@ -149,10 +149,12 @@ export const makeExecutionRateLimiter = ( · 123 unmodified lines
149	149	// FAIL OPEN: the backstop must never block executions because its
150	150	// counter is unreachable or slow.
151	151	Effect.catch((error: unknown) =>
152		− Effect.sync((): GateDecision => {
153		− console.warn("[rate-limit] execution rate limit check failed open:", error);
154		− Sentry.captureException(error);
155		− return { blocked: false };
	152	+ Effect.gen(function* () {
	153	+ yield* Effect.sync(() => {
	154	+ console.warn("[rate-limit] execution rate limit check failed open:", error);
	155	+ });
	156	+ yield* captureCauseEffect(error);
	157	+ return { blocked: false } as const satisfies GateDecision;
156	158	}),
157	159	),
158	160	);

apps/cloud/src/env-augment.d.ts+2−0

@@ -11,6 +11,8 @@ declare global { · 10 unmodified lines
11	11	AXIOM_TRACES_URL?: string;
12	12	AXIOM_TRACES_SAMPLE_RATIO?: string;
13	13	SENTRY_DSN?: string;
	14	+ SENTRY_OTEL_LOG_PAYLOAD?: string;
	15	+ SENTRY_OTEL_VERIFY?: string;
14	16	VITE_PUBLIC_SENTRY_DSN?: string;
15	17	VITE_PUBLIC_POSTHOG_KEY?: string;
16	18	VITE_PUBLIC_POSTHOG_HOST?: string;

apps/cloud/src/extensions/billing/route.ts+6−2

@@ -5,7 +5,11 @@ import { autumnHandler } from "autumn-js/backend"; · 4 unmodified lines
5	5
6	6	import { WorkOSClient } from "../../auth/workos";
7	7	import { ORG_SELECTOR_HEADER, authorizeOrganizationSelector } from "../../auth/organization";
8		−import { HttpResponseError, isServerError, toErrorServerResponse } from "../../api/error-response";
	8	+import {
	9	+ HttpResponseError,
	10	+ isServerError,
	11	+ toErrorServerResponseEffect,
	12	+} from "../../api/error-response";
9	13
10	14	type BillingSession = {
11	15	readonly userId: string;
@@ -111,7 +115,7 @@ const handler = Effect.gen(function* () { · 99 unmodified lines
111	115	if (isServerError(err)) {
112	116	console.error("[autumn] request failed:", Cause.pretty(err));
113	117	}
114		− return Effect.succeed(toErrorServerResponse(err));
	118	+ return toErrorServerResponseEffect(err);
115	119	}),
116	120	);
117	121

apps/cloud/src/extensions/billing/service.ts+4−3

@@ -3,10 +3,11 @@ · 2 unmodified lines
3	3	// ---------------------------------------------------------------------------
4	4
5	5	import { env } from "cloudflare:workers";
6		−import * as Sentry from "@sentry/cloudflare";
7	6	import { Autumn } from "autumn-js";
8	7	import { Context, Data, Effect, Layer } from "effect";
9	8
	9	+import { captureCauseEffect } from "../../observability";
	10	+
10	11	// ---------------------------------------------------------------------------
11	12	// Errors
12	13	// ---------------------------------------------------------------------------
@@ -72,12 +73,12 @@ const make = Effect.sync(() => { · 59 unmodified lines
72	73	).pipe(
73	74	Effect.catchTag("AutumnError", (error) =>
74	75	Effect.gen(function* () {
75		− // Silent billing data loss is worth paging on — autumn.trackExecution
	76	+ // Silent billing data loss is worth paging on: autumn.trackExecution
76	77	// is fire-and-forget so the caller doesn't handle it themselves.
77	78	yield* Effect.sync(() => {
78	79	console.error("[billing] track failed:", error);
79		− Sentry.captureException(error);
80	80	});
	81	+ yield* captureCauseEffect(error);
81	82	yield* Effect.annotateCurrentSpan({ "autumn.track.failed": true });
82	83	}),
83	84	),

apps/cloud/src/mcp/session-durable-object.ts+15−1

@@ -68,7 +68,11 @@ import { makeExecutionStack } from "../engine/execution-stack"; · 67 unmodified lines
68	68	import { CloudMeteredExecutionStackLayer } from "../engine/execution-stack-metered";
69	69	import { AutumnService } from "../extensions/billing/service";
70	70	import { DoTelemetryLive, flushTracerProvider } from "../observability/telemetry";
71		−import { captureCause as reportCause } from "../observability";
	71	+import {
	72	+ captureCause as reportCause,
	73	+ captureCauseEffect as reportCauseEffect,
	74	+ tagCurrentSentryScopeWithCurrentOtelSpan,
	75	+} from "../observability";
72	76
73	77	// Re-export the shared types so existing cloud importers
74	78	// (`auth/handlers.ts`, etc.) keep their `../mcp/session-durable-object` path.
@@ -314,6 +318,16 @@ export class McpSessionDOSqlite extends McpAgentSessionDOBase<Env, CloudSessionD · 239 unmodified lines
314	318	reportCause(cause);
315	319	}
316	320
	321	+ protected override captureCauseEffect(
	322	+ cause: Cause.Cause<unknown>,
	323	+ ): Effect.Effect<string \| undefined> {
	324	+ return reportCauseEffect(cause);
	325	+ }
	326	+
	327	+ protected override prepareErrorCaptureScope(): Effect.Effect<void> {
	328	+ return Effect.asVoid(tagCurrentSentryScopeWithCurrentOtelSpan);
	329	+ }
	330	+
317	331	// Best-effort export the DO isolate's buffered spans after the RPC settles,
318	332	// so a dying init/handleRequest can ship its own spans (and the exception +
319	333	// stack recorded on them) — not just the worker-side `mcp.do.*` span. Keep it

apps/cloud/src/observability/index.ts+114−3

@@ -11,7 +11,9 @@ · 10 unmodified lines
11	11	// ---------------------------------------------------------------------------
12	12
13	13	import * as Sentry from "@sentry/cloudflare";
	14	+import type { ErrorEvent, Scope } from "@sentry/cloudflare";
14	15	import { Cause, Effect, Layer } from "effect";
	16	+import type * as Tracer from "effect/Tracer";
15	17
16	18	import { ErrorCapture } from "@executor-js/api";
17	19
@@ -21,12 +23,106 @@ import { ErrorCapture } from "@executor-js/api"; · 3 unmodified lines
21	23	// error. Sentry still receives the full, untruncated cause via
22	24	// `setExtra`; only the dev-console mirror is capped.
23	25	const MAX_CONSOLE_CAUSE_CHARS = 4_000;
	26	+const OTEL_TRACE_ID_PATTERN = /^[0-9a-f]{32}$/;
	27	+const OTEL_SPAN_ID_PATTERN = /^[0-9a-f]{16}$/;
	28	+
	29	+export const OTEL_TRACE_ID_TAG = "otel_trace_id";
	30	+export const OTEL_SPAN_ID_TAG = "otel_span_id";
	31	+export const SENTRY_EVENT_ID_ATTRIBUTE = "sentry.event_id";
	32	+
	33	+export type OtelCorrelationContext = {
	34	+ readonly traceId: string;
	35	+ readonly spanId: string;
	36	+};
24	37
25	38	const truncate = (s: string): string =>
26	39	s.length <= MAX_CONSOLE_CAUSE_CHARS
27	40	? s
28	41	: `${s.slice(0, MAX_CONSOLE_CAUSE_CHARS)}\n…[truncated ${s.length - MAX_CONSOLE_CAUSE_CHARS} chars]`;
29	42
	43	+const validOtelContext = (context: OtelCorrelationContext): boolean =>
	44	+ OTEL_TRACE_ID_PATTERN.test(context.traceId) && OTEL_SPAN_ID_PATTERN.test(context.spanId);
	45	+
	46	+export const otelCorrelationContextFromEffectSpan = (
	47	+ span: Tracer.Span,
	48	+): OtelCorrelationContext \| null => {
	49	+ const context = { traceId: span.traceId, spanId: span.spanId };
	50	+ return validOtelContext(context) ? context : null;
	51	+};
	52	+
	53	+export const otelCorrelationContextFromOpenTelemetrySpan = (span: {
	54	+ readonly spanContext: () => { readonly traceId: string; readonly spanId: string };
	55	+}): OtelCorrelationContext \| null => {
	56	+ const { traceId, spanId } = span.spanContext();
	57	+ const context = { traceId, spanId };
	58	+ return validOtelContext(context) ? context : null;
	59	+};
	60	+
	61	+export const addOtelCorrelationTags = <T extends { readonly tags?: Record<string, unknown> }>(
	62	+ event: T,
	63	+ context: OtelCorrelationContext \| null,
	64	+): T => {
	65	+ if (!context) return event;
	66	+ return {
	67	+ ...event,
	68	+ tags: {
	69	+ ...event.tags,
	70	+ [OTEL_TRACE_ID_TAG]: context.traceId,
	71	+ [OTEL_SPAN_ID_TAG]: context.spanId,
	72	+ },
	73	+ };
	74	+};
	75	+
	76	+export const tagSentryScopeWithOtelContext = (
	77	+ scope: Scope,
	78	+ context: OtelCorrelationContext \| null,
	79	+): void => {
	80	+ if (!context) return;
	81	+ scope.setTag(OTEL_TRACE_ID_TAG, context.traceId);
	82	+ scope.setTag(OTEL_SPAN_ID_TAG, context.spanId);
	83	+};
	84	+
	85	+export const tagCurrentSentryScopeWithOtelContext = (
	86	+ context: OtelCorrelationContext \| null,
	87	+): void => {
	88	+ tagSentryScopeWithOtelContext(Sentry.getCurrentScope(), context);
	89	+};
	90	+
	91	+const currentOtelContext = Effect.map(
	92	+ Effect.currentSpan,
	93	+ otelCorrelationContextFromEffectSpan,
	94	+).pipe(Effect.orElseSucceed(() => null));
	95	+
	96	+export const tagCurrentSentryScopeWithCurrentOtelSpan: Effect.Effect<OtelCorrelationContext \| null> =
	97	+ Effect.map(currentOtelContext, (context) => {
	98	+ tagCurrentSentryScopeWithOtelContext(context);
	99	+ return context;
	100	+ });
	101	+
	102	+export const beforeSendWithOtelCorrelation = (
	103	+ event: ErrorEvent,
	104	+ options?: { readonly logPayload?: boolean },
	105	+): ErrorEvent => {
	106	+ if (options?.logPayload) {
	107	+ console.info(
	108	+ JSON.stringify({
	109	+ event: "sentry_before_send_otel_correlation",
	110	+ sentry_event_id: event.event_id ?? "",
	111	+ otel_trace_id: String(event.tags?.[OTEL_TRACE_ID_TAG] ?? ""),
	112	+ otel_span_id: String(event.tags?.[OTEL_SPAN_ID_TAG] ?? ""),
	113	+ }),
	114	+ );
	115	+ }
	116	+ return event;
	117	+};
	118	+
	119	+export const addCurrentOtelCorrelationTags = <
	120	+ T extends { readonly tags?: Record<string, unknown> },
	121	+>(
	122	+ event: T,
	123	+): Effect.Effect<T> =>
	124	+ Effect.map(currentOtelContext, (context) => addOtelCorrelationTags(event, context));
	125	+
30	126	// Sentry's `captureException` can't serialize Effect's `CauseImpl` (it logs
31	127	// `'CauseImpl' captured as exception with keys: reasons, ~effect/Cause` and
32	128	// drops the real failure). `Cause.squash` isn't enough on its own: when an
@@ -48,21 +144,36 @@ export const sentryPayloadForCause = ( · 15 unmodified lines
48	144	return { primary: input, pretty: null };
49	145	};
50	146
51		−export const captureCause = (input: unknown): string \| undefined => {
	147	+export const captureCause = (
	148	+ input: unknown,
	149	+ context: OtelCorrelationContext \| null = null,
	150	+): string \| undefined => {
52	151	const { primary, pretty } = sentryPayloadForCause(input);
	152	+ tagCurrentSentryScopeWithOtelContext(context);
53	153	return Sentry.captureException(primary, (scope) => {
	154	+ tagSentryScopeWithOtelContext(scope, context);
54	155	if (pretty !== null) scope.setExtra("cause", pretty);
55	156	return scope;
56	157	});
57	158	};
58	159
	160	+export const captureCauseEffect = (input: unknown): Effect.Effect<string \| undefined> =>
	161	+ Effect.gen(function* () {
	162	+ const context = yield* tagCurrentSentryScopeWithCurrentOtelSpan;
	163	+ const eventId = yield* Effect.sync(() => captureCause(input, context));
	164	+ if (eventId && context) {
	165	+ yield* Effect.annotateCurrentSpan(SENTRY_EVENT_ID_ATTRIBUTE, eventId);
	166	+ }
	167	+ return eventId;
	168	+ });
	169	+
59	170	export const ErrorCaptureLive: Layer.Layer<ErrorCapture> = Layer.succeed(
60	171	ErrorCapture,
61	172	ErrorCapture.of({
62	173	captureException: (cause) =>
63		− Effect.sync(() => {
	174	+ Effect.gen(function* () {
64	175	console.error("[api] unhandled cause:", truncate(Cause.pretty(cause)));
65		− return captureCause(cause) ?? "";
	176	+ return (yield* captureCauseEffect(cause)) ?? "";
66	177	}),
67	178	}),
68	179	);

apps/cloud/src/observability/observability.test.ts+54−2

@@ -1,7 +1,13 @@
1	1	import { describe, expect, it } from "@effect/vitest";
2		−import { Cause } from "effect";
	2	+import { Cause, Effect } from "effect";
	3	+import type * as Tracer from "effect/Tracer";
3	4
4		−import { sentryPayloadForCause } from "./index";
	5	+import {
	6	+ addCurrentOtelCorrelationTags,
	7	+ OTEL_SPAN_ID_TAG,
	8	+ OTEL_TRACE_ID_TAG,
	9	+ sentryPayloadForCause,
	10	+} from "./index";
5	11
6	12	// Mirrors Sentry core's `is.isError`: it picks the proper-Error path iff
7	13	// `Object.prototype.toString.call(x) === "[object Error]"`. Anything that
@@ -10,6 +16,39 @@ import { sentryPayloadForCause } from "./index"; · 2 unmodified lines
10	16	const looksLikeErrorToSentry = (value: unknown): boolean =>
11	17	Object.prototype.toString.call(value) === "[object Error]";
12	18
	19	+const traceId = "4268a606000000000000000000000000";
	20	+const spanId = "1234567890abcdef";
	21	+
	22	+const makeFixedTracer = (): Tracer.Tracer => ({
	23	+ span: (options) => {
	24	+ const attributes = new Map<string, unknown>();
	25	+ let status: Tracer.SpanStatus = { _tag: "Started", startTime: options.startTime };
	26	+ return {
	27	+ _tag: "Span",
	28	+ name: options.name,
	29	+ spanId,
	30	+ traceId,
	31	+ parent: options.parent,
	32	+ annotations: options.annotations,
	33	+ get status() {
	34	+ return status;
	35	+ },
	36	+ attributes,
	37	+ links: options.links,
	38	+ sampled: options.sampled,
	39	+ kind: options.kind,
	40	+ end: (endTime, exit) => {
	41	+ status = { _tag: "Ended", startTime: options.startTime, endTime, exit };
	42	+ },
	43	+ attribute: (key, value) => {
	44	+ attributes.set(key, value);
	45	+ },
	46	+ event: () => undefined,
	47	+ addLinks: () => undefined,
	48	+ };
	49	+ },
	50	+});
	51	+
13	52	describe("sentryPayloadForCause", () => {
14	53	it("hands Sentry a real Error when the defect is itself a Cause", () => {
15	54	// Reproduces the production chain: an inner runPromise rejects with a
@@ -39,3 +78,16 @@ describe("sentryPayloadForCause", () => { · 23 unmodified lines
39	78	expect(pretty).toBeNull();
40	79	});
41	80	});
	81	+
	82	+describe("Sentry OTel correlation", () => {
	83	+ it.effect("adds tags from the active Effect span", () =>
	84	+ Effect.gen(function* () {
	85	+ const baseEvent: { readonly tags: Record<string, unknown> } = { tags: { existing: "tag" } };
	86	+ const event = yield* addCurrentOtelCorrelationTags(baseEvent);
	87	+
	88	+ expect(event.tags.existing).toBe("tag");
	89	+ expect(event.tags[OTEL_TRACE_ID_TAG]).toBe(traceId);
	90	+ expect(event.tags[OTEL_SPAN_ID_TAG]).toBe(spanId);
	91	+ }).pipe(Effect.withSpan("test.sentry_capture"), Effect.withTracer(makeFixedTracer())),
	92	+ );
	93	+});

apps/cloud/src/server.ts+30−0

@@ -1,5 +1,6 @@
1	1	import { DurableObject } from "cloudflare:workers";
2	2	import { SpanKind, SpanStatusCode, context, trace } from "@opentelemetry/api";
	3	+import type { ErrorEvent } from "@sentry/cloudflare";
3	4	import {
4	5	ATTR_HTTP_REQUEST_METHOD,
5	6	ATTR_HTTP_RESPONSE_STATUS_CODE,
@@ -14,6 +15,13 @@ import { isAppOwnedPath } from "./app-paths"; · 8 unmodified lines
14	15	import { makeCloudMcpAgentHandler } from "./mcp/agent-handler";
15	16	import { classifyMcpPath, prepareMcpOrgScope } from "./mcp/mount";
16	17	import { McpSessionDOSqlite as McpSessionDOBase } from "./mcp/session-durable-object";
	18	+import {
	19	+ beforeSendWithOtelCorrelation,
	20	+ captureCause,
	21	+ otelCorrelationContextFromOpenTelemetrySpan,
	22	+ SENTRY_EVENT_ID_ATTRIBUTE,
	23	+ tagCurrentSentryScopeWithOtelContext,
	24	+} from "./observability";
17	25	import { browserTracesResponse } from "./observability/browser-traces";
18	26	import { flushTracerProvider, installTracerProvider } from "./observability/telemetry";
19	27
@@ -27,6 +35,10 @@ const sentryOptions = (env: Env) => ({ · 7 unmodified lines
27	35	enableLogs: true,
28	36	sendDefaultPii: true,
29	37	skipOpenTelemetrySetup: true,
	38	+ beforeSend: (event: ErrorEvent) =>
	39	+ beforeSendWithOtelCorrelation(event, {
	40	+ logPayload: !env.SENTRY_DSN \|\| env.SENTRY_OTEL_LOG_PAYLOAD === "true",
	41	+ }),
30	42	// NOTE: do NOT enable `instrumentPrototypeMethods`. It walks the DO prototype
31	43	// and reads every property — including accessors — to find methods to wrap,
32	44	// which invokes the `sessionId` getter with `this` bound to the prototype
@@ -158,6 +170,8 @@ const cloudflareHandler: ExportedHandler<Env> = { · 125 unmodified lines
158	170	{ kind: SpanKind.SERVER },
159	171	parentContext,
160	172	async (span) => {
	173	+ const otelContext = otelCorrelationContextFromOpenTelemetrySpan(span);
	174	+ tagCurrentSentryScopeWithOtelContext(otelContext);
161	175	span.setAttribute(ATTR_HTTP_REQUEST_METHOD, request.method);
162	176	span.setAttribute(ATTR_URL_FULL, request.url);
163	177	span.setAttribute(ATTR_URL_PATH, url.pathname);
@@ -168,6 +182,22 @@ const cloudflareHandler: ExportedHandler<Env> = { · 4 unmodified lines
168	182	// outer wrapper still captures the exception; we only mark span status.
169	183	// oxlint-disable-next-line executor/no-try-catch-or-throw -- adapter boundary
170	184	try {
	185	+ if (env.SENTRY_OTEL_VERIFY === "true" && url.pathname === "/__sentry-otel-verify") {
	186	+ // oxlint-disable-next-line executor/no-error-constructor -- boundary: synthetic verification needs an Error payload for Sentry grouping
	187	+ const eventId = captureCause(new Error("sentry otel verification"), otelContext) ?? "";
	188	+ if (eventId) span.setAttribute(SENTRY_EVENT_ID_ATTRIBUTE, eventId);
	189	+ span.setAttribute("sentry_otel.verify", true);
	190	+ span.setAttribute(ATTR_HTTP_RESPONSE_STATUS_CODE, 500);
	191	+ span.setStatus({ code: SpanStatusCode.ERROR });
	192	+ return Response.json(
	193	+ {
	194	+ sentryEventId: eventId,
	195	+ otelTraceId: otelContext?.traceId ?? "",
	196	+ otelSpanId: otelContext?.spanId ?? "",
	197	+ },
	198	+ { status: 500 },
	199	+ );
	200	+ }
171	201	const response = await fetchHandler(request, env, ctx);
172	202	span.setAttribute(ATTR_HTTP_RESPONSE_STATUS_CODE, response.status);
173	203	if (response.status >= 500) {

e2e/cloud/sentry-otel-correlation.test.ts+70−0

@@ -0,0 +1,70 @@
	1	+import { readFileSync } from "node:fs";
	2	+import { resolve } from "node:path";
	3	+
	4	+import { expect } from "@effect/vitest";
	5	+import { Effect, Schedule } from "effect";
	6	+
	7	+import { RUNS_DIR, scenario } from "../src/scenario";
	8	+import { Target, Telemetry } from "../src/services";
	9	+
	10	+type VerificationResponse = {
	11	+ readonly sentryEventId: string;
	12	+ readonly otelTraceId: string;
	13	+ readonly otelSpanId: string;
	14	+};
	15	+
	16	+const beforeSendLogFile = resolve(RUNS_DIR, "cloud", "server-logs", "boot.log");
	17	+
	18	+const hasCorrelationPayload = (line: string, body: VerificationResponse): boolean =>
	19	+ line.includes('"event":"sentry_before_send_otel_correlation"') &&
	20	+ line.includes(`"sentry_event_id":"${body.sentryEventId}"`) &&
	21	+ line.includes(`"otel_trace_id":"${body.otelTraceId}"`) &&
	22	+ line.includes(`"otel_span_id":"${body.otelSpanId}"`);
	23	+
	24	+const findBeforeSendPayload = (body: VerificationResponse): Effect.Effect<string, string> =>
	25	+ Effect.sync(() => {
	26	+ const text = readFileSync(beforeSendLogFile, "utf8");
	27	+ return text
	28	+ .split("\n")
	29	+ .slice()
	30	+ .reverse()
	31	+ .find((line) => hasCorrelationPayload(line, body));
	32	+ }).pipe(
	33	+ Effect.filterOrFail(
	34	+ (line): line is string => line !== undefined,
	35	+ () =>
	36	+ `no Sentry beforeSend correlation payload matched ${body.sentryEventId} in ${beforeSendLogFile}`,
	37	+ ),
	38	+ Effect.retry(Schedule.both(Schedule.spaced("500 millis"), Schedule.recurs(20))),
	39	+ );
	40	+
	41	+scenario(
	42	+ "Telemetry · Sentry error payload joins to exported OTel span",
	43	+ { timeout: 120_000 },
	44	+ Effect.gen(function* () {
	45	+ const target = yield* Target;
	46	+ const telemetry = yield* Telemetry;
	47	+
	48	+ const response = yield* Effect.promise(() =>
	49	+ fetch(new URL("/__sentry-otel-verify", target.baseUrl)),
	50	+ );
	51	+ expect(response.status).toBe(500);
	52	+
	53	+ const body = (yield* Effect.promise(() => response.json())) as VerificationResponse;
	54	+ expect(body.sentryEventId).toMatch(/^[0-9a-f]{32}$/);
	55	+ expect(body.otelTraceId).toMatch(/^[0-9a-f]{32}$/);
	56	+ expect(body.otelSpanId).toMatch(/^[0-9a-f]{16}$/);
	57	+
	58	+ const beforeSendLine = yield* findBeforeSendPayload(body);
	59	+ expect(beforeSendLine).toContain(body.otelTraceId);
	60	+
	61	+ const span = yield* telemetry.expectSpan({
	62	+ traceId: body.otelTraceId,
	63	+ attributes: {
	64	+ "sentry.event_id": body.sentryEventId,
	65	+ "sentry_otel.verify": "true",
	66	+ },
	67	+ });
	68	+ expect(span.span.spanId).toBe(body.otelSpanId);
	69	+ }),
	70	+);

e2e/setup/cloud.globalsetup.ts+16−1

@@ -25,6 +25,21 @@ const bootLogFile = process.env.E2E_VERBOSE · 24 unmodified lines
25	25	? undefined
26	26	: resolve(RUNS_DIR, "cloud", "server-logs", "boot.log");
27	27
	28	+const optionalCloudEnv = (): Record<string, string> => {
	29	+ // The Sentry/OTel correlation scenario always runs in this suite, so the
	30	+ // verification route and beforeSend payload logging default on here; the
	31	+ // env vars remain overridable for local runs against a different setup.
	32	+ const env: Record<string, string> = {
	33	+ SENTRY_OTEL_VERIFY: "true",
	34	+ SENTRY_OTEL_LOG_PAYLOAD: "true",
	35	+ };
	36	+ for (const key of ["SENTRY_DSN", "SENTRY_OTEL_LOG_PAYLOAD", "SENTRY_OTEL_VERIFY"]) {
	37	+ const value = process.env[key];
	38	+ if (value) env[key] = value;
	39	+ }
	40	+ return env;
	41	+};
	42	+
28	43	export default async function setup(): Promise<(() => Promise<void>) \| void> {
29	44	if (process.env.E2E_CLOUD_URL) {
30	45	await waitForHttp(process.env.E2E_CLOUD_URL);
@@ -74,7 +89,7 @@ export default async function setup(): Promise<(() => Promise<void>) \| void> { · 43 unmodified lines
74	89	// endpoint-agnostic, so the same layer that ships prod traces to
75	90	// Axiom ships e2e traces to the suite store — "why was that page
76	91	// slow" gets a span waterfall, not a guess.
77		− extraEnv: motelExporterEnv(motel, publicUrl),
	92	+ extraEnv: { ...motelExporterEnv(motel, publicUrl), ...optionalCloudEnv() },
78	93	});
79	94	return { teardown: cloud.teardown, value: cloud };
80	95	},

packages/hosts/cloudflare/src/mcp/agent-session-durable-object.ts+36−10

@@ -236,6 +236,17 @@ export abstract class McpAgentSessionDOBase< · 235 unmodified lines
236	236
237	237	protected captureCause(_cause: Cause.Cause<unknown>): void {}
238	238
	239	+ protected captureCauseEffect(cause: Cause.Cause<unknown>): Effect.Effect<string \| undefined> {
	240	+ return Effect.sync(() => {
	241	+ this.captureCause(cause);
	242	+ return undefined;
	243	+ });
	244	+ }
	245	+
	246	+ protected prepareErrorCaptureScope(): Effect.Effect<void> {
	247	+ return Effect.void;
	248	+ }
	249	+
239	250	protected flushTelemetry(): Promise<void> {
240	251	return Promise.resolve();
241	252	}
@@ -604,6 +615,7 @@ export abstract class McpAgentSessionDOBase< · 362 unmodified lines
604	615	}
605	616	const self = this;
606	617	const program = Effect.gen(function* () {
	618	+ yield* self.prepareErrorCaptureScope();
607	619	const sessionMeta = yield* self.resolveAndStoreSessionMeta(props.session);
608	620	const dbHandle = yield* self.openSessionDbHandle();
609	621	const { mcpServer, engine } = yield* self.buildRuntime(sessionMeta, dbHandle);
@@ -618,7 +630,7 @@ export abstract class McpAgentSessionDOBase< · 8 unmodified lines
618	630	Effect.tapCause((cause) =>
619	631	Effect.gen(function* () {
620	632	console.error("[mcp-session] init failed:", Cause.pretty(cause));
621		− self.captureCause(cause);
	633	+ yield* self.captureCauseEffect(cause);
622	634	yield* self.recordCauseOnSpan(cause);
623	635	}),
624	636	),
@@ -650,6 +662,7 @@ export abstract class McpAgentSessionDOBase< · 25 unmodified lines
650	662	const self = this;
651	663	return Effect.runPromise(
652	664	Effect.gen(function* () {
	665	+ yield* self.prepareErrorCaptureScope();
653	666	const sessionMeta = yield* self.loadSessionMeta();
654	667	if (!sessionMeta) return "not_found" as const;
655	668	if (self.initialized) {
@@ -681,6 +694,7 @@ export abstract class McpAgentSessionDOBase< · 25 unmodified lines
681	694	const self = this;
682	695	return Effect.runPromise(
683	696	Effect.gen(function* () {
	697	+ yield* self.prepareErrorCaptureScope();
684	698	const owner = yield* self.validateApprovalIdentity(identity);
685	699	if (owner !== "ok") return { status: owner } as const;
686	700
@@ -718,6 +732,7 @@ export abstract class McpAgentSessionDOBase< · 31 unmodified lines
718	732	const self = this;
719	733	return Effect.runPromise(
720	734	Effect.gen(function* () {
	735	+ yield* self.prepareErrorCaptureScope();
721	736	const owner = yield* self.validateApprovalIdentity(identity);
722	737	if (owner === "forbidden") return { status: "execution_forbidden" } as const;
723	738	if (owner === "not_found") {
@@ -775,6 +790,7 @@ export abstract class McpAgentSessionDOBase< · 51 unmodified lines
775	790	const self = this;
776	791	return Effect.runPromise(
777	792	Effect.gen(function* () {
	793	+ yield* self.prepareErrorCaptureScope();
778	794	const owner = yield* self.validateApprovalIdentity(identity);
779	795	if (owner !== "ok") return { status: owner } as const;
780	796
@@ -1015,6 +1031,7 @@ export abstract class McpAgentSessionDOBase< · 234 unmodified lines
1015	1031	): Effect.Effect<void> {
1016	1032	const self = this;
1017	1033	return Effect.gen(function* () {
	1034	+ yield* self.prepareErrorCaptureScope();
1018	1035	if (self.pendingApprovalLeases.has(executionId)) return;
1019	1036
1020	1037	yield* Effect.promise(() => self.markActivity()).pipe(
@@ -1031,9 +1048,14 @@ export abstract class McpAgentSessionDOBase< · 10 unmodified lines
1031	1048	attributes: { "mcp.execution.id": executionId },
1032	1049	}),
1033	1050	Effect.tapCause((cause) =>
1034		− Effect.sync(() => {
1035		− console.error("[mcp-session] pending approval lease start failed:", Cause.pretty(cause));
1036		− self.captureCause(cause);
	1051	+ Effect.gen(function* () {
	1052	+ yield* Effect.sync(() => {
	1053	+ console.error(
	1054	+ "[mcp-session] pending approval lease start failed:",
	1055	+ Cause.pretty(cause),
	1056	+ );
	1057	+ });
	1058	+ yield* self.captureCauseEffect(cause);
1037	1059	}),
1038	1060	),
1039	1061	Effect.ignore,
@@ -1048,16 +1070,19 @@ export abstract class McpAgentSessionDOBase< · 8 unmodified lines
1048	1070	}
1049	1071
1050	1072	private queuePendingApprovalLeaseExpiration(executionId: string): void {
	1073	+ const self = this;
1051	1074	this.ctx.waitUntil(
1052	1075	Effect.runPromise(
1053	1076	this.expirePendingApproval(executionId).pipe(
1054	1077	Effect.tapCause((cause) =>
1055		− Effect.sync(() => {
1056		− console.error(
1057		− "[mcp-session] pending approval lease expiration failed:",
1058		− Cause.pretty(cause),
1059		− );
1060		− this.captureCause(cause);
	1078	+ Effect.gen(function* () {
	1079	+ yield* Effect.sync(() => {
	1080	+ console.error(
	1081	+ "[mcp-session] pending approval lease expiration failed:",
	1082	+ Cause.pretty(cause),
	1083	+ );
	1084	+ });
	1085	+ yield* self.captureCauseEffect(cause);
1061	1086	}),
1062	1087	),
1063	1088	Effect.ignore,
@@ -1134,6 +1159,7 @@ export abstract class McpAgentSessionDOBase< · 70 unmodified lines
1134	1159	private expirePendingApproval(executionId: string): Effect.Effect<void> {
1135	1160	const self = this;
1136	1161	return Effect.gen(function* () {
	1162	+ yield* self.prepareErrorCaptureScope();
1137	1163	const lease = self.pendingApprovalLeases.get(executionId);
1138	1164	if (!lease \|\| lease.expiring) return;
1139	1165	lease.expiring = true;