History

Correlate Sentry errors with OTel traces (#1320)

* Correlate Sentry errors with OTel traces * Default the Sentry OTel verification flags on in the e2e cloud stack

Rhys Sullivan committed 4 months ago

Commit 4a84b0d1 parent dfbc10413 files · +382 −40Browse files

Unified · Split

13 files
  1. apps/cloud/src/api/error-response.ts +21 −8
  2. apps/cloud/src/engine/execution-gate.ts +7 −5
  3. apps/cloud/src/engine/execution-rate-limit.ts +7 −5
  4. apps/cloud/src/env-augment.d.ts +2 −0
  5. apps/cloud/src/extensions/billing/route.ts +6 −2
  6. apps/cloud/src/extensions/billing/service.ts +4 −3
  7. apps/cloud/src/mcp/session-durable-object.ts +15 −1
  8. apps/cloud/src/observability/index.ts +114 −3
  9. apps/cloud/src/observability/observability.test.ts +54 −2
  10. apps/cloud/src/server.ts +30 −0
  11. e2e/cloud/sentry-otel-correlation.test.ts +70 −0
  12. e2e/setup/cloud.globalsetup.ts +16 −1
  13. packages/hosts/cloudflare/src/mcp/agent-session-durable-object.ts +36 −10
apps/cloud/src/api/error-response.ts+21−8
@@ -1,7 +1,7 @@
1import { Cause, Data, Effect, Predicate, Result } from "effect";1import { Cause, Data, Effect, Predicate, Result } from "effect";
2import { HttpServerRespondable, HttpServerResponse } from "effect/unstable/http";2import { HttpServerRespondable, HttpServerResponse } from "effect/unstable/http";
3 3
4import { captureCause } from "../observability";4import { captureCause, captureCauseEffect } from "../observability";
5 5
6// Implements `Respondable` so the framework's default cause→response6// Implements `Respondable` so the framework's default cause→response
7// pipeline (`HttpServerRespondable.toResponseOrElse`) renders this as the7// pipeline (`HttpServerRespondable.toResponseOrElse`) renders this as the
@@ -13,13 +13,14 @@ export class HttpResponseError extends Data.TaggedError("HttpResponseError")<{ · 5 unmodified lines
13 readonly message: string;13 readonly message: string;
14}> {14}> {
15 [HttpServerRespondable.symbol](): Effect.Effect<HttpServerResponse.HttpServerResponse> {15 [HttpServerRespondable.symbol](): Effect.Effect<HttpServerResponse.HttpServerResponse> {
16 if (this.status >= 500) captureCause(this);16 const self = this;
17 return Effect.succeed(17 return Effect.gen(function* () {
18 HttpServerResponse.jsonUnsafe(18 if (self.status >= 500) yield* captureCauseEffect(self);
19 { error: this.message, code: this.code },19 return HttpServerResponse.jsonUnsafe(
20 { status: this.status },20 { error: self.message, code: self.code },
21 ),21 { status: self.status },
22 );22 );
23 });
23 }24 }
24}25}
25 26
@@ -65,3 +66,15 @@ export const toErrorServerResponse = (error: unknown): HttpServerResponse.HttpSe · 39 unmodified lines
65 { status: mapped.status },66 { status: mapped.status },
66 );67 );
67};68};
69
70export const toErrorServerResponseEffect = (
71 error: unknown,
72): Effect.Effect<HttpServerResponse.HttpServerResponse> =>
73 Effect.gen(function* () {
74 const mapped = toHttpResponseError(error);
75 if (mapped.status >= 500) yield* captureCauseEffect(mapped);
76 return HttpServerResponse.jsonUnsafe(
77 { error: mapped.message, code: mapped.code },
78 { status: mapped.status },
79 );
80 });
apps/cloud/src/engine/execution-gate.ts+7−5
@@ -24,12 +24,12 @@ · 23 unmodified lines
24// `CodeCompilationError` / `SandboxRuntimeError` into `ExecuteResult.error`.24// `CodeCompilationError` / `SandboxRuntimeError` into `ExecuteResult.error`.
25// ---------------------------------------------------------------------------25// ---------------------------------------------------------------------------
26 26
27import * as Sentry from "@sentry/cloudflare";
28import { Data, Effect } from "effect";27import { Data, Effect } from "effect";
29import type * as Cause from "effect/Cause";28import type * as Cause from "effect/Cause";
30 29
31import type { ExecutionEngine, ExecutionResult } from "@executor-js/execution";30import type { ExecutionEngine, ExecutionResult } from "@executor-js/execution";
32 31
32import { captureCauseEffect } from "../observability";
33import { EXECUTION_LIMIT_BLOCKED_MESSAGE } from "./execution-limit-messages";33import { EXECUTION_LIMIT_BLOCKED_MESSAGE } from "./execution-limit-messages";
34 34
35// The engine's completed-result payload (`ExecuteResult` in codemode-core),35// The engine's completed-result payload (`ExecuteResult` in codemode-core),
@@ -166,10 +166,12 @@ export const makeExecutionLimitGate = (checkBalance: ExecutionBalanceCheck) => { · 130 unmodified lines
166 // must never block executions. Reported like `trackExecution` so a166 // must never block executions. Reported like `trackExecution` so a
167 // billing outage still pages; the error outcome is never cached.167 // billing outage still pages; the error outcome is never cached.
168 Effect.catch((error: unknown) =>168 Effect.catch((error: unknown) =>
169 Effect.sync((): GateDecision => {169 Effect.gen(function* () {
170 console.warn("[billing] execution balance check failed open:", error);170 yield* Effect.sync(() => {
171 Sentry.captureException(error);171 console.warn("[billing] execution balance check failed open:", error);
172 return { blocked: false };172 });
173 yield* captureCauseEffect(error);
174 return { blocked: false } as const satisfies GateDecision;
173 }),175 }),
174 ),176 ),
175 );177 );
apps/cloud/src/engine/execution-rate-limit.ts+7−5
@@ -14,12 +14,12 @@ · 13 unmodified lines
14// ---------------------------------------------------------------------------14// ---------------------------------------------------------------------------
15 15
16import { DurableObject, env } from "cloudflare:workers";16import { DurableObject, env } from "cloudflare:workers";
17import * as Sentry from "@sentry/cloudflare";
18import { Data, Effect } from "effect";17import { Data, Effect } from "effect";
19import type * as Cause from "effect/Cause";18import type * as Cause from "effect/Cause";
20 19
21import type { ExecutionEngine } from "@executor-js/execution";20import type { ExecutionEngine } from "@executor-js/execution";
22 21
22import { captureCauseEffect } from "../observability";
23import { withPreExecutionGate, type GateDecision } from "./execution-gate";23import { withPreExecutionGate, type GateDecision } from "./execution-gate";
24import { RATE_LIMIT_BLOCKED_MESSAGE } from "./execution-limit-messages";24import { RATE_LIMIT_BLOCKED_MESSAGE } from "./execution-limit-messages";
25 25
@@ -149,10 +149,12 @@ export const makeExecutionRateLimiter = ( · 123 unmodified lines
149 // FAIL OPEN: the backstop must never block executions because its149 // FAIL OPEN: the backstop must never block executions because its
150 // counter is unreachable or slow.150 // counter is unreachable or slow.
151 Effect.catch((error: unknown) =>151 Effect.catch((error: unknown) =>
152 Effect.sync((): GateDecision => {152 Effect.gen(function* () {
153 console.warn("[rate-limit] execution rate limit check failed open:", error);153 yield* Effect.sync(() => {
154 Sentry.captureException(error);154 console.warn("[rate-limit] execution rate limit check failed open:", error);
155 return { blocked: false };155 });
156 yield* captureCauseEffect(error);
157 return { blocked: false } as const satisfies GateDecision;
156 }),158 }),
157 ),159 ),
158 );160 );
apps/cloud/src/env-augment.d.ts+2−0
@@ -11,6 +11,8 @@ declare global { · 10 unmodified lines
11 AXIOM_TRACES_URL?: string;11 AXIOM_TRACES_URL?: string;
12 AXIOM_TRACES_SAMPLE_RATIO?: string;12 AXIOM_TRACES_SAMPLE_RATIO?: string;
13 SENTRY_DSN?: string;13 SENTRY_DSN?: string;
14 SENTRY_OTEL_LOG_PAYLOAD?: string;
15 SENTRY_OTEL_VERIFY?: string;
14 VITE_PUBLIC_SENTRY_DSN?: string;16 VITE_PUBLIC_SENTRY_DSN?: string;
15 VITE_PUBLIC_POSTHOG_KEY?: string;17 VITE_PUBLIC_POSTHOG_KEY?: string;
16 VITE_PUBLIC_POSTHOG_HOST?: string;18 VITE_PUBLIC_POSTHOG_HOST?: string;
apps/cloud/src/extensions/billing/route.ts+6−2
@@ -5,7 +5,11 @@ import { autumnHandler } from "autumn-js/backend"; · 4 unmodified lines
5 5
6import { WorkOSClient } from "../../auth/workos";6import { WorkOSClient } from "../../auth/workos";
7import { ORG_SELECTOR_HEADER, authorizeOrganizationSelector } from "../../auth/organization";7import { ORG_SELECTOR_HEADER, authorizeOrganizationSelector } from "../../auth/organization";
8import { HttpResponseError, isServerError, toErrorServerResponse } from "../../api/error-response";8import {
9 HttpResponseError,
10 isServerError,
11 toErrorServerResponseEffect,
12} from "../../api/error-response";
9 13
10type BillingSession = {14type BillingSession = {
11 readonly userId: string;15 readonly userId: string;
@@ -111,7 +115,7 @@ const handler = Effect.gen(function* () { · 99 unmodified lines
111 if (isServerError(err)) {115 if (isServerError(err)) {
112 console.error("[autumn] request failed:", Cause.pretty(err));116 console.error("[autumn] request failed:", Cause.pretty(err));
113 }117 }
114 return Effect.succeed(toErrorServerResponse(err));118 return toErrorServerResponseEffect(err);
115 }),119 }),
116);120);
117 121
apps/cloud/src/extensions/billing/service.ts+4−3
@@ -3,10 +3,11 @@ · 2 unmodified lines
3// ---------------------------------------------------------------------------3// ---------------------------------------------------------------------------
4 4
5import { env } from "cloudflare:workers";5import { env } from "cloudflare:workers";
6import * as Sentry from "@sentry/cloudflare";
7import { Autumn } from "autumn-js";6import { Autumn } from "autumn-js";
8import { Context, Data, Effect, Layer } from "effect";7import { Context, Data, Effect, Layer } from "effect";
9 8
9import { captureCauseEffect } from "../../observability";
10
10// ---------------------------------------------------------------------------11// ---------------------------------------------------------------------------
11// Errors12// Errors
12// ---------------------------------------------------------------------------13// ---------------------------------------------------------------------------
@@ -72,12 +73,12 @@ const make = Effect.sync(() => { · 59 unmodified lines
72 ).pipe(73 ).pipe(
73 Effect.catchTag("AutumnError", (error) =>74 Effect.catchTag("AutumnError", (error) =>
74 Effect.gen(function* () {75 Effect.gen(function* () {
75 // Silent billing data loss is worth paging on — autumn.trackExecution76 // Silent billing data loss is worth paging on: autumn.trackExecution
76 // is fire-and-forget so the caller doesn't handle it themselves.77 // is fire-and-forget so the caller doesn't handle it themselves.
77 yield* Effect.sync(() => {78 yield* Effect.sync(() => {
78 console.error("[billing] track failed:", error);79 console.error("[billing] track failed:", error);
79 Sentry.captureException(error);
80 });80 });
81 yield* captureCauseEffect(error);
81 yield* Effect.annotateCurrentSpan({ "autumn.track.failed": true });82 yield* Effect.annotateCurrentSpan({ "autumn.track.failed": true });
82 }),83 }),
83 ),84 ),
apps/cloud/src/mcp/session-durable-object.ts+15−1
@@ -68,7 +68,11 @@ import { makeExecutionStack } from "../engine/execution-stack"; · 67 unmodified lines
68import { CloudMeteredExecutionStackLayer } from "../engine/execution-stack-metered";68import { CloudMeteredExecutionStackLayer } from "../engine/execution-stack-metered";
69import { AutumnService } from "../extensions/billing/service";69import { AutumnService } from "../extensions/billing/service";
70import { DoTelemetryLive, flushTracerProvider } from "../observability/telemetry";70import { DoTelemetryLive, flushTracerProvider } from "../observability/telemetry";
71import { captureCause as reportCause } from "../observability";71import {
72 captureCause as reportCause,
73 captureCauseEffect as reportCauseEffect,
74 tagCurrentSentryScopeWithCurrentOtelSpan,
75} from "../observability";
72 76
73// Re-export the shared types so existing cloud importers77// Re-export the shared types so existing cloud importers
74// (`auth/handlers.ts`, etc.) keep their `../mcp/session-durable-object` path.78// (`auth/handlers.ts`, etc.) keep their `../mcp/session-durable-object` path.
@@ -314,6 +318,16 @@ export class McpSessionDOSqlite extends McpAgentSessionDOBase<Env, CloudSessionD · 239 unmodified lines
314 reportCause(cause);318 reportCause(cause);
315 }319 }
316 320
321 protected override captureCauseEffect(
322 cause: Cause.Cause<unknown>,
323 ): Effect.Effect<string | undefined> {
324 return reportCauseEffect(cause);
325 }
326
327 protected override prepareErrorCaptureScope(): Effect.Effect<void> {
328 return Effect.asVoid(tagCurrentSentryScopeWithCurrentOtelSpan);
329 }
330
317 // Best-effort export the DO isolate's buffered spans after the RPC settles,331 // Best-effort export the DO isolate's buffered spans after the RPC settles,
318 // so a dying init/handleRequest can ship its own spans (and the exception +332 // so a dying init/handleRequest can ship its own spans (and the exception +
319 // stack recorded on them) — not just the worker-side `mcp.do.*` span. Keep it333 // stack recorded on them) — not just the worker-side `mcp.do.*` span. Keep it
apps/cloud/src/observability/index.ts+114−3
@@ -11,7 +11,9 @@ · 10 unmodified lines
11// ---------------------------------------------------------------------------11// ---------------------------------------------------------------------------
12 12
13import * as Sentry from "@sentry/cloudflare";13import * as Sentry from "@sentry/cloudflare";
14import type { ErrorEvent, Scope } from "@sentry/cloudflare";
14import { Cause, Effect, Layer } from "effect";15import { Cause, Effect, Layer } from "effect";
16import type * as Tracer from "effect/Tracer";
15 17
16import { ErrorCapture } from "@executor-js/api";18import { ErrorCapture } from "@executor-js/api";
17 19
@@ -21,12 +23,106 @@ import { ErrorCapture } from "@executor-js/api"; · 3 unmodified lines
21// error. Sentry still receives the full, untruncated cause via23// error. Sentry still receives the full, untruncated cause via
22// `setExtra`; only the dev-console mirror is capped.24// `setExtra`; only the dev-console mirror is capped.
23const MAX_CONSOLE_CAUSE_CHARS = 4_000;25const MAX_CONSOLE_CAUSE_CHARS = 4_000;
26const OTEL_TRACE_ID_PATTERN = /^[0-9a-f]{32}$/;
27const OTEL_SPAN_ID_PATTERN = /^[0-9a-f]{16}$/;
28
29export const OTEL_TRACE_ID_TAG = "otel_trace_id";
30export const OTEL_SPAN_ID_TAG = "otel_span_id";
31export const SENTRY_EVENT_ID_ATTRIBUTE = "sentry.event_id";
32
33export type OtelCorrelationContext = {
34 readonly traceId: string;
35 readonly spanId: string;
36};
24 37
25const truncate = (s: string): string =>38const truncate = (s: string): string =>
26 s.length <= MAX_CONSOLE_CAUSE_CHARS39 s.length <= MAX_CONSOLE_CAUSE_CHARS
27 ? s40 ? s
28 : `${s.slice(0, MAX_CONSOLE_CAUSE_CHARS)}\n…[truncated ${s.length - MAX_CONSOLE_CAUSE_CHARS} chars]`;41 : `${s.slice(0, MAX_CONSOLE_CAUSE_CHARS)}\n…[truncated ${s.length - MAX_CONSOLE_CAUSE_CHARS} chars]`;
29 42
43const validOtelContext = (context: OtelCorrelationContext): boolean =>
44 OTEL_TRACE_ID_PATTERN.test(context.traceId) && OTEL_SPAN_ID_PATTERN.test(context.spanId);
45
46export const otelCorrelationContextFromEffectSpan = (
47 span: Tracer.Span,
48): OtelCorrelationContext | null => {
49 const context = { traceId: span.traceId, spanId: span.spanId };
50 return validOtelContext(context) ? context : null;
51};
52
53export const otelCorrelationContextFromOpenTelemetrySpan = (span: {
54 readonly spanContext: () => { readonly traceId: string; readonly spanId: string };
55}): OtelCorrelationContext | null => {
56 const { traceId, spanId } = span.spanContext();
57 const context = { traceId, spanId };
58 return validOtelContext(context) ? context : null;
59};
60
61export const addOtelCorrelationTags = <T extends { readonly tags?: Record<string, unknown> }>(
62 event: T,
63 context: OtelCorrelationContext | null,
64): T => {
65 if (!context) return event;
66 return {
67 ...event,
68 tags: {
69 ...event.tags,
70 [OTEL_TRACE_ID_TAG]: context.traceId,
71 [OTEL_SPAN_ID_TAG]: context.spanId,
72 },
73 };
74};
75
76export const tagSentryScopeWithOtelContext = (
77 scope: Scope,
78 context: OtelCorrelationContext | null,
79): void => {
80 if (!context) return;
81 scope.setTag(OTEL_TRACE_ID_TAG, context.traceId);
82 scope.setTag(OTEL_SPAN_ID_TAG, context.spanId);
83};
84
85export const tagCurrentSentryScopeWithOtelContext = (
86 context: OtelCorrelationContext | null,
87): void => {
88 tagSentryScopeWithOtelContext(Sentry.getCurrentScope(), context);
89};
90
91const currentOtelContext = Effect.map(
92 Effect.currentSpan,
93 otelCorrelationContextFromEffectSpan,
94).pipe(Effect.orElseSucceed(() => null));
95
96export const tagCurrentSentryScopeWithCurrentOtelSpan: Effect.Effect<OtelCorrelationContext | null> =
97 Effect.map(currentOtelContext, (context) => {
98 tagCurrentSentryScopeWithOtelContext(context);
99 return context;
100 });
101
102export const beforeSendWithOtelCorrelation = (
103 event: ErrorEvent,
104 options?: { readonly logPayload?: boolean },
105): ErrorEvent => {
106 if (options?.logPayload) {
107 console.info(
108 JSON.stringify({
109 event: "sentry_before_send_otel_correlation",
110 sentry_event_id: event.event_id ?? "",
111 otel_trace_id: String(event.tags?.[OTEL_TRACE_ID_TAG] ?? ""),
112 otel_span_id: String(event.tags?.[OTEL_SPAN_ID_TAG] ?? ""),
113 }),
114 );
115 }
116 return event;
117};
118
119export const addCurrentOtelCorrelationTags = <
120 T extends { readonly tags?: Record<string, unknown> },
121>(
122 event: T,
123): Effect.Effect<T> =>
124 Effect.map(currentOtelContext, (context) => addOtelCorrelationTags(event, context));
125
30// Sentry's `captureException` can't serialize Effect's `CauseImpl` (it logs126// Sentry's `captureException` can't serialize Effect's `CauseImpl` (it logs
31// `'CauseImpl' captured as exception with keys: reasons, ~effect/Cause` and127// `'CauseImpl' captured as exception with keys: reasons, ~effect/Cause` and
32// drops the real failure). `Cause.squash` isn't enough on its own: when an128// drops the real failure). `Cause.squash` isn't enough on its own: when an
@@ -48,21 +144,36 @@ export const sentryPayloadForCause = ( · 15 unmodified lines
48 return { primary: input, pretty: null };144 return { primary: input, pretty: null };
49};145};
50 146
51export const captureCause = (input: unknown): string | undefined => {147export const captureCause = (
148 input: unknown,
149 context: OtelCorrelationContext | null = null,
150): string | undefined => {
52 const { primary, pretty } = sentryPayloadForCause(input);151 const { primary, pretty } = sentryPayloadForCause(input);
152 tagCurrentSentryScopeWithOtelContext(context);
53 return Sentry.captureException(primary, (scope) => {153 return Sentry.captureException(primary, (scope) => {
154 tagSentryScopeWithOtelContext(scope, context);
54 if (pretty !== null) scope.setExtra("cause", pretty);155 if (pretty !== null) scope.setExtra("cause", pretty);
55 return scope;156 return scope;
56 });157 });
57};158};
58 159
160export const captureCauseEffect = (input: unknown): Effect.Effect<string | undefined> =>
161 Effect.gen(function* () {
162 const context = yield* tagCurrentSentryScopeWithCurrentOtelSpan;
163 const eventId = yield* Effect.sync(() => captureCause(input, context));
164 if (eventId && context) {
165 yield* Effect.annotateCurrentSpan(SENTRY_EVENT_ID_ATTRIBUTE, eventId);
166 }
167 return eventId;
168 });
169
59export const ErrorCaptureLive: Layer.Layer<ErrorCapture> = Layer.succeed(170export const ErrorCaptureLive: Layer.Layer<ErrorCapture> = Layer.succeed(
60 ErrorCapture,171 ErrorCapture,
61 ErrorCapture.of({172 ErrorCapture.of({
62 captureException: (cause) =>173 captureException: (cause) =>
63 Effect.sync(() => {174 Effect.gen(function* () {
64 console.error("[api] unhandled cause:", truncate(Cause.pretty(cause)));175 console.error("[api] unhandled cause:", truncate(Cause.pretty(cause)));
65 return captureCause(cause) ?? "";176 return (yield* captureCauseEffect(cause)) ?? "";
66 }),177 }),
67 }),178 }),
68);179);
apps/cloud/src/observability/observability.test.ts+54−2
@@ -1,7 +1,13 @@
1import { describe, expect, it } from "@effect/vitest";1import { describe, expect, it } from "@effect/vitest";
2import { Cause } from "effect";2import { Cause, Effect } from "effect";
3import type * as Tracer from "effect/Tracer";
3 4
4import { sentryPayloadForCause } from "./index";5import {
6 addCurrentOtelCorrelationTags,
7 OTEL_SPAN_ID_TAG,
8 OTEL_TRACE_ID_TAG,
9 sentryPayloadForCause,
10} from "./index";
5 11
6// Mirrors Sentry core's `is.isError`: it picks the proper-Error path iff12// Mirrors Sentry core's `is.isError`: it picks the proper-Error path iff
7// `Object.prototype.toString.call(x) === "[object Error]"`. Anything that13// `Object.prototype.toString.call(x) === "[object Error]"`. Anything that
@@ -10,6 +16,39 @@ import { sentryPayloadForCause } from "./index"; · 2 unmodified lines
10const looksLikeErrorToSentry = (value: unknown): boolean =>16const looksLikeErrorToSentry = (value: unknown): boolean =>
11 Object.prototype.toString.call(value) === "[object Error]";17 Object.prototype.toString.call(value) === "[object Error]";
12 18
19const traceId = "4268a606000000000000000000000000";
20const spanId = "1234567890abcdef";
21
22const makeFixedTracer = (): Tracer.Tracer => ({
23 span: (options) => {
24 const attributes = new Map<string, unknown>();
25 let status: Tracer.SpanStatus = { _tag: "Started", startTime: options.startTime };
26 return {
27 _tag: "Span",
28 name: options.name,
29 spanId,
30 traceId,
31 parent: options.parent,
32 annotations: options.annotations,
33 get status() {
34 return status;
35 },
36 attributes,
37 links: options.links,
38 sampled: options.sampled,
39 kind: options.kind,
40 end: (endTime, exit) => {
41 status = { _tag: "Ended", startTime: options.startTime, endTime, exit };
42 },
43 attribute: (key, value) => {
44 attributes.set(key, value);
45 },
46 event: () => undefined,
47 addLinks: () => undefined,
48 };
49 },
50});
51
13describe("sentryPayloadForCause", () => {52describe("sentryPayloadForCause", () => {
14 it("hands Sentry a real Error when the defect is itself a Cause", () => {53 it("hands Sentry a real Error when the defect is itself a Cause", () => {
15 // Reproduces the production chain: an inner runPromise rejects with a54 // Reproduces the production chain: an inner runPromise rejects with a
@@ -39,3 +78,16 @@ describe("sentryPayloadForCause", () => { · 23 unmodified lines
39 expect(pretty).toBeNull();78 expect(pretty).toBeNull();
40 });79 });
41});80});
81
82describe("Sentry OTel correlation", () => {
83 it.effect("adds tags from the active Effect span", () =>
84 Effect.gen(function* () {
85 const baseEvent: { readonly tags: Record<string, unknown> } = { tags: { existing: "tag" } };
86 const event = yield* addCurrentOtelCorrelationTags(baseEvent);
87
88 expect(event.tags.existing).toBe("tag");
89 expect(event.tags[OTEL_TRACE_ID_TAG]).toBe(traceId);
90 expect(event.tags[OTEL_SPAN_ID_TAG]).toBe(spanId);
91 }).pipe(Effect.withSpan("test.sentry_capture"), Effect.withTracer(makeFixedTracer())),
92 );
93});
apps/cloud/src/server.ts+30−0
@@ -1,5 +1,6 @@
1import { DurableObject } from "cloudflare:workers";1import { DurableObject } from "cloudflare:workers";
2import { SpanKind, SpanStatusCode, context, trace } from "@opentelemetry/api";2import { SpanKind, SpanStatusCode, context, trace } from "@opentelemetry/api";
3import type { ErrorEvent } from "@sentry/cloudflare";
3import {4import {
4 ATTR_HTTP_REQUEST_METHOD,5 ATTR_HTTP_REQUEST_METHOD,
5 ATTR_HTTP_RESPONSE_STATUS_CODE,6 ATTR_HTTP_RESPONSE_STATUS_CODE,
@@ -14,6 +15,13 @@ import { isAppOwnedPath } from "./app-paths"; · 8 unmodified lines
14import { makeCloudMcpAgentHandler } from "./mcp/agent-handler";15import { makeCloudMcpAgentHandler } from "./mcp/agent-handler";
15import { classifyMcpPath, prepareMcpOrgScope } from "./mcp/mount";16import { classifyMcpPath, prepareMcpOrgScope } from "./mcp/mount";
16import { McpSessionDOSqlite as McpSessionDOBase } from "./mcp/session-durable-object";17import { McpSessionDOSqlite as McpSessionDOBase } from "./mcp/session-durable-object";
18import {
19 beforeSendWithOtelCorrelation,
20 captureCause,
21 otelCorrelationContextFromOpenTelemetrySpan,
22 SENTRY_EVENT_ID_ATTRIBUTE,
23 tagCurrentSentryScopeWithOtelContext,
24} from "./observability";
17import { browserTracesResponse } from "./observability/browser-traces";25import { browserTracesResponse } from "./observability/browser-traces";
18import { flushTracerProvider, installTracerProvider } from "./observability/telemetry";26import { flushTracerProvider, installTracerProvider } from "./observability/telemetry";
19 27
@@ -27,6 +35,10 @@ const sentryOptions = (env: Env) => ({ · 7 unmodified lines
27 enableLogs: true,35 enableLogs: true,
28 sendDefaultPii: true,36 sendDefaultPii: true,
29 skipOpenTelemetrySetup: true,37 skipOpenTelemetrySetup: true,
38 beforeSend: (event: ErrorEvent) =>
39 beforeSendWithOtelCorrelation(event, {
40 logPayload: !env.SENTRY_DSN || env.SENTRY_OTEL_LOG_PAYLOAD === "true",
41 }),
30 // NOTE: do NOT enable `instrumentPrototypeMethods`. It walks the DO prototype42 // NOTE: do NOT enable `instrumentPrototypeMethods`. It walks the DO prototype
31 // and reads every property — including accessors — to find methods to wrap,43 // and reads every property — including accessors — to find methods to wrap,
32 // which invokes the `sessionId` getter with `this` bound to the prototype44 // which invokes the `sessionId` getter with `this` bound to the prototype
@@ -158,6 +170,8 @@ const cloudflareHandler: ExportedHandler<Env> = { · 125 unmodified lines
158 { kind: SpanKind.SERVER },170 { kind: SpanKind.SERVER },
159 parentContext,171 parentContext,
160 async (span) => {172 async (span) => {
173 const otelContext = otelCorrelationContextFromOpenTelemetrySpan(span);
174 tagCurrentSentryScopeWithOtelContext(otelContext);
161 span.setAttribute(ATTR_HTTP_REQUEST_METHOD, request.method);175 span.setAttribute(ATTR_HTTP_REQUEST_METHOD, request.method);
162 span.setAttribute(ATTR_URL_FULL, request.url);176 span.setAttribute(ATTR_URL_FULL, request.url);
163 span.setAttribute(ATTR_URL_PATH, url.pathname);177 span.setAttribute(ATTR_URL_PATH, url.pathname);
@@ -168,6 +182,22 @@ const cloudflareHandler: ExportedHandler<Env> = { · 4 unmodified lines
168 // outer wrapper still captures the exception; we only mark span status.182 // outer wrapper still captures the exception; we only mark span status.
169 // oxlint-disable-next-line executor/no-try-catch-or-throw -- adapter boundary183 // oxlint-disable-next-line executor/no-try-catch-or-throw -- adapter boundary
170 try {184 try {
185 if (env.SENTRY_OTEL_VERIFY === "true" && url.pathname === "/__sentry-otel-verify") {
186 // oxlint-disable-next-line executor/no-error-constructor -- boundary: synthetic verification needs an Error payload for Sentry grouping
187 const eventId = captureCause(new Error("sentry otel verification"), otelContext) ?? "";
188 if (eventId) span.setAttribute(SENTRY_EVENT_ID_ATTRIBUTE, eventId);
189 span.setAttribute("sentry_otel.verify", true);
190 span.setAttribute(ATTR_HTTP_RESPONSE_STATUS_CODE, 500);
191 span.setStatus({ code: SpanStatusCode.ERROR });
192 return Response.json(
193 {
194 sentryEventId: eventId,
195 otelTraceId: otelContext?.traceId ?? "",
196 otelSpanId: otelContext?.spanId ?? "",
197 },
198 { status: 500 },
199 );
200 }
171 const response = await fetchHandler(request, env, ctx);201 const response = await fetchHandler(request, env, ctx);
172 span.setAttribute(ATTR_HTTP_RESPONSE_STATUS_CODE, response.status);202 span.setAttribute(ATTR_HTTP_RESPONSE_STATUS_CODE, response.status);
173 if (response.status >= 500) {203 if (response.status >= 500) {
e2e/cloud/sentry-otel-correlation.test.ts+70−0
@@ -0,0 +1,70 @@
1import { readFileSync } from "node:fs";
2import { resolve } from "node:path";
3
4import { expect } from "@effect/vitest";
5import { Effect, Schedule } from "effect";
6
7import { RUNS_DIR, scenario } from "../src/scenario";
8import { Target, Telemetry } from "../src/services";
9
10type VerificationResponse = {
11 readonly sentryEventId: string;
12 readonly otelTraceId: string;
13 readonly otelSpanId: string;
14};
15
16const beforeSendLogFile = resolve(RUNS_DIR, "cloud", "server-logs", "boot.log");
17
18const hasCorrelationPayload = (line: string, body: VerificationResponse): boolean =>
19 line.includes('"event":"sentry_before_send_otel_correlation"') &&
20 line.includes(`"sentry_event_id":"${body.sentryEventId}"`) &&
21 line.includes(`"otel_trace_id":"${body.otelTraceId}"`) &&
22 line.includes(`"otel_span_id":"${body.otelSpanId}"`);
23
24const findBeforeSendPayload = (body: VerificationResponse): Effect.Effect<string, string> =>
25 Effect.sync(() => {
26 const text = readFileSync(beforeSendLogFile, "utf8");
27 return text
28 .split("\n")
29 .slice()
30 .reverse()
31 .find((line) => hasCorrelationPayload(line, body));
32 }).pipe(
33 Effect.filterOrFail(
34 (line): line is string => line !== undefined,
35 () =>
36 `no Sentry beforeSend correlation payload matched ${body.sentryEventId} in ${beforeSendLogFile}`,
37 ),
38 Effect.retry(Schedule.both(Schedule.spaced("500 millis"), Schedule.recurs(20))),
39 );
40
41scenario(
42 "Telemetry · Sentry error payload joins to exported OTel span",
43 { timeout: 120_000 },
44 Effect.gen(function* () {
45 const target = yield* Target;
46 const telemetry = yield* Telemetry;
47
48 const response = yield* Effect.promise(() =>
49 fetch(new URL("/__sentry-otel-verify", target.baseUrl)),
50 );
51 expect(response.status).toBe(500);
52
53 const body = (yield* Effect.promise(() => response.json())) as VerificationResponse;
54 expect(body.sentryEventId).toMatch(/^[0-9a-f]{32}$/);
55 expect(body.otelTraceId).toMatch(/^[0-9a-f]{32}$/);
56 expect(body.otelSpanId).toMatch(/^[0-9a-f]{16}$/);
57
58 const beforeSendLine = yield* findBeforeSendPayload(body);
59 expect(beforeSendLine).toContain(body.otelTraceId);
60
61 const span = yield* telemetry.expectSpan({
62 traceId: body.otelTraceId,
63 attributes: {
64 "sentry.event_id": body.sentryEventId,
65 "sentry_otel.verify": "true",
66 },
67 });
68 expect(span.span.spanId).toBe(body.otelSpanId);
69 }),
70);
e2e/setup/cloud.globalsetup.ts+16−1
@@ -25,6 +25,21 @@ const bootLogFile = process.env.E2E_VERBOSE · 24 unmodified lines
25 ? undefined25 ? undefined
26 : resolve(RUNS_DIR, "cloud", "server-logs", "boot.log");26 : resolve(RUNS_DIR, "cloud", "server-logs", "boot.log");
27 27
28const optionalCloudEnv = (): Record<string, string> => {
29 // The Sentry/OTel correlation scenario always runs in this suite, so the
30 // verification route and beforeSend payload logging default on here; the
31 // env vars remain overridable for local runs against a different setup.
32 const env: Record<string, string> = {
33 SENTRY_OTEL_VERIFY: "true",
34 SENTRY_OTEL_LOG_PAYLOAD: "true",
35 };
36 for (const key of ["SENTRY_DSN", "SENTRY_OTEL_LOG_PAYLOAD", "SENTRY_OTEL_VERIFY"]) {
37 const value = process.env[key];
38 if (value) env[key] = value;
39 }
40 return env;
41};
42
28export default async function setup(): Promise<(() => Promise<void>) | void> {43export default async function setup(): Promise<(() => Promise<void>) | void> {
29 if (process.env.E2E_CLOUD_URL) {44 if (process.env.E2E_CLOUD_URL) {
30 await waitForHttp(process.env.E2E_CLOUD_URL);45 await waitForHttp(process.env.E2E_CLOUD_URL);
@@ -74,7 +89,7 @@ export default async function setup(): Promise<(() => Promise<void>) | void> { · 43 unmodified lines
74 // endpoint-agnostic, so the same layer that ships prod traces to89 // endpoint-agnostic, so the same layer that ships prod traces to
75 // Axiom ships e2e traces to the suite store — "why was that page90 // Axiom ships e2e traces to the suite store — "why was that page
76 // slow" gets a span waterfall, not a guess.91 // slow" gets a span waterfall, not a guess.
77 extraEnv: motelExporterEnv(motel, publicUrl),92 extraEnv: { ...motelExporterEnv(motel, publicUrl), ...optionalCloudEnv() },
78 });93 });
79 return { teardown: cloud.teardown, value: cloud };94 return { teardown: cloud.teardown, value: cloud };
80 },95 },
packages/hosts/cloudflare/src/mcp/agent-session-durable-object.ts+36−10
@@ -236,6 +236,17 @@ export abstract class McpAgentSessionDOBase< · 235 unmodified lines
236 236
237 protected captureCause(_cause: Cause.Cause<unknown>): void {}237 protected captureCause(_cause: Cause.Cause<unknown>): void {}
238 238
239 protected captureCauseEffect(cause: Cause.Cause<unknown>): Effect.Effect<string | undefined> {
240 return Effect.sync(() => {
241 this.captureCause(cause);
242 return undefined;
243 });
244 }
245
246 protected prepareErrorCaptureScope(): Effect.Effect<void> {
247 return Effect.void;
248 }
249
239 protected flushTelemetry(): Promise<void> {250 protected flushTelemetry(): Promise<void> {
240 return Promise.resolve();251 return Promise.resolve();
241 }252 }
@@ -604,6 +615,7 @@ export abstract class McpAgentSessionDOBase< · 362 unmodified lines
604 }615 }
605 const self = this;616 const self = this;
606 const program = Effect.gen(function* () {617 const program = Effect.gen(function* () {
618 yield* self.prepareErrorCaptureScope();
607 const sessionMeta = yield* self.resolveAndStoreSessionMeta(props.session);619 const sessionMeta = yield* self.resolveAndStoreSessionMeta(props.session);
608 const dbHandle = yield* self.openSessionDbHandle();620 const dbHandle = yield* self.openSessionDbHandle();
609 const { mcpServer, engine } = yield* self.buildRuntime(sessionMeta, dbHandle);621 const { mcpServer, engine } = yield* self.buildRuntime(sessionMeta, dbHandle);
@@ -618,7 +630,7 @@ export abstract class McpAgentSessionDOBase< · 8 unmodified lines
618 Effect.tapCause((cause) =>630 Effect.tapCause((cause) =>
619 Effect.gen(function* () {631 Effect.gen(function* () {
620 console.error("[mcp-session] init failed:", Cause.pretty(cause));632 console.error("[mcp-session] init failed:", Cause.pretty(cause));
621 self.captureCause(cause);633 yield* self.captureCauseEffect(cause);
622 yield* self.recordCauseOnSpan(cause);634 yield* self.recordCauseOnSpan(cause);
623 }),635 }),
624 ),636 ),
@@ -650,6 +662,7 @@ export abstract class McpAgentSessionDOBase< · 25 unmodified lines
650 const self = this;662 const self = this;
651 return Effect.runPromise(663 return Effect.runPromise(
652 Effect.gen(function* () {664 Effect.gen(function* () {
665 yield* self.prepareErrorCaptureScope();
653 const sessionMeta = yield* self.loadSessionMeta();666 const sessionMeta = yield* self.loadSessionMeta();
654 if (!sessionMeta) return "not_found" as const;667 if (!sessionMeta) return "not_found" as const;
655 if (self.initialized) {668 if (self.initialized) {
@@ -681,6 +694,7 @@ export abstract class McpAgentSessionDOBase< · 25 unmodified lines
681 const self = this;694 const self = this;
682 return Effect.runPromise(695 return Effect.runPromise(
683 Effect.gen(function* () {696 Effect.gen(function* () {
697 yield* self.prepareErrorCaptureScope();
684 const owner = yield* self.validateApprovalIdentity(identity);698 const owner = yield* self.validateApprovalIdentity(identity);
685 if (owner !== "ok") return { status: owner } as const;699 if (owner !== "ok") return { status: owner } as const;
686 700
@@ -718,6 +732,7 @@ export abstract class McpAgentSessionDOBase< · 31 unmodified lines
718 const self = this;732 const self = this;
719 return Effect.runPromise(733 return Effect.runPromise(
720 Effect.gen(function* () {734 Effect.gen(function* () {
735 yield* self.prepareErrorCaptureScope();
721 const owner = yield* self.validateApprovalIdentity(identity);736 const owner = yield* self.validateApprovalIdentity(identity);
722 if (owner === "forbidden") return { status: "execution_forbidden" } as const;737 if (owner === "forbidden") return { status: "execution_forbidden" } as const;
723 if (owner === "not_found") {738 if (owner === "not_found") {
@@ -775,6 +790,7 @@ export abstract class McpAgentSessionDOBase< · 51 unmodified lines
775 const self = this;790 const self = this;
776 return Effect.runPromise(791 return Effect.runPromise(
777 Effect.gen(function* () {792 Effect.gen(function* () {
793 yield* self.prepareErrorCaptureScope();
778 const owner = yield* self.validateApprovalIdentity(identity);794 const owner = yield* self.validateApprovalIdentity(identity);
779 if (owner !== "ok") return { status: owner } as const;795 if (owner !== "ok") return { status: owner } as const;
780 796
@@ -1015,6 +1031,7 @@ export abstract class McpAgentSessionDOBase< · 234 unmodified lines
1015 ): Effect.Effect<void> {1031 ): Effect.Effect<void> {
1016 const self = this;1032 const self = this;
1017 return Effect.gen(function* () {1033 return Effect.gen(function* () {
1034 yield* self.prepareErrorCaptureScope();
1018 if (self.pendingApprovalLeases.has(executionId)) return;1035 if (self.pendingApprovalLeases.has(executionId)) return;
1019 1036
1020 yield* Effect.promise(() => self.markActivity()).pipe(1037 yield* Effect.promise(() => self.markActivity()).pipe(
@@ -1031,9 +1048,14 @@ export abstract class McpAgentSessionDOBase< · 10 unmodified lines
1031 attributes: { "mcp.execution.id": executionId },1048 attributes: { "mcp.execution.id": executionId },
1032 }),1049 }),
1033 Effect.tapCause((cause) =>1050 Effect.tapCause((cause) =>
1034 Effect.sync(() => {1051 Effect.gen(function* () {
1035 console.error("[mcp-session] pending approval lease start failed:", Cause.pretty(cause));1052 yield* Effect.sync(() => {
1036 self.captureCause(cause);1053 console.error(
1054 "[mcp-session] pending approval lease start failed:",
1055 Cause.pretty(cause),
1056 );
1057 });
1058 yield* self.captureCauseEffect(cause);
1037 }),1059 }),
1038 ),1060 ),
1039 Effect.ignore,1061 Effect.ignore,
@@ -1048,16 +1070,19 @@ export abstract class McpAgentSessionDOBase< · 8 unmodified lines
1048 }1070 }
1049 1071
1050 private queuePendingApprovalLeaseExpiration(executionId: string): void {1072 private queuePendingApprovalLeaseExpiration(executionId: string): void {
1073 const self = this;
1051 this.ctx.waitUntil(1074 this.ctx.waitUntil(
1052 Effect.runPromise(1075 Effect.runPromise(
1053 this.expirePendingApproval(executionId).pipe(1076 this.expirePendingApproval(executionId).pipe(
1054 Effect.tapCause((cause) =>1077 Effect.tapCause((cause) =>
1055 Effect.sync(() => {1078 Effect.gen(function* () {
1056 console.error(1079 yield* Effect.sync(() => {
1057 "[mcp-session] pending approval lease expiration failed:",1080 console.error(
1058 Cause.pretty(cause),1081 "[mcp-session] pending approval lease expiration failed:",
1059 );1082 Cause.pretty(cause),
1060 this.captureCause(cause);1083 );
1084 });
1085 yield* self.captureCauseEffect(cause);
1061 }),1086 }),
1062 ),1087 ),
1063 Effect.ignore,1088 Effect.ignore,
@@ -1134,6 +1159,7 @@ export abstract class McpAgentSessionDOBase< · 70 unmodified lines
1134 private expirePendingApproval(executionId: string): Effect.Effect<void> {1159 private expirePendingApproval(executionId: string): Effect.Effect<void> {
1135 const self = this;1160 const self = this;
1136 return Effect.gen(function* () {1161 return Effect.gen(function* () {
1162 yield* self.prepareErrorCaptureScope();
1137 const lease = self.pendingApprovalLeases.get(executionId);1163 const lease = self.pendingApprovalLeases.get(executionId);
1138 if (!lease || lease.expiring) return;1164 if (!lease || lease.expiring) return;
1139 lease.expiring = true;1165 lease.expiring = true;