lael-labs/executor

apps/cloud/src/observability/index.ts+48−0

@@ -16,6 +16,7 @@ import { Cause, Effect, Layer, Predicate } from "effect"; · 15 unmodified lines
16	import type * as Tracer from "effect/Tracer";	16	import type * as Tracer from "effect/Tracer";
17		17
18	import { ErrorCapture } from "@executor-js/api";	18	import { ErrorCapture } from "@executor-js/api";
		19	import { withStableGroupingFingerprint } from "@executor-js/sdk/sentry-grouping";
19		20
20	// Drizzle/postgres-js include the failing SQL (params + bound values) in	21	// Drizzle/postgres-js include the failing SQL (params + bound values) in
21	// their error message. For OpenAPI source inserts that's 1MB+ of spec	22	// their error message. For OpenAPI source inserts that's 1MB+ of spec
@@ -146,6 +147,53 @@ export const beforeSendWithOtelCorrelation = ( · 124 unmodified lines
146	return event;	147	return event;
147	};	148	};
148		149
		150	/**
		151	* The single `beforeSend` the worker and its Durable Objects install.
		152	*
		153	* The worker ships as content-hashed chunks and its frames are not resolved
		154	* back to source, so Sentry's default grouping keys on names like
		155	* `execution-rate-limit-<hash>` and re-opens every issue on the next deploy.
		156	* `withStableGroupingFingerprint` pins a key with the hash normalized out;
		157	* events with no hashed grouping input are left on the default algorithm.
		158	*
		159	* The two stages are independent and compose in this order: the capture-owner
		160	* pass decides WHETHER the event is reported at all (a cause the Durable
		161	* Object already claimed is dropped, and a dropped event is never
		162	* fingerprinted), and the grouping pass then decides HOW whatever survives is
		163	* grouped.
		164	*/
		165	export const beforeSendCloudEvent = (
		166	event: ErrorEvent,
		167	options?: { readonly logPayload?: boolean },
		168	): ErrorEvent \| null => {
		169	const reported = beforeSendWithOtelCorrelation(event, options);
		170	return reported === null ? null : withStableGroupingFingerprint(reported);
		171	};
		172
		173	/**
		174	* The Sentry options the worker and every Durable Object install. It lives
		175	* beside the `beforeSend` it wires so the composition is covered by
		176	* observability.test.ts; `server.ts` only passes this through.
		177	*
		178	* NOTE: do NOT enable `instrumentPrototypeMethods`. It walks the DO prototype
		179	* and reads every property — including accessors — to find methods to wrap,
		180	* which invokes the `sessionId` getter with `this` bound to the prototype
		181	* (where `ctx` is undefined) and throws during construction, 500ing every
		182	* session create / cold restore. The DO captures its own errors via the
		183	* `captureCause` seam (→ Sentry) instead.
		184	*/
		185	export const cloudSentryOptions = (env: Env) => ({
		186	dsn: env.SENTRY_DSN,
		187	tracesSampleRate: 0,
		188	enableLogs: true,
		189	sendDefaultPii: true,
		190	skipOpenTelemetrySetup: true,
		191	beforeSend: (event: ErrorEvent) =>
		192	beforeSendCloudEvent(event, {
		193	logPayload: !env.SENTRY_DSN \|\| env.SENTRY_OTEL_LOG_PAYLOAD === "true",
		194	}),
		195	});
		196
149	export const addCurrentOtelCorrelationTags = <	197	export const addCurrentOtelCorrelationTags = <
150	T extends { readonly tags?: Record<string, unknown> },	198	T extends { readonly tags?: Record<string, unknown> },
151	>(	199	>(

apps/cloud/src/observability/observability.test.ts+137−0

@@ -6,7 +6,9 @@ import type { ErrorEvent } from "@sentry/cloudflare"; · 5 unmodified lines
6		6
7	import {	7	import {
8	addCurrentOtelCorrelationTags,	8	addCurrentOtelCorrelationTags,
		9	beforeSendCloudEvent,
9	beforeSendWithOtelCorrelation,	10	beforeSendWithOtelCorrelation,
		11	cloudSentryOptions,
10	DO_CAUSE_OWNER_TAG,	12	DO_CAUSE_OWNER_TAG,
11	DO_CAUSE_OWNER_VALUE,	13	DO_CAUSE_OWNER_VALUE,
12	OTEL_SPAN_ID_TAG,	14	OTEL_SPAN_ID_TAG,
@@ -84,6 +86,83 @@ describe("sentryPayloadForCause", () => { · 71 unmodified lines
84	});	86	});
85	});	87	});
86		88
		89	// Grouping keys are decided inside the Sentry SDK and never appear on any
		90	// product surface, so the e2e harness cannot observe them. The split is:
		91	// e2e/cloud/sentry-otel-correlation.test.ts proves the worker really installs
		92	// `cloudSentryOptions.beforeSend` (its correlation payload only exists if that
		93	// hook ran), and the tests here prove the hook it installs fingerprints.
		94	describe("Sentry grouping", () => {
		95	// The worker bundle ships as content-hashed chunks, so the only module name
		96	// Sentry ever sees for a given frame changes on every deploy.
		97	const workerEvent = (chunkHash: string): ErrorEvent => ({
		98	type: undefined,
		99	exception: {
		100	values: [
		101	{
		102	type: "GateCheckTimeoutError",
		103	value: "balance check timed out",
		104	stacktrace: {
		105	frames: [
		106	{
		107	filename: `/assets/execution-rate-limit-${chunkHash}.js`,
		108	module: `execution-rate-limit-${chunkHash}`,
		109	function: "timeoutOrElse",
		110	in_app: true,
		111	},
		112	],
		113	},
		114	},
		115	],
		116	},
		117	});
		118
		119	it("pins one fingerprint across two deploys of the same chunk", () => {
		120	const before = beforeSendCloudEvent(workerEvent("BAuwphPA"), {});
		121	const after = beforeSendCloudEvent(workerEvent("DkcPBbWe"), {});
		122
		123	expect(before?.fingerprint).toBeDefined();
		124	expect(before?.fingerprint).toEqual(after?.fingerprint);
		125	});
		126
		127	it("leaves unhashed events on Sentry's default grouping", () => {
		128	const event: ErrorEvent = {
		129	type: undefined,
		130	exception: {
		131	values: [
		132	{
		133	type: "AutumnError",
		134	stacktrace: {
		135	frames: [
		136	{ filename: "/src/engine/execution-gate.ts", function: "checkExecutionBalance" },
		137	],
		138	},
		139	},
		140	],
		141	},
		142	};
		143
		144	const sent = beforeSendCloudEvent(event, {});
		145
		146	expect(sent).not.toBeNull();
		147	expect(sent?.fingerprint).toBeUndefined();
		148	});
		149
		150	// The wiring check: this is the exact object handed to `Sentry.withSentry`
		151	// and `instrumentDurableObjectWithSentry` in server.ts. If the normalizer is
		152	// ever dropped from the hook the worker installs, this fails.
		153	it("the options the worker and DOs install carry the fingerprinting hook", () => {
		154	const options = cloudSentryOptions({ SENTRY_DSN: "https://public@example.invalid/1" } as Env);
		155	const sent = options.beforeSend(workerEvent("BAuwphPA"));
		156
		157	expect(sent?.fingerprint).toEqual([
		158	"GateCheckTimeoutError",
		159	"timeoutOrElse@execution-rate-limit",
		160	]);
		161	// Same source, next deploy, new chunk hash — one issue, not two.
		162	expect(options.beforeSend(workerEvent("DkcPBbWe"))?.fingerprint).toEqual(sent?.fingerprint);
		163	});
		164	});
		165
87	describe("Sentry OTel correlation", () => {	166	describe("Sentry OTel correlation", () => {
88	it.effect("adds tags from the active Effect span", () =>	167	it.effect("adds tags from the active Effect span", () =>
89	Effect.gen(function* () {	168	Effect.gen(function* () {
@@ -162,4 +241,62 @@ describe("Durable Object capture ownership", () => { · 72 unmodified lines
162	};	241	};
163	expect(beforeSendWithOtelCorrelation(workerEvent)).not.toBeNull();	242	expect(beforeSendWithOtelCorrelation(workerEvent)).not.toBeNull();
164	});	243	});
		244
		245	// The two stages of the installed `beforeSend` answer different questions and
		246	// must both keep working: capture ownership decides WHETHER an event is
		247	// reported, stable grouping decides HOW a reported one is grouped. A dropped
		248	// event is never fingerprinted, and a surviving one still is.
		249	describe("composed with stable grouping", () => {
		250	const hashedFrames = (chunkHash: string) => ({
		251	stacktrace: {
		252	frames: [
		253	{
		254	filename: `/assets/session-durable-object-${chunkHash}.js`,
		255	module: `session-durable-object-${chunkHash}`,
		256	function: "handleSessionRequest",
		257	in_app: true,
		258	},
		259	],
		260	},
		261	});
		262
		263	it("drops a claimed echo rather than fingerprinting it", () => {
		264	const echo = doEcho({
		265	exception: {
		266	values: [
		267	{
		268	type: "Error",
		269	value: "Durable Object reset because its code was updated.",
		270	mechanism: { type: "auto.faas.cloudflare.durable_object", handled: false },
		271	...hashedFrames("BAuwphPA"),
		272	},
		273	],
		274	},
		275	});
		276
		277	expect(beforeSendCloudEvent(echo, {})).toBeNull();
		278	});
		279
		280	it("pins a stable fingerprint on the report the DO itself owns", () => {
		281	const ownReport = (chunkHash: string): ErrorEvent =>
		282	doEcho({
		283	exception: {
		284	values: [
		285	{
		286	type: "Error",
		287	value: "Durable Object reset because its code was updated.",
		288	mechanism: { type: "generic", handled: true },
		289	...hashedFrames(chunkHash),
		290	},
		291	],
		292	},
		293	});
		294
		295	const before = beforeSendCloudEvent(ownReport("BAuwphPA"), {});
		296	const after = beforeSendCloudEvent(ownReport("DkcPBbWe"), {});
		297
		298	expect(before?.fingerprint).toBeDefined();
		299	expect(before?.fingerprint).toEqual(after?.fingerprint);
		300	});
		301	});
165	});	302	});

apps/cloud/src/server.ts+3−26

 import { DurableObject } from "cloudflare:workers";
 import { SpanKind, SpanStatusCode, context, trace, type SpanContext } from "@opentelemetry/api";
-import type { ErrorEvent } from "@sentry/cloudflare";
 import {
   ATTR_HTTP_REQUEST_METHOD,
   ATTR_HTTP_RESPONSE_STATUS_CODE,
 import { parseTraceparent } from "./mcp/traceparent";
 import { McpSessionDOSqlite as McpSessionDOBase } from "./mcp/session-durable-object";
 import {
-  beforeSendWithOtelCorrelation,
+  cloudSentryOptions,
   captureCause,
   otelCorrelationContextFromOpenTelemetrySpan,
   SENTRY_EVENT_ID_ATTRIBUTE,
 import { browserTracesResponse } from "./observability/browser-traces";
 import { flushTracerProvider, installTracerProvider } from "./observability/telemetry";
-// ---------------------------------------------------------------------------
-// Sentry config
-// ---------------------------------------------------------------------------
-const sentryOptions = (env: Env) => ({
-  dsn: env.SENTRY_DSN,
-  tracesSampleRate: 0,
-  enableLogs: true,
-  sendDefaultPii: true,
-  skipOpenTelemetrySetup: true,
-  beforeSend: (event: ErrorEvent) =>
-    beforeSendWithOtelCorrelation(event, {
-      logPayload: !env.SENTRY_DSN || env.SENTRY_OTEL_LOG_PAYLOAD === "true",
-    }),
-  // NOTE: do NOT enable `instrumentPrototypeMethods`. It walks the DO prototype
-  // and reads every property — including accessors — to find methods to wrap,
-  // which invokes the `sessionId` getter with `this` bound to the prototype
-  // (where `ctx` is undefined) and throws during construction, 500ing every
-  // session create / cold restore. The DO captures its own errors via the
-  // `captureCause` seam (→ Sentry) instead.
-});
 // ---------------------------------------------------------------------------
 // Durable Object — wrapped with Sentry so DO errors land in Sentry (inits the
 // client inside the DO isolate, which plain `Sentry.captureException` cannot
 // ---------------------------------------------------------------------------
 export const McpSessionDOSqlite = Sentry.instrumentDurableObjectWithSentry(
-  sentryOptions,
+  cloudSentryOptions,
   McpSessionDOBase,
 );
   },
 };
-export default Sentry.withSentry(sentryOptions, cloudflareHandler);
+export default Sentry.withSentry(cloudSentryOptions, cloudflareHandler);

apps/desktop/src/main/crash-fingerprint.test.ts+143−0

@@ -0,0 +1,143 @@
	1	import { expect, test } from "@effect/vitest";
	2
	3	import {
	4	crashReportFingerprint,
	5	mainCrashReportingOptions,
	6	withCrashReportFingerprint,
	7	type CrashEvent,
	8	} from "./crash-fingerprint";
	9
	10	/** A crash event as Sentry hands it to `beforeSend` — with the grouping key
	11	* slot the hook is allowed to fill. */
	12	type SentCrashEvent = CrashEvent & { readonly fingerprint?: readonly string[] \| undefined };
	13
	14	// Chromium's soft-assert path (`NOTREACHED()`/`DCHECK`) dumps and keeps
	15	// running. Sentry titles each one with the faulting load address, so one
	16	// condition arrives as a new issue every time the address moves.
	17	const softAssertEvent = (address: string): SentCrashEvent => ({
	18	exception: {
	19	values: [
	20	{
	21	type: "Fatal Error",
	22	value: `Simulated Exception / ${address}`,
	23	mechanism: { type: "minidump" },
	24	stacktrace: {
	25	frames: [
	26	{ function: "logging::NotReachedLogMessage::~NotReachedLogMessage" },
	27	{ function: "logging::HandleCheckErrorLogMessage" },
	28	{ function: "base::debug::DumpWithoutCrashing" },
	29	{ function: "crash_reporter::DumpWithoutCrashing" },
	30	],
	31	},
	32	},
	33	],
	34	},
	35	});
	36
	37	test("address-keyed Chromium soft asserts collapse to one fingerprint", () => {
	38	const first = crashReportFingerprint(softAssertEvent("0x00000001a2b3c4d5"));
	39	const second = crashReportFingerprint(softAssertEvent("0x00000007e8f9a0b1"));
	40
	41	expect(first).toEqual(["chromium-dump-without-crashing"]);
	42	expect(first).toEqual(second);
	43	});
	44
	45	test("a real native abort keeps Sentry's own grouping", () => {
	46	const abortEvent: CrashEvent = {
	47	exception: {
	48	values: [
	49	{
	50	type: "EXC_CRASH",
	51	value: "SIGABRT",
	52	mechanism: { type: "minidump" },
	53	stacktrace: {
	54	frames: [
	55	{ function: "abort" },
	56	{ function: "pthread_kill" },
	57	{ function: "__pthread_kill" },
	58	],
	59	},
	60	},
	61	],
	62	},
	63	};
	64
	65	expect(crashReportFingerprint(abortEvent)).toBeUndefined();
	66	});
	67
	68	test("renderer chunk hashes are normalized out of the fingerprint", () => {
	69	const rendererEvent = (chunkHash: string): CrashEvent => ({
	70	culprit: `loadConnections(assets/atoms-${chunkHash})`,
	71	exception: {
	72	values: [
	73	{
	74	type: "TypeError",
	75	value: "cannot read properties of undefined",
	76	stacktrace: {
	77	frames: [
	78	{
	79	filename: `http://127.0.0.1:4789/assets/atoms-${chunkHash}.js`,
	80	module: `atoms-${chunkHash}`,
	81	function: "loadConnections",
	82	in_app: true,
	83	},
	84	],
	85	},
	86	},
	87	],
	88	},
	89	});
	90
	91	const release1 = crashReportFingerprint(rendererEvent("Yemn7yhP"));
	92	const release2 = crashReportFingerprint(rendererEvent("CeCENfWa"));
	93
	94	expect(release1).toEqual(["TypeError", "loadConnections@atoms"]);
	95	expect(release1).toEqual(release2);
	96	});
	97
	98	// `withCrashReportFingerprint` is the function object diagnostics.ts installs
	99	// as its `beforeSend`, so these assertions cover the main-process wiring and
	100	// not just the classifier behind it.
	101	test("the main-process beforeSend pins the key and forwards the event", () => {
	102	const collapsed = withCrashReportFingerprint(softAssertEvent("0x00000001a2b3c4d5"));
	103	expect(collapsed.fingerprint).toEqual(["chromium-dump-without-crashing"]);
	104	// Nothing else about the event is touched — this is grouping only.
	105	expect(collapsed.exception?.values?.[0]?.value).toBe("Simulated Exception / 0x00000001a2b3c4d5");
	106
	107	const untouched: SentCrashEvent = {
	108	exception: { values: [{ type: "EXC_CRASH", stacktrace: { frames: [{ function: "abort" }] } }] },
	109	};
	110	expect(withCrashReportFingerprint(untouched)).toBe(untouched);
	111	});
	112
	113	// The wiring check: this is the exact object handed to `Sentry.init` in
	114	// diagnostics.ts. If the hook is ever dropped from the main process, this
	115	// fails.
	116	test("the options the main process installs carry the fingerprinting hook", () => {
	117	const options = mainCrashReportingOptions({
	118	dsn: "https://public@example.invalid/1",
	119	release: "executor-desktop@0.0.0",
	120	environment: "production",
	121	runId: "abcdef123456",
	122	});
	123	const sent = options.beforeSend(softAssertEvent("0x00000001a2b3c4d5"));
	124	expect(sent.fingerprint).toEqual(["chromium-dump-without-crashing"]);
	125	});
	126
	127	test("events with no volatile grouping input are left alone", () => {
	128	expect(crashReportFingerprint({})).toBeUndefined();
	129	expect(
	130	crashReportFingerprint({
	131	exception: {
	132	values: [
	133	{
	134	type: "Error",
	135	stacktrace: {
	136	frames: [{ filename: "/src/main/sidecar.ts", function: "startSidecar" }],
	137	},
	138	},
	139	],
	140	},
	141	}),
	142	).toBeUndefined();
	143	});

apps/desktop/src/main/crash-fingerprint.ts+80−0

@@ -0,0 +1,80 @@
	1	/**
	2	* Grouping keys for desktop crash reports.
	3	*
	4	* Two things split one desktop problem across many Sentry issues:
	5	*
	6	* - Chromium's soft-assert path (`NOTREACHED()`/`DCHECK`) calls
	7	* `DumpWithoutCrashing`, which files a minidump and lets the process carry
	8	* on. Sentry titles each one with the faulting load address, so the same
	9	* condition arrives as a new issue every time the address moves.
	10	* - Renderer and main bundles ship as content-hashed chunks, so unresolved
	11	* frames name `atoms-<hash>` and re-group on every release.
	12	*
	13	* Both are grouping problems only: nothing here drops, downgrades or edits an
	14	* event, and the frames keep their hashes so sourcemap resolution still works.
	15	*/
	16
	17	import {
	18	stableGroupingFingerprint,
	19	withStableGroupingFingerprint,
	20	type GroupingEvent,
	21	type GroupingFrame,
	22	} from "@executor-js/sdk/sentry-grouping";
	23
	24	export type CrashEvent = GroupingEvent;
	25
	26	export const CHROMIUM_SOFT_ASSERT_FINGERPRINT = "chromium-dump-without-crashing";
	27
	28	const isSoftAssertFrame = (frame: GroupingFrame): boolean =>
	29	(frame.function ?? "").includes("DumpWithoutCrashing");
	30
	31	/**
	32	* The fingerprint to attach to a crash event, or `undefined` to keep Sentry's
	33	* default grouping (which is right for a real native abort — one issue per
	34	* distinct stack is what we want there).
	35	*/
	36	export const crashReportFingerprint = (event: CrashEvent): readonly string[] \| undefined => {
	37	const frames = (event.exception?.values ?? []).flatMap((value) => value.stacktrace?.frames ?? []);
	38	if (frames.some(isSoftAssertFrame)) return [CHROMIUM_SOFT_ASSERT_FINGERPRINT];
	39	return stableGroupingFingerprint(event);
	40	};
	41
	42	/**
	43	* The `beforeSend` the Electron main process installs. Grouping only — an
	44	* event with no volatile grouping input is forwarded unchanged.
	45	*/
	46	export const withCrashReportFingerprint = <T extends CrashEvent>(event: T): T => {
	47	const frames = (event.exception?.values ?? []).flatMap((value) => value.stacktrace?.frames ?? []);
	48	if (frames.some(isSoftAssertFrame)) {
	49	return { ...event, fingerprint: [CHROMIUM_SOFT_ASSERT_FINGERPRINT] };
	50	}
	51	return withStableGroupingFingerprint(event);
	52	};
	53
	54	/**
	55	* The Sentry options the Electron main process installs, assembled here rather
	56	* than inline at the `Sentry.init` call so the `beforeSend` wiring is covered
	57	* by crash-fingerprint.test.ts. `diagnostics.ts` only passes this through.
	58	*
	59	* Typed structurally on purpose: this module stays importable without pulling
	60	* in electron, which is what keeps it unit-testable at all.
	61	*/
	62	export const mainCrashReportingOptions = (config: {
	63	readonly dsn: string;
	64	readonly release: string;
	65	readonly environment: string;
	66	readonly runId: string;
	67	}) => ({
	68	dsn: config.dsn,
	69	release: config.release,
	70	environment: config.environment,
	71	initialScope: {
	72	tags: {
	73	platform: process.platform,
	74	arch: process.arch,
	75	runId: config.runId,
	76	},
	77	},
	78	// Grouping only — the event is forwarded untouched otherwise.
	79	beforeSend: withCrashReportFingerprint,
	80	});

apps/desktop/src/main/diagnostics.ts+9−12

 import { app, crashReporter, dialog, shell } from "electron";
 import log from "electron-log/main.js";
 import * as Sentry from "@sentry/electron/main";
+import { mainCrashReportingOptions } from "./crash-fingerprint";
 import { getServerSettings } from "./settings";
 const sentryDsn = __EXECUTOR_SENTRY_DSN__;
  */
 export const initErrorReporting = () => {
   if (errorReportingEnabled) {
-    Sentry.init({
+    Sentry.init(
-      dsn: sentryDsn,
+      mainCrashReportingOptions({
-      release: releaseTag(),
+        dsn: sentryDsn,
-      environment: environmentTag(),
+        release: releaseTag(),
-      initialScope: {
+        environment: environmentTag(),
-        tags: {
+        runId,
-          platform: process.platform,
+      }),
-          arch: process.arch,
+    );
-          runId,
-        },
-      },
-    });
   } else {
     // No DSN baked in — keep native crash dumps local so a user-reported
     // crash still leaves minidumps for the diagnostics zip to collect.

packages/app/src/crash-reporting.ts+7−0

  * once initialized — no reporter rewiring needed.
  */
+import { withStableGroupingFingerprint } from "@executor-js/sdk/sentry-grouping";
 interface CrashReportingConfig {
   readonly dsn: string;
   readonly release: string;
             runId: config.runId,
           },
         },
+        // Route chunks are content-hashed, so an unresolved frame names
+        // `atoms-<hash>` and one bug re-groups on every release. Pin a
+        // fingerprint with the hash normalized out; the event itself keeps its
+        // hashed filenames so sourcemap resolution is unaffected.
+        beforeSend: withStableGroupingFingerprint,
       });
     } catch {
       // Reporting failures stay silent — there is nowhere left to report them.

packages/core/sdk/package.json+8−1

     "./testing": "./src/testing.ts",
     "./migration": "./src/migration-spec.ts",
     "./http-auth": "./src/http-auth/index.ts",
-    "./public-origin": "./src/public-origin.ts"
+    "./public-origin": "./src/public-origin.ts",
+    "./sentry-grouping": "./src/sentry-grouping.ts"
   },
   "publishConfig": {
     "access": "public",
           "types": "./dist/public-origin.d.ts",
           "default": "./dist/public-origin.js"
         }
+      },
+      "./sentry-grouping": {
+        "import": {
+          "types": "./dist/sentry-grouping.d.ts",
+          "default": "./dist/sentry-grouping.js"
+        }
       }
     }
   },

packages/core/sdk/src/sentry-grouping.test.ts+269−0

@@ -0,0 +1,269 @@
	1	import { expect, test } from "@effect/vitest";
	2
	3	import {
	4	containsContentHash,
	5	stableGroupingFingerprint,
	6	stripContentHashes,
	7	withStableGroupingFingerprint,
	8	type GroupingEvent,
	9	} from "./sentry-grouping";
	10
	11	// Two builds of the same source: only the Vite content hash differs.
	12	const workerFrame = (hash: string) => ({
	13	filename: `/assets/execution-rate-limit-${hash}.js`,
	14	module: `execution-rate-limit-${hash}`,
	15	function: "timeoutOrElse",
	16	in_app: true,
	17	});
	18
	19	const workerEvent = (hash: string): GroupingEvent => ({
	20	culprit: `timeoutOrElse(execution-rate-limit-${hash})`,
	21	exception: {
	22	values: [
	23	{
	24	type: "GateCheckTimeoutError",
	25	value: "balance check timed out",
	26	stacktrace: { frames: [workerFrame(hash)] },
	27	},
	28	],
	29	},
	30	});
	31
	32	test("stripContentHashes rewrites chunk hashes and leaves everything else alone", () => {
	33	expect(stripContentHashes("/assets/execution-rate-limit-BAuwphPA.js")).toBe(
	34	"/assets/execution-rate-limit.js",
	35	);
	36	expect(stripContentHashes("/assets/execution-rate-limit-DkcPBbWe.js")).toBe(
	37	"/assets/execution-rate-limit.js",
	38	);
	39	// Rollup hashes may themselves contain a dash — the whole 8-char tail goes.
	40	expect(stripContentHashes("http://127.0.0.1:4789/assets/AddGraphqlIntegration-BCr-oWx4.js")).toBe(
	41	"http://127.0.0.1:4789/assets/AddGraphqlIntegration.js",
	42	);
	43	// Culprit strings wrap the chunk name in parentheses.
	44	expect(stripContentHashes("U(assets/atoms-Yemn7yhP)")).toBe("U(assets/atoms)");
	45	expect(stripContentHashes("W(assets/atoms-CeCENfWa)")).toBe("W(assets/atoms)");
	46	});
	47
	48	test("stripContentHashes does not eat real name segments", () => {
	49	// 8 lowercase letters is a word, not a hash.
	50	expect(stripContentHashes("/src/auth/oauth-callback.ts")).toBe("/src/auth/oauth-callback.ts");
	51	expect(stripContentHashes("apps/cloud/src/engine/execution-gate.ts")).toBe(
	52	"apps/cloud/src/engine/execution-gate.ts",
	53	);
	54	expect(stripContentHashes("/assets/execution-rate-limit.js")).toBe(
	55	"/assets/execution-rate-limit.js",
	56	);
	57	expect(stripContentHashes("")).toBe("");
	58	});
	59
	60	test("stripContentHashes keeps genuinely different modules distinct", () => {
	61	expect(stripContentHashes("/assets/atoms-Yemn7yhP.js")).not.toBe(
	62	stripContentHashes("/assets/router-CeCENfWa.js"),
	63	);
	64	expect(stripContentHashes("/assets/atoms-Yemn7yhP.js")).not.toBe(
	65	stripContentHashes("/assets/atoms-shell-CeCENfWa.js"),
	66	);
	67	});
	68
	69	// Merging two unrelated bugs into one issue is unrecoverable, so the "is this
	70	// a hash?" threshold is pinned from both sides: a segment needs two of the
	71	// three hash signals (uppercase, uppercase+digit, digits) before it is eaten.
	72	// Loosening it to one signal — the tempting fix for the word-shaped hashes
	73	// this deliberately misses — silently collapses real chunk names.
	74	test("a name segment one signal short of a hash is left in place", () => {
	75	// One capital, no digits.
	76	expect(stripContentHashes("/assets/connections-Settings.js")).toBe(
	77	"/assets/connections-Settings.js",
	78	);
	79	// One digit, no capitals.
	80	expect(stripContentHashes("/assets/storage-s3client.js")).toBe("/assets/storage-s3client.js");
	81	// ...and neither may be mistaken for the chunk it would collapse onto.
	82	expect(stripContentHashes("/assets/connections-Settings.js")).not.toBe(
	83	stripContentHashes("/assets/connections-BAuwphPA.js"),
	84	);
	85	});
	86
	87	test("a segment with two hash signals is eaten", () => {
	88	expect(stripContentHashes("/assets/chunk-AbcdefgH.js")).toBe("/assets/chunk.js");
	89	expect(stripContentHashes("/assets/chunk-Abcdefg1.js")).toBe("/assets/chunk.js");
	90	expect(stripContentHashes("/assets/chunk-abcdef12.js")).toBe("/assets/chunk.js");
	91	});
	92
	93	test("containsContentHash detects only hashed paths", () => {
	94	expect(containsContentHash("/assets/atoms-Yemn7yhP.js")).toBe(true);
	95	expect(containsContentHash("apps/cloud/src/engine/execution-gate.ts")).toBe(false);
	96	});
	97
	98	test("the same logical frame fingerprints identically across two deploys", () => {
	99	const first = stableGroupingFingerprint(workerEvent("BAuwphPA"));
	100	const second = stableGroupingFingerprint(workerEvent("DkcPBbWe"));
	101	expect(first).toBeDefined();
	102	expect(first).toEqual(second);
	103	expect(first).toEqual(["GateCheckTimeoutError", "timeoutOrElse@execution-rate-limit"]);
	104	});
	105
	106	test("different modules keep different fingerprints", () => {
	107	const other: GroupingEvent = {
	108	exception: {
	109	values: [
	110	{
	111	type: "GateCheckTimeoutError",
	112	stacktrace: {
	113	frames: [
	114	{
	115	filename: "/assets/atoms-Yemn7yhP.js",
	116	module: "atoms-Yemn7yhP",
	117	function: "loadConnections",
	118	in_app: true,
	119	},
	120	],
	121	},
	122	},
	123	],
	124	},
	125	};
	126	expect(stableGroupingFingerprint(other)).not.toEqual(
	127	stableGroupingFingerprint(workerEvent("BAuwphPA")),
	128	);
	129	});
	130
	131	test("minified single-letter frame functions are left out of the fingerprint", () => {
	132	// Minified identifiers rotate with every build exactly like chunk hashes, so
	133	// keeping them would re-split the issue on the next deploy.
	134	const atoms = (fn: string, hash: string): GroupingEvent => ({
	135	exception: {
	136	values: [
	137	{
	138	type: "TypeError",
	139	stacktrace: {
	140	frames: [
	141	{ filename: `/assets/atoms-${hash}.js`, module: `atoms-${hash}`, function: fn },
	142	],
	143	},
	144	},
	145	],
	146	},
	147	});
	148	expect(stableGroupingFingerprint(atoms("U", "Yemn7yhP"))).toEqual(
	149	stableGroupingFingerprint(atoms("W", "CeCENfWa")),
	150	);
	151	expect(stableGroupingFingerprint(atoms("U", "Yemn7yhP"))).toEqual(["TypeError", "atoms"]);
	152	});
	153
	154	// A single build emits many `dist-<hash>.js` chunks from unrelated packages,
	155	// so the crashing frame alone is not a safe key — two different bugs would
	156	// land in one issue. Frames are oldest-first, so the shared chunk is last.
	157	const vendorEvent = (caller: string, hash: string): GroupingEvent => ({
	158	exception: {
	159	values: [
	160	{
	161	type: "TypeError",
	162	stacktrace: {
	163	frames: [
	164	{ module: `${caller}-${hash}`, function: caller, in_app: true },
	165	{ module: `dist-${hash}`, function: "throwHelper", in_app: true },
	166	],
	167	},
	168	},
	169	],
	170	},
	171	});
	172
	173	test("two vendor chunks sharing a name stay apart on their callers", () => {
	174	const upload = stableGroupingFingerprint(vendorEvent("uploadArtifact", "BQZkXWT2"));
	175	const render = stableGroupingFingerprint(vendorEvent("renderMarkdown", "BQZkXWT2"));
	176
	177	// The crashing frame is identical in both — only the caller chain separates
	178	// them, so a fingerprint built from the top frame alone would over-merge.
	179	expect(upload?.slice(0, 2)).toEqual(["TypeError", "throwHelper@dist"]);
	180	expect(render?.slice(0, 2)).toEqual(upload?.slice(0, 2));
	181	expect(upload).not.toEqual(render);
	182	expect(upload).toEqual(["TypeError", "throwHelper@dist", "uploadArtifact@uploadArtifact"]);
	183	});
	184
	185	test("the caller chain is itself deploy-stable", () => {
	186	// Every frame in the chain must lose its hash, not just the crashing one.
	187	expect(stableGroupingFingerprint(vendorEvent("uploadArtifact", "BQZkXWT2"))).toEqual(
	188	stableGroupingFingerprint(vendorEvent("uploadArtifact", "Yemn7yhP")),
	189	);
	190	});
	191
	192	test("events without a chunk hash keep Sentry's default grouping", () => {
	193	const unhashed: GroupingEvent = {
	194	culprit: "checkExecutionBalance(execution-gate.ts)",
	195	exception: {
	196	values: [
	197	{
	198	type: "AutumnError",
	199	stacktrace: {
	200	frames: [
	201	{
	202	filename: "/apps/cloud/src/engine/execution-gate.ts",
	203	function: "checkExecutionBalance",
	204	in_app: true,
	205	},
	206	],
	207	},
	208	},
	209	],
	210	},
	211	};
	212	expect(stableGroupingFingerprint(unhashed)).toBeUndefined();
	213	expect(stableGroupingFingerprint({})).toBeUndefined();
	214	expect(stableGroupingFingerprint({ exception: { values: [] } })).toBeUndefined();
	215	});
	216
	217	test("a hashed culprit alone is enough to pin the fingerprint", () => {
	218	const culpritOnly: GroupingEvent = {
	219	culprit: "orElse(execution-rate-limit-BAuwphPA)",
	220	exception: { values: [{ type: "TypeError", value: "destroyed" }] },
	221	};
	222	expect(stableGroupingFingerprint(culpritOnly)).toEqual([
	223	"TypeError",
	224	"orElse(execution-rate-limit)",
	225	]);
	226	});
	227
	228	// The wrapper every process installs as its `beforeSend`. `fingerprint` and
	229	// `tags` stand in for the fields a real Sentry event carries around it.
	230	type SentEvent = GroupingEvent & {
	231	readonly fingerprint?: readonly string[] \| undefined;
	232	readonly tags?: Record<string, string> \| undefined;
	233	};
	234
	235	test("withStableGroupingFingerprint pins the key and changes nothing else", () => {
	236	const input: SentEvent = { ...workerEvent("BAuwphPA"), tags: { a: "b" } };
	237	const hashed = withStableGroupingFingerprint(input);
	238	expect(hashed.fingerprint).toEqual([
	239	"GateCheckTimeoutError",
	240	"timeoutOrElse@execution-rate-limit",
	241	]);
	242	// The event still carries its hashed filename, or server-side sourcemap
	243	// resolution would stop finding the release's artifacts.
	244	expect(hashed.exception?.values?.[0]?.stacktrace?.frames?.[0]?.filename).toBe(
	245	"/assets/execution-rate-limit-BAuwphPA.js",
	246	);
	247	expect(hashed.tags).toEqual({ a: "b" });
	248	});
	249
	250	test("withStableGroupingFingerprint forwards an unhashed event untouched", () => {
	251	const plain: SentEvent = { culprit: "checkExecutionBalance(execution-gate.ts)" };
	252	const out = withStableGroupingFingerprint(plain);
	253	expect(out).toBe(plain);
	254	expect("fingerprint" in out).toBe(false);
	255	});
	256
	257	test("the outermost exception drives the fingerprint", () => {
	258	// Sentry orders `values` innermost-first; the last entry is the one whose
	259	// type the issue is titled with.
	260	const chained: GroupingEvent = {
	261	exception: {
	262	values: [
	263	{ type: "InnerError", stacktrace: { frames: [workerFrame("BAuwphPA")] } },
	264	{ type: "OuterError", stacktrace: { frames: [workerFrame("BAuwphPA")] } },
	265	],
	266	},
	267	};
	268	expect(stableGroupingFingerprint(chained)?.[0]).toBe("OuterError");
	269	});

packages/core/sdk/src/sentry-grouping.ts+169−0

@@ -0,0 +1,169 @@
	1	// ---------------------------------------------------------------------------
	2	// Stable Sentry grouping across deploys.
	3	//
	4	// Every bundle we ship names its chunks `<name>-<contentHash>.js`, and the
	5	// hash rotates on every build. When a stack frame is not resolved back to
	6	// source (no uploaded sourcemap for that release), Sentry groups on those
	7	// minified names, so the SAME error opens a brand-new issue after each
	8	// deploy — one bug spread over N issues, no counts, no regression history.
	9	//
	10	// The fix is a `beforeSend` that computes an explicit `fingerprint` from the
	11	// event's grouping inputs with the volatile hash segment removed. The event
	12	// itself is left untouched: filenames still carry their hashes so server-side
	13	// sourcemap resolution keeps working — only the grouping key is normalized.
	14	//
	15	// Deliberately narrow: a fingerprint is returned ONLY for events that actually
	16	// carry a content hash. Everything else keeps Sentry's default grouping.
	17	// ---------------------------------------------------------------------------
	18
	19	/** The shape of a Sentry stack frame this module reads. Structural on purpose:
	20	* the same helper serves @sentry/cloudflare, /electron and /browser events. */
	21	export type GroupingFrame = {
	22	readonly filename?: string \| undefined;
	23	readonly abs_path?: string \| undefined;
	24	readonly module?: string \| undefined;
	25	readonly function?: string \| undefined;
	26	readonly in_app?: boolean \| undefined;
	27	};
	28
	29	export type GroupingExceptionValue = {
	30	readonly type?: string \| undefined;
	31	readonly value?: string \| undefined;
	32	readonly stacktrace?: { readonly frames?: readonly GroupingFrame[] \| undefined } \| undefined;
	33	readonly mechanism?: { readonly type?: string \| undefined } \| undefined;
	34	};
	35
	36	export type GroupingEvent = {
	37	readonly culprit?: string \| undefined;
	38	readonly exception?: { readonly values?: readonly GroupingExceptionValue[] \| undefined };
	39	};
	40
	41	/**
	42	* A `-XXXXXXXX` tail that ends a path segment — i.e. is followed by nothing,
	43	* by file extensions, or by a delimiter such as the `)` in a Sentry culprit
	44	* (`orElse(execution-rate-limit-BAuwphPA)`). Eight characters is the Vite and
	45	* Rollup default; the hash alphabet includes `-` and `_`, so a hash can itself
	46	* contain a dash (`BCr-oWx4`) and the whole 8-char tail must go at once.
	47	*/
	48	const HASH_TAIL = /-([A-Za-z0-9_-]{8})(?=(?:\.[A-Za-z0-9]+)*(?:$\|[)\]}'"\s,:;?#]))/g;
	49
	50	/**
	51	* Whether an 8-character segment reads like a content hash rather than a word.
	52	* Build hashes are random base64url, so they mix cases and digits; real name
	53	* segments (`callback`, `grouping`, `provider`) are all lowercase. Requiring
	54	* that mix is what keeps `oauth-callback` from collapsing to `oauth`. The cost
	55	* is that the occasional word-shaped hash (one capital, no digits) is left
	56	* alone and keeps re-grouping — 1 of 147 chunks in a sample desktop build. A
	57	* missed normalization is recoverable; a wrongly merged issue is not.
	58	*/
	59	const looksLikeContentHash = (segment: string): boolean => {
	60	let upper = 0;
	61	let digits = 0;
	62	for (const char of segment) {
	63	if (char >= "A" && char <= "Z") upper += 1;
	64	else if (char >= "0" && char <= "9") digits += 1;
	65	}
	66	return upper >= 2 \|\| (upper >= 1 && digits >= 1) \|\| digits >= 2;
	67	};
	68
	69	/**
	70	* Remove build content hashes from a path, module name or culprit string,
	71	* leaving every other character in place.
	72	*/
	73	export const stripContentHashes = (value: string): string =>
	74	value.replace(HASH_TAIL, (match, segment: string) =>
	75	looksLikeContentHash(segment) ? "" : match,
	76	);
	77
	78	/** True when the value carries at least one build content hash. */
	79	export const containsContentHash = (value: string): boolean => stripContentHashes(value) !== value;
	80
	81	/**
	82	* Minified identifiers rotate with every build exactly like chunk hashes, so a
	83	* fingerprint containing one is no more stable than the hash it replaced. Names
	84	* of three characters or fewer are treated as minified and dropped; the frame
	85	* still contributes its module.
	86	*/
	87	const MINIFIED_FUNCTION = /^[A-Za-z_$][A-Za-z0-9_$]{0,2}$/;
	88
	89	const usableFunction = (name: string \| undefined): string \| undefined => {
	90	if (!name \|\| name === "<anonymous>" \|\| name === "?") return undefined;
	91	return MINIFIED_FUNCTION.test(name) ? undefined : name;
	92	};
	93
	94	/** Last path segment, without query or fragment. Chunk paths are served from
	95	* host- and port-specific origins (`http://127.0.0.1:4789/assets/…`), which are
	96	* volatile in their own right. */
	97	const basename = (path: string): string => {
	98	const clean = path.split(/[?#]/)[0] ?? path;
	99	const segments = clean.split(/[/\\]/);
	100	return segments[segments.length - 1] \|\| clean;
	101	};
	102
	103	const frameLocation = (frame: GroupingFrame): string \| undefined => {
	104	if (frame.module) return stripContentHashes(frame.module);
	105	const path = frame.filename ?? frame.abs_path;
	106	return path ? stripContentHashes(basename(path)) : undefined;
	107	};
	108
	109	const frameKey = (frame: GroupingFrame): string \| undefined => {
	110	const location = frameLocation(frame);
	111	if (!location) return undefined;
	112	const fn = usableFunction(frame.function);
	113	return fn ? `${fn}@${location}` : location;
	114	};
	115
	116	const frameHasContentHash = (frame: GroupingFrame): boolean =>
	117	containsContentHash(frame.module ?? "") \|\|
	118	containsContentHash(frame.filename ?? "") \|\|
	119	containsContentHash(frame.abs_path ?? "");
	120
	121	/**
	122	* How many frames from the top of the stack enter the fingerprint. Bundlers
	123	* emit several unrelated chunks under the same name (nine different
	124	* `dist-<hash>.js` in one desktop build), so the top frame alone would merge
	125	* unrelated vendor code; the caller chain is what keeps them apart. The
	126	* resulting key is still strictly coarser than Sentry's default input — same
	127	* frames, minus the volatile hash and minified identifiers — so this can only
	128	* merge issues, never split one further.
	129	*/
	130	const FINGERPRINT_FRAMES = 8;
	131
	132	/**
	133	* A deploy-stable fingerprint for an event whose grouping input carries a build
	134	* content hash, or `undefined` to leave Sentry's default grouping in place.
	135	*/
	136	export const stableGroupingFingerprint = (event: GroupingEvent): readonly string[] \| undefined => {
	137	const values = event.exception?.values ?? [];
	138	// Sentry orders chained exceptions innermost-first; the last entry is the one
	139	// the issue is titled with.
	140	const primary = values[values.length - 1];
	141	if (!primary) return undefined;
	142
	143	const frames = primary.stacktrace?.frames ?? [];
	144	const culprit = event.culprit ?? "";
	145	if (!frames.some(frameHasContentHash) && !containsContentHash(culprit)) return undefined;
	146
	147	const inApp = frames.filter((frame) => frame.in_app !== false);
	148	const considered = (inApp.length > 0 ? inApp : frames).slice(-FINGERPRINT_FRAMES).reverse();
	149	const keys = considered.flatMap((frame) => {
	150	const key = frameKey(frame);
	151	return key ? [key] : [];
	152	});
	153
	154	const type = primary.type ?? "Error";
	155	if (keys.length > 0) return [type, ...keys];
	156	return culprit ? [type, stripContentHashes(culprit)] : undefined;
	157	};
	158
	159	/**
	160	* The whole `beforeSend` contract: pin the deploy-stable fingerprint when the
	161	* event has a volatile grouping input, and forward the event untouched when it
	162	* does not. Every call site (cloud worker/DO, desktop main, desktop renderer)
	163	* installs this one symbol rather than its own copy, so the wiring is covered
	164	* by the tests below instead of being retyped per process.
	165	*/
	166	export const withStableGroupingFingerprint = <T extends GroupingEvent>(event: T): T => {
	167	const fingerprint = stableGroupingFingerprint(event);
	168	return fingerprint ? { ...event, fingerprint: [...fingerprint] } : event;
	169	};

packages/core/sdk/tsup.config.ts+1−0

@@ -10,6 +10,7 @@ export default defineConfig({ · 9 unmodified lines
10	"migration-spec": "src/migration-spec.ts",	10	"migration-spec": "src/migration-spec.ts",
11	"http-auth": "src/http-auth/index.ts",	11	"http-auth": "src/http-auth/index.ts",
12	testing: "src/testing.ts",	12	testing: "src/testing.ts",
		13	"sentry-grouping": "src/sentry-grouping.ts",
13	},	14	},
14	format: ["esm"],	15	format: ["esm"],
15	dts: false,	16	dts: false,

Stable Sentry error grouping across deploys (#1781)

Keyboard shortcuts

General

Issues and pull requests

Issues, pull requests and Inbox

Repository code