History

Stable Sentry error grouping across deploys (#1781)

* Keep Sentry issue grouping stable across deploys Normalize build content hashes out of the grouping key in the cloud, desktop main and desktop renderer Sentry inits, and merge address-keyed Chromium soft-assert minidumps into one fingerprint. * Cover the fingerprint wiring and pin the hash threshold * Assemble desktop main Sentry options where they are tested

Rhys Sullivan committed 2 months ago

Commit 36c901b1 parent c59e23c11 files · +874 −39Browse files

Unified · Split

11 files
  1. apps/cloud/src/observability/index.ts +48 −0
  2. apps/cloud/src/observability/observability.test.ts +137 −0
  3. apps/cloud/src/server.ts +3 −26
  4. apps/desktop/src/main/crash-fingerprint.test.ts +143 −0
  5. apps/desktop/src/main/crash-fingerprint.ts +80 −0
  6. apps/desktop/src/main/diagnostics.ts +9 −12
  7. packages/app/src/crash-reporting.ts +7 −0
  8. packages/core/sdk/package.json +8 −1
  9. packages/core/sdk/src/sentry-grouping.test.ts +269 −0
  10. packages/core/sdk/src/sentry-grouping.ts +169 −0
  11. packages/core/sdk/tsup.config.ts +1 −0
apps/cloud/src/observability/index.ts+48−0
@@ -16,6 +16,7 @@ import { Cause, Effect, Layer, Predicate } from "effect"; · 15 unmodified lines
1616 import type * as Tracer from "effect/Tracer";
1717
1818 import { ErrorCapture } from "@executor-js/api";
19+import { withStableGroupingFingerprint } from "@executor-js/sdk/sentry-grouping";
1920
2021 // Drizzle/postgres-js include the failing SQL (params + bound values) in
2122 // their error message. For OpenAPI source inserts that's 1MB+ of spec
@@ -146,6 +147,53 @@ export const beforeSendWithOtelCorrelation = ( · 124 unmodified lines
146147 return event;
147148 };
148149
150+/**
151+ * The single `beforeSend` the worker and its Durable Objects install.
152+ *
153+ * The worker ships as content-hashed chunks and its frames are not resolved
154+ * back to source, so Sentry's default grouping keys on names like
155+ * `execution-rate-limit-<hash>` and re-opens every issue on the next deploy.
156+ * `withStableGroupingFingerprint` pins a key with the hash normalized out;
157+ * events with no hashed grouping input are left on the default algorithm.
158+ *
159+ * The two stages are independent and compose in this order: the capture-owner
160+ * pass decides WHETHER the event is reported at all (a cause the Durable
161+ * Object already claimed is dropped, and a dropped event is never
162+ * fingerprinted), and the grouping pass then decides HOW whatever survives is
163+ * grouped.
164+ */
165+export const beforeSendCloudEvent = (
166+ event: ErrorEvent,
167+ options?: { readonly logPayload?: boolean },
168+): ErrorEvent | null => {
169+ const reported = beforeSendWithOtelCorrelation(event, options);
170+ return reported === null ? null : withStableGroupingFingerprint(reported);
171+};
172+
173+/**
174+ * The Sentry options the worker and every Durable Object install. It lives
175+ * beside the `beforeSend` it wires so the composition is covered by
176+ * observability.test.ts; `server.ts` only passes this through.
177+ *
178+ * NOTE: do NOT enable `instrumentPrototypeMethods`. It walks the DO prototype
179+ * and reads every property — including accessors — to find methods to wrap,
180+ * which invokes the `sessionId` getter with `this` bound to the prototype
181+ * (where `ctx` is undefined) and throws during construction, 500ing every
182+ * session create / cold restore. The DO captures its own errors via the
183+ * `captureCause` seam (→ Sentry) instead.
184+ */
185+export const cloudSentryOptions = (env: Env) => ({
186+ dsn: env.SENTRY_DSN,
187+ tracesSampleRate: 0,
188+ enableLogs: true,
189+ sendDefaultPii: true,
190+ skipOpenTelemetrySetup: true,
191+ beforeSend: (event: ErrorEvent) =>
192+ beforeSendCloudEvent(event, {
193+ logPayload: !env.SENTRY_DSN || env.SENTRY_OTEL_LOG_PAYLOAD === "true",
194+ }),
195+});
196+
149197 export const addCurrentOtelCorrelationTags = <
150198 T extends { readonly tags?: Record<string, unknown> },
151199 >(
apps/cloud/src/observability/observability.test.ts+137−0
@@ -6,7 +6,9 @@ import type { ErrorEvent } from "@sentry/cloudflare"; · 5 unmodified lines
66
77 import {
88 addCurrentOtelCorrelationTags,
9+ beforeSendCloudEvent,
910 beforeSendWithOtelCorrelation,
11+ cloudSentryOptions,
1012 DO_CAUSE_OWNER_TAG,
1113 DO_CAUSE_OWNER_VALUE,
1214 OTEL_SPAN_ID_TAG,
@@ -84,6 +86,83 @@ describe("sentryPayloadForCause", () => { · 71 unmodified lines
8486 });
8587 });
8688
89+// Grouping keys are decided inside the Sentry SDK and never appear on any
90+// product surface, so the e2e harness cannot observe them. The split is:
91+// e2e/cloud/sentry-otel-correlation.test.ts proves the worker really installs
92+// `cloudSentryOptions.beforeSend` (its correlation payload only exists if that
93+// hook ran), and the tests here prove the hook it installs fingerprints.
94+describe("Sentry grouping", () => {
95+ // The worker bundle ships as content-hashed chunks, so the only module name
96+ // Sentry ever sees for a given frame changes on every deploy.
97+ const workerEvent = (chunkHash: string): ErrorEvent => ({
98+ type: undefined,
99+ exception: {
100+ values: [
101+ {
102+ type: "GateCheckTimeoutError",
103+ value: "balance check timed out",
104+ stacktrace: {
105+ frames: [
106+ {
107+ filename: `/assets/execution-rate-limit-${chunkHash}.js`,
108+ module: `execution-rate-limit-${chunkHash}`,
109+ function: "timeoutOrElse",
110+ in_app: true,
111+ },
112+ ],
113+ },
114+ },
115+ ],
116+ },
117+ });
118+
119+ it("pins one fingerprint across two deploys of the same chunk", () => {
120+ const before = beforeSendCloudEvent(workerEvent("BAuwphPA"), {});
121+ const after = beforeSendCloudEvent(workerEvent("DkcPBbWe"), {});
122+
123+ expect(before?.fingerprint).toBeDefined();
124+ expect(before?.fingerprint).toEqual(after?.fingerprint);
125+ });
126+
127+ it("leaves unhashed events on Sentry's default grouping", () => {
128+ const event: ErrorEvent = {
129+ type: undefined,
130+ exception: {
131+ values: [
132+ {
133+ type: "AutumnError",
134+ stacktrace: {
135+ frames: [
136+ { filename: "/src/engine/execution-gate.ts", function: "checkExecutionBalance" },
137+ ],
138+ },
139+ },
140+ ],
141+ },
142+ };
143+
144+ const sent = beforeSendCloudEvent(event, {});
145+
146+ expect(sent).not.toBeNull();
147+ expect(sent?.fingerprint).toBeUndefined();
148+ });
149+
150+ // The wiring check: this is the exact object handed to `Sentry.withSentry`
151+ // and `instrumentDurableObjectWithSentry` in server.ts. If the normalizer is
152+ // ever dropped from the hook the worker installs, this fails.
153+ it("the options the worker and DOs install carry the fingerprinting hook", () => {
154+ const options = cloudSentryOptions({ SENTRY_DSN: "https://public@example.invalid/1" } as Env);
155+ const sent = options.beforeSend(workerEvent("BAuwphPA"));
156+
157+ expect(sent?.fingerprint).toEqual([
158+ "GateCheckTimeoutError",
159+ "timeoutOrElse@execution-rate-limit",
160+ ]);
161+ // Same source, next deploy, new chunk hash — one issue, not two.
162+ expect(options.beforeSend(workerEvent("DkcPBbWe"))?.fingerprint).toEqual(sent?.fingerprint);
163+ });
164+});
165+
87166 describe("Sentry OTel correlation", () => {
88167 it.effect("adds tags from the active Effect span", () =>
89168 Effect.gen(function* () {
@@ -162,4 +241,62 @@ describe("Durable Object capture ownership", () => { · 72 unmodified lines
162241 };
163242 expect(beforeSendWithOtelCorrelation(workerEvent)).not.toBeNull();
164243 });
244+
245+ // The two stages of the installed `beforeSend` answer different questions and
246+ // must both keep working: capture ownership decides WHETHER an event is
247+ // reported, stable grouping decides HOW a reported one is grouped. A dropped
248+ // event is never fingerprinted, and a surviving one still is.
249+ describe("composed with stable grouping", () => {
250+ const hashedFrames = (chunkHash: string) => ({
251+ stacktrace: {
252+ frames: [
253+ {
254+ filename: `/assets/session-durable-object-${chunkHash}.js`,
255+ module: `session-durable-object-${chunkHash}`,
256+ function: "handleSessionRequest",
257+ in_app: true,
258+ },
259+ ],
260+ },
261+ });
262+
263+ it("drops a claimed echo rather than fingerprinting it", () => {
264+ const echo = doEcho({
265+ exception: {
266+ values: [
267+ {
268+ type: "Error",
269+ value: "Durable Object reset because its code was updated.",
270+ mechanism: { type: "auto.faas.cloudflare.durable_object", handled: false },
271+ ...hashedFrames("BAuwphPA"),
272+ },
273+ ],
274+ },
275+ });
276+
277+ expect(beforeSendCloudEvent(echo, {})).toBeNull();
278+ });
279+
280+ it("pins a stable fingerprint on the report the DO itself owns", () => {
281+ const ownReport = (chunkHash: string): ErrorEvent =>
282+ doEcho({
283+ exception: {
284+ values: [
285+ {
286+ type: "Error",
287+ value: "Durable Object reset because its code was updated.",
288+ mechanism: { type: "generic", handled: true },
289+ ...hashedFrames(chunkHash),
290+ },
291+ ],
292+ },
293+ });
294+
295+ const before = beforeSendCloudEvent(ownReport("BAuwphPA"), {});
296+ const after = beforeSendCloudEvent(ownReport("DkcPBbWe"), {});
297+
298+ expect(before?.fingerprint).toBeDefined();
299+ expect(before?.fingerprint).toEqual(after?.fingerprint);
300+ });
301+ });
165302 });
apps/cloud/src/server.ts+3−26
@@ -1,6 +1,5 @@
11 import { DurableObject } from "cloudflare:workers";
22 import { SpanKind, SpanStatusCode, context, trace, type SpanContext } from "@opentelemetry/api";
3−import type { ErrorEvent } from "@sentry/cloudflare";
43 import {
54 ATTR_HTTP_REQUEST_METHOD,
65 ATTR_HTTP_RESPONSE_STATUS_CODE,
@@ -19,7 +18,7 @@ import { classifyMcpPath, prepareMcpOrgScope } from "./mcp/mount"; · 12 unmodified lines
1918 import { parseTraceparent } from "./mcp/traceparent";
2019 import { McpSessionDOSqlite as McpSessionDOBase } from "./mcp/session-durable-object";
2120 import {
22− beforeSendWithOtelCorrelation,
21+ cloudSentryOptions,
2322 captureCause,
2423 otelCorrelationContextFromOpenTelemetrySpan,
2524 SENTRY_EVENT_ID_ATTRIBUTE,
@@ -28,28 +27,6 @@ import { · 2 unmodified lines
2827 import { browserTracesResponse } from "./observability/browser-traces";
2928 import { flushTracerProvider, installTracerProvider } from "./observability/telemetry";
3029
31−// ---------------------------------------------------------------------------
32−// Sentry config
33−// ---------------------------------------------------------------------------
34−
35−const sentryOptions = (env: Env) => ({
36− dsn: env.SENTRY_DSN,
37− tracesSampleRate: 0,
38− enableLogs: true,
39− sendDefaultPii: true,
40− skipOpenTelemetrySetup: true,
41− beforeSend: (event: ErrorEvent) =>
42− beforeSendWithOtelCorrelation(event, {
43− logPayload: !env.SENTRY_DSN || env.SENTRY_OTEL_LOG_PAYLOAD === "true",
44− }),
45− // NOTE: do NOT enable `instrumentPrototypeMethods`. It walks the DO prototype
46− // and reads every property — including accessors — to find methods to wrap,
47− // which invokes the `sessionId` getter with `this` bound to the prototype
48− // (where `ctx` is undefined) and throws during construction, 500ing every
49− // session create / cold restore. The DO captures its own errors via the
50− // `captureCause` seam (→ Sentry) instead.
51−});
52−
5330 // ---------------------------------------------------------------------------
5431 // Durable Object — wrapped with Sentry so DO errors land in Sentry (inits the
5532 // client inside the DO isolate, which plain `Sentry.captureException` cannot
@@ -58,7 +35,7 @@ const sentryOptions = (env: Env) => ({ · 2 unmodified lines
5835 // ---------------------------------------------------------------------------
5936
6037 export const McpSessionDOSqlite = Sentry.instrumentDurableObjectWithSentry(
61− sentryOptions,
38+ cloudSentryOptions,
6239 McpSessionDOBase,
6340 );
6441
@@ -458,4 +435,4 @@ const cloudflareHandler: ExportedHandler<Env> = { · 393 unmodified lines
458435 },
459436 };
460437
461−export default Sentry.withSentry(sentryOptions, cloudflareHandler);
438+export default Sentry.withSentry(cloudSentryOptions, cloudflareHandler);
apps/desktop/src/main/crash-fingerprint.test.ts+143−0
@@ -0,0 +1,143 @@
1+import { expect, test } from "@effect/vitest";
2+
3+import {
4+ crashReportFingerprint,
5+ mainCrashReportingOptions,
6+ withCrashReportFingerprint,
7+ type CrashEvent,
8+} from "./crash-fingerprint";
9+
10+/** A crash event as Sentry hands it to `beforeSend` — with the grouping key
11+ * slot the hook is allowed to fill. */
12+type SentCrashEvent = CrashEvent & { readonly fingerprint?: readonly string[] | undefined };
13+
14+// Chromium's soft-assert path (`NOTREACHED()`/`DCHECK`) dumps and keeps
15+// running. Sentry titles each one with the faulting load address, so one
16+// condition arrives as a new issue every time the address moves.
17+const softAssertEvent = (address: string): SentCrashEvent => ({
18+ exception: {
19+ values: [
20+ {
21+ type: "Fatal Error",
22+ value: `Simulated Exception / ${address}`,
23+ mechanism: { type: "minidump" },
24+ stacktrace: {
25+ frames: [
26+ { function: "logging::NotReachedLogMessage::~NotReachedLogMessage" },
27+ { function: "logging::HandleCheckErrorLogMessage" },
28+ { function: "base::debug::DumpWithoutCrashing" },
29+ { function: "crash_reporter::DumpWithoutCrashing" },
30+ ],
31+ },
32+ },
33+ ],
34+ },
35+});
36+
37+test("address-keyed Chromium soft asserts collapse to one fingerprint", () => {
38+ const first = crashReportFingerprint(softAssertEvent("0x00000001a2b3c4d5"));
39+ const second = crashReportFingerprint(softAssertEvent("0x00000007e8f9a0b1"));
40+
41+ expect(first).toEqual(["chromium-dump-without-crashing"]);
42+ expect(first).toEqual(second);
43+});
44+
45+test("a real native abort keeps Sentry's own grouping", () => {
46+ const abortEvent: CrashEvent = {
47+ exception: {
48+ values: [
49+ {
50+ type: "EXC_CRASH",
51+ value: "SIGABRT",
52+ mechanism: { type: "minidump" },
53+ stacktrace: {
54+ frames: [
55+ { function: "abort" },
56+ { function: "pthread_kill" },
57+ { function: "__pthread_kill" },
58+ ],
59+ },
60+ },
61+ ],
62+ },
63+ };
64+
65+ expect(crashReportFingerprint(abortEvent)).toBeUndefined();
66+});
67+
68+test("renderer chunk hashes are normalized out of the fingerprint", () => {
69+ const rendererEvent = (chunkHash: string): CrashEvent => ({
70+ culprit: `loadConnections(assets/atoms-${chunkHash})`,
71+ exception: {
72+ values: [
73+ {
74+ type: "TypeError",
75+ value: "cannot read properties of undefined",
76+ stacktrace: {
77+ frames: [
78+ {
79+ filename: `http://127.0.0.1:4789/assets/atoms-${chunkHash}.js`,
80+ module: `atoms-${chunkHash}`,
81+ function: "loadConnections",
82+ in_app: true,
83+ },
84+ ],
85+ },
86+ },
87+ ],
88+ },
89+ });
90+
91+ const release1 = crashReportFingerprint(rendererEvent("Yemn7yhP"));
92+ const release2 = crashReportFingerprint(rendererEvent("CeCENfWa"));
93+
94+ expect(release1).toEqual(["TypeError", "loadConnections@atoms"]);
95+ expect(release1).toEqual(release2);
96+});
97+
98+// `withCrashReportFingerprint` is the function object diagnostics.ts installs
99+// as its `beforeSend`, so these assertions cover the main-process wiring and
100+// not just the classifier behind it.
101+test("the main-process beforeSend pins the key and forwards the event", () => {
102+ const collapsed = withCrashReportFingerprint(softAssertEvent("0x00000001a2b3c4d5"));
103+ expect(collapsed.fingerprint).toEqual(["chromium-dump-without-crashing"]);
104+ // Nothing else about the event is touched — this is grouping only.
105+ expect(collapsed.exception?.values?.[0]?.value).toBe("Simulated Exception / 0x00000001a2b3c4d5");
106+
107+ const untouched: SentCrashEvent = {
108+ exception: { values: [{ type: "EXC_CRASH", stacktrace: { frames: [{ function: "abort" }] } }] },
109+ };
110+ expect(withCrashReportFingerprint(untouched)).toBe(untouched);
111+});
112+
113+// The wiring check: this is the exact object handed to `Sentry.init` in
114+// diagnostics.ts. If the hook is ever dropped from the main process, this
115+// fails.
116+test("the options the main process installs carry the fingerprinting hook", () => {
117+ const options = mainCrashReportingOptions({
118+ dsn: "https://public@example.invalid/1",
119+ release: "executor-desktop@0.0.0",
120+ environment: "production",
121+ runId: "abcdef123456",
122+ });
123+ const sent = options.beforeSend(softAssertEvent("0x00000001a2b3c4d5"));
124+ expect(sent.fingerprint).toEqual(["chromium-dump-without-crashing"]);
125+});
126+
127+test("events with no volatile grouping input are left alone", () => {
128+ expect(crashReportFingerprint({})).toBeUndefined();
129+ expect(
130+ crashReportFingerprint({
131+ exception: {
132+ values: [
133+ {
134+ type: "Error",
135+ stacktrace: {
136+ frames: [{ filename: "/src/main/sidecar.ts", function: "startSidecar" }],
137+ },
138+ },
139+ ],
140+ },
141+ }),
142+ ).toBeUndefined();
143+});
apps/desktop/src/main/crash-fingerprint.ts+80−0
@@ -0,0 +1,80 @@
1+/**
2+ * Grouping keys for desktop crash reports.
3+ *
4+ * Two things split one desktop problem across many Sentry issues:
5+ *
6+ * - Chromium's soft-assert path (`NOTREACHED()`/`DCHECK`) calls
7+ * `DumpWithoutCrashing`, which files a minidump and lets the process carry
8+ * on. Sentry titles each one with the faulting load address, so the same
9+ * condition arrives as a new issue every time the address moves.
10+ * - Renderer and main bundles ship as content-hashed chunks, so unresolved
11+ * frames name `atoms-<hash>` and re-group on every release.
12+ *
13+ * Both are grouping problems only: nothing here drops, downgrades or edits an
14+ * event, and the frames keep their hashes so sourcemap resolution still works.
15+ */
16+
17+import {
18+ stableGroupingFingerprint,
19+ withStableGroupingFingerprint,
20+ type GroupingEvent,
21+ type GroupingFrame,
22+} from "@executor-js/sdk/sentry-grouping";
23+
24+export type CrashEvent = GroupingEvent;
25+
26+export const CHROMIUM_SOFT_ASSERT_FINGERPRINT = "chromium-dump-without-crashing";
27+
28+const isSoftAssertFrame = (frame: GroupingFrame): boolean =>
29+ (frame.function ?? "").includes("DumpWithoutCrashing");
30+
31+/**
32+ * The fingerprint to attach to a crash event, or `undefined` to keep Sentry's
33+ * default grouping (which is right for a real native abort — one issue per
34+ * distinct stack is what we want there).
35+ */
36+export const crashReportFingerprint = (event: CrashEvent): readonly string[] | undefined => {
37+ const frames = (event.exception?.values ?? []).flatMap((value) => value.stacktrace?.frames ?? []);
38+ if (frames.some(isSoftAssertFrame)) return [CHROMIUM_SOFT_ASSERT_FINGERPRINT];
39+ return stableGroupingFingerprint(event);
40+};
41+
42+/**
43+ * The `beforeSend` the Electron main process installs. Grouping only — an
44+ * event with no volatile grouping input is forwarded unchanged.
45+ */
46+export const withCrashReportFingerprint = <T extends CrashEvent>(event: T): T => {
47+ const frames = (event.exception?.values ?? []).flatMap((value) => value.stacktrace?.frames ?? []);
48+ if (frames.some(isSoftAssertFrame)) {
49+ return { ...event, fingerprint: [CHROMIUM_SOFT_ASSERT_FINGERPRINT] };
50+ }
51+ return withStableGroupingFingerprint(event);
52+};
53+
54+/**
55+ * The Sentry options the Electron main process installs, assembled here rather
56+ * than inline at the `Sentry.init` call so the `beforeSend` wiring is covered
57+ * by crash-fingerprint.test.ts. `diagnostics.ts` only passes this through.
58+ *
59+ * Typed structurally on purpose: this module stays importable without pulling
60+ * in electron, which is what keeps it unit-testable at all.
61+ */
62+export const mainCrashReportingOptions = (config: {
63+ readonly dsn: string;
64+ readonly release: string;
65+ readonly environment: string;
66+ readonly runId: string;
67+}) => ({
68+ dsn: config.dsn,
69+ release: config.release,
70+ environment: config.environment,
71+ initialScope: {
72+ tags: {
73+ platform: process.platform,
74+ arch: process.arch,
75+ runId: config.runId,
76+ },
77+ },
78+ // Grouping only — the event is forwarded untouched otherwise.
79+ beforeSend: withCrashReportFingerprint,
80+});
apps/desktop/src/main/diagnostics.ts+9−12
@@ -23,6 +23,7 @@ import { dirname, join } from "node:path"; · 22 unmodified lines
2323 import { app, crashReporter, dialog, shell } from "electron";
2424 import log from "electron-log/main.js";
2525 import * as Sentry from "@sentry/electron/main";
26+import { mainCrashReportingOptions } from "./crash-fingerprint";
2627 import { getServerSettings } from "./settings";
2728
2829 const sentryDsn = __EXECUTOR_SENTRY_DSN__;
@@ -78,18 +79,14 @@ export const sidecarCrashReportingEnv = (): Record<string, string> => · 49 unmodified lines
7879 */
7980 export const initErrorReporting = () => {
8081 if (errorReportingEnabled) {
81− Sentry.init({
82− dsn: sentryDsn,
83− release: releaseTag(),
84− environment: environmentTag(),
85− initialScope: {
86− tags: {
87− platform: process.platform,
88− arch: process.arch,
89− runId,
90− },
91− },
92− });
82+ Sentry.init(
83+ mainCrashReportingOptions({
84+ dsn: sentryDsn,
85+ release: releaseTag(),
86+ environment: environmentTag(),
87+ runId,
88+ }),
89+ );
9390 } else {
9491 // No DSN baked in — keep native crash dumps local so a user-reported
9592 // crash still leaves minidumps for the diagnostics zip to collect.
packages/app/src/crash-reporting.ts+7−0
@@ -12,6 +12,8 @@ · 11 unmodified lines
1212 * once initialized — no reporter rewiring needed.
1313 */
1414
15+import { withStableGroupingFingerprint } from "@executor-js/sdk/sentry-grouping";
16+
1517 interface CrashReportingConfig {
1618 readonly dsn: string;
1719 readonly release: string;
@@ -45,6 +47,11 @@ export const initDesktopCrashReporting = (): void => { · 27 unmodified lines
4547 runId: config.runId,
4648 },
4749 },
50+ // Route chunks are content-hashed, so an unresolved frame names
51+ // `atoms-<hash>` and one bug re-groups on every release. Pin a
52+ // fingerprint with the hash normalized out; the event itself keeps its
53+ // hashed filenames so sourcemap resolution is unaffected.
54+ beforeSend: withStableGroupingFingerprint,
4855 });
4956 } catch {
5057 // Reporting failures stay silent — there is nowhere left to report them.
packages/core/sdk/package.json+8−1
@@ -25,7 +25,8 @@ · 24 unmodified lines
2525 "./testing": "./src/testing.ts",
2626 "./migration": "./src/migration-spec.ts",
2727 "./http-auth": "./src/http-auth/index.ts",
28− "./public-origin": "./src/public-origin.ts"
28+ "./public-origin": "./src/public-origin.ts",
29+ "./sentry-grouping": "./src/sentry-grouping.ts"
2930 },
3031 "publishConfig": {
3132 "access": "public",
@@ -83,6 +84,12 @@ · 51 unmodified lines
8384 "types": "./dist/public-origin.d.ts",
8485 "default": "./dist/public-origin.js"
8586 }
87+ },
88+ "./sentry-grouping": {
89+ "import": {
90+ "types": "./dist/sentry-grouping.d.ts",
91+ "default": "./dist/sentry-grouping.js"
92+ }
8693 }
8794 }
8895 },
packages/core/sdk/src/sentry-grouping.test.ts+269−0
@@ -0,0 +1,269 @@
1+import { expect, test } from "@effect/vitest";
2+
3+import {
4+ containsContentHash,
5+ stableGroupingFingerprint,
6+ stripContentHashes,
7+ withStableGroupingFingerprint,
8+ type GroupingEvent,
9+} from "./sentry-grouping";
10+
11+// Two builds of the same source: only the Vite content hash differs.
12+const workerFrame = (hash: string) => ({
13+ filename: `/assets/execution-rate-limit-${hash}.js`,
14+ module: `execution-rate-limit-${hash}`,
15+ function: "timeoutOrElse",
16+ in_app: true,
17+});
18+
19+const workerEvent = (hash: string): GroupingEvent => ({
20+ culprit: `timeoutOrElse(execution-rate-limit-${hash})`,
21+ exception: {
22+ values: [
23+ {
24+ type: "GateCheckTimeoutError",
25+ value: "balance check timed out",
26+ stacktrace: { frames: [workerFrame(hash)] },
27+ },
28+ ],
29+ },
30+});
31+
32+test("stripContentHashes rewrites chunk hashes and leaves everything else alone", () => {
33+ expect(stripContentHashes("/assets/execution-rate-limit-BAuwphPA.js")).toBe(
34+ "/assets/execution-rate-limit.js",
35+ );
36+ expect(stripContentHashes("/assets/execution-rate-limit-DkcPBbWe.js")).toBe(
37+ "/assets/execution-rate-limit.js",
38+ );
39+ // Rollup hashes may themselves contain a dash — the whole 8-char tail goes.
40+ expect(stripContentHashes("http://127.0.0.1:4789/assets/AddGraphqlIntegration-BCr-oWx4.js")).toBe(
41+ "http://127.0.0.1:4789/assets/AddGraphqlIntegration.js",
42+ );
43+ // Culprit strings wrap the chunk name in parentheses.
44+ expect(stripContentHashes("U(assets/atoms-Yemn7yhP)")).toBe("U(assets/atoms)");
45+ expect(stripContentHashes("W(assets/atoms-CeCENfWa)")).toBe("W(assets/atoms)");
46+});
47+
48+test("stripContentHashes does not eat real name segments", () => {
49+ // 8 lowercase letters is a word, not a hash.
50+ expect(stripContentHashes("/src/auth/oauth-callback.ts")).toBe("/src/auth/oauth-callback.ts");
51+ expect(stripContentHashes("apps/cloud/src/engine/execution-gate.ts")).toBe(
52+ "apps/cloud/src/engine/execution-gate.ts",
53+ );
54+ expect(stripContentHashes("/assets/execution-rate-limit.js")).toBe(
55+ "/assets/execution-rate-limit.js",
56+ );
57+ expect(stripContentHashes("")).toBe("");
58+});
59+
60+test("stripContentHashes keeps genuinely different modules distinct", () => {
61+ expect(stripContentHashes("/assets/atoms-Yemn7yhP.js")).not.toBe(
62+ stripContentHashes("/assets/router-CeCENfWa.js"),
63+ );
64+ expect(stripContentHashes("/assets/atoms-Yemn7yhP.js")).not.toBe(
65+ stripContentHashes("/assets/atoms-shell-CeCENfWa.js"),
66+ );
67+});
68+
69+// Merging two unrelated bugs into one issue is unrecoverable, so the "is this
70+// a hash?" threshold is pinned from both sides: a segment needs two of the
71+// three hash signals (uppercase, uppercase+digit, digits) before it is eaten.
72+// Loosening it to one signal — the tempting fix for the word-shaped hashes
73+// this deliberately misses — silently collapses real chunk names.
74+test("a name segment one signal short of a hash is left in place", () => {
75+ // One capital, no digits.
76+ expect(stripContentHashes("/assets/connections-Settings.js")).toBe(
77+ "/assets/connections-Settings.js",
78+ );
79+ // One digit, no capitals.
80+ expect(stripContentHashes("/assets/storage-s3client.js")).toBe("/assets/storage-s3client.js");
81+ // ...and neither may be mistaken for the chunk it would collapse onto.
82+ expect(stripContentHashes("/assets/connections-Settings.js")).not.toBe(
83+ stripContentHashes("/assets/connections-BAuwphPA.js"),
84+ );
85+});
86+
87+test("a segment with two hash signals is eaten", () => {
88+ expect(stripContentHashes("/assets/chunk-AbcdefgH.js")).toBe("/assets/chunk.js");
89+ expect(stripContentHashes("/assets/chunk-Abcdefg1.js")).toBe("/assets/chunk.js");
90+ expect(stripContentHashes("/assets/chunk-abcdef12.js")).toBe("/assets/chunk.js");
91+});
92+
93+test("containsContentHash detects only hashed paths", () => {
94+ expect(containsContentHash("/assets/atoms-Yemn7yhP.js")).toBe(true);
95+ expect(containsContentHash("apps/cloud/src/engine/execution-gate.ts")).toBe(false);
96+});
97+
98+test("the same logical frame fingerprints identically across two deploys", () => {
99+ const first = stableGroupingFingerprint(workerEvent("BAuwphPA"));
100+ const second = stableGroupingFingerprint(workerEvent("DkcPBbWe"));
101+ expect(first).toBeDefined();
102+ expect(first).toEqual(second);
103+ expect(first).toEqual(["GateCheckTimeoutError", "timeoutOrElse@execution-rate-limit"]);
104+});
105+
106+test("different modules keep different fingerprints", () => {
107+ const other: GroupingEvent = {
108+ exception: {
109+ values: [
110+ {
111+ type: "GateCheckTimeoutError",
112+ stacktrace: {
113+ frames: [
114+ {
115+ filename: "/assets/atoms-Yemn7yhP.js",
116+ module: "atoms-Yemn7yhP",
117+ function: "loadConnections",
118+ in_app: true,
119+ },
120+ ],
121+ },
122+ },
123+ ],
124+ },
125+ };
126+ expect(stableGroupingFingerprint(other)).not.toEqual(
127+ stableGroupingFingerprint(workerEvent("BAuwphPA")),
128+ );
129+});
130+
131+test("minified single-letter frame functions are left out of the fingerprint", () => {
132+ // Minified identifiers rotate with every build exactly like chunk hashes, so
133+ // keeping them would re-split the issue on the next deploy.
134+ const atoms = (fn: string, hash: string): GroupingEvent => ({
135+ exception: {
136+ values: [
137+ {
138+ type: "TypeError",
139+ stacktrace: {
140+ frames: [
141+ { filename: `/assets/atoms-${hash}.js`, module: `atoms-${hash}`, function: fn },
142+ ],
143+ },
144+ },
145+ ],
146+ },
147+ });
148+ expect(stableGroupingFingerprint(atoms("U", "Yemn7yhP"))).toEqual(
149+ stableGroupingFingerprint(atoms("W", "CeCENfWa")),
150+ );
151+ expect(stableGroupingFingerprint(atoms("U", "Yemn7yhP"))).toEqual(["TypeError", "atoms"]);
152+});
153+
154+// A single build emits many `dist-<hash>.js` chunks from unrelated packages,
155+// so the crashing frame alone is not a safe key — two different bugs would
156+// land in one issue. Frames are oldest-first, so the shared chunk is last.
157+const vendorEvent = (caller: string, hash: string): GroupingEvent => ({
158+ exception: {
159+ values: [
160+ {
161+ type: "TypeError",
162+ stacktrace: {
163+ frames: [
164+ { module: `${caller}-${hash}`, function: caller, in_app: true },
165+ { module: `dist-${hash}`, function: "throwHelper", in_app: true },
166+ ],
167+ },
168+ },
169+ ],
170+ },
171+});
172+
173+test("two vendor chunks sharing a name stay apart on their callers", () => {
174+ const upload = stableGroupingFingerprint(vendorEvent("uploadArtifact", "BQZkXWT2"));
175+ const render = stableGroupingFingerprint(vendorEvent("renderMarkdown", "BQZkXWT2"));
176+
177+ // The crashing frame is identical in both — only the caller chain separates
178+ // them, so a fingerprint built from the top frame alone would over-merge.
179+ expect(upload?.slice(0, 2)).toEqual(["TypeError", "throwHelper@dist"]);
180+ expect(render?.slice(0, 2)).toEqual(upload?.slice(0, 2));
181+ expect(upload).not.toEqual(render);
182+ expect(upload).toEqual(["TypeError", "throwHelper@dist", "uploadArtifact@uploadArtifact"]);
183+});
184+
185+test("the caller chain is itself deploy-stable", () => {
186+ // Every frame in the chain must lose its hash, not just the crashing one.
187+ expect(stableGroupingFingerprint(vendorEvent("uploadArtifact", "BQZkXWT2"))).toEqual(
188+ stableGroupingFingerprint(vendorEvent("uploadArtifact", "Yemn7yhP")),
189+ );
190+});
191+
192+test("events without a chunk hash keep Sentry's default grouping", () => {
193+ const unhashed: GroupingEvent = {
194+ culprit: "checkExecutionBalance(execution-gate.ts)",
195+ exception: {
196+ values: [
197+ {
198+ type: "AutumnError",
199+ stacktrace: {
200+ frames: [
201+ {
202+ filename: "/apps/cloud/src/engine/execution-gate.ts",
203+ function: "checkExecutionBalance",
204+ in_app: true,
205+ },
206+ ],
207+ },
208+ },
209+ ],
210+ },
211+ };
212+ expect(stableGroupingFingerprint(unhashed)).toBeUndefined();
213+ expect(stableGroupingFingerprint({})).toBeUndefined();
214+ expect(stableGroupingFingerprint({ exception: { values: [] } })).toBeUndefined();
215+});
216+
217+test("a hashed culprit alone is enough to pin the fingerprint", () => {
218+ const culpritOnly: GroupingEvent = {
219+ culprit: "orElse(execution-rate-limit-BAuwphPA)",
220+ exception: { values: [{ type: "TypeError", value: "destroyed" }] },
221+ };
222+ expect(stableGroupingFingerprint(culpritOnly)).toEqual([
223+ "TypeError",
224+ "orElse(execution-rate-limit)",
225+ ]);
226+});
227+
228+// The wrapper every process installs as its `beforeSend`. `fingerprint` and
229+// `tags` stand in for the fields a real Sentry event carries around it.
230+type SentEvent = GroupingEvent & {
231+ readonly fingerprint?: readonly string[] | undefined;
232+ readonly tags?: Record<string, string> | undefined;
233+};
234+
235+test("withStableGroupingFingerprint pins the key and changes nothing else", () => {
236+ const input: SentEvent = { ...workerEvent("BAuwphPA"), tags: { a: "b" } };
237+ const hashed = withStableGroupingFingerprint(input);
238+ expect(hashed.fingerprint).toEqual([
239+ "GateCheckTimeoutError",
240+ "timeoutOrElse@execution-rate-limit",
241+ ]);
242+ // The event still carries its hashed filename, or server-side sourcemap
243+ // resolution would stop finding the release's artifacts.
244+ expect(hashed.exception?.values?.[0]?.stacktrace?.frames?.[0]?.filename).toBe(
245+ "/assets/execution-rate-limit-BAuwphPA.js",
246+ );
247+ expect(hashed.tags).toEqual({ a: "b" });
248+});
249+
250+test("withStableGroupingFingerprint forwards an unhashed event untouched", () => {
251+ const plain: SentEvent = { culprit: "checkExecutionBalance(execution-gate.ts)" };
252+ const out = withStableGroupingFingerprint(plain);
253+ expect(out).toBe(plain);
254+ expect("fingerprint" in out).toBe(false);
255+});
256+
257+test("the outermost exception drives the fingerprint", () => {
258+ // Sentry orders `values` innermost-first; the last entry is the one whose
259+ // type the issue is titled with.
260+ const chained: GroupingEvent = {
261+ exception: {
262+ values: [
263+ { type: "InnerError", stacktrace: { frames: [workerFrame("BAuwphPA")] } },
264+ { type: "OuterError", stacktrace: { frames: [workerFrame("BAuwphPA")] } },
265+ ],
266+ },
267+ };
268+ expect(stableGroupingFingerprint(chained)?.[0]).toBe("OuterError");
269+});
packages/core/sdk/src/sentry-grouping.ts+169−0
@@ -0,0 +1,169 @@
1+// ---------------------------------------------------------------------------
2+// Stable Sentry grouping across deploys.
3+//
4+// Every bundle we ship names its chunks `<name>-<contentHash>.js`, and the
5+// hash rotates on every build. When a stack frame is not resolved back to
6+// source (no uploaded sourcemap for that release), Sentry groups on those
7+// minified names, so the SAME error opens a brand-new issue after each
8+// deploy — one bug spread over N issues, no counts, no regression history.
9+//
10+// The fix is a `beforeSend` that computes an explicit `fingerprint` from the
11+// event's grouping inputs with the volatile hash segment removed. The event
12+// itself is left untouched: filenames still carry their hashes so server-side
13+// sourcemap resolution keeps working — only the grouping key is normalized.
14+//
15+// Deliberately narrow: a fingerprint is returned ONLY for events that actually
16+// carry a content hash. Everything else keeps Sentry's default grouping.
17+// ---------------------------------------------------------------------------
18+
19+/** The shape of a Sentry stack frame this module reads. Structural on purpose:
20+ * the same helper serves @sentry/cloudflare, /electron and /browser events. */
21+export type GroupingFrame = {
22+ readonly filename?: string | undefined;
23+ readonly abs_path?: string | undefined;
24+ readonly module?: string | undefined;
25+ readonly function?: string | undefined;
26+ readonly in_app?: boolean | undefined;
27+};
28+
29+export type GroupingExceptionValue = {
30+ readonly type?: string | undefined;
31+ readonly value?: string | undefined;
32+ readonly stacktrace?: { readonly frames?: readonly GroupingFrame[] | undefined } | undefined;
33+ readonly mechanism?: { readonly type?: string | undefined } | undefined;
34+};
35+
36+export type GroupingEvent = {
37+ readonly culprit?: string | undefined;
38+ readonly exception?: { readonly values?: readonly GroupingExceptionValue[] | undefined };
39+};
40+
41+/**
42+ * A `-XXXXXXXX` tail that ends a path segment — i.e. is followed by nothing,
43+ * by file extensions, or by a delimiter such as the `)` in a Sentry culprit
44+ * (`orElse(execution-rate-limit-BAuwphPA)`). Eight characters is the Vite and
45+ * Rollup default; the hash alphabet includes `-` and `_`, so a hash can itself
46+ * contain a dash (`BCr-oWx4`) and the whole 8-char tail must go at once.
47+ */
48+const HASH_TAIL = /-([A-Za-z0-9_-]{8})(?=(?:\.[A-Za-z0-9]+)*(?:$|[)\]}'"\s,:;?#]))/g;
49+
50+/**
51+ * Whether an 8-character segment reads like a content hash rather than a word.
52+ * Build hashes are random base64url, so they mix cases and digits; real name
53+ * segments (`callback`, `grouping`, `provider`) are all lowercase. Requiring
54+ * that mix is what keeps `oauth-callback` from collapsing to `oauth`. The cost
55+ * is that the occasional word-shaped hash (one capital, no digits) is left
56+ * alone and keeps re-grouping — 1 of 147 chunks in a sample desktop build. A
57+ * missed normalization is recoverable; a wrongly merged issue is not.
58+ */
59+const looksLikeContentHash = (segment: string): boolean => {
60+ let upper = 0;
61+ let digits = 0;
62+ for (const char of segment) {
63+ if (char >= "A" && char <= "Z") upper += 1;
64+ else if (char >= "0" && char <= "9") digits += 1;
65+ }
66+ return upper >= 2 || (upper >= 1 && digits >= 1) || digits >= 2;
67+};
68+
69+/**
70+ * Remove build content hashes from a path, module name or culprit string,
71+ * leaving every other character in place.
72+ */
73+export const stripContentHashes = (value: string): string =>
74+ value.replace(HASH_TAIL, (match, segment: string) =>
75+ looksLikeContentHash(segment) ? "" : match,
76+ );
77+
78+/** True when the value carries at least one build content hash. */
79+export const containsContentHash = (value: string): boolean => stripContentHashes(value) !== value;
80+
81+/**
82+ * Minified identifiers rotate with every build exactly like chunk hashes, so a
83+ * fingerprint containing one is no more stable than the hash it replaced. Names
84+ * of three characters or fewer are treated as minified and dropped; the frame
85+ * still contributes its module.
86+ */
87+const MINIFIED_FUNCTION = /^[A-Za-z_$][A-Za-z0-9_$]{0,2}$/;
88+
89+const usableFunction = (name: string | undefined): string | undefined => {
90+ if (!name || name === "<anonymous>" || name === "?") return undefined;
91+ return MINIFIED_FUNCTION.test(name) ? undefined : name;
92+};
93+
94+/** Last path segment, without query or fragment. Chunk paths are served from
95+ * host- and port-specific origins (`http://127.0.0.1:4789/assets/…`), which are
96+ * volatile in their own right. */
97+const basename = (path: string): string => {
98+ const clean = path.split(/[?#]/)[0] ?? path;
99+ const segments = clean.split(/[/\\]/);
100+ return segments[segments.length - 1] || clean;
101+};
102+
103+const frameLocation = (frame: GroupingFrame): string | undefined => {
104+ if (frame.module) return stripContentHashes(frame.module);
105+ const path = frame.filename ?? frame.abs_path;
106+ return path ? stripContentHashes(basename(path)) : undefined;
107+};
108+
109+const frameKey = (frame: GroupingFrame): string | undefined => {
110+ const location = frameLocation(frame);
111+ if (!location) return undefined;
112+ const fn = usableFunction(frame.function);
113+ return fn ? `${fn}@${location}` : location;
114+};
115+
116+const frameHasContentHash = (frame: GroupingFrame): boolean =>
117+ containsContentHash(frame.module ?? "") ||
118+ containsContentHash(frame.filename ?? "") ||
119+ containsContentHash(frame.abs_path ?? "");
120+
121+/**
122+ * How many frames from the top of the stack enter the fingerprint. Bundlers
123+ * emit several unrelated chunks under the same name (nine different
124+ * `dist-<hash>.js` in one desktop build), so the top frame alone would merge
125+ * unrelated vendor code; the caller chain is what keeps them apart. The
126+ * resulting key is still strictly coarser than Sentry's default input — same
127+ * frames, minus the volatile hash and minified identifiers — so this can only
128+ * merge issues, never split one further.
129+ */
130+const FINGERPRINT_FRAMES = 8;
131+
132+/**
133+ * A deploy-stable fingerprint for an event whose grouping input carries a build
134+ * content hash, or `undefined` to leave Sentry's default grouping in place.
135+ */
136+export const stableGroupingFingerprint = (event: GroupingEvent): readonly string[] | undefined => {
137+ const values = event.exception?.values ?? [];
138+ // Sentry orders chained exceptions innermost-first; the last entry is the one
139+ // the issue is titled with.
140+ const primary = values[values.length - 1];
141+ if (!primary) return undefined;
142+
143+ const frames = primary.stacktrace?.frames ?? [];
144+ const culprit = event.culprit ?? "";
145+ if (!frames.some(frameHasContentHash) && !containsContentHash(culprit)) return undefined;
146+
147+ const inApp = frames.filter((frame) => frame.in_app !== false);
148+ const considered = (inApp.length > 0 ? inApp : frames).slice(-FINGERPRINT_FRAMES).reverse();
149+ const keys = considered.flatMap((frame) => {
150+ const key = frameKey(frame);
151+ return key ? [key] : [];
152+ });
153+
154+ const type = primary.type ?? "Error";
155+ if (keys.length > 0) return [type, ...keys];
156+ return culprit ? [type, stripContentHashes(culprit)] : undefined;
157+};
158+
159+/**
160+ * The whole `beforeSend` contract: pin the deploy-stable fingerprint when the
161+ * event has a volatile grouping input, and forward the event untouched when it
162+ * does not. Every call site (cloud worker/DO, desktop main, desktop renderer)
163+ * installs this one symbol rather than its own copy, so the wiring is covered
164+ * by the tests below instead of being retyped per process.
165+ */
166+export const withStableGroupingFingerprint = <T extends GroupingEvent>(event: T): T => {
167+ const fingerprint = stableGroupingFingerprint(event);
168+ return fingerprint ? { ...event, fingerprint: [...fingerprint] } : event;
169+};
packages/core/sdk/tsup.config.ts+1−0
@@ -10,6 +10,7 @@ export default defineConfig({ · 9 unmodified lines
1010 "migration-spec": "src/migration-spec.ts",
1111 "http-auth": "src/http-auth/index.ts",
1212 testing: "src/testing.ts",
13+ "sentry-grouping": "src/sentry-grouping.ts",
1314 },
1415 format: ["esm"],
1516 dts: false,

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t