History

[codex] Structure analytics batch delivery failures (#3311)

Julius Marminge committed 4 months ago

Commit ff76f761 parent 973d3c12 files · +153 −14Browse files

Unified · Split

2 files
  1. apps/server/src/telemetry/AnalyticsService.test.ts +86 −7
  2. apps/server/src/telemetry/AnalyticsService.ts +67 −7
apps/server/src/telemetry/AnalyticsService.test.ts+86−7
@@ -4,7 +4,11 @@ import { assert, it } from "@effect/vitest"; · 3 unmodified lines
44 import * as ConfigProvider from "effect/ConfigProvider";
55 import * as Effect from "effect/Effect";
66 import * as Layer from "effect/Layer";
7+import * as Logger from "effect/Logger";
8+import * as References from "effect/References";
9+import * as Schema from "effect/Schema";
710 import * as HttpServer from "effect/unstable/http/HttpServer";
11+import * as HttpClientError from "effect/unstable/http/HttpClientError";
812 import * as HttpServerRequest from "effect/unstable/http/HttpServerRequest";
913 import * as HttpServerResponse from "effect/unstable/http/HttpServerResponse";
1014
@@ -35,10 +39,25 @@ interface RecordedBatchBody { · 24 unmodified lines
3539 }>;
3640 }
3741
42+interface CapturedLog {
43+ readonly message: unknown;
44+ readonly annotations: Readonly<Record<string, unknown>>;
45+}
46+
47+const isAnalyticsBatchDeliveryError = Schema.is(AnalyticsService.AnalyticsBatchDeliveryError);
48+
3849 it.layer(NodeServices.layer)("AnalyticsService test", (it) => {
39− it.effect("flush drains all buffered events across multiple batches", () =>
50+ it.effect("flush drains buffered events and retries failed batches with structured context", () =>
4051 Effect.gen(function* () {
4152 const capturedRequests: Array<RecordedBatchRequest> = [];
53+ const capturedLogs: CapturedLog[] = [];
54+ let rejectNextBatch = false;
55+ const logger = Logger.make(({ fiber, message }) => {
56+ capturedLogs.push({
57+ message,
58+ annotations: fiber.getRef(References.CurrentLogAnnotations),
59+ });
60+ });
4261 const serverConfigLayer = ServerConfig.ServerConfig.layerTest(process.cwd(), {
4362 prefix: "t3-telemetry-base-",
4463 });
@@ -66,12 +85,20 @@ it.layer(NodeServices.layer)("AnalyticsService test", (it) => { · 21 unmodified lines
6685
6786 capturedRequests.push({ path: request.url, body: payload });
6887
88+ if (rejectNextBatch) {
89+ rejectNextBatch = false;
90+ return HttpServerResponse.empty({ status: 503 });
91+ }
92+
6993 return HttpServerResponse.jsonUnsafe({});
7094 }),
7195 );
72− const runtimeLayer = telemetryLayer.pipe(
73− Layer.provide(configLayer),
74− Layer.provideMerge(NodeHttpServer.layerTest),
96+ const runtimeLayer = Layer.merge(
97+ telemetryLayer.pipe(
98+ Layer.provide(configLayer),
99+ Layer.provideMerge(NodeHttpServer.layerTest),
100+ ),
101+ Logger.layer([logger], { mergeWithExisting: false }),
75102 );
76103
77104 yield* Effect.gen(function* () {
@@ -85,12 +112,17 @@ it.layer(NodeServices.layer)("AnalyticsService test", (it) => { · 7 unmodified lines
85112 }
86113
87114 yield* analytics.flush;
115+ yield* analytics.record("test.flush.retry", { index: 45 });
116+ rejectNextBatch = true;
117+ yield* analytics.flush;
118+ yield* analytics.flush;
88119 }).pipe(Effect.provide(runtimeLayer));
89120
90− const batchRequests = capturedRequests.filter(
91− (request): request is RecordedBatchRequest & { readonly body: RecordedBatchBody } =>
121+ const batchRequests = capturedRequests
122+ .slice(0, 3)
123+ .filter((request): request is RecordedBatchRequest & { readonly body: RecordedBatchBody } =>
92124 Array.isArray(request.body?.batch),
93− );
125+ );
94126 assert.equal(batchRequests.length, 3);
95127 assert.equal(
96128 batchRequests.every((request) => request.path === "/batch/" || request.path === "/batch"),
@@ -115,6 +147,53 @@ it.layer(NodeServices.layer)("AnalyticsService test", (it) => { · 18 unmodified lines
115147 ),
116148 true,
117149 );
150+
151+ const retryRequests = capturedRequests.slice(3);
152+ assert.equal(retryRequests.length, 2);
153+ assert.deepEqual(retryRequests[0]?.body, retryRequests[1]?.body);
154+
155+ const deliveryLog = capturedLogs.find((log) =>
156+ isAnalyticsBatchDeliveryError(log.annotations.cause),
157+ );
158+ assert.isDefined(deliveryLog);
159+ assert.equal(
160+ deliveryLog?.message,
161+ "Failed to deliver 1 analytics event to PostHog (endpoint input length 7).",
162+ );
163+ assert.equal("endpoint" in deliveryLog.annotations, false);
164+ assert.equal(deliveryLog.annotations.endpointInputLength, 7);
165+
166+ const error = deliveryLog?.annotations.cause;
167+ assert.instanceOf(error, AnalyticsService.AnalyticsBatchDeliveryError);
168+ if (isAnalyticsBatchDeliveryError(error)) {
169+ assert.equal(error.endpointInputLength, 7);
170+ assert.equal(error.endpointProtocol, undefined);
171+ assert.equal(error.endpointHostname, undefined);
172+ assert.equal(error.eventCount, 1);
173+ assert.instanceOf(error.cause, HttpClientError.HttpClientError);
174+ if (error.cause instanceof HttpClientError.HttpClientError) {
175+ assert.equal(error.cause.reason._tag, "StatusCodeError");
176+ assert.equal(error.cause.response?.status, 503);
177+ }
178+ }
118179 }),
119180 );
120181 });
182+
183+it("keeps configured PostHog endpoint secrets out of direct error and log context", () => {
184+ const endpoint =
185+ "https://user:password@posthog.example.test/private/project?api_key=secret#fragment";
186+ const cause = new Error("delivery failed");
187+ const error = AnalyticsService.AnalyticsBatchDeliveryError.fromEndpoint({
188+ endpoint,
189+ eventCount: 2,
190+ cause,
191+ });
192+
193+ assert.equal(error.endpointInputLength, endpoint.length);
194+ assert.equal(error.endpointProtocol, "https:");
195+ assert.equal(error.endpointHostname, "posthog.example.test");
196+ assert.equal(error.cause, cause);
197+ assert.equal("endpoint" in error, false);
198+ assert.equal(/user|password|private|project|api_key|secret|fragment/.test(error.message), false);
199+});
apps/server/src/telemetry/AnalyticsService.ts+67−7
@@ -7,6 +7,7 @@ · 6 unmodified lines
77 * @module AnalyticsService
88 */
99 import { HostProcessArchitecture, HostProcessPlatform } from "@t3tools/shared/hostProcess";
10+import { getUrlDiagnostics } from "@t3tools/shared/urlDiagnostics";
1011 import * as Config from "effect/Config";
1112 import * as Context from "effect/Context";
1213 import * as DateTime from "effect/DateTime";
@@ -14,6 +15,7 @@ import * as Effect from "effect/Effect"; · 1 unmodified line
1415 import * as Layer from "effect/Layer";
1516 import * as Option from "effect/Option";
1617 import * as Ref from "effect/Ref";
18+import * as Schema from "effect/Schema";
1719 import * as HttpClient from "effect/unstable/http/HttpClient";
1820 import * as HttpClientRequest from "effect/unstable/http/HttpClientRequest";
1921 import * as HttpClientResponse from "effect/unstable/http/HttpClientResponse";
@@ -43,6 +45,38 @@ const TelemetryEnvConfig = Config.all({ · 23 unmodified lines
4345 wslDistroName: Config.string("WSL_DISTRO_NAME").pipe(Config.option),
4446 });
4547
48+export class AnalyticsBatchDeliveryError extends Schema.TaggedErrorClass<AnalyticsBatchDeliveryError>()(
49+ "AnalyticsBatchDeliveryError",
50+ {
51+ endpointInputLength: Schema.Number,
52+ endpointProtocol: Schema.optionalKey(Schema.String),
53+ endpointHostname: Schema.optionalKey(Schema.String),
54+ eventCount: Schema.Int.check(Schema.isGreaterThan(0)),
55+ cause: Schema.Defect(),
56+ },
57+) {
58+ static fromEndpoint(input: {
59+ readonly endpoint: string;
60+ readonly eventCount: number;
61+ readonly cause: unknown;
62+ }): AnalyticsBatchDeliveryError {
63+ const diagnostics = getUrlDiagnostics(input.endpoint);
64+ return new AnalyticsBatchDeliveryError({
65+ endpointInputLength: diagnostics.inputLength,
66+ ...(diagnostics.protocol === undefined ? {} : { endpointProtocol: diagnostics.protocol }),
67+ ...(diagnostics.hostname === undefined ? {} : { endpointHostname: diagnostics.hostname }),
68+ eventCount: input.eventCount,
69+ cause: input.cause,
70+ });
71+ }
72+
73+ override get message(): string {
74+ const eventLabel = this.eventCount === 1 ? "event" : "events";
75+ const destination = this.endpointHostname ? ` at ${this.endpointHostname}` : "";
76+ return `Failed to deliver ${this.eventCount} analytics ${eventLabel} to PostHog${destination} (endpoint input length ${this.endpointInputLength}).`;
77+ }
78+}
79+
4680 export class AnalyticsService extends Context.Service<
4781 AnalyticsService,
4882 {
@@ -75,6 +109,7 @@ export const make = Effect.gen(function* () { · 26 unmodified lines
75109 const clientType = serverConfig.mode === "desktop" ? "desktop-app" : "cli-web-client";
76110 const hostPlatform = yield* HostProcessPlatform;
77111 const hostArchitecture = yield* HostProcessArchitecture;
112+ const batchEndpoint = `${telemetryConfig.posthogHost}/batch/`;
78113
79114 const enqueueBufferedEvent = (event: string, properties?: Readonly<Record<string, unknown>>) =>
80115 Effect.flatMap(DateTime.now, (now) =>
@@ -126,10 +161,17 @@ export const make = Effect.gen(function* () { · 45 unmodified lines
126161 })),
127162 };
128163
129− yield* HttpClientRequest.post(`${telemetryConfig.posthogHost}/batch/`).pipe(
164+ yield* HttpClientRequest.post(batchEndpoint).pipe(
130165 HttpClientRequest.bodyJson(payload),
131166 Effect.flatMap(httpClient.execute),
132167 Effect.flatMap(HttpClientResponse.filterStatusOk),
168+ Effect.mapError((cause) =>
169+ AnalyticsBatchDeliveryError.fromEndpoint({
170+ endpoint: batchEndpoint,
171+ eventCount: events.length,
172+ cause,
173+ }),
174+ ),
133175 );
134176 });
135177
@@ -149,14 +191,32 @@ export const make = Effect.gen(function* () { · 13 unmodified lines
149191 }
150192
151193 yield* sendBatch(batch).pipe(
152− Effect.catch((error) =>
153− Ref.update(bufferRef, (current) => [...batch, ...current]).pipe(
154− Effect.flatMap(() => Effect.fail(error)),
155− ),
156− ),
194+ Effect.catchTags({
195+ AnalyticsBatchDeliveryError: (error) =>
196+ Ref.update(bufferRef, (current) => [...batch, ...current]).pipe(
197+ Effect.flatMap(() => Effect.fail(error)),
198+ ),
199+ }),
157200 );
158201 }
159− }).pipe(Effect.catch((cause) => Effect.logError("Failed to flush telemetry", { cause })));
202+ }).pipe(
203+ Effect.catchTags({
204+ AnalyticsBatchDeliveryError: (error) =>
205+ Effect.logError(error.message).pipe(
206+ Effect.annotateLogs({
207+ endpointInputLength: error.endpointInputLength,
208+ ...(error.endpointProtocol === undefined
209+ ? {}
210+ : { endpointProtocol: error.endpointProtocol }),
211+ ...(error.endpointHostname === undefined
212+ ? {}
213+ : { endpointHostname: error.endpointHostname }),
214+ eventCount: error.eventCount,
215+ cause: error,
216+ }),
217+ ),
218+ }),
219+ );
160220
161221 const record: AnalyticsService["Service"]["record"] = Effect.fn("AnalyticsService.record")(
162222 function* (event, properties) {

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t