History

[codex] Structure analytics batch delivery failures (#3311)

Julius Marminge committed 4 months ago

Commit ff76f761 parent 973d3c12 files · +153 −14Browse files

Unified · Split

2 files
  1. apps/server/src/telemetry/AnalyticsService.test.ts +86 −7
  2. apps/server/src/telemetry/AnalyticsService.ts +67 −7
apps/server/src/telemetry/AnalyticsService.test.ts+86−7
@@ -4,7 +4,11 @@ import { assert, it } from "@effect/vitest"; · 3 unmodified lines
4import * as ConfigProvider from "effect/ConfigProvider";4import * as ConfigProvider from "effect/ConfigProvider";
5import * as Effect from "effect/Effect";5import * as Effect from "effect/Effect";
6import * as Layer from "effect/Layer";6import * as Layer from "effect/Layer";
7import * as Logger from "effect/Logger";
8import * as References from "effect/References";
9import * as Schema from "effect/Schema";
7import * as HttpServer from "effect/unstable/http/HttpServer";10import * as HttpServer from "effect/unstable/http/HttpServer";
11import * as HttpClientError from "effect/unstable/http/HttpClientError";
8import * as HttpServerRequest from "effect/unstable/http/HttpServerRequest";12import * as HttpServerRequest from "effect/unstable/http/HttpServerRequest";
9import * as HttpServerResponse from "effect/unstable/http/HttpServerResponse";13import * as HttpServerResponse from "effect/unstable/http/HttpServerResponse";
10 14
@@ -35,10 +39,25 @@ interface RecordedBatchBody { · 24 unmodified lines
35 }>;39 }>;
36}40}
37 41
42interface CapturedLog {
43 readonly message: unknown;
44 readonly annotations: Readonly<Record<string, unknown>>;
45}
46
47const isAnalyticsBatchDeliveryError = Schema.is(AnalyticsService.AnalyticsBatchDeliveryError);
48
38it.layer(NodeServices.layer)("AnalyticsService test", (it) => {49it.layer(NodeServices.layer)("AnalyticsService test", (it) => {
39 it.effect("flush drains all buffered events across multiple batches", () =>50 it.effect("flush drains buffered events and retries failed batches with structured context", () =>
40 Effect.gen(function* () {51 Effect.gen(function* () {
41 const capturedRequests: Array<RecordedBatchRequest> = [];52 const capturedRequests: Array<RecordedBatchRequest> = [];
53 const capturedLogs: CapturedLog[] = [];
54 let rejectNextBatch = false;
55 const logger = Logger.make(({ fiber, message }) => {
56 capturedLogs.push({
57 message,
58 annotations: fiber.getRef(References.CurrentLogAnnotations),
59 });
60 });
42 const serverConfigLayer = ServerConfig.ServerConfig.layerTest(process.cwd(), {61 const serverConfigLayer = ServerConfig.ServerConfig.layerTest(process.cwd(), {
43 prefix: "t3-telemetry-base-",62 prefix: "t3-telemetry-base-",
44 });63 });
@@ -66,12 +85,20 @@ it.layer(NodeServices.layer)("AnalyticsService test", (it) => { · 21 unmodified lines
66 85
67 capturedRequests.push({ path: request.url, body: payload });86 capturedRequests.push({ path: request.url, body: payload });
68 87
88 if (rejectNextBatch) {
89 rejectNextBatch = false;
90 return HttpServerResponse.empty({ status: 503 });
91 }
92
69 return HttpServerResponse.jsonUnsafe({});93 return HttpServerResponse.jsonUnsafe({});
70 }),94 }),
71 );95 );
72 const runtimeLayer = telemetryLayer.pipe(96 const runtimeLayer = Layer.merge(
73 Layer.provide(configLayer),97 telemetryLayer.pipe(
74 Layer.provideMerge(NodeHttpServer.layerTest),98 Layer.provide(configLayer),
99 Layer.provideMerge(NodeHttpServer.layerTest),
100 ),
101 Logger.layer([logger], { mergeWithExisting: false }),
75 );102 );
76 103
77 yield* Effect.gen(function* () {104 yield* Effect.gen(function* () {
@@ -85,12 +112,17 @@ it.layer(NodeServices.layer)("AnalyticsService test", (it) => { · 7 unmodified lines
85 }112 }
86 113
87 yield* analytics.flush;114 yield* analytics.flush;
115 yield* analytics.record("test.flush.retry", { index: 45 });
116 rejectNextBatch = true;
117 yield* analytics.flush;
118 yield* analytics.flush;
88 }).pipe(Effect.provide(runtimeLayer));119 }).pipe(Effect.provide(runtimeLayer));
89 120
90 const batchRequests = capturedRequests.filter(121 const batchRequests = capturedRequests
91 (request): request is RecordedBatchRequest & { readonly body: RecordedBatchBody } =>122 .slice(0, 3)
123 .filter((request): request is RecordedBatchRequest & { readonly body: RecordedBatchBody } =>
92 Array.isArray(request.body?.batch),124 Array.isArray(request.body?.batch),
93 );125 );
94 assert.equal(batchRequests.length, 3);126 assert.equal(batchRequests.length, 3);
95 assert.equal(127 assert.equal(
96 batchRequests.every((request) => request.path === "/batch/" || request.path === "/batch"),128 batchRequests.every((request) => request.path === "/batch/" || request.path === "/batch"),
@@ -115,6 +147,53 @@ it.layer(NodeServices.layer)("AnalyticsService test", (it) => { · 18 unmodified lines
115 ),147 ),
116 true,148 true,
117 );149 );
150
151 const retryRequests = capturedRequests.slice(3);
152 assert.equal(retryRequests.length, 2);
153 assert.deepEqual(retryRequests[0]?.body, retryRequests[1]?.body);
154
155 const deliveryLog = capturedLogs.find((log) =>
156 isAnalyticsBatchDeliveryError(log.annotations.cause),
157 );
158 assert.isDefined(deliveryLog);
159 assert.equal(
160 deliveryLog?.message,
161 "Failed to deliver 1 analytics event to PostHog (endpoint input length 7).",
162 );
163 assert.equal("endpoint" in deliveryLog.annotations, false);
164 assert.equal(deliveryLog.annotations.endpointInputLength, 7);
165
166 const error = deliveryLog?.annotations.cause;
167 assert.instanceOf(error, AnalyticsService.AnalyticsBatchDeliveryError);
168 if (isAnalyticsBatchDeliveryError(error)) {
169 assert.equal(error.endpointInputLength, 7);
170 assert.equal(error.endpointProtocol, undefined);
171 assert.equal(error.endpointHostname, undefined);
172 assert.equal(error.eventCount, 1);
173 assert.instanceOf(error.cause, HttpClientError.HttpClientError);
174 if (error.cause instanceof HttpClientError.HttpClientError) {
175 assert.equal(error.cause.reason._tag, "StatusCodeError");
176 assert.equal(error.cause.response?.status, 503);
177 }
178 }
118 }),179 }),
119 );180 );
120});181});
182
183it("keeps configured PostHog endpoint secrets out of direct error and log context", () => {
184 const endpoint =
185 "https://user:password@posthog.example.test/private/project?api_key=secret#fragment";
186 const cause = new Error("delivery failed");
187 const error = AnalyticsService.AnalyticsBatchDeliveryError.fromEndpoint({
188 endpoint,
189 eventCount: 2,
190 cause,
191 });
192
193 assert.equal(error.endpointInputLength, endpoint.length);
194 assert.equal(error.endpointProtocol, "https:");
195 assert.equal(error.endpointHostname, "posthog.example.test");
196 assert.equal(error.cause, cause);
197 assert.equal("endpoint" in error, false);
198 assert.equal(/user|password|private|project|api_key|secret|fragment/.test(error.message), false);
199});
apps/server/src/telemetry/AnalyticsService.ts+67−7
@@ -7,6 +7,7 @@ · 6 unmodified lines
7 * @module AnalyticsService7 * @module AnalyticsService
8 */8 */
9import { HostProcessArchitecture, HostProcessPlatform } from "@t3tools/shared/hostProcess";9import { HostProcessArchitecture, HostProcessPlatform } from "@t3tools/shared/hostProcess";
10import { getUrlDiagnostics } from "@t3tools/shared/urlDiagnostics";
10import * as Config from "effect/Config";11import * as Config from "effect/Config";
11import * as Context from "effect/Context";12import * as Context from "effect/Context";
12import * as DateTime from "effect/DateTime";13import * as DateTime from "effect/DateTime";
@@ -14,6 +15,7 @@ import * as Effect from "effect/Effect"; · 1 unmodified line
14import * as Layer from "effect/Layer";15import * as Layer from "effect/Layer";
15import * as Option from "effect/Option";16import * as Option from "effect/Option";
16import * as Ref from "effect/Ref";17import * as Ref from "effect/Ref";
18import * as Schema from "effect/Schema";
17import * as HttpClient from "effect/unstable/http/HttpClient";19import * as HttpClient from "effect/unstable/http/HttpClient";
18import * as HttpClientRequest from "effect/unstable/http/HttpClientRequest";20import * as HttpClientRequest from "effect/unstable/http/HttpClientRequest";
19import * as HttpClientResponse from "effect/unstable/http/HttpClientResponse";21import * as HttpClientResponse from "effect/unstable/http/HttpClientResponse";
@@ -43,6 +45,38 @@ const TelemetryEnvConfig = Config.all({ · 23 unmodified lines
43 wslDistroName: Config.string("WSL_DISTRO_NAME").pipe(Config.option),45 wslDistroName: Config.string("WSL_DISTRO_NAME").pipe(Config.option),
44});46});
45 47
48export class AnalyticsBatchDeliveryError extends Schema.TaggedErrorClass<AnalyticsBatchDeliveryError>()(
49 "AnalyticsBatchDeliveryError",
50 {
51 endpointInputLength: Schema.Number,
52 endpointProtocol: Schema.optionalKey(Schema.String),
53 endpointHostname: Schema.optionalKey(Schema.String),
54 eventCount: Schema.Int.check(Schema.isGreaterThan(0)),
55 cause: Schema.Defect(),
56 },
57) {
58 static fromEndpoint(input: {
59 readonly endpoint: string;
60 readonly eventCount: number;
61 readonly cause: unknown;
62 }): AnalyticsBatchDeliveryError {
63 const diagnostics = getUrlDiagnostics(input.endpoint);
64 return new AnalyticsBatchDeliveryError({
65 endpointInputLength: diagnostics.inputLength,
66 ...(diagnostics.protocol === undefined ? {} : { endpointProtocol: diagnostics.protocol }),
67 ...(diagnostics.hostname === undefined ? {} : { endpointHostname: diagnostics.hostname }),
68 eventCount: input.eventCount,
69 cause: input.cause,
70 });
71 }
72
73 override get message(): string {
74 const eventLabel = this.eventCount === 1 ? "event" : "events";
75 const destination = this.endpointHostname ? ` at ${this.endpointHostname}` : "";
76 return `Failed to deliver ${this.eventCount} analytics ${eventLabel} to PostHog${destination} (endpoint input length ${this.endpointInputLength}).`;
77 }
78}
79
46export class AnalyticsService extends Context.Service<80export class AnalyticsService extends Context.Service<
47 AnalyticsService,81 AnalyticsService,
48 {82 {
@@ -75,6 +109,7 @@ export const make = Effect.gen(function* () { · 26 unmodified lines
75 const clientType = serverConfig.mode === "desktop" ? "desktop-app" : "cli-web-client";109 const clientType = serverConfig.mode === "desktop" ? "desktop-app" : "cli-web-client";
76 const hostPlatform = yield* HostProcessPlatform;110 const hostPlatform = yield* HostProcessPlatform;
77 const hostArchitecture = yield* HostProcessArchitecture;111 const hostArchitecture = yield* HostProcessArchitecture;
112 const batchEndpoint = `${telemetryConfig.posthogHost}/batch/`;
78 113
79 const enqueueBufferedEvent = (event: string, properties?: Readonly<Record<string, unknown>>) =>114 const enqueueBufferedEvent = (event: string, properties?: Readonly<Record<string, unknown>>) =>
80 Effect.flatMap(DateTime.now, (now) =>115 Effect.flatMap(DateTime.now, (now) =>
@@ -126,10 +161,17 @@ export const make = Effect.gen(function* () { · 45 unmodified lines
126 })),161 })),
127 };162 };
128 163
129 yield* HttpClientRequest.post(`${telemetryConfig.posthogHost}/batch/`).pipe(164 yield* HttpClientRequest.post(batchEndpoint).pipe(
130 HttpClientRequest.bodyJson(payload),165 HttpClientRequest.bodyJson(payload),
131 Effect.flatMap(httpClient.execute),166 Effect.flatMap(httpClient.execute),
132 Effect.flatMap(HttpClientResponse.filterStatusOk),167 Effect.flatMap(HttpClientResponse.filterStatusOk),
168 Effect.mapError((cause) =>
169 AnalyticsBatchDeliveryError.fromEndpoint({
170 endpoint: batchEndpoint,
171 eventCount: events.length,
172 cause,
173 }),
174 ),
133 );175 );
134 });176 });
135 177
@@ -149,14 +191,32 @@ export const make = Effect.gen(function* () { · 13 unmodified lines
149 }191 }
150 192
151 yield* sendBatch(batch).pipe(193 yield* sendBatch(batch).pipe(
152 Effect.catch((error) =>194 Effect.catchTags({
153 Ref.update(bufferRef, (current) => [...batch, ...current]).pipe(195 AnalyticsBatchDeliveryError: (error) =>
154 Effect.flatMap(() => Effect.fail(error)),196 Ref.update(bufferRef, (current) => [...batch, ...current]).pipe(
155 ),197 Effect.flatMap(() => Effect.fail(error)),
156 ),198 ),
199 }),
157 );200 );
158 }201 }
159 }).pipe(Effect.catch((cause) => Effect.logError("Failed to flush telemetry", { cause })));202 }).pipe(
203 Effect.catchTags({
204 AnalyticsBatchDeliveryError: (error) =>
205 Effect.logError(error.message).pipe(
206 Effect.annotateLogs({
207 endpointInputLength: error.endpointInputLength,
208 ...(error.endpointProtocol === undefined
209 ? {}
210 : { endpointProtocol: error.endpointProtocol }),
211 ...(error.endpointHostname === undefined
212 ? {}
213 : { endpointHostname: error.endpointHostname }),
214 eventCount: error.eventCount,
215 cause: error,
216 }),
217 ),
218 }),
219 );
160 220
161 const record: AnalyticsService["Service"]["record"] = Effect.fn("AnalyticsService.record")(221 const record: AnalyticsService["Service"]["record"] = Effect.fn("AnalyticsService.record")(
162 function* (event, properties) {222 function* (event, properties) {

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t