Unified · Split
| @@ -4,7 +4,11 @@ import { assert, it } from "@effect/vitest"; · 3 unmodified lines | |||
| 4 | import * as ConfigProvider from "effect/ConfigProvider"; | 4 | import * as ConfigProvider from "effect/ConfigProvider"; |
| 5 | import * as Effect from "effect/Effect"; | 5 | import * as Effect from "effect/Effect"; |
| 6 | import * as Layer from "effect/Layer"; | 6 | import * as Layer from "effect/Layer"; |
| 7 | import * as Logger from "effect/Logger"; | ||
| 8 | import * as References from "effect/References"; | ||
| 9 | import * as Schema from "effect/Schema"; | ||
| 7 | import * as HttpServer from "effect/unstable/http/HttpServer"; | 10 | import * as HttpServer from "effect/unstable/http/HttpServer"; |
| 11 | import * as HttpClientError from "effect/unstable/http/HttpClientError"; | ||
| 8 | import * as HttpServerRequest from "effect/unstable/http/HttpServerRequest"; | 12 | import * as HttpServerRequest from "effect/unstable/http/HttpServerRequest"; |
| 9 | import * as HttpServerResponse from "effect/unstable/http/HttpServerResponse"; | 13 | import * as HttpServerResponse from "effect/unstable/http/HttpServerResponse"; |
| 10 | 14 | ||
| @@ -35,10 +39,25 @@ interface RecordedBatchBody { · 24 unmodified lines | |||
| 35 | }>; | 39 | }>; |
| 36 | } | 40 | } |
| 37 | 41 | ||
| 42 | interface CapturedLog { | ||
| 43 | readonly message: unknown; | ||
| 44 | readonly annotations: Readonly<Record<string, unknown>>; | ||
| 45 | } | ||
| 46 | |||
| 47 | const isAnalyticsBatchDeliveryError = Schema.is(AnalyticsService.AnalyticsBatchDeliveryError); | ||
| 48 | |||
| 38 | it.layer(NodeServices.layer)("AnalyticsService test", (it) => { | 49 | it.layer(NodeServices.layer)("AnalyticsService test", (it) => { |
| 39 | it.effect("flush drains all buffered events across multiple batches", () => | 50 | it.effect("flush drains buffered events and retries failed batches with structured context", () => |
| 40 | Effect.gen(function* () { | 51 | Effect.gen(function* () { |
| 41 | const capturedRequests: Array<RecordedBatchRequest> = []; | 52 | const capturedRequests: Array<RecordedBatchRequest> = []; |
| 53 | const capturedLogs: CapturedLog[] = []; | ||
| 54 | let rejectNextBatch = false; | ||
| 55 | const logger = Logger.make(({ fiber, message }) => { | ||
| 56 | capturedLogs.push({ | ||
| 57 | message, | ||
| 58 | annotations: fiber.getRef(References.CurrentLogAnnotations), | ||
| 59 | }); | ||
| 60 | }); | ||
| 42 | const serverConfigLayer = ServerConfig.ServerConfig.layerTest(process.cwd(), { | 61 | const serverConfigLayer = ServerConfig.ServerConfig.layerTest(process.cwd(), { |
| 43 | prefix: "t3-telemetry-base-", | 62 | prefix: "t3-telemetry-base-", |
| 44 | }); | 63 | }); |
| @@ -66,12 +85,20 @@ it.layer(NodeServices.layer)("AnalyticsService test", (it) => { · 21 unmodified lines | |||
| 66 | 85 | ||
| 67 | capturedRequests.push({ path: request.url, body: payload }); | 86 | capturedRequests.push({ path: request.url, body: payload }); |
| 68 | 87 | ||
| 88 | if (rejectNextBatch) { | ||
| 89 | rejectNextBatch = false; | ||
| 90 | return HttpServerResponse.empty({ status: 503 }); | ||
| 91 | } | ||
| 92 | |||
| 69 | return HttpServerResponse.jsonUnsafe({}); | 93 | return HttpServerResponse.jsonUnsafe({}); |
| 70 | }), | 94 | }), |
| 71 | ); | 95 | ); |
| 72 | const runtimeLayer = telemetryLayer.pipe( | 96 | const runtimeLayer = Layer.merge( |
| 73 | Layer.provide(configLayer), | 97 | telemetryLayer.pipe( |
| 74 | Layer.provideMerge(NodeHttpServer.layerTest), | 98 | Layer.provide(configLayer), |
| 99 | Layer.provideMerge(NodeHttpServer.layerTest), | ||
| 100 | ), | ||
| 101 | Logger.layer([logger], { mergeWithExisting: false }), | ||
| 75 | ); | 102 | ); |
| 76 | 103 | ||
| 77 | yield* Effect.gen(function* () { | 104 | yield* Effect.gen(function* () { |
| @@ -85,12 +112,17 @@ it.layer(NodeServices.layer)("AnalyticsService test", (it) => { · 7 unmodified lines | |||
| 85 | } | 112 | } |
| 86 | 113 | ||
| 87 | yield* analytics.flush; | 114 | yield* analytics.flush; |
| 115 | yield* analytics.record("test.flush.retry", { index: 45 }); | ||
| 116 | rejectNextBatch = true; | ||
| 117 | yield* analytics.flush; | ||
| 118 | yield* analytics.flush; | ||
| 88 | }).pipe(Effect.provide(runtimeLayer)); | 119 | }).pipe(Effect.provide(runtimeLayer)); |
| 89 | 120 | ||
| 90 | const batchRequests = capturedRequests.filter( | 121 | const batchRequests = capturedRequests |
| 91 | (request): request is RecordedBatchRequest & { readonly body: RecordedBatchBody } => | 122 | .slice(0, 3) |
| 123 | .filter((request): request is RecordedBatchRequest & { readonly body: RecordedBatchBody } => | ||
| 92 | Array.isArray(request.body?.batch), | 124 | Array.isArray(request.body?.batch), |
| 93 | ); | 125 | ); |
| 94 | assert.equal(batchRequests.length, 3); | 126 | assert.equal(batchRequests.length, 3); |
| 95 | assert.equal( | 127 | assert.equal( |
| 96 | batchRequests.every((request) => request.path === "/batch/" || request.path === "/batch"), | 128 | batchRequests.every((request) => request.path === "/batch/" || request.path === "/batch"), |
| @@ -115,6 +147,53 @@ it.layer(NodeServices.layer)("AnalyticsService test", (it) => { · 18 unmodified lines | |||
| 115 | ), | 147 | ), |
| 116 | true, | 148 | true, |
| 117 | ); | 149 | ); |
| 150 | |||
| 151 | const retryRequests = capturedRequests.slice(3); | ||
| 152 | assert.equal(retryRequests.length, 2); | ||
| 153 | assert.deepEqual(retryRequests[0]?.body, retryRequests[1]?.body); | ||
| 154 | |||
| 155 | const deliveryLog = capturedLogs.find((log) => | ||
| 156 | isAnalyticsBatchDeliveryError(log.annotations.cause), | ||
| 157 | ); | ||
| 158 | assert.isDefined(deliveryLog); | ||
| 159 | assert.equal( | ||
| 160 | deliveryLog?.message, | ||
| 161 | "Failed to deliver 1 analytics event to PostHog (endpoint input length 7).", | ||
| 162 | ); | ||
| 163 | assert.equal("endpoint" in deliveryLog.annotations, false); | ||
| 164 | assert.equal(deliveryLog.annotations.endpointInputLength, 7); | ||
| 165 | |||
| 166 | const error = deliveryLog?.annotations.cause; | ||
| 167 | assert.instanceOf(error, AnalyticsService.AnalyticsBatchDeliveryError); | ||
| 168 | if (isAnalyticsBatchDeliveryError(error)) { | ||
| 169 | assert.equal(error.endpointInputLength, 7); | ||
| 170 | assert.equal(error.endpointProtocol, undefined); | ||
| 171 | assert.equal(error.endpointHostname, undefined); | ||
| 172 | assert.equal(error.eventCount, 1); | ||
| 173 | assert.instanceOf(error.cause, HttpClientError.HttpClientError); | ||
| 174 | if (error.cause instanceof HttpClientError.HttpClientError) { | ||
| 175 | assert.equal(error.cause.reason._tag, "StatusCodeError"); | ||
| 176 | assert.equal(error.cause.response?.status, 503); | ||
| 177 | } | ||
| 178 | } | ||
| 118 | }), | 179 | }), |
| 119 | ); | 180 | ); |
| 120 | }); | 181 | }); |
| 182 | |||
| 183 | it("keeps configured PostHog endpoint secrets out of direct error and log context", () => { | ||
| 184 | const endpoint = | ||
| 185 | "https://user:password@posthog.example.test/private/project?api_key=secret#fragment"; | ||
| 186 | const cause = new Error("delivery failed"); | ||
| 187 | const error = AnalyticsService.AnalyticsBatchDeliveryError.fromEndpoint({ | ||
| 188 | endpoint, | ||
| 189 | eventCount: 2, | ||
| 190 | cause, | ||
| 191 | }); | ||
| 192 | |||
| 193 | assert.equal(error.endpointInputLength, endpoint.length); | ||
| 194 | assert.equal(error.endpointProtocol, "https:"); | ||
| 195 | assert.equal(error.endpointHostname, "posthog.example.test"); | ||
| 196 | assert.equal(error.cause, cause); | ||
| 197 | assert.equal("endpoint" in error, false); | ||
| 198 | assert.equal(/user|password|private|project|api_key|secret|fragment/.test(error.message), false); | ||
| 199 | }); | ||
| @@ -7,6 +7,7 @@ · 6 unmodified lines | |||
| 7 | * @module AnalyticsService | 7 | * @module AnalyticsService |
| 8 | */ | 8 | */ |
| 9 | import { HostProcessArchitecture, HostProcessPlatform } from "@t3tools/shared/hostProcess"; | 9 | import { HostProcessArchitecture, HostProcessPlatform } from "@t3tools/shared/hostProcess"; |
| 10 | import { getUrlDiagnostics } from "@t3tools/shared/urlDiagnostics"; | ||
| 10 | import * as Config from "effect/Config"; | 11 | import * as Config from "effect/Config"; |
| 11 | import * as Context from "effect/Context"; | 12 | import * as Context from "effect/Context"; |
| 12 | import * as DateTime from "effect/DateTime"; | 13 | import * as DateTime from "effect/DateTime"; |
| @@ -14,6 +15,7 @@ import * as Effect from "effect/Effect"; · 1 unmodified line | |||
| 14 | import * as Layer from "effect/Layer"; | 15 | import * as Layer from "effect/Layer"; |
| 15 | import * as Option from "effect/Option"; | 16 | import * as Option from "effect/Option"; |
| 16 | import * as Ref from "effect/Ref"; | 17 | import * as Ref from "effect/Ref"; |
| 18 | import * as Schema from "effect/Schema"; | ||
| 17 | import * as HttpClient from "effect/unstable/http/HttpClient"; | 19 | import * as HttpClient from "effect/unstable/http/HttpClient"; |
| 18 | import * as HttpClientRequest from "effect/unstable/http/HttpClientRequest"; | 20 | import * as HttpClientRequest from "effect/unstable/http/HttpClientRequest"; |
| 19 | import * as HttpClientResponse from "effect/unstable/http/HttpClientResponse"; | 21 | import * as HttpClientResponse from "effect/unstable/http/HttpClientResponse"; |
| @@ -43,6 +45,38 @@ const TelemetryEnvConfig = Config.all({ · 23 unmodified lines | |||
| 43 | wslDistroName: Config.string("WSL_DISTRO_NAME").pipe(Config.option), | 45 | wslDistroName: Config.string("WSL_DISTRO_NAME").pipe(Config.option), |
| 44 | }); | 46 | }); |
| 45 | 47 | ||
| 48 | export class AnalyticsBatchDeliveryError extends Schema.TaggedErrorClass<AnalyticsBatchDeliveryError>()( | ||
| 49 | "AnalyticsBatchDeliveryError", | ||
| 50 | { | ||
| 51 | endpointInputLength: Schema.Number, | ||
| 52 | endpointProtocol: Schema.optionalKey(Schema.String), | ||
| 53 | endpointHostname: Schema.optionalKey(Schema.String), | ||
| 54 | eventCount: Schema.Int.check(Schema.isGreaterThan(0)), | ||
| 55 | cause: Schema.Defect(), | ||
| 56 | }, | ||
| 57 | ) { | ||
| 58 | static fromEndpoint(input: { | ||
| 59 | readonly endpoint: string; | ||
| 60 | readonly eventCount: number; | ||
| 61 | readonly cause: unknown; | ||
| 62 | }): AnalyticsBatchDeliveryError { | ||
| 63 | const diagnostics = getUrlDiagnostics(input.endpoint); | ||
| 64 | return new AnalyticsBatchDeliveryError({ | ||
| 65 | endpointInputLength: diagnostics.inputLength, | ||
| 66 | ...(diagnostics.protocol === undefined ? {} : { endpointProtocol: diagnostics.protocol }), | ||
| 67 | ...(diagnostics.hostname === undefined ? {} : { endpointHostname: diagnostics.hostname }), | ||
| 68 | eventCount: input.eventCount, | ||
| 69 | cause: input.cause, | ||
| 70 | }); | ||
| 71 | } | ||
| 72 | |||
| 73 | override get message(): string { | ||
| 74 | const eventLabel = this.eventCount === 1 ? "event" : "events"; | ||
| 75 | const destination = this.endpointHostname ? ` at ${this.endpointHostname}` : ""; | ||
| 76 | return `Failed to deliver ${this.eventCount} analytics ${eventLabel} to PostHog${destination} (endpoint input length ${this.endpointInputLength}).`; | ||
| 77 | } | ||
| 78 | } | ||
| 79 | |||
| 46 | export class AnalyticsService extends Context.Service< | 80 | export class AnalyticsService extends Context.Service< |
| 47 | AnalyticsService, | 81 | AnalyticsService, |
| 48 | { | 82 | { |
| @@ -75,6 +109,7 @@ export const make = Effect.gen(function* () { · 26 unmodified lines | |||
| 75 | const clientType = serverConfig.mode === "desktop" ? "desktop-app" : "cli-web-client"; | 109 | const clientType = serverConfig.mode === "desktop" ? "desktop-app" : "cli-web-client"; |
| 76 | const hostPlatform = yield* HostProcessPlatform; | 110 | const hostPlatform = yield* HostProcessPlatform; |
| 77 | const hostArchitecture = yield* HostProcessArchitecture; | 111 | const hostArchitecture = yield* HostProcessArchitecture; |
| 112 | const batchEndpoint = `${telemetryConfig.posthogHost}/batch/`; | ||
| 78 | 113 | ||
| 79 | const enqueueBufferedEvent = (event: string, properties?: Readonly<Record<string, unknown>>) => | 114 | const enqueueBufferedEvent = (event: string, properties?: Readonly<Record<string, unknown>>) => |
| 80 | Effect.flatMap(DateTime.now, (now) => | 115 | Effect.flatMap(DateTime.now, (now) => |
| @@ -126,10 +161,17 @@ export const make = Effect.gen(function* () { · 45 unmodified lines | |||
| 126 | })), | 161 | })), |
| 127 | }; | 162 | }; |
| 128 | 163 | ||
| 129 | yield* HttpClientRequest.post(`${telemetryConfig.posthogHost}/batch/`).pipe( | 164 | yield* HttpClientRequest.post(batchEndpoint).pipe( |
| 130 | HttpClientRequest.bodyJson(payload), | 165 | HttpClientRequest.bodyJson(payload), |
| 131 | Effect.flatMap(httpClient.execute), | 166 | Effect.flatMap(httpClient.execute), |
| 132 | Effect.flatMap(HttpClientResponse.filterStatusOk), | 167 | Effect.flatMap(HttpClientResponse.filterStatusOk), |
| 168 | Effect.mapError((cause) => | ||
| 169 | AnalyticsBatchDeliveryError.fromEndpoint({ | ||
| 170 | endpoint: batchEndpoint, | ||
| 171 | eventCount: events.length, | ||
| 172 | cause, | ||
| 173 | }), | ||
| 174 | ), | ||
| 133 | ); | 175 | ); |
| 134 | }); | 176 | }); |
| 135 | 177 | ||
| @@ -149,14 +191,32 @@ export const make = Effect.gen(function* () { · 13 unmodified lines | |||
| 149 | } | 191 | } |
| 150 | 192 | ||
| 151 | yield* sendBatch(batch).pipe( | 193 | yield* sendBatch(batch).pipe( |
| 152 | Effect.catch((error) => | 194 | Effect.catchTags({ |
| 153 | Ref.update(bufferRef, (current) => [...batch, ...current]).pipe( | 195 | AnalyticsBatchDeliveryError: (error) => |
| 154 | Effect.flatMap(() => Effect.fail(error)), | 196 | Ref.update(bufferRef, (current) => [...batch, ...current]).pipe( |
| 155 | ), | 197 | Effect.flatMap(() => Effect.fail(error)), |
| 156 | ), | 198 | ), |
| 199 | }), | ||
| 157 | ); | 200 | ); |
| 158 | } | 201 | } |
| 159 | }).pipe(Effect.catch((cause) => Effect.logError("Failed to flush telemetry", { cause }))); | 202 | }).pipe( |
| 203 | Effect.catchTags({ | ||
| 204 | AnalyticsBatchDeliveryError: (error) => | ||
| 205 | Effect.logError(error.message).pipe( | ||
| 206 | Effect.annotateLogs({ | ||
| 207 | endpointInputLength: error.endpointInputLength, | ||
| 208 | ...(error.endpointProtocol === undefined | ||
| 209 | ? {} | ||
| 210 | : { endpointProtocol: error.endpointProtocol }), | ||
| 211 | ...(error.endpointHostname === undefined | ||
| 212 | ? {} | ||
| 213 | : { endpointHostname: error.endpointHostname }), | ||
| 214 | eventCount: error.eventCount, | ||
| 215 | cause: error, | ||
| 216 | }), | ||
| 217 | ), | ||
| 218 | }), | ||
| 219 | ); | ||
| 160 | 220 | ||
| 161 | const record: AnalyticsService["Service"]["record"] = Effect.fn("AnalyticsService.record")( | 221 | const record: AnalyticsService["Service"]["record"] = Effect.fn("AnalyticsService.record")( |
| 162 | function* (event, properties) { | 222 | function* (event, properties) { |