Unified · Split
| @@ -0,0 +1,8 @@ | |||
| 1 | --- | ||
| 2 | "@executor-js/sdk": patch | ||
| 3 | "@executor-js/api": patch | ||
| 4 | --- | ||
| 5 | |||
| 6 | Build `StorageError.message` from the call-site label plus the driver's error code instead of the driver's raw text. The driver text is drizzle's `Failed query: <sql>\nparams: <bound values>`, so error reporting grouped one storage defect by statement shape and printed bound parameters into issue titles. The full driver error stays on `cause`. | ||
| 7 | |||
| 8 | Add `StorageConnectionError`, a `StorageFailure` variant for postgres.js connection faults (`CONNECTION_ENDED`, `CONNECTION_CLOSED`, `CONNECTION_DESTROYED`, `CONNECT_TIMEOUT`, `ECONNRESET`) and workerd's cross-request I/O rejection. It carries the fault `code` and a `retryable` flag so a lost socket can be told apart from a pool-lifetime bug. | ||
| @@ -7,7 +7,7 @@ · 6 unmodified lines | |||
| 7 | 7 | ||
| 8 | import { describe, expect, it } from "@effect/vitest"; | 8 | import { describe, expect, it } from "@effect/vitest"; |
| 9 | import { Cause, Effect, Exit, Layer, Ref, Result } from "effect"; | 9 | import { Cause, Effect, Exit, Layer, Ref, Result } from "effect"; |
| 10 | import { StorageError, UniqueViolationError } from "@executor-js/sdk/core"; | 10 | import { StorageConnectionError, StorageError, UniqueViolationError } from "@executor-js/sdk/core"; |
| 11 | 11 | ||
| 12 | import { capture, ErrorCapture, InternalError } from "./observability"; | 12 | import { capture, ErrorCapture, InternalError } from "./observability"; |
| 13 | 13 | ||
| @@ -47,6 +47,30 @@ describe("capture", () => { · 33 unmodified lines | |||
| 47 | }), | 47 | }), |
| 48 | ); | 48 | ); |
| 49 | 49 | ||
| 50 | it.effect("translates StorageConnectionError the same way as StorageError", () => | ||
| 51 | Effect.gen(function* () { | ||
| 52 | const { layer, seen } = yield* makeRecorder("trace-conn"); | ||
| 53 | const err = new StorageConnectionError({ | ||
| 54 | message: "FumaDB plugin_storage.findFirst failed: CONNECTION_ENDED", | ||
| 55 | label: "plugin_storage.findFirst", | ||
| 56 | code: "CONNECTION_ENDED", | ||
| 57 | retryable: false, | ||
| 58 | cause: "driver", | ||
| 59 | }); | ||
| 60 | |||
| 61 | const result = yield* Effect.flip(capture(Effect.fail(err))).pipe(Effect.provide(layer)); | ||
| 62 | |||
| 63 | expect(result).toBeInstanceOf(InternalError); | ||
| 64 | expect(result.traceId).toBe("trace-conn"); | ||
| 65 | |||
| 66 | const causes = yield* Ref.get(seen); | ||
| 67 | expect(causes.length).toBe(1); | ||
| 68 | const squashed = Cause.squash(causes[0]!) as StorageConnectionError; | ||
| 69 | expect(squashed).toBeInstanceOf(StorageConnectionError); | ||
| 70 | expect(squashed.code).toBe("CONNECTION_ENDED"); | ||
| 71 | }), | ||
| 72 | ); | ||
| 73 | |||
| 50 | it.effect("empty traceId when no ErrorCapture is wired", () => | 74 | it.effect("empty traceId when no ErrorCapture is wired", () => |
| 51 | Effect.gen(function* () { | 75 | Effect.gen(function* () { |
| 52 | const err = new StorageError({ message: "nope", cause: undefined }); | 76 | const err = new StorageError({ message: "nope", cause: undefined }); |
| @@ -13,8 +13,9 @@ · 12 unmodified lines | |||
| 13 | // the cloud Worker, console in the CLI, in-memory in tests) to | 13 | // the cloud Worker, console in the CLI, in-memory in tests) to |
| 14 | // record causes and return correlation ids. Optional; absent → | 14 | // record causes and return correlation ids. Optional; absent → |
| 15 | // empty trace ids, nothing breaks. | 15 | // empty trace ids, nothing breaks. |
| 16 | // 3. `capture(eff)` — the one translator. Catches `StorageError` and | 16 | // 3. `capture(eff)` — the one translator. Catches `StorageError`, |
| 17 | // `UniqueViolationError` in the typed channel: the former is | 17 | // `StorageConnectionError` and `UniqueViolationError` in the typed |
| 18 | // channel: the storage failures are | ||
| 18 | // captured via `ErrorCapture` and re-failed as `InternalError({ | 19 | // captured via `ErrorCapture` and re-failed as `InternalError({ |
| 19 | // traceId })`; the latter dies as a defect (plugins that want to | 20 | // traceId })`; the latter dies as a defect (plugins that want to |
| 20 | // surface it as a typed domain error should `Effect.catchTag` | 21 | // surface it as a typed domain error should `Effect.catchTag` |
| @@ -75,6 +76,10 @@ const resolveCapture = Effect.serviceOption(ErrorCapture).pipe( · 54 unmodified lines | |||
| 75 | * | 76 | * |
| 76 | * - `StorageError` — known backend failure. Capture the cause via | 77 | * - `StorageError` — known backend failure. Capture the cause via |
| 77 | * `ErrorCapture`, fail with `InternalError({ traceId })`. | 78 | * `ErrorCapture`, fail with `InternalError({ traceId })`. |
| 79 | * - `StorageConnectionError` — the database connection failed, so the | ||
| 80 | * statement never got a verdict. Same edge treatment as | ||
| 81 | * `StorageError` (capture, opaque 500); the tag exists so callers | ||
| 82 | * that can retry on a fresh pool are able to tell the two apart. | ||
| 78 | * - `UniqueViolationError` — invariant violation at the HTTP edge: | 83 | * - `UniqueViolationError` — invariant violation at the HTTP edge: |
| 79 | * if a plugin wanted to surface a unique-conflict as a typed | 84 | * if a plugin wanted to surface a unique-conflict as a typed |
| 80 | * domain error (e.g. "source already exists") it should | 85 | * domain error (e.g. "source already exists") it should |
| @@ -97,6 +102,12 @@ export const capture = <A, E, R>( · 16 unmodified lines | |||
| 97 | Effect.flatMap((traceId) => Effect.fail(new InternalError({ traceId }))), | 102 | Effect.flatMap((traceId) => Effect.fail(new InternalError({ traceId }))), |
| 98 | ), | 103 | ), |
| 99 | ), | 104 | ), |
| 105 | Effect.catchTag("StorageConnectionError", (err) => | ||
| 106 | resolveCapture.pipe( | ||
| 107 | Effect.flatMap((c) => c.captureException(Cause.fail(err))), | ||
| 108 | Effect.flatMap((traceId) => Effect.fail(new InternalError({ traceId }))), | ||
| 109 | ), | ||
| 110 | ), | ||
| 100 | ) as Effect.Effect<A, Exclude<E, StorageFailure> | InternalError, R>; | 111 | ) as Effect.Effect<A, Exclude<E, StorageFailure> | InternalError, R>; |
| 101 | 112 | ||
| 102 | /** | 113 | /** |
| @@ -32,7 +32,8 @@ export const isUserActionableError = (value: unknown): value is UserActionableEr · 31 unmodified lines | |||
| 32 | /* The failure set the SDK surfaces. `execute`'s invoke failures are ported from | 32 | /* The failure set the SDK surfaces. `execute`'s invoke failures are ported from |
| 33 | * v1 but re-keyed by `address` (the full `tools.<integration>.<owner>.<connection>.<tool>` | 33 | * v1 but re-keyed by `address` (the full `tools.<integration>.<owner>.<connection>.<tool>` |
| 34 | * handle) instead of an opaque tool id. Storage failures reuse fuma-runtime's | 34 | * handle) instead of an opaque tool id. Storage failures reuse fuma-runtime's |
| 35 | * `StorageError`/`UniqueViolationError` (`StorageFailure`) — not redefined here. */ | 35 | * `StorageError`/`StorageConnectionError`/`UniqueViolationError` |
| 36 | * (`StorageFailure`) — not redefined here. */ | ||
| 36 | 37 | ||
| 37 | // --------------------------------------------------------------------------- | 38 | // --------------------------------------------------------------------------- |
| 38 | // Tool lifecycle | 39 | // Tool lifecycle |
| @@ -0,0 +1,260 @@ | |||
| 1 | // Regression for the "StorageError: Failed query: …" reports in production. | ||
| 2 | // One root cause fanned out into a report per table and WHERE-clause because | ||
| 3 | // `fumaFailureFromCause` copied the driver's error text verbatim into | ||
| 4 | // `StorageError.message`: drizzle's `DrizzleQueryError` message is | ||
| 5 | // `Failed query: <sql>\nparams: <bound values>`, so error reports grouped by | ||
| 6 | // SQL statement AND printed bound parameters (org ids, user ids, connection | ||
| 7 | // names) in their TITLES. | ||
| 8 | // | ||
| 9 | // Two contracts are pinned here: | ||
| 10 | // 1. `StorageError.message` is built from a stable label plus the driver's | ||
| 11 | // error CODE. Never the statement text, never the bound parameters. | ||
| 12 | // 2. postgres.js connection faults are classified as a distinct | ||
| 13 | // `StorageConnectionError` rather than melting into a generic | ||
| 14 | // `StorageError`, so a pool-lifetime bug is not indistinguishable from a | ||
| 15 | // malformed query. | ||
| 16 | // | ||
| 17 | // The fixtures below are synthetic reconstructions of the driver error shapes | ||
| 18 | // (verified against node_modules/postgres/src/errors.js and | ||
| 19 | // node_modules/drizzle-orm/errors.js); all identifiers are placeholders. | ||
| 20 | |||
| 21 | import { describe, expect, it } from "@effect/vitest"; | ||
| 22 | import { Cause, Effect, Exit, Predicate } from "effect"; | ||
| 23 | |||
| 24 | import { | ||
| 25 | fumaEffect, | ||
| 26 | fumaFailureFromCause, | ||
| 27 | isStorageFailure, | ||
| 28 | StorageError, | ||
| 29 | UniqueViolationError, | ||
| 30 | } from "./fuma-runtime"; | ||
| 31 | |||
| 32 | /** Shape of `postgres.js` `Errors.connection(code, options, socket)`. */ | ||
| 33 | const postgresConnectionError = (code: string): Error => | ||
| 34 | Object.assign( | ||
| 35 | // oxlint-disable-next-line executor/no-error-constructor -- boundary: reconstructs the native driver error this module has to classify | ||
| 36 | new Error(`write ${code} db-placeholder.hyperdrive.local:5432`), | ||
| 37 | { code, errno: code, address: "db-placeholder.hyperdrive.local", port: 5432 }, | ||
| 38 | ); | ||
| 39 | |||
| 40 | /** Shape of `postgres.js` `Errors.postgres(x)` — a server-side error report. */ | ||
| 41 | const postgresServerError = (code: string, message: string): Error => | ||
| 42 | // oxlint-disable-next-line executor/no-error-constructor -- boundary: reconstructs the native driver error this module has to classify | ||
| 43 | Object.assign(new Error(message), { code, severity: "ERROR" }); | ||
| 44 | |||
| 45 | /** | ||
| 46 | * Shape of drizzle's `DrizzleQueryError`: the statement text and the bound | ||
| 47 | * parameters are baked into `message`, and the driver error hangs off `cause`. | ||
| 48 | */ | ||
| 49 | const drizzleQueryError = (sql: string, params: readonly unknown[], cause: unknown): Error => { | ||
| 50 | // oxlint-disable-next-line executor/no-error-constructor -- boundary: reconstructs the native driver error this module has to classify | ||
| 51 | const error = new Error(`Failed query: ${sql}\nparams: ${params.join(",")}`); | ||
| 52 | return Object.assign(error, { query: sql, params, cause }); | ||
| 53 | }; | ||
| 54 | |||
| 55 | const SQL = 'select "key", "value" from "plugin_storage" where "scope_id" = $1 and "key" = $2'; | ||
| 56 | // Synthetic placeholders standing in for the real bound values (a WorkOS org id | ||
| 57 | // and a connection-scoped storage key) that leaked into error-report titles. | ||
| 58 | const PARAMS = ["org_placeholder_0000", "oauth:integration-placeholder:refresh"] as const; | ||
| 59 | |||
| 60 | const expectNoDriverTextIn = (message: string): void => { | ||
| 61 | expect(message).not.toContain("Failed query"); | ||
| 62 | expect(message).not.toContain("params:"); | ||
| 63 | expect(message).not.toContain("select"); | ||
| 64 | expect(message).not.toContain('plugin_storage"'); | ||
| 65 | for (const param of PARAMS) expect(message).not.toContain(param); | ||
| 66 | }; | ||
| 67 | |||
| 68 | describe("fumaFailureFromCause — message shaping", () => { | ||
| 69 | it("does not put the statement text or bound parameters in the message", () => { | ||
| 70 | const failure = fumaFailureFromCause( | ||
| 71 | "plugin_storage.findFirst", | ||
| 72 | drizzleQueryError(SQL, PARAMS, postgresConnectionError("CONNECTION_ENDED")), | ||
| 73 | ); | ||
| 74 | |||
| 75 | expectNoDriverTextIn(failure.message ?? ""); | ||
| 76 | }); | ||
| 77 | |||
| 78 | it("builds a stable message from the label and the driver error code", () => { | ||
| 79 | const failure = fumaFailureFromCause( | ||
| 80 | "plugin_storage.findFirst", | ||
| 81 | drizzleQueryError( | ||
| 82 | SQL, | ||
| 83 | PARAMS, | ||
| 84 | postgresServerError("42P01", 'relation "nope" does not exist'), | ||
| 85 | ), | ||
| 86 | ); | ||
| 87 | |||
| 88 | expect(Predicate.isTagged(failure, "StorageError")).toBe(true); | ||
| 89 | expect(failure.message).toContain("plugin_storage.findFirst"); | ||
| 90 | expect(failure.message).toContain("42P01"); | ||
| 91 | expectNoDriverTextIn(failure.message ?? ""); | ||
| 92 | }); | ||
| 93 | |||
| 94 | it("groups two different statements failing the same way onto one message", () => { | ||
| 95 | const a = fumaFailureFromCause( | ||
| 96 | "integration.findFirst", | ||
| 97 | drizzleQueryError( | ||
| 98 | 'select * from "integration" where "slug" = $1', | ||
| 99 | ["integration-placeholder"], | ||
| 100 | postgresServerError("57P01", "terminating connection due to administrator command"), | ||
| 101 | ), | ||
| 102 | ); | ||
| 103 | const b = fumaFailureFromCause( | ||
| 104 | "integration.findFirst", | ||
| 105 | drizzleQueryError( | ||
| 106 | 'select * from "integration" where "owner" = $1 and "slug" = $2', | ||
| 107 | ["org_placeholder_0000", "other-integration-placeholder"], | ||
| 108 | postgresServerError("57P01", "terminating connection due to administrator command"), | ||
| 109 | ), | ||
| 110 | ); | ||
| 111 | |||
| 112 | expect(a.message).toBe(b.message); | ||
| 113 | }); | ||
| 114 | |||
| 115 | it("still yields a stable message when the driver reports no code", () => { | ||
| 116 | const failure = fumaFailureFromCause( | ||
| 117 | "tool.findMany", | ||
| 118 | drizzleQueryError( | ||
| 119 | SQL, | ||
| 120 | PARAMS, | ||
| 121 | // oxlint-disable-next-line executor/no-error-constructor -- boundary: reconstructs a codeless native driver error | ||
| 122 | new Error("something went wrong in the driver"), | ||
| 123 | ), | ||
| 124 | ); | ||
| 125 | |||
| 126 | expect(Predicate.isTagged(failure, "StorageError")).toBe(true); | ||
| 127 | expect(failure.message).toContain("tool.findMany"); | ||
| 128 | expectNoDriverTextIn(failure.message ?? ""); | ||
| 129 | expect(failure.message).not.toContain("something went wrong"); | ||
| 130 | }); | ||
| 131 | |||
| 132 | it("keeps the full driver error reachable on `cause`", () => { | ||
| 133 | const driver = drizzleQueryError(SQL, PARAMS, postgresConnectionError("CONNECTION_ENDED")); | ||
| 134 | const failure = fumaFailureFromCause("plugin_storage.findFirst", driver); | ||
| 135 | |||
| 136 | expect((failure as { readonly cause?: unknown }).cause).toBe(driver); | ||
| 137 | }); | ||
| 138 | }); | ||
| 139 | |||
| 140 | describe("fumaFailureFromCause — classification", () => { | ||
| 141 | it.each([ | ||
| 142 | ["CONNECTION_ENDED"], | ||
| 143 | ["CONNECTION_CLOSED"], | ||
| 144 | ["CONNECTION_DESTROYED"], | ||
| 145 | ["CONNECT_TIMEOUT"], | ||
| 146 | ])("classifies postgres.js %s as a storage connection fault", (code) => { | ||
| 147 | const failure = fumaFailureFromCause( | ||
| 148 | "plugin_storage.findFirst", | ||
| 149 | drizzleQueryError(SQL, PARAMS, postgresConnectionError(code)), | ||
| 150 | ); | ||
| 151 | |||
| 152 | expect(Predicate.isTagged(failure, "StorageConnectionError")).toBe(true); | ||
| 153 | expect(failure).toEqual( | ||
| 154 | expect.objectContaining({ | ||
| 155 | _tag: "StorageConnectionError", | ||
| 156 | code, | ||
| 157 | label: "plugin_storage.findFirst", | ||
| 158 | }), | ||
| 159 | ); | ||
| 160 | expect(typeof (failure as { readonly retryable?: unknown }).retryable).toBe("boolean"); | ||
| 161 | expectNoDriverTextIn(failure.message ?? ""); | ||
| 162 | }); | ||
| 163 | |||
| 164 | it("classifies the workerd cross-request I/O rejection as a connection fault", () => { | ||
| 165 | const failure = fumaFailureFromCause( | ||
| 166 | "integration.findFirst", | ||
| 167 | drizzleQueryError( | ||
| 168 | 'select * from "integration" where "slug" = $1', | ||
| 169 | ["integration-placeholder"], | ||
| 170 | // oxlint-disable-next-line executor/no-error-constructor -- boundary: reconstructs workerd's cross-request I/O rejection, which carries no code | ||
| 171 | new Error( | ||
| 172 | "Cannot perform I/O on behalf of a different request. I/O objects (such as streams, request/response bodies, and others) created in the context of one request handler cannot be accessed from a different request's handler.", | ||
| 173 | ), | ||
| 174 | ), | ||
| 175 | ); | ||
| 176 | |||
| 177 | expect(Predicate.isTagged(failure, "StorageConnectionError")).toBe(true); | ||
| 178 | expectNoDriverTextIn(failure.message ?? ""); | ||
| 179 | expect(failure.message).not.toContain("Cannot perform I/O"); | ||
| 180 | }); | ||
| 181 | |||
| 182 | it("marks transient socket loss retryable and pool-lifetime faults not retryable", () => { | ||
| 183 | const transient = fumaFailureFromCause( | ||
| 184 | "tool.findMany", | ||
| 185 | postgresConnectionError("CONNECTION_CLOSED"), | ||
| 186 | ) as { readonly retryable?: boolean }; | ||
| 187 | const lifetime = fumaFailureFromCause( | ||
| 188 | "tool.findMany", | ||
| 189 | postgresConnectionError("CONNECTION_ENDED"), | ||
| 190 | ) as { readonly retryable?: boolean }; | ||
| 191 | |||
| 192 | expect(transient.retryable).toBe(true); | ||
| 193 | expect(lifetime.retryable).toBe(false); | ||
| 194 | }); | ||
| 195 | |||
| 196 | it("still recognises unique violations by SQLSTATE", () => { | ||
| 197 | const failure = fumaFailureFromCause( | ||
| 198 | "connection.create", | ||
| 199 | drizzleQueryError( | ||
| 200 | 'insert into "connection" ("owner","name") values ($1,$2)', | ||
| 201 | ["org_placeholder_0000", "connection-placeholder"], | ||
| 202 | postgresServerError( | ||
| 203 | "23505", | ||
| 204 | 'duplicate key value violates unique constraint "connection_pkey"', | ||
| 205 | ), | ||
| 206 | ), | ||
| 207 | ); | ||
| 208 | |||
| 209 | expect(failure).toBeInstanceOf(UniqueViolationError); | ||
| 210 | expect(Predicate.isTagged(failure, "UniqueViolationError")).toBe(true); | ||
| 211 | }); | ||
| 212 | |||
| 213 | it("does not misread a unique violation as a connection fault", () => { | ||
| 214 | const failure = fumaFailureFromCause( | ||
| 215 | "connection.create", | ||
| 216 | postgresServerError("23505", "duplicate key value violates unique constraint"), | ||
| 217 | ); | ||
| 218 | |||
| 219 | expect(Predicate.isTagged(failure, "StorageConnectionError")).toBe(false); | ||
| 220 | }); | ||
| 221 | |||
| 222 | it("passes an already-typed storage failure through untouched", () => { | ||
| 223 | const original = new StorageError({ | ||
| 224 | message: 'FumaDB table "secret" is not available through this storage boundary.', | ||
| 225 | cause: undefined, | ||
| 226 | }); | ||
| 227 | |||
| 228 | expect(fumaFailureFromCause("plugin_storage.findFirst", original)).toBe(original); | ||
| 229 | }); | ||
| 230 | |||
| 231 | it("treats a connection fault as a storage failure at the boundary", () => { | ||
| 232 | const failure = fumaFailureFromCause( | ||
| 233 | "plugin_storage.findFirst", | ||
| 234 | postgresConnectionError("CONNECTION_ENDED"), | ||
| 235 | ); | ||
| 236 | |||
| 237 | expect(isStorageFailure(failure)).toBe(true); | ||
| 238 | expect(fumaFailureFromCause("plugin_storage.findFirst", failure)).toBe(failure); | ||
| 239 | }); | ||
| 240 | }); | ||
| 241 | |||
| 242 | describe("fumaEffect", () => { | ||
| 243 | it("maps a rejected driver promise onto the classified failure", async () => { | ||
| 244 | const driverRejection = () => | ||
| 245 | // oxlint-disable-next-line executor/no-promise-reject -- boundary: fumaEffect's whole contract is adapting a rejected driver promise | ||
| 246 | Promise.reject(drizzleQueryError(SQL, PARAMS, postgresConnectionError("CONNECTION_ENDED"))); | ||
| 247 | |||
| 248 | const exit = await Effect.runPromiseExit( | ||
| 249 | fumaEffect("plugin_storage.findFirst", driverRejection), | ||
| 250 | ); | ||
| 251 | |||
| 252 | expect(Exit.isFailure(exit)).toBe(true); | ||
| 253 | const failure = Exit.isFailure(exit) | ||
| 254 | ? exit.cause.reasons.find(Cause.isFailReason)?.error | ||
| 255 | : undefined; | ||
| 256 | |||
| 257 | expect(Predicate.isTagged(failure, "StorageConnectionError")).toBe(true); | ||
| 258 | expectNoDriverTextIn(failure?.message ?? ""); | ||
| 259 | }); | ||
| 260 | }); | ||
| @@ -11,7 +11,35 @@ export class UniqueViolationError extends Data.TaggedError("UniqueViolationError · 10 unmodified lines | |||
| 11 | readonly model?: string; | 11 | readonly model?: string; |
| 12 | }> {} | 12 | }> {} |
| 13 | 13 | ||
| 14 | export type StorageFailure = StorageError | UniqueViolationError; | 14 | /** |
| 15 | * The database connection itself failed — the statement never got a verdict. | ||
| 16 | * Distinct from `StorageError` (a query the backend answered with an error) | ||
| 17 | * because the two need different responses: a connection fault is either a | ||
| 18 | * transient socket loss worth retrying on a FRESH pool, or a pool-lifetime | ||
| 19 | * bug that must stay loud. | ||
| 20 | * | ||
| 21 | * `retryable` says which. It is a property of the fault, not a policy: | ||
| 22 | * - `true` — the socket died underneath a live pool (`CONNECTION_CLOSED`, | ||
| 23 | * `CONNECT_TIMEOUT`, `ECONNRESET`). Reconnecting can succeed. | ||
| 24 | * - `false` — the pool was already torn down or the socket belongs to a | ||
| 25 | * different request context (`CONNECTION_ENDED`, `CONNECTION_DESTROYED`, | ||
| 26 | * the workerd cross-request I/O rejection). Retrying is futile by | ||
| 27 | * construction — postgres.js rejects every query once `end()` has been | ||
| 28 | * called — so these must surface rather than be papered over. | ||
| 29 | * | ||
| 30 | * No retry consumes this yet; classification lands first so the retry seam | ||
| 31 | * (and the request-scope fix behind these faults) can be designed against a | ||
| 32 | * typed signal instead of driver strings. | ||
| 33 | */ | ||
| 34 | export class StorageConnectionError extends Data.TaggedError("StorageConnectionError")<{ | ||
| 35 | readonly message: string; | ||
| 36 | readonly label: string; | ||
| 37 | readonly code: string; | ||
| 38 | readonly retryable: boolean; | ||
| 39 | readonly cause: unknown; | ||
| 40 | }> {} | ||
| 41 | |||
| 42 | export type StorageFailure = StorageError | StorageConnectionError | UniqueViolationError; | ||
| 15 | 43 | ||
| 16 | export type FumaTables = Record<string, AnyTable>; | 44 | export type FumaTables = Record<string, AnyTable>; |
| 17 | type EmptyFumaSchema = FumaSchema<"latest", Record<never, never>>; | 45 | type EmptyFumaSchema = FumaSchema<"latest", Record<never, never>>; |
| @@ -57,23 +85,111 @@ const isUniqueViolation = (cause: unknown): boolean => { · 39 unmodified lines | |||
| 57 | return false; | 85 | return false; |
| 58 | }; | 86 | }; |
| 59 | 87 | ||
| 60 | const causeMessage = (cause: unknown): string | undefined => { | 88 | /** |
| 61 | const message = | 89 | * postgres.js raises connection faults through `Errors.connection(code, …)`, |
| 62 | cause && typeof cause === "object" | 90 | * which puts the code on `error.code` (see `postgres/src/errors.js`). Workerd's |
| 63 | ? // oxlint-disable-next-line executor/no-unknown-error-message -- boundary: preserve database driver error text inside typed StorageError | 91 | * cross-request I/O rejection is a plain `Error` with no code, so it is matched |
| 64 | (cause as Record<string, unknown>)["message"] | 92 | * on its fixed runtime text and given a synthetic code. |
| 65 | : undefined; | 93 | */ |
| 66 | return typeof message === "string" && message.length > 0 ? message : undefined; | 94 | const RETRYABLE_CONNECTION_CODES: ReadonlySet<string> = new Set([ |
| 95 | "CONNECTION_CLOSED", | ||
| 96 | "CONNECT_TIMEOUT", | ||
| 97 | "ECONNRESET", | ||
| 98 | ]); | ||
| 99 | const FATAL_CONNECTION_CODES: ReadonlySet<string> = new Set([ | ||
| 100 | "CONNECTION_ENDED", | ||
| 101 | "CONNECTION_DESTROYED", | ||
| 102 | ]); | ||
| 103 | const CROSS_REQUEST_IO_CODE = "CROSS_REQUEST_IO"; | ||
| 104 | const CROSS_REQUEST_IO_PATTERN = /cannot perform i\/o on behalf of a different request/i; | ||
| 105 | |||
| 106 | /** Walk a cause chain, newest first, at most 5 links deep (as `isUniqueViolation` does). */ | ||
| 107 | const walkCauses = (cause: unknown, visit: (err: Record<string, unknown>) => boolean): boolean => { | ||
| 108 | let current = cause; | ||
| 109 | for (let i = 0; i < 5; i += 1) { | ||
| 110 | const err = | ||
| 111 | current && typeof current === "object" ? (current as Record<string, unknown>) : null; | ||
| 112 | if (!err) return false; | ||
| 113 | if (visit(err)) return true; | ||
| 114 | const innerCause = err["cause"]; | ||
| 115 | if (!innerCause || innerCause === current) return false; | ||
| 116 | current = innerCause; | ||
| 117 | } | ||
| 118 | return false; | ||
| 119 | }; | ||
| 120 | |||
| 121 | /** First driver error code in the cause chain, if any. Never the error text. */ | ||
| 122 | const causeCode = (cause: unknown): string | undefined => { | ||
| 123 | let found: string | undefined; | ||
| 124 | walkCauses(cause, (err) => { | ||
| 125 | const code = err["code"]; | ||
| 126 | if (typeof code === "string" && code.length > 0) { | ||
| 127 | found = code; | ||
| 128 | return true; | ||
| 129 | } | ||
| 130 | return false; | ||
| 131 | }); | ||
| 132 | return found; | ||
| 67 | }; | 133 | }; |
| 68 | 134 | ||
| 135 | /** Connection-fault code in the cause chain, if this is a connection fault at all. */ | ||
| 136 | const connectionFaultCode = (cause: unknown): string | undefined => { | ||
| 137 | let found: string | undefined; | ||
| 138 | walkCauses(cause, (err) => { | ||
| 139 | const code = err["code"]; | ||
| 140 | if ( | ||
| 141 | typeof code === "string" && | ||
| 142 | (RETRYABLE_CONNECTION_CODES.has(code) || FATAL_CONNECTION_CODES.has(code)) | ||
| 143 | ) { | ||
| 144 | found = code; | ||
| 145 | return true; | ||
| 146 | } | ||
| 147 | // oxlint-disable-next-line executor/no-unknown-error-message -- boundary: workerd's cross-request I/O rejection carries no code, only this fixed message | ||
| 148 | const message = err["message"]; | ||
| 149 | if (typeof message === "string" && CROSS_REQUEST_IO_PATTERN.test(message)) { | ||
| 150 | found = CROSS_REQUEST_IO_CODE; | ||
| 151 | return true; | ||
| 152 | } | ||
| 153 | return false; | ||
| 154 | }); | ||
| 155 | return found; | ||
| 156 | }; | ||
| 157 | |||
| 158 | /** | ||
| 159 | * Build the failure message from stable inputs only — the call-site label and | ||
| 160 | * the driver's error code — never the driver's text. | ||
| 161 | * | ||
| 162 | * The driver text is `Failed query: <sql>\nparams: <bound values>` | ||
| 163 | * (`drizzle-orm/errors.js`). Copying it into the message made error reporting | ||
| 164 | * group by statement shape, splitting one defect across a report per table and | ||
| 165 | * WHERE-clause, and printed bound parameters (organization ids, user ids, | ||
| 166 | * user-chosen connection names) into report titles. The full driver error is | ||
| 167 | * preserved on `cause`, which the reporter still receives as a chained | ||
| 168 | * exception. | ||
| 169 | */ | ||
| 170 | const stableMessage = (label: string, code: string | undefined): string => | ||
| 171 | code ? `FumaDB ${label} failed: ${code}` : `FumaDB ${label} failed`; | ||
| 172 | |||
| 69 | export const isStorageFailure = (error: unknown): error is StorageFailure => | 173 | export const isStorageFailure = (error: unknown): error is StorageFailure => |
| 70 | Predicate.isTagged(error, "StorageError") || Predicate.isTagged(error, "UniqueViolationError"); | 174 | Predicate.isTagged(error, "StorageError") || |
| 175 | Predicate.isTagged(error, "StorageConnectionError") || | ||
| 176 | Predicate.isTagged(error, "UniqueViolationError"); | ||
| 71 | 177 | ||
| 72 | export const fumaFailureFromCause = (label: string, cause: unknown): StorageFailure => { | 178 | export const fumaFailureFromCause = (label: string, cause: unknown): StorageFailure => { |
| 73 | if (isStorageFailure(cause)) return cause; | 179 | if (isStorageFailure(cause)) return cause; |
| 74 | if (isUniqueViolation(cause)) return new UniqueViolationError({ model: label }); | 180 | if (isUniqueViolation(cause)) return new UniqueViolationError({ model: label }); |
| 181 | const connectionCode = connectionFaultCode(cause); | ||
| 182 | if (connectionCode !== undefined) { | ||
| 183 | return new StorageConnectionError({ | ||
| 184 | message: stableMessage(label, connectionCode), | ||
| 185 | label, | ||
| 186 | code: connectionCode, | ||
| 187 | retryable: RETRYABLE_CONNECTION_CODES.has(connectionCode), | ||
| 188 | cause, | ||
| 189 | }); | ||
| 190 | } | ||
| 75 | return new StorageError({ | 191 | return new StorageError({ |
| 76 | message: causeMessage(cause) ?? `FumaDB operation failed: ${label}`, | 192 | message: stableMessage(label, causeCode(cause)), |
| 77 | cause, | 193 | cause, |
| 78 | }); | 194 | }); |
| 79 | }; | 195 | }; |
| @@ -36,7 +36,12 @@ export type { · 35 unmodified lines | |||
| 36 | IFumaClient, | 36 | IFumaClient, |
| 37 | StorageFailure, | 37 | StorageFailure, |
| 38 | } from "./fuma-runtime"; | 38 | } from "./fuma-runtime"; |
| 39 | export { StorageError, UniqueViolationError, isStorageFailure } from "./fuma-runtime"; | 39 | export { |
| 40 | StorageError, | ||
| 41 | StorageConnectionError, | ||
| 42 | UniqueViolationError, | ||
| 43 | isStorageFailure, | ||
| 44 | } from "./fuma-runtime"; | ||
| 40 | 45 | ||
| 41 | // IDs (branded) — the v2 set. | 46 | // IDs (branded) — the v2 set. |
| 42 | export { | 47 | export { |