// Cloud-only: what an operator SEES when a write is rejected by the database. // // The product guarantee: a storage failure is reported under a stable headline // built from the operation and the database's error code — never the statement // text, never the values that were bound into it. Two consequences, both of // them things production got wrong: // // - The values bound into a rejected statement are customer data (the // organization id, the connection name, whatever the user typed into the // description). They must not appear in the report's headline. // - The headline is the grouping key of the error reporter, so one defect that // hits several tables — or the same table through several WHERE shapes — // must arrive as ONE report, not one per statement. // // The failure is induced through the public typed API only: PostgreSQL cannot // store a NUL byte in a text column, so a connection whose description carries // one is rejected by the driver with SQLSTATE 22021 while the statement and its // bound parameters are already assembled. That is the same class of failure the // production reports came from, reachable without touching the database. // // Two surfaces are asserted, both public: // 1. What the CALLER gets — an opaque `InternalError` carrying only a trace // id, with no driver text anywhere in the payload. // 2. What the OPERATOR gets — the server's own error log, where the trace id // the caller received joins to the report the server filed. Its headline — // the captured exception's type and message — is what the error reporter // files the report under, and groups by. import { randomBytes } from "node:crypto"; import { readFileSync } from "node:fs"; import { resolve } from "node:path"; import { expect } from "@effect/vitest"; import { Cause, Effect, Exit, Schedule } from "effect"; import type { HttpApiClient } from "effect/unstable/httpapi"; import { composePluginApi } from "@executor-js/api/server"; import { openApiHttpPlugin } from "@executor-js/plugin-openapi/api"; import { AuthTemplateSlug, ConnectionName, IntegrationSlug } from "@executor-js/sdk/shared"; import { RUNS_DIR, scenario } from "../src/scenario"; import { Api, Target } from "../src/services"; const api = composePluginApi([openApiHttpPlugin()] as const); type Client = HttpApiClient.ForApi; /** A text value PostgreSQL cannot store — the driver rejects it as 22021. */ const NUL = String.fromCharCode(0); const SLUG = "storage-error-report-shape"; /** Minimal OpenAPI spec with a single GET /ping — never contacted here. */ const pingSpec = JSON.stringify({ openapi: "3.0.3", info: { title: "Ping API", version: "1.0.0" }, paths: { "/ping": { get: { operationId: "ping", summary: "Ping", responses: { "200": { description: "pong" } } }, }, }, }); /** Registers a fresh apiKey-authenticated integration for connections to bind to. */ const registerIntegration = (client: Client) => Effect.gen(function* () { const slug = IntegrationSlug.make(`${SLUG}-${randomBytes(4).toString("hex")}`); yield* client.openapi.addSpec({ payload: { spec: { kind: "blob", value: pingSpec }, slug, baseUrl: "http://127.0.0.1:59999", // never contacted during registration authenticationTemplate: [ { slug: "apiKey", type: "apiKey", headers: { authorization: ["Bearer ", { type: "variable", name: "token" }] }, }, ], }, }); return slug; }); interface RejectedWrite { /** The trace id the caller was handed; joins to the server's report. */ readonly traceId: string; /** The name the caller chose for the connection — customer data. */ readonly name: string; /** The free text the caller typed — customer data, and the NUL carrier. */ readonly description: string; /** The whole client-visible failure, serialized. */ readonly payload: string; } /** * Create a connection whose description PostgreSQL will refuse, and return what * the caller can see about the failure. */ const rejectedConnectionWrite = ( client: Client, integration: IntegrationSlug, ): Effect.Effect => Effect.gen(function* () { const name = ConnectionName.make( `${SLUG.replaceAll("-", "")}${randomBytes(4).toString("hex")}`, ); const description = `desc-${randomBytes(6).toString("hex")}${NUL}tail`; const exit = yield* Effect.exit( client.connections.create({ payload: { owner: "org", name, integration, template: AuthTemplateSlug.make("apiKey"), description, value: `sk-${randomBytes(4).toString("hex")}`, }, }), ); const failure = Exit.isFailure(exit) ? exit.cause.reasons.find(Cause.isFailReason)?.error : exit.value; const error = failure as { readonly _tag?: string; readonly traceId?: string } | undefined; // PostgreSQL refuses the NUL byte, so the write cannot succeed, and what // comes back is the opaque internal failure — never the storage error. expect(error?._tag, "the caller sees an opaque internal failure").toBe("InternalError"); expect(error?.traceId ?? "", "the caller is handed a trace id to quote").toMatch( /^[0-9a-f]{32}$/, ); return { traceId: error?.traceId ?? "", name, description, payload: JSON.stringify(failure), }; }); /** * The dev stack's stdout. The suite's globalsetup funnels it into the run * artifacts; a scenario run against an already-booted instance (`cli up cloud`) * reads that instance's log instead. */ const serverLogCandidates = [ resolve(RUNS_DIR, "cloud", "server-logs", "boot.log"), resolve(RUNS_DIR, "..", ".dev", "cloud.log"), ]; const readServerLog = (): string => { const texts = serverLogCandidates.flatMap((path) => { // oxlint-disable-next-line executor/no-try-catch-or-throw -- boundary: probing which of the two stdout sinks this run uses try { return [readFileSync(path, "utf8")]; } catch { return []; } }); return texts.join("\n"); }; const REPORT_PREFIX = "[api] unhandled cause: "; /** A stack frame in the logged cause — where the report's headline stops. */ const STACK_FRAME = /^\s+at /; interface FiledReport { /** Type + message: what the reporter names and groups the report by. */ readonly headline: string; /** The whole record, headline and chained cause — what a diagnosis reads. */ readonly full: string; } /** * The report the server filed for one request. * * The headline is the captured cause's type and message up to the first stack * frame — exactly what `Cause.prettyErrors` hands the reporter as the * exception. The message is multi-line whenever the driver's text is * (`Failed query: …\nparams: …`), so the whole headline has to be read, not * just its first line. * * Found by walking back from the correlation record carrying the caller's trace * id, so it is THIS request's report and not a neighbour's. */ const reportFor = (traceId: string): Effect.Effect => Effect.sync(() => { const lines = readServerLog().split("\n"); const correlated = lines.findLastIndex( (line) => line.includes('"event":"sentry_before_send_otel_correlation"') && line.includes(`"sentry_event_id":"${traceId}"`), ); if (correlated === -1) return undefined; const reported = lines .slice(0, correlated) .findLastIndex((line) => line.startsWith(REPORT_PREFIX)); if (reported === -1) return undefined; const block = [ lines[reported]!.slice(REPORT_PREFIX.length), ...lines.slice(reported + 1, correlated), ]; const end = block.slice(1).findIndex((line) => STACK_FRAME.test(line)); return { headline: block .slice(0, end === -1 ? 1 : end + 1) .join("\n") .trimEnd(), full: block.join("\n"), }; }).pipe( Effect.filterOrFail( (report): report is FiledReport => report !== undefined, () => `no error report joined to trace id ${traceId} in the server log`, ), // The log is a file the dev stack appends to; the write lands moments after // the response. Poll rather than sleep (~20s ceiling). Effect.retry(Schedule.both(Schedule.spaced("500 millis"), Schedule.recurs(40))), ); scenario( "Storage · a rejected write is reported without its SQL or the caller's data", { timeout: 120_000 }, Effect.gen(function* () { const target = yield* Target; const { client: apiClient } = yield* Api; const identity = yield* target.newIdentity(); const client = yield* apiClient(api, identity); const integration = yield* registerIntegration(client); const first = yield* rejectedConnectionWrite(client, integration); const second = yield* rejectedConnectionWrite(client, integration); for (const write of [first, second]) { expect(write.payload, "the client payload carries no driver text").not.toContain( "Failed query", ); expect(write.payload, "the client payload carries no bound parameter").not.toContain( write.description, ); } const report = yield* reportFor(first.traceId); const headline = report.headline; // The symptom: the statement and everything bound into it used to BE the // headline, so the report was named after the customer's data. expect(headline, "the report headline carries no statement text").not.toContain("Failed query"); expect(headline, "the report headline carries no statement text").not.toContain("insert into"); expect(headline, "the report headline carries no bound parameters").not.toContain("params:"); expect(headline, "the report headline carries no user-typed description").not.toContain( first.description, ); expect(headline, "the report headline carries no user-chosen connection name").not.toContain( first.name, ); expect(headline, "the report headline carries no organization id").not.toContain("org_"); // What it says instead: which operation failed, and how the database // refused it — enough to act on, stable across calls. expect(headline, "the report still names the failing operation").toContain("connection.create"); expect(headline, "the report still names the database's error code").toContain("22021"); // Shaping the headline must not mean throwing the diagnosis away: the // driver's own text is still filed with the report, one level down, where // it informs a fix instead of naming the report. expect(report.full, "the driver's statement is still filed under the report").toContain( "Failed query", ); // The fan-out: the two writes bound different names, descriptions and // secrets, so their statements differ in every parameter. One defect, one // report — not one report per set of values. const secondReport = yield* reportFor(second.traceId); expect( secondReport.headline, "a second rejected write with different values files the same report", ).toBe(headline); }), );