112 lines · 5.0 KB · TypeScript
RawDownload
// ---------------------------------------------------------------------------// Runs one reconciler pass (`syncWorkOsEvents`) from a Worker entry that is// not an HTTP request handled by the Effect app: the every-minute cron// (`scheduled` in server.ts) and the webhook poke (`workos-webhook.ts`,// detached past the response with `waitUntil`).//// Both entries build the request-scoped services FRESH for the run — the// same reason `mcp/auth.ts` does: a postgres socket belongs to one Workers// invocation, and the webhook route's own per-request layer is closed the// moment its response is returned, so a detached run cannot borrow it. The// run is its own scope; the socket is released when it ends.//// A failing run is captured (Sentry + structured log) and swallowed here:// neither entry has a caller to report to, and the run is retried by the// next cron tick from the last committed cursor.//// A run that does not fail outright can still leave the mirror stale: one// that stops short of `"drained"` (page budget, cursor contention) means the// reconciler did not catch up this tick, and one stuck on// `"awaiting_backfill"` past the budget means the boundary row the backfill// was supposed to hand off was lost — both are read from `drainedAt` after// the run and, past `MIRROR_RECONCILER_LAG_BUDGET`, reported the same way a// thrown failure is: a structured error log plus a Sentry capture. This is// the ONLY place a stale reconciler surfaces now; the request path// (`auth/organization.ts`) no longer checks readiness or falls back.// --------------------------------------------------------------------------- import { Clock, Data, Duration, Effect, Layer } from "effect"; import { captureCauseEffect } from "../observability";import { WorkerTelemetryLive } from "../observability/telemetry";import { makeDbLayer } from "../db/db";import { makeUserStoreLayer } from "./context";import { MIRROR_RECONCILER_LAG_BUDGET } from "./mirror-readiness-store";import { CoreSharedServices } from "./workos";import { syncWorkOsEvents, type WorkOsEventsSyncReport } from "./workos-events-sync";import { makeWorkOsMirrorLayer, WorkOsMirror } from "./workos-mirror"; const makeSyncServices = () => {  const dbLive = makeDbLayer();  return Layer.mergeAll(    makeUserStoreLayer().pipe(Layer.provide(dbLive)),    makeWorkOsMirrorLayer().pipe(Layer.provide(dbLive)),    CoreSharedServices,  );}; const LAG_BUDGET_MS = Duration.toMillis(MIRROR_RECONCILER_LAG_BUDGET); /** * The mirror has not drained the WorkOS events stream inside its lag budget. * Its own tagged error so Sentry groups every occurrence under one issue, * with the run's outcome and the heartbeat's age as the fields to read. */export class WorkOsReconcilerStale extends Data.TaggedError("WorkOsReconcilerStale")<{  readonly stopped: WorkOsEventsSyncReport["stopped"];  readonly drainedAt: string | null;  readonly ageMs: number | null;}> {} /** * After a run that did not end `"drained"`, check whether the mirror has * fallen behind its lag budget and, if so, report it the way a failed run is * reported: a structured error log plus a Sentry capture. A run ending * `"drained"` just wrote `markDrained` and is healthy by definition, so it is * never checked. Cheap by design: one `drainedAt` read per run (about once a * minute). */export const alertOnStaleReconciler = Effect.fn("workos_events.alert_on_stale_reconciler")(  function* (report: WorkOsEventsSyncReport) {    if (report.stopped === "drained") return;     const mirror = yield* WorkOsMirror;    const drainedAt = yield* mirror.drainedAt();    const now = yield* Clock.currentTimeMillis;    const ageMs = drainedAt === null ? null : now - drainedAt.getTime();    if (ageMs !== null && ageMs <= LAG_BUDGET_MS) return;     const stale = new WorkOsReconcilerStale({      stopped: report.stopped,      drainedAt: drainedAt === null ? null : drainedAt.toISOString(),      ageMs,    });    yield* Effect.logError("workos_events: reconciler stale", stale);    yield* captureCauseEffect(stale);  },); /** * One reconciler pass over fresh request-scoped services. Resolves when the * pass ends, whether it drained the stream, stopped at the page budget, * yielded to another run, or failed (a failure is reported, never thrown). * A run that ends short of `"drained"` and leaves `drainedAt` past the lag * budget is ALSO reported (see {@link alertOnStaleReconciler}) — a stalled * reconciler is now an operational alert, not a per-request fallback. */export const runWorkOsEventsSync = (): Promise<void> =>  Effect.runPromise(    syncWorkOsEvents().pipe(      Effect.tap((report) => alertOnStaleReconciler(report)),      Effect.asVoid,      Effect.provide(makeSyncServices()),      Effect.scoped,      Effect.catchCause((cause) =>        Effect.gen(function* () {          yield* Effect.logError("workos_events: sync run failed", cause);          yield* captureCauseEffect(cause);        }),      ),      Effect.provide(WorkerTelemetryLive),    ),  );