History

Feed the cloud membership mirror from login and member writes (#2004)

Rhys Sullivan committed 1 month ago

Commit ddfe3f51 parent 89f424633 files · +5374 −606Browse files

Showing the changes to apps/cloud/src (27 of 33 files). Show all

Unified · Split

27 files
  1. apps/cloud/src/account/account-api.ts +7 −3
  2. apps/cloud/src/account/org-api-key-revoke.node.test.ts +27 −0
  3. apps/cloud/src/account/workos-account-service.ts +61 −11
  4. apps/cloud/src/api/layers.ts +3 −1
  5. apps/cloud/src/api/protected-api-key-auth.node.test.ts +9 −0
  6. apps/cloud/src/api/protected-jwt-auth.node.test.ts +9 −0
  7. apps/cloud/src/api/router.ts +4 −1
  8. apps/cloud/src/auth/api.ts +4 −2
  9. apps/cloud/src/auth/errors.ts +21 −0
  10. apps/cloud/src/auth/handlers.ts +81 −27
  11. apps/cloud/src/auth/mirror-feeders.node.test.ts +1040 −0
  12. apps/cloud/src/auth/mirror-feeders.ts +80 −0
  13. apps/cloud/src/auth/org-api-key-auth.node.test.ts +13 −1
  14. apps/cloud/src/auth/org-selector-auth.node.test.ts +9 −0
  15. apps/cloud/src/auth/organization.ts +5 −1
  16. apps/cloud/src/auth/user-store.ts +49 −12
  17. apps/cloud/src/auth/workos-callback-state.node.test.ts +28 −0
  18. apps/cloud/src/auth/workos-mirror-backfill.ts +274 −0
  19. apps/cloud/src/auth/workos-mirror-store.ts +999 −0
  20. apps/cloud/src/auth/workos-mirror.node.test.ts +543 −19
  21. apps/cloud/src/auth/workos-mirror.ts +33 −497
  22. apps/cloud/src/db/db.test.ts +42 −4
  23. apps/cloud/src/db/org-deletion.test.ts +28 −13
  24. apps/cloud/src/db/org-deletion.ts +30 −7
  25. apps/cloud/src/db/schema.ts +71 −5
  26. apps/cloud/src/extensions/billing/route.node.test.ts +9 −0
  27. apps/cloud/src/extensions/routes.ts +4 −1
apps/cloud/src/account/account-api.ts+7−3
@@ -9,6 +9,7 @@ import { · 8 unmodified lines
99
1010 import { ApiKeyService } from "../auth/api-keys";
1111 import { UserStoreService } from "../auth/context";
12+import { WorkOsMirror } from "../auth/workos-mirror";
1213 import { sessionFromSealed, type Session } from "../auth/middleware";
1314 import { WorkOSClient } from "../auth/workos";
1415 import { AutumnService } from "../extensions/billing/service";
@@ -95,10 +96,13 @@ const AccountProviderMiddleware = HttpRouter.middleware<{ provides: AccountProvi · 80 unmodified lines
9596 * account service closes over the per-request postgres socket). `AutumnService`
9697 * (the seat-gate) stays a residual requirement, satisfied by the app `boot`.
9798 */
98−export const workosAccountMiddleware = (rsLive: Layer.Layer<DbService | UserStoreService>) =>
99− AccountProviderMiddleware.combine(requestScopedMiddleware(rsLive)).layer;
99+export const workosAccountMiddleware = (
100+ rsLive: Layer.Layer<DbService | UserStoreService | WorkOsMirror>,
101+) => AccountProviderMiddleware.combine(requestScopedMiddleware(rsLive)).layer;
100102
101−export const makeAccountApiLive = (rsLive: Layer.Layer<DbService | UserStoreService>) => {
103+export const makeAccountApiLive = (
104+ rsLive: Layer.Layer<DbService | UserStoreService | WorkOsMirror>,
105+) => {
102106 // Cloud builds the WorkOS `AccountProvider` INSIDE the request body (so it
103107 // closes over the per-request postgres socket), so it can't be a self-
104108 // contained `Layer<AccountProvider>` — it combines its own middleware with
apps/cloud/src/account/org-api-key-revoke.node.test.ts+27−0
@@ -8,6 +8,7 @@ import { ApiKeyService, OrgApiKeyNotFound } from "../auth/api-keys"; · 7 unmodified lines
88 import { UserStoreService } from "../auth/context";
99 import { ORG_SELECTOR_HEADER } from "../auth/organization";
1010 import { WorkOSClient, type WorkOSClientService } from "../auth/workos";
11+import { WorkOsMirror } from "../auth/workos-mirror";
1112 import { AutumnService } from "../extensions/billing/service";
1213 import { AccountCaller, workosAccountProvider } from "./workos-account-service";
1314
@@ -100,18 +101,27 @@ const stubUsers = Layer.succeed(UserStoreService)({ · 86 unmodified lines
100101 upsertOrganization: async (org: { id: string; name: string }) => ({
101102 ...org,
102103 slug: org.id,
104+ backfilledAt: null,
105+ deletedAt: null,
106+ workosUpdatedAt: null,
103107 createdAt,
104108 }),
105109 getOrganization: async (id: string) => ({
106110 id,
107111 name: `Org ${id}`,
108112 slug: id,
113+ backfilledAt: null,
114+ deletedAt: null,
115+ workosUpdatedAt: null,
109116 createdAt,
110117 }),
111118 getOrganizationBySlug: async (slug: string) => ({
112119 id: slug,
113120 name: `Org ${slug}`,
114121 slug,
122+ backfilledAt: null,
123+ deletedAt: null,
124+ workosUpdatedAt: null,
115125 createdAt,
116126 }),
117127 deleteOrganizationCascade: async () => {},
@@ -119,6 +129,22 @@ const stubUsers = Layer.succeed(UserStoreService)({ · 1 unmodified line
119129 ),
120130 });
121131
132+// Revoke changes no membership, so the mirror is never written.
133+const stubMirror = Layer.succeed(WorkOsMirror)({
134+ upsertUser: () => Effect.die("revoke does not write the membership mirror"),
135+ upsertMembership: () => Effect.die("revoke does not write the membership mirror"),
136+ deleteMembership: () => Effect.die("revoke does not write the membership mirror"),
137+ deleteUser: () => Effect.die("revoke does not write the membership mirror"),
138+ getCursor: () => Effect.die("revoke does not read the events cursor"),
139+ setCursor: () => Effect.die("revoke does not move the events cursor"),
140+ applyOrganizationScan: () => Effect.die("revoke does not run the backfill"),
141+ replayBoundary: () => Effect.die("revoke does not run the reconciler"),
142+ setReplayBoundary: () => Effect.die("revoke does not run the backfill"),
143+ backfillCompletedAt: () => Effect.die("revoke does not check mirror readiness"),
144+ markBackfillCompleted: () => Effect.die("revoke does not run the backfill"),
145+ organizationBackfilledAt: () => Effect.die("revoke does not report seats"),
146+});
147+
122148 const stubAutumn = Layer.succeed(AutumnService)({
123149 use: () => Effect.die("revoke does not touch billing"),
124150 ensureCustomer: () => Effect.die("revoke does not touch billing"),
@@ -156,6 +182,7 @@ const providerWith = (accountId: string) => { · 31 unmodified lines
156182 Layer.mergeAll(
157183 stubWorkOS,
158184 stubUsers,
185+ stubMirror,
159186 stubApiKeys,
160187 stubAutumn,
161188 Layer.succeed(AccountCaller)({ session: session(accountId) }),
apps/cloud/src/account/workos-account-service.ts+61−11
@@ -12,6 +12,7 @@ import { ApiKeyService } from "../auth/api-keys"; · 11 unmodified lines
1212 import { UserStoreService } from "../auth/context";
1313 import type { Session } from "../auth/middleware";
1414 import { WorkOSClient } from "../auth/workos";
15+import { WorkOsMirror, mirrorMembershipFromWorkOs } from "../auth/workos-mirror";
1516 import { ORG_SELECTOR_HEADER, authorizeOrganizationSelector } from "../auth/organization";
1617 import { AutumnService } from "../extensions/billing/service";
1718 import { forkReportMemberSeats } from "../extensions/billing/member-seats";
@@ -50,7 +51,7 @@ export class AccountCaller extends Context.Service< · 32 unmodified lines
5051 // (me / API keys) and `org/handlers.ts` (members / roles / invite / role /
5152 // name). Native WorkOS / store failures are mapped at this boundary onto the
5253 // neutral account errors so the shared UI sees one shape:
53−// WorkOSError | UserStoreError | ApiKeyManagementError → AccountError
54+// WorkOSError | UserStoreError | ApiKeyManagementError | WorkOsMirrorError → AccountError
5455 // no organization in session → AccountNoOrganization
5556 // not-an-admin / over-seat-limit / not-allowed → AccountForbidden
5657 // ---------------------------------------------------------------------------
@@ -65,13 +66,17 @@ const toAccountError = () => Effect.fail(new AccountError({ message: "Account re · 8 unmodified lines
6566 export const workosAccountProvider: Layer.Layer<
6667 AccountProvider,
6768 never,
68− WorkOSClient | UserStoreService | ApiKeyService | AutumnService | AccountCaller
69+ WorkOSClient | UserStoreService | WorkOsMirror | ApiKeyService | AutumnService | AccountCaller
6970 > = Layer.effect(AccountProvider)(
7071 Effect.gen(function* () {
7172 const workos = yield* WorkOSClient;
7273 const apiKeys = yield* ApiKeyService;
7374 const autumn = yield* AutumnService;
7475 const users = yield* UserStoreService;
76+ // Membership writes below go to WorkOS FIRST (the authority), then are
77+ // written through to the local mirror so the member list and the seat
78+ // count read the change without waiting for the Events reconciler.
79+ const mirror = yield* WorkOsMirror;
7580
7681 // The caller, resolved once per request by the cookie-only session
7782 // middleware (account-api.ts) — the same credential `SessionAuthLive`
@@ -137,6 +142,7 @@ export const workosAccountProvider: Layer.Layer< · 59 unmodified lines
137142 if (!membership || membership.organizationId !== organizationId) {
138143 return yield* new AccountForbidden();
139144 }
145+ return membership;
140146 });
141147
142148 // Mirror of org/handlers `getMemberSeats` — live seat usage from WorkOS.
@@ -215,7 +221,10 @@ export const workosAccountProvider: Layer.Layer< · 72 unmodified lines
215221 Effect.gen(function* () {
216222 const { session, org } = yield* requireOrganization(headers);
217223 const keys = yield* apiKeys
218− .listUserKeys({ accountId: session.accountId, organizationId: org.id })
224+ .listUserKeys({
225+ accountId: session.accountId,
226+ organizationId: org.id,
227+ })
219228 .pipe(Effect.catchTag("ApiKeyManagementError", toAccountError));
220229 return { apiKeys: keys };
221230 }),
@@ -225,10 +234,16 @@ export const workosAccountProvider: Layer.Layer< · 3 unmodified lines
225234 const { session, org } = yield* requireOrganization(headers);
226235 const trimmed = name.trim().slice(0, MAX_API_KEY_NAME_LENGTH);
227236 if (!trimmed) {
228− return yield* new AccountError({ message: "API key name is required" });
237+ return yield* new AccountError({
238+ message: "API key name is required",
239+ });
229240 }
230241 return yield* apiKeys
231− .createUserKey({ accountId: session.accountId, organizationId: org.id, name: trimmed })
242+ .createUserKey({
243+ accountId: session.accountId,
244+ organizationId: org.id,
245+ name: trimmed,
246+ })
232247 .pipe(Effect.catchTag("ApiKeyManagementError", toAccountError));
233248 }),
234249
@@ -236,7 +251,10 @@ export const workosAccountProvider: Layer.Layer< · 1 unmodified line
236251 Effect.gen(function* () {
237252 const { session, org } = yield* requireOrganization(headers);
238253 const ownedKeys = yield* apiKeys
239− .listUserKeys({ accountId: session.accountId, organizationId: org.id })
254+ .listUserKeys({
255+ accountId: session.accountId,
256+ organizationId: org.id,
257+ })
240258 .pipe(Effect.catchTag("ApiKeyManagementError", toAccountError));
241259 if (!ownedKeys.some((key) => key.id === apiKeyId)) {
242260 return yield* new AccountError({ message: "API key not found" });
@@ -266,7 +284,9 @@ export const workosAccountProvider: Layer.Layer< · 23 unmodified lines
266284 yield* requireAdmin(session.accountId, org.id);
267285 const trimmed = name.trim().slice(0, MAX_API_KEY_NAME_LENGTH);
268286 if (!trimmed) {
269− return yield* new AccountError({ message: "API key name is required" });
287+ return yield* new AccountError({
288+ message: "API key name is required",
289+ });
270290 }
271291 return yield* apiKeys
272292 .createOrgKey({ organizationId: org.id, name: trimmed })
@@ -287,7 +307,11 @@ export const workosAccountProvider: Layer.Layer< · 14 unmodified lines
287307 yield* apiKeys.revokeOrgKey({ organizationId: org.id, keyId: apiKeyId }).pipe(
288308 Effect.catchTag("ApiKeyManagementError", toAccountError),
289309 Effect.catchTag("OrgApiKeyNotFound", () =>
290− Effect.fail(new AccountError({ message: "Organization API key not found" })),
310+ Effect.fail(
311+ new AccountError({
312+ message: "Organization API key not found",
313+ }),
314+ ),
291315 ),
292316 );
293317 return { success: true };
@@ -361,10 +385,29 @@ export const workosAccountProvider: Layer.Layer< · 67 unmodified lines
361385 Effect.gen(function* () {
362386 const { session, org } = yield* requireOrganization(headers);
363387 yield* requireAdmin(session.accountId, org.id);
364− yield* assertMembershipInOrg(org.id, membershipId);
388+ const membership = yield* assertMembershipInOrg(org.id, membershipId);
365389 yield* workos
366390 .deleteOrgMembership(membershipId)
367391 .pipe(Effect.catchTag("WorkOSError", toAccountError));
392+ // Tombstoned by identity: the deleted WorkOS id never returns, so
393+ // a login or backfill that fetched this membership before the
394+ // delete — or a role change issued before it and delivered after
395+ // — is refused however it is stamped, while a replacement
396+ // membership WorkOS creates for the same member (a new id) is
397+ // not. No WorkOS instant is in hand (WorkOS answers a delete with
398+ // no time): the row keeps its own stamp, never the local clock,
399+ // which read after WorkOS answered could post-date that
400+ // replacement.
401+ yield* mirror
402+ .deleteMembership(
403+ {
404+ id: membershipId,
405+ accountId: membership.userId,
406+ organizationId: membership.organizationId,
407+ },
408+ new Date(membership.updatedAt),
409+ )
410+ .pipe(Effect.catchTag("WorkOsMirrorError", toAccountError));
368411 yield* forkReportMemberSeats(org.id).pipe(Effect.provideContext(ctx));
369412 return { success: true };
370413 }),
@@ -374,9 +417,12 @@ export const workosAccountProvider: Layer.Layer< · 3 unmodified lines
374417 const { session, org } = yield* requireOrganization(headers);
375418 yield* requireAdmin(session.accountId, org.id);
376419 yield* assertMembershipInOrg(org.id, membershipId);
377− yield* workos
420+ const updated = yield* workos
378421 .updateOrgMembershipRole(membershipId, roleSlug)
379422 .pipe(Effect.catchTag("WorkOSError", toAccountError));
423+ yield* mirror
424+ .upsertMembership(mirrorMembershipFromWorkOs(updated))
425+ .pipe(Effect.catchTag("WorkOsMirrorError", toAccountError));
380426 return { success: true };
381427 }),
382428
@@ -389,7 +435,11 @@ export const workosAccountProvider: Layer.Layer< · 6 unmodified lines
389435 .pipe(Effect.catchTag("WorkOSError", toAccountError));
390436 yield* users
391437 .use("upsertOrganization", (s) =>
392− s.upsertOrganization({ id: updated.id, name: updated.name }),
438+ s.upsertOrganization({
439+ id: updated.id,
440+ name: updated.name,
441+ updatedAt: new Date(updated.updatedAt),
442+ }),
393443 )
394444 .pipe(Effect.catchTag("UserStoreError", toAccountError));
395445 return { name: updated.name };
apps/cloud/src/api/layers.ts+3−1
@@ -56,7 +56,9 @@ export const BootSharedServices = Layer.mergeAll( · 55 unmodified lines
5656 // `AutumnService.Default` is provided HERE because the `createOrganization`
5757 // handler reads it for the free-organizations-per-user limit gate — one of the
5858 // few app-only billing touchpoints. (It is NOT on the neutral boot core.)
59−export const makeNonProtectedApiLive = (rsLive: Layer.Layer<DbService | UserStoreService>) =>
59+export const makeNonProtectedApiLive = (
60+ rsLive: Layer.Layer<DbService | UserStoreService | WorkOsMirror>,
61+) =>
6062 HttpApiBuilder.layer(NonProtectedApi).pipe(
6163 Layer.provide(Layer.mergeAll(CloudAuthPublicHandlers, CloudSessionAuthHandlers)),
6264 Layer.provide(requestScopedMiddleware(rsLive).layer),
apps/cloud/src/api/protected-api-key-auth.node.test.ts+9−0
@@ -68,18 +68,27 @@ const stubUsers = Layer.succeed(UserStoreService)({ · 67 unmodified lines
6868 upsertOrganization: async (org: { id: string; name: string }) => ({
6969 ...org,
7070 slug: `org-slug-${org.id}`,
71+ backfilledAt: null,
72+ deletedAt: null,
73+ workosUpdatedAt: null,
7174 createdAt,
7275 }),
7376 getOrganization: async (id: string) => ({
7477 id,
7578 name: `Org ${id}`,
7679 slug: `org-slug-${id}`,
80+ backfilledAt: null,
81+ deletedAt: null,
82+ workosUpdatedAt: null,
7783 createdAt,
7884 }),
7985 getOrganizationBySlug: async (slug: string) => ({
8086 id: "org_by_slug",
8187 name: `Org ${slug}`,
8288 slug,
89+ backfilledAt: null,
90+ deletedAt: null,
91+ workosUpdatedAt: null,
8392 createdAt,
8493 }),
8594 deleteOrganizationCascade: async () => {},
apps/cloud/src/api/protected-jwt-auth.node.test.ts+9−0
@@ -85,18 +85,27 @@ const stubUsers = Layer.succeed(UserStoreService)({ · 84 unmodified lines
8585 upsertOrganization: async (org: { id: string; name: string }) => ({
8686 ...org,
8787 slug: `org-slug-${org.id}`,
88+ backfilledAt: null,
89+ deletedAt: null,
90+ workosUpdatedAt: null,
8891 createdAt,
8992 }),
9093 getOrganization: async (id: string) => ({
9194 id,
9295 name: `Org ${id}`,
9396 slug: `org-slug-${id}`,
97+ backfilledAt: null,
98+ deletedAt: null,
99+ workosUpdatedAt: null,
94100 createdAt,
95101 }),
96102 getOrganizationBySlug: async (slug: string) => ({
97103 id: "org_by_slug",
98104 name: `Org ${slug}`,
99105 slug,
106+ backfilledAt: null,
107+ deletedAt: null,
108+ workosUpdatedAt: null,
100109 createdAt,
101110 }),
102111 deleteOrganizationCascade: async () => {},
apps/cloud/src/api/router.ts+4−1
@@ -4,6 +4,7 @@ import { HttpRouter } from "effect/unstable/http"; · 3 unmodified lines
44 import { RouterConfigLive, requestScopedMiddleware } from "@executor-js/api/server";
55
66 import { UserStoreService } from "../auth/context";
7+import { WorkOsMirror } from "../auth/workos-mirror";
78 import { DbService } from "../db/db";
89 import { makeAccountApiLive } from "../account/account-api";
910
@@ -29,7 +30,9 @@ import { makeProtectedApiLive } from "./protected"; · 19 unmodified lines
2930 // so tests can substitute a counting fake for `DbService.Live` and
3031 // assert per-request semantics — see
3132 // `apps/cloud/src/api.request-scope.node.test.ts`.
32−export const makeApiLive = (requestScopedLive: Layer.Layer<DbService | UserStoreService>) => {
33+export const makeApiLive = (
34+ requestScopedLive: Layer.Layer<DbService | UserStoreService | WorkOsMirror>,
35+) => {
3336 const BillingRoutesLive = AutumnRoutesLive.pipe(
3437 Layer.provide(requestScopedMiddleware(requestScopedLive).layer),
3538 );
apps/cloud/src/auth/api.ts+4−2
@@ -1,6 +1,6 @@
11 import { HttpApiEndpoint, HttpApiGroup } from "effect/unstable/httpapi";
22 import { Schema } from "effect";
3−import { UserStoreError, WorkOSError } from "./errors";
3+import { UserStoreError, WorkOSError, WorkOsMirrorError } from "./errors";
44 import { NoOrganization } from "@executor-js/api/server";
55 import { SessionAuth } from "./middleware";
66
@@ -172,7 +172,9 @@ export const AUTH_PATHS = { · 165 unmodified lines
172172 callback: "/api/auth/callback",
173173 } as const;
174174
175−const AuthErrors = [UserStoreError, WorkOSError] as const;
175+// The login callback and the org handlers feed the membership mirror, so a
176+// mirror write failure is one of their wire errors (same 500 as a store failure).
177+const AuthErrors = [UserStoreError, WorkOSError, WorkOsMirrorError] as const;
176178 const McpApprovalErrors = [
177179 NoOrganization,
178180 McpExecutionNotFoundError,
apps/cloud/src/auth/errors.ts+21−0
@@ -38,6 +38,27 @@ export class UserStoreError extends Schema.TaggedErrorClass<UserStoreError>()( · 37 unmodified lines
3838 }
3939 }
4040
41+/**
42+ * The public failure of every cloud membership-mirror write (`WorkOsMirror`).
43+ * Same two diagnosable fields as `UserStoreError` — which mirror call failed,
44+ * and how — classified from the driver cause the same way. Declared here,
45+ * beside `UserStoreError`, because the auth API (`auth/api.ts`, in the SPA
46+ * bundle) names it on the wire for the login and org handlers that feed the
47+ * mirror; the service itself lives in `workos-mirror.ts`.
48+ */
49+export class WorkOsMirrorError extends Schema.TaggedErrorClass<WorkOsMirrorError>()(
50+ "WorkOsMirrorError",
51+ {
52+ operation: Schema.String,
53+ reason: Schema.Literals(USER_STORE_FAILURE_REASONS),
54+ },
55+ { httpApiStatus: 500 },
56+) {
57+ override get message(): string {
58+ return `workos mirror ${this.operation} failed: ${this.reason}`;
59+ }
60+}
61+
4162 /** Reasons a retry can plausibly clear: the query never reached a healthy
4263 * server. A `query` failure is deterministic and must not be retried. */
4364 export const isTransientUserStoreReason = (reason: UserStoreFailureReason): boolean =>
apps/cloud/src/auth/handlers.ts+81−27
@@ -1,6 +1,6 @@
11 import { HttpApi, HttpApiBuilder } from "effect/unstable/httpapi";
22 import { HttpServerRequest, HttpServerResponse } from "effect/unstable/http";
3−import { Duration, Effect, Predicate } from "effect";
3+import { Clock, Duration, Effect, Predicate } from "effect";
44 import { isValidOrgSlug } from "@executor-js/api";
55
66 import {
@@ -18,6 +18,7 @@ import { SessionContext, SessionCookies } from "./middleware"; · 11 unmodified lines
1818 import { encodeLoginState, decodeLoginState } from "./login-state";
1919 import { safeReturnTo } from "./return-to";
2020 import { UserStoreService } from "./context";
21+import { mirrorMembership, mirrorSignIn } from "./mirror-feeders";
2122 import { env } from "cloudflare:workers";
2223 import { WorkOSError } from "./errors";
2324 import { WorkOSClient } from "./workos";
@@ -209,8 +210,16 @@ export const CloudAuthPublicHandlers = HttpApiBuilder.group( · 185 unmodified lines
209210
210211 const result = yield* workos.authenticateWithCode(query.code);
211212
212− // Mirror the account locally
213− yield* users.use("ensureAccount", (s) => s.ensureAccount(result.user.id));
213+ // ONE membership list for the whole callback. It feeds the mirror
214+ // (the user + every org they hold a membership in, all already in
215+ // hand) and it is the membership check for every landing-org
216+ // candidate below, so the callback makes no per-candidate WorkOS
217+ // call. The user's account row is minted by the mirror's user
218+ // upsert. The list's fetch instant, taken before the read, stamps
219+ // the organization names it carries (see `mirrorSignIn`).
220+ const fetchedAt = new Date(yield* Clock.currentTimeMillis);
221+ const memberships = yield* workos.listUserMemberships(result.user.id);
222+ yield* mirrorSignIn(result.user, memberships.data, fetchedAt);
214223
215224 let sealedSession = result.sealedSession;
216225
@@ -220,34 +229,43 @@ export const CloudAuthPublicHandlers = HttpApiBuilder.group( · 3 unmodified lines
220229 // any other untrusted path.
221230 const returnTo = safeReturnTo(decodeLoginState(query.state)?.returnTo) ?? "/";
222231 const requestedOrgSelector = requestedOrgSelectorFromReturnTo(returnTo);
223− const requestedOrg = requestedOrgSelector
224− ? yield* authorizeOrganizationSelector(result.user.id, requestedOrgSelector).pipe(
232+
233+ // An org SLUG (both candidate sources below are slug-validated, so
234+ // an `org_…` id never reaches here) resolves to its id only when the
235+ // list above holds an ACTIVE membership in it. Pending memberships
236+ // are skipped because refreshing into one 400s and would bypass
237+ // invite consent. A slug that fails to resolve (unknown, or a store
238+ // hiccup) is not a candidate, the same as an org the user is not in.
239+ const activeOrganizationIds = new Set(
240+ memberships.data.filter((m) => m.status === "active").map((m) => m.organizationId),
241+ );
242+ const activeOrganizationFor = (slug: string) =>
243+ users
244+ .use("getOrganizationBySlug", (s) => s.getOrganizationBySlug(slug))
245+ .pipe(
246+ Effect.map((org) => (org && activeOrganizationIds.has(org.id) ? org.id : null)),
225247 Effect.orElseSucceed(() => null),
226− )
227− : null;
248+ );
228249
229250 // Prefer the org in the URL that sent the user to login. If the URL
230251 // is bare, or not an org route, prefer the org this browser last
231252 // worked in (the last-org cookie — it outlives the session precisely
232253 // so a fresh login lands where the user left off), then WorkOS's
233254 // org, then the first active membership for org-less sessions.
234− // Pending memberships are skipped because refreshing into one 400s
235− // and would bypass invite consent. The cookie is membership-checked
236− // like any selector, so a stale one just falls through.
237− let targetOrganizationId = requestedOrg?.id ?? null;
255+ // The cookie is membership-checked like any selector, so a stale
256+ // one just falls through.
257+ let targetOrganizationId = requestedOrgSelector
258+ ? yield* activeOrganizationFor(requestedOrgSelector)
259+ : null;
238260 if (!targetOrganizationId && !requestedOrgSelector) {
239261 const lastOrgSlug = request.cookies[LAST_ORG_COOKIE];
240− const lastOrg =
262+ targetOrganizationId =
241263 lastOrgSlug && isValidOrgSlug(lastOrgSlug)
242− ? yield* authorizeOrganizationSelector(result.user.id, lastOrgSlug).pipe(
243− Effect.orElseSucceed(() => null),
244− )
264+ ? yield* activeOrganizationFor(lastOrgSlug)
245265 : null;
246− targetOrganizationId = lastOrg?.id ?? null;
247266 }
248267 targetOrganizationId ??= result.organizationId ?? null;
249268 if (!targetOrganizationId && !requestedOrgSelector) {
250− const memberships = yield* workos.listUserMemberships(result.user.id);
251269 const existingActive = memberships.data.find((m) => m.status === "active");
252270 targetOrganizationId = existingActive?.organizationId ?? null;
253271 }
@@ -311,7 +329,9 @@ export const CloudAuthPublicHandlers = HttpApiBuilder.group( · 57 unmodified lines
311329 ? yield* workos.logoutUrl(sealedSession, origin ? `${origin}/` : undefined)
312330 : null;
313331
314− const response = HttpServerResponse.redirect(logoutUrl ?? "/", { status: 302 });
332+ const response = HttpServerResponse.redirect(logoutUrl ?? "/", {
333+ status: 302,
334+ });
315335
316336 // Drop only what this browser actually presented. Both cookies are
317337 // SameSite=Lax, so a cross-site form POST carries neither — it gets
@@ -447,11 +467,18 @@ export const CloudSessionAuthHandlers = HttpApiBuilder.group( · 129 unmodified lines
447467 }
448468
449469 const org = yield* workos.createOrganization(name);
450− yield* workos.createMembership(org.id, session.accountId, "admin");
470+ const membership = yield* workos.createMembership(org.id, session.accountId, "admin");
451471 // `upsertOrganization` mints the slug at insert — no separate heal step.
452472 const mirrored = yield* users.use("upsertOrganization", (s) =>
453− s.upsertOrganization({ id: org.id, name: org.name }),
473+ s.upsertOrganization({
474+ id: org.id,
475+ name: org.name,
476+ updatedAt: new Date(org.updatedAt),
477+ }),
454478 );
479+ // Write-through: the creator's admin membership, from the create
480+ // response, lands in the mirror before anything reads it.
481+ yield* mirrorMembership(membership);
455482
456483 // Provision the org's billing customer while we're the ones creating
457484 // the org. Without this the first billing call an org ever makes is a
@@ -535,13 +562,18 @@ export const CloudSessionAuthHandlers = HttpApiBuilder.group( · 77 unmodified lines
535562 // empty workspace when a later request re-mirrors it with a new slug.
536563 yield* workos.deleteOrganization(organizationId);
537564
538− // Purge all local tenant data, secrets, and the identity mirror
539− // (cascades local memberships) in one transaction. If this fails
540− // after the WorkOS delete already succeeded, the org is gone for
541− // everyone (unreachable) but its secrets/tenant rows linger orphaned —
542− // alert loudly so that window gets swept, then surface the failure.
565+ // Purge all local tenant data, secrets, and the org's memberships in
566+ // one transaction, leaving the org row as a tombstone marked deleted
567+ // (so a login that fetched its membership list before the deletion
568+ // cannot re-mint the org afterwards). If this fails after the WorkOS
569+ // delete already succeeded, the org is gone for everyone
570+ // (unreachable) but its secrets/tenant rows linger orphaned — alert
571+ // loudly so that window gets swept, then surface the failure.
572+ const deletedAt = new Date(yield* Clock.currentTimeMillis);
543573 yield* users
544− .use("deleteOrganizationCascade", (s) => s.deleteOrganizationCascade(organizationId))
574+ .use("deleteOrganizationCascade", (s) =>
575+ s.deleteOrganizationCascade(organizationId, deletedAt),
576+ )
545577 .pipe(
546578 Effect.tapError((error) =>
547579 Effect.logError(
@@ -645,9 +677,31 @@ export const CloudSessionAuthHandlers = HttpApiBuilder.group( · 97 unmodified lines
645677 // upsert mints the slug at insert — no separate heal step.
646678 const org = yield* workos.getOrganization(invitation.organizationId);
647679 const mirrored = yield* users.use("upsertOrganization", (s) =>
648− s.upsertOrganization({ id: org.id, name: org.name }),
680+ s.upsertOrganization({
681+ id: org.id,
682+ name: org.name,
683+ updatedAt: new Date(org.updatedAt),
684+ }),
649685 );
650686
687+ // Write-through: acceptance returns the invitation, not the
688+ // membership it activated, so this is the one feeder that reads the
689+ // membership back (a rare path; one extra call). WorkOS activates it
690+ // as part of acceptance, so its absence is worth a warning — the
691+ // Events reconciler will still land it.
692+ const membership = yield* workos.getUserOrgMembership(org.id, session.accountId);
693+ if (membership) {
694+ yield* mirrorMembership(membership);
695+ } else {
696+ yield* Effect.logWarning(
697+ "acceptInvitation: accepted invitation has no membership yet",
698+ {
699+ userId: session.accountId,
700+ organizationId: org.id,
701+ },
702+ );
703+ }
704+
651705 // The membership is active in WorkOS from this point even if
652706 // attaching the session below fails, so reconcile the org's billed
653707 // seat count now.
apps/cloud/src/auth/mirror-feeders.node.test.ts+1040−0
@@ -0,0 +1,1040 @@
1+// ---------------------------------------------------------------------------
2+// The membership mirror's FEEDERS, end to end through the code that runs in
3+// production, against the real PGlite Postgres every cloud unit test runs on
4+// (scripts/test-globalsetup.ts). WorkOS is a fake `WorkOSClient` (the
5+// emulator has no list-users / events routes); the mirror, the user store,
6+// and the directory read are the live layers over `DbService.Live`.
7+//
8+// What this pins:
9+// - the login callback records the signed-in user and EVERY membership
10+// WorkOS lists (active and pending), with the org row minted so the FK
11+// holds — from the one membership list it already fetches
12+// - the callback picks the landing org from that same list: a returnTo
13+// slug or last-org cookie lands only in an ACTIVE membership, an unknown
14+// or pending one falls through
15+// - `removeMember` tombstones the mirror row after the WorkOS delete,
16+// stamped with the membership's last WorkOS state (never a local clock),
17+// so a replay of the membership as it was before the delete cannot
18+// restore it while a replacement WorkOS created meanwhile is accepted
19+// - `updateMemberRole` writes the role WorkOS returned
20+// - the backfill mirrors every org's members and counts what it wrote,
21+// writes nothing on a dry run, converges on a re-run, tombstones a
22+// membership WorkOS no longer lists — but never one written after its
23+// listing was taken — marks each org backfilled as of its listing, and
24+// records the events replay boundary only when it completes and only
25+// ONCE: a run that fails part-way keeps the marks of the orgs it
26+// finished and records no boundary, and a later completed run keeps the
27+// first boundary (the org renames and user deletions between two runs
28+// are the events stream's to replay)
29+// - two scans of one org that overlap cannot resurrect a membership: a scan
30+// that listed it, stalled, and resumed after a later listing (which no
31+// longer had it) was applied is refused whole
32+// - a login whose membership list was fetched BEFORE the org was purged and
33+// written after cannot re-mint the org or its membership, one fetched
34+// before a rename cannot revert the rename, and one fetched before a
35+// revocation the backfill has since scanned cannot reinstate the
36+// membership
37+// ---------------------------------------------------------------------------
38+
39+import { describe, expect, it } from "@effect/vitest";
40+import { sql } from "drizzle-orm";
41+import { Effect, Exit, Fiber, Latch, Layer } from "effect";
42+import { HttpRouter, HttpServer } from "effect/unstable/http";
43+import { HttpApiBuilder } from "effect/unstable/httpapi";
44+
45+import {
46+ AccountProvider,
47+ MemberDirectory,
48+ RouterConfigLive,
49+ requestScopedMiddleware,
50+} from "@executor-js/api/server";
51+
52+import { AccountCaller, workosAccountProvider } from "../account/workos-account-service";
53+import { RequestScopedServicesLive } from "../api/layers";
54+import { DbService } from "../db/db";
55+import { AutumnService } from "../extensions/billing/service";
56+import { ApiKeyService } from "./api-keys";
57+import { UserStoreService } from "./context";
58+import { WorkOSError } from "./errors";
59+import { CloudAuthPublicHandlers, CloudSessionAuthHandlers, NonProtectedApi } from "./handlers";
60+import { LAST_ORG_COOKIE } from "./last-org-cookie";
61+import { encodeLoginState } from "./login-state";
62+import { cloudMemberDirectoryLayer } from "./member-directory";
63+import { SessionAuthLive } from "./middleware-live";
64+import { mirrorSignIn } from "./mirror-feeders";
65+import { ORG_SELECTOR_HEADER } from "./organization";
66+import { WorkOSClient, type WorkOSClientService } from "./workos";
67+import { WorkOsMirror, type WorkOsMirrorShape } from "./workos-mirror";
68+import { backfillOrganization, backfillWorkOsMirror } from "./workos-mirror-backfill";
69+import type { WorkOsMembershipPayload, WorkOsUserPayload } from "./workos-mirror-store";
70+
71+const T1 = "2026-01-01T00:00:00.000Z";
72+const T2 = "2026-01-02T00:00:00.000Z";
73+
74+// Synthetic identities only. Every test mints its own org ids so the shared
75+// test database never couples two tests.
76+const freshId = (prefix: string) => `${prefix}_${crypto.randomUUID().replaceAll("-", "")}`;
77+
78+const workosUser = (id: string, overrides: Partial<WorkOsUserPayload> = {}) => ({
79+ object: "user" as const,
80+ id,
81+ email: `${id}@placeholder.test`,
82+ emailVerified: true,
83+ firstName: "Ada",
84+ lastName: "Placeholder",
85+ profilePictureUrl: null,
86+ lastSignInAt: T1,
87+ locale: null,
88+ createdAt: T1,
89+ updatedAt: T1,
90+ externalId: null,
91+ metadata: {},
92+ ...overrides,
93+});
94+
95+interface FakeMembership extends WorkOsMembershipPayload {
96+ readonly organizationName: string;
97+}
98+
99+const workosMembership = (
100+ userId: string,
101+ organizationId: string,
102+ overrides: Partial<FakeMembership> = {},
103+): FakeMembership => ({
104+ id: `om_${userId}_${organizationId}`,
105+ userId,
106+ organizationId,
107+ organizationName: `Org ${organizationId}`,
108+ role: { slug: "member" },
109+ status: "active",
110+ updatedAt: T1,
111+ ...overrides,
112+});
113+
114+/** Mirrored rows for one org, read through the live cloud `MemberDirectory`. */
115+const readMembers = (organizationId: string) =>
116+ Effect.runPromise(
117+ Effect.flatMap(MemberDirectory.asEffect(), (directory) =>
118+ directory.members(organizationId, {
119+ statuses: ["active", "pending", "inactive"],
120+ }),
121+ ).pipe(
122+ Effect.provide(cloudMemberDirectoryLayer.pipe(Layer.provide(DbService.Live))),
123+ Effect.scoped,
124+ ),
125+ );
126+
127+/** Mirror an org row (named as of T1) and return the URL slug the store minted for it. */
128+const seedOrganization = (id: string) =>
129+ Effect.runPromise(
130+ Effect.flatMap(UserStoreService.asEffect(), (users) =>
131+ users.use("upsertOrganization", (s) =>
132+ s.upsertOrganization({
133+ id,
134+ name: `Org ${id}`,
135+ updatedAt: new Date(T1),
136+ }),
137+ ),
138+ ).pipe(
139+ Effect.map((org) => org.slug),
140+ Effect.provide(UserStoreService.Live.pipe(Layer.provide(DbService.Live))),
141+ Effect.scoped,
142+ ),
143+ );
144+
145+const stubAutumn = Layer.succeed(AutumnService)({
146+ use: () => Effect.die("feeders do not read billing"),
147+ ensureCustomer: () => Effect.void,
148+ checkExecutionBalance: () => Effect.die("feeders do not check balances"),
149+ trackExecution: () => Effect.void,
150+ setMemberSeats: () => Effect.void,
151+});
152+
153+/**
154+ * A `WorkOSClient` whose every method is one of `methods`; anything else is
155+ * an unexpected call and dies, so a feeder that silently adds a WorkOS read
156+ * fails the test instead of passing on a fake.
157+ */
158+const stubWorkOS = (methods: Partial<WorkOSClientService>) =>
159+ Layer.succeed(
160+ WorkOSClient,
161+ new Proxy({} as WorkOSClientService, {
162+ get: (_target, prop) =>
163+ (methods as Record<PropertyKey, unknown>)[prop] ??
164+ (() => Effect.die(`unexpected WorkOSClient.${String(prop)} call`)),
165+ }),
166+ );
167+
168+describe("login callback", () => {
169+ const callbackHandler = (workos: Layer.Layer<WorkOSClient>) =>
170+ HttpRouter.toWebHandler(
171+ HttpApiBuilder.layer(NonProtectedApi).pipe(
172+ Layer.provide(Layer.mergeAll(CloudAuthPublicHandlers, CloudSessionAuthHandlers)),
173+ Layer.provide(requestScopedMiddleware(RequestScopedServicesLive).layer),
174+ Layer.provideMerge(SessionAuthLive),
175+ Layer.provideMerge(stubAutumn),
176+ Layer.provideMerge(workos),
177+ Layer.provideMerge(HttpServer.layerServices),
178+ Layer.provideMerge(RouterConfigLive),
179+ ),
180+ { disableLogger: true },
181+ ).handler;
182+
183+ const STATE_COOKIE = "wos-login-state";
184+
185+ /**
186+ * A callback handler over a fake WorkOS that authenticates `user` with the
187+ * memberships `listed`, recording every WorkOS read (`calls`) and every
188+ * session refresh (`refreshedInto`, the org ids) so the landing-org choice
189+ * is assertable from the outside.
190+ */
191+ const signIn = (user: ReturnType<typeof workosUser>, listed: readonly FakeMembership[]) => {
192+ const calls: string[] = [];
193+ const refreshedInto: (string | undefined)[] = [];
194+ const handler = callbackHandler(
195+ stubWorkOS({
196+ authenticateWithCode: () =>
197+ Effect.succeed({
198+ user,
199+ organizationId: undefined,
200+ accessToken: "access",
201+ refreshToken: "refresh",
202+ sealedSession: "sealed",
203+ }),
204+ listUserMemberships: (id) => {
205+ calls.push(`listUserMemberships:${id}`);
206+ return Effect.succeed({
207+ object: "list" as const,
208+ data: listed as never[],
209+ listMetadata: { before: null, after: null },
210+ });
211+ },
212+ // The forked seat recount after login.
213+ listOrgMembers: () =>
214+ Effect.succeed({
215+ object: "list" as const,
216+ data: [] as never[],
217+ listMetadata: { before: null, after: null },
218+ }),
219+ refreshSession: (_sealed, organizationId) => {
220+ refreshedInto.push(organizationId);
221+ return Effect.succeed("sealed-refreshed");
222+ },
223+ }),
224+ );
225+ return { handler, calls, refreshedInto };
226+ };
227+
228+ /**
229+ * `GET /auth/callback` with the CSRF-matched login `state` (the callback
230+ * refuses any request without one) and any extra cookies; `returnTo`
231+ * rides inside the state as /login mints it.
232+ */
233+ const callbackRequest = (options: { returnTo?: string; cookies?: Record<string, string> }) => {
234+ const url = new URL("http://test.local/auth/callback");
235+ url.searchParams.set("code", "code_1");
236+ const state = encodeLoginState({
237+ nonce: "nonce",
238+ ...(options.returnTo === undefined ? {} : { returnTo: options.returnTo }),
239+ });
240+ url.searchParams.set("state", state);
241+ const cookies = { ...options.cookies, [STATE_COOKIE]: state };
242+ const cookie = Object.entries(cookies)
243+ .map(([name, value]) => `${name}=${value}`)
244+ .join("; ");
245+ return new Request(url, { headers: cookie ? { cookie } : {} });
246+ };
247+
248+ it("records the user and every listed membership from the one list it already fetches", async () => {
249+ const userId = freshId("user");
250+ const activeOrg = freshId("org");
251+ const pendingOrg = freshId("org");
252+ const { handler, calls } = signIn(
253+ workosUser(userId, {
254+ firstName: "Grace",
255+ lastName: "Hopper",
256+ updatedAt: T2,
257+ }),
258+ [
259+ workosMembership(userId, activeOrg, {
260+ role: { slug: "admin" },
261+ updatedAt: T2,
262+ }),
263+ workosMembership(userId, pendingOrg, { status: "pending" }),
264+ ],
265+ );
266+
267+ const response = await handler(callbackRequest({}));
268+
269+ expect(response.status).toBe(302);
270+ expect(calls, "one membership list for the whole callback").toEqual([
271+ `listUserMemberships:${userId}`,
272+ ]);
273+
274+ const active = await readMembers(activeOrg);
275+ expect(active).toHaveLength(1);
276+ expect(active[0]).toMatchObject({
277+ accountId: userId,
278+ membershipId: `om_${userId}_${activeOrg}`,
279+ email: `${userId}@placeholder.test`,
280+ name: "Grace Hopper",
281+ role: "admin",
282+ status: "active",
283+ lastActiveAt: new Date(T1).getTime(),
284+ });
285+ const pending = await readMembers(pendingOrg);
286+ expect(
287+ pending.map((m) => m.status),
288+ "pending memberships are mirrored too",
289+ ).toEqual(["pending"]);
290+ });
291+
292+ describe("lands in the org the returnTo slug names", () => {
293+ it("when the user holds an active membership there", async () => {
294+ const userId = freshId("user");
295+ const requested = freshId("org");
296+ const other = freshId("org");
297+ const slug = await seedOrganization(requested);
298+ const { handler, refreshedInto } = signIn(workosUser(userId), [
299+ workosMembership(userId, other),
300+ workosMembership(userId, requested),
301+ ]);
302+
303+ const response = await handler(callbackRequest({ returnTo: `/${slug}/settings` }));
304+
305+ expect(response.status).toBe(302);
306+ expect(response.headers.get("location")).toBe(`/${slug}/settings`);
307+ expect(refreshedInto, "the session is switched into the requested org").toEqual([requested]);
308+ });
309+
310+ it("never when the membership there is only pending", async () => {
311+ const userId = freshId("user");
312+ const requested = freshId("org");
313+ const other = freshId("org");
314+ const slug = await seedOrganization(requested);
315+ const { handler, refreshedInto } = signIn(workosUser(userId), [
316+ workosMembership(userId, other),
317+ workosMembership(userId, requested, { status: "pending" }),
318+ ]);
319+
320+ const response = await handler(callbackRequest({ returnTo: `/${slug}` }));
321+
322+ expect(response.status).toBe(302);
323+ expect(response.headers.get("location")).toBe(`/${slug}`);
324+ expect(
325+ refreshedInto,
326+ "a pending membership is not a landing candidate, and an explicit slug does not fall back to another org",
327+ ).toEqual([]);
328+ });
329+ });
330+
331+ describe("without a returnTo org", () => {
332+ it("lands in the last-org cookie's org when the user is active there", async () => {
333+ const userId = freshId("user");
334+ const last = freshId("org");
335+ const other = freshId("org");
336+ const slug = await seedOrganization(last);
337+ const { handler, refreshedInto } = signIn(workosUser(userId), [
338+ workosMembership(userId, other),
339+ workosMembership(userId, last),
340+ ]);
341+
342+ const response = await handler(callbackRequest({ cookies: { [LAST_ORG_COOKIE]: slug } }));
343+
344+ expect(response.status).toBe(302);
345+ expect(response.headers.get("location")).toBe("/");
346+ expect(refreshedInto).toEqual([last]);
347+ });
348+
349+ it("falls through an unknown last-org slug to the first active membership", async () => {
350+ const userId = freshId("user");
351+ const pendingOrg = freshId("org");
352+ const activeOrg = freshId("org");
353+ const { handler, refreshedInto } = signIn(workosUser(userId), [
354+ workosMembership(userId, pendingOrg, { status: "pending" }),
355+ workosMembership(userId, activeOrg),
356+ ]);
357+
358+ const response = await handler(
359+ // Valid slug grammar, never minted: the store finds no org for it.
360+ callbackRequest({ cookies: { [LAST_ORG_COOKIE]: "no-such-org-slug" } }),
361+ );
362+
363+ expect(response.status).toBe(302);
364+ expect(refreshedInto).toEqual([activeOrg]);
365+ });
366+ });
367+});
368+
369+describe("a delayed sign-in feeder", () => {
370+ /** The live mirror, user store, and directory over one test-db socket. */
371+ const Services = Layer.mergeAll(
372+ UserStoreService.Live,
373+ WorkOsMirror.Live,
374+ cloudMemberDirectoryLayer,
375+ ).pipe(Layer.provideMerge(DbService.Live));
376+
377+ const run = <A, E>(
378+ body: Effect.Effect<A, E, UserStoreService | WorkOsMirror | MemberDirectory | DbService>,
379+ ) => Effect.runPromise(body.pipe(Effect.provide(Services), Effect.scoped));
380+
381+ const readOrganization = (org: string) =>
382+ Effect.flatMap(UserStoreService.asEffect(), (users) =>
383+ users.use("getOrganization", (s) => s.getOrganization(org)),
384+ );
385+
386+ const readMembership = (userId: string, org: string) =>
387+ Effect.flatMap(MemberDirectory.asEffect(), (directory) =>
388+ directory.membership(userId, org, ["active", "pending", "inactive"]),
389+ );
390+
391+ it("cannot re-mint a purged organization or its membership from a list fetched before the purge", async () => {
392+ const userId = freshId("user");
393+ const org = freshId("org");
394+ await seedOrganization(org);
395+ const result = await run(
396+ Effect.gen(function* () {
397+ const users = yield* UserStoreService;
398+ // The login fetched its membership list at T1, while the org lived...
399+ const fetchedAt = new Date(T1);
400+ const listed = [workosMembership(userId, org)];
401+ // ...then stalled while cloud's deletion flow purged the org at T2.
402+ yield* users.use("deleteOrganizationCascade", (s) =>
403+ s.deleteOrganizationCascade(org, new Date(T2)),
404+ );
405+ // The stalled login resumes and writes what it holds.
406+ yield* mirrorSignIn(workosUser(userId), listed, fetchedAt);
407+ return {
408+ organization: yield* readOrganization(org),
409+ membership: yield* readMembership(userId, org),
410+ };
411+ }),
412+ );
413+ expect(result.organization?.deletedAt, "the org stays a deleted tombstone").toEqual(
414+ new Date(T2),
415+ );
416+ expect(result.membership, "and holds no membership: nothing to authorize").toBeNull();
417+ });
418+
419+ it("cannot reinstate a membership from a list fetched before a revocation the backfill has since scanned", async () => {
420+ const userId = freshId("user");
421+ const org = freshId("org");
422+ await seedOrganization(org);
423+ const result = await run(
424+ Effect.gen(function* () {
425+ const mirror = yield* WorkOsMirror;
426+ // The login fetched its membership list at T1, while the user was
427+ // a member, then stalled...
428+ const fetchedAt = new Date(T1);
429+ const listed = [workosMembership(userId, org)];
430+ // ...WorkOS revoked the membership before the mirror was ever
431+ // backfilled, and the backfill then scanned the org at T2 without
432+ // it: no tombstone, the row was never there — and the revocation
433+ // predates the events replay boundary, so no event will land it.
434+ yield* mirror.applyOrganizationScan({
435+ organizationId: org,
436+ listedAt: new Date(T2),
437+ members: [],
438+ });
439+ // The stalled login resumes and writes what it holds.
440+ yield* mirrorSignIn(workosUser(userId), listed, fetchedAt);
441+ const membership = yield* readMembership(userId, org);
442+ // A login after the scan, carrying a membership WorkOS created
443+ // since (stamped past the scan), is recorded.
444+ const rejoinedAt = "2026-01-03T00:00:00.000Z";
445+ yield* mirrorSignIn(
446+ workosUser(userId),
447+ [workosMembership(userId, org, { id: `om_${userId}_${org}_2`, updatedAt: rejoinedAt })],
448+ new Date(rejoinedAt),
449+ );
450+ return { membership, rejoined: yield* readMembership(userId, org) };
451+ }),
452+ );
453+ expect(result.membership, "the pre-scan list reinstates nothing").toBeNull();
454+ expect(result.rejoined?.membershipId, "a membership newer than the scan is recorded").toBe(
455+ `om_${userId}_${org}_2`,
456+ );
457+ });
458+
459+ it("cannot revert a rename from a list fetched before it, and applies a newer name", async () => {
460+ const userId = freshId("user");
461+ const org = freshId("org");
462+ await seedOrganization(org);
463+ const result = await run(
464+ Effect.gen(function* () {
465+ const users = yield* UserStoreService;
466+ // The org is renamed through Executor (write-through of the WorkOS
467+ // organization payload, stamped T2)...
468+ yield* users.use("upsertOrganization", (s) =>
469+ s.upsertOrganization({
470+ id: org,
471+ name: "Renamed Org",
472+ updatedAt: new Date(T2),
473+ }),
474+ );
475+ // ...after a login had fetched a list still carrying the old name at T1.
476+ yield* mirrorSignIn(
477+ workosUser(userId),
478+ [workosMembership(userId, org, { organizationName: `Org ${org}` })],
479+ new Date(T1),
480+ );
481+ const afterStale = yield* readOrganization(org);
482+ // A login whose list was fetched after the rename carries the new name.
483+ yield* mirrorSignIn(
484+ workosUser(userId),
485+ [
486+ workosMembership(userId, org, {
487+ organizationName: "Renamed Again",
488+ }),
489+ ],
490+ new Date("2026-01-03T00:00:00.000Z"),
491+ );
492+ const afterNewer = yield* readOrganization(org);
493+ return {
494+ afterStale,
495+ afterNewer,
496+ membership: yield* readMembership(userId, org),
497+ };
498+ }),
499+ );
500+ expect(result.afterStale?.name, "the stale list does not revert the rename").toBe(
501+ "Renamed Org",
502+ );
503+ expect(result.afterStale?.slug, "and the slug is untouched").toBe(result.afterNewer?.slug);
504+ expect(result.afterNewer?.name, "a list fetched after the rename is applied").toBe(
505+ "Renamed Again",
506+ );
507+ expect(result.membership?.status, "the membership itself is recorded either way").toBe(
508+ "active",
509+ );
510+ });
511+});
512+
513+describe("account service writes through to the mirror", () => {
514+ const ADMIN = freshId("user");
515+ const TARGET = freshId("user");
516+
517+ const session = (accountId: string) => ({
518+ accountId,
519+ email: `${accountId}@placeholder.test`,
520+ name: null,
521+ avatarUrl: null,
522+ organizationId: null,
523+ sealedSession: "sealed",
524+ refreshedSession: null,
525+ });
526+
527+ const stubApiKeys = Layer.succeed(ApiKeyService)({
528+ validate: () => Effect.die("membership writes do not validate keys"),
529+ listUserKeys: () => Effect.die("membership writes do not list keys"),
530+ createUserKey: () => Effect.die("membership writes do not create keys"),
531+ revokeUserKey: () => Effect.die("membership writes do not revoke keys"),
532+ listOrgKeys: () => Effect.die("membership writes do not list keys"),
533+ createOrgKey: () => Effect.die("membership writes do not create keys"),
534+ revokeOrgKey: () => Effect.die("membership writes do not revoke keys"),
535+ });
536+
537+ /**
538+ * The provider layer over the LIVE mirror + user store (test db) and a fake
539+ * WorkOS in which ADMIN administers `org` and TARGET is a plain member.
540+ * `deleted` records the WorkOS-side deletes so "WorkOS first" is assertable.
541+ * Provided around the WHOLE test body so the postgres socket outlives the
542+ * provider call under test.
543+ */
544+ const providerLayer = (org: string, deleted: string[]) => {
545+ const list = (data: readonly unknown[]) =>
546+ Effect.succeed({
547+ object: "list" as const,
548+ data: data as never[],
549+ listMetadata: { before: null, after: null },
550+ });
551+ const workos = stubWorkOS({
552+ listUserMemberships: (userId) => list([workosMembership(userId, org)]),
553+ getUserOrgMembership: (organizationId, userId) =>
554+ Effect.succeed(
555+ workosMembership(userId, organizationId, {
556+ role: { slug: userId === ADMIN ? "admin" : "member" },
557+ }) as never,
558+ ),
559+ getOrgMembership: (membershipId) =>
560+ Effect.succeed(workosMembership(TARGET, org, { id: membershipId }) as never),
561+ deleteOrgMembership: (membershipId) =>
562+ Effect.sync(() => {
563+ deleted.push(membershipId);
564+ }),
565+ updateOrgMembershipRole: (membershipId, roleSlug) =>
566+ Effect.succeed(
567+ workosMembership(TARGET, org, {
568+ id: membershipId,
569+ role: { slug: roleSlug },
570+ updatedAt: T2,
571+ }) as never,
572+ ),
573+ listOrgMembers: () => list([]),
574+ });
575+ // The test database serves ONE connection at a time, so the seed, the
576+ // provider, and the directory read all share this layer's socket.
577+ const stores = Layer.mergeAll(
578+ UserStoreService.Live,
579+ WorkOsMirror.Live,
580+ cloudMemberDirectoryLayer,
581+ );
582+ return workosAccountProvider.pipe(
583+ Layer.provide(
584+ Layer.mergeAll(
585+ workos,
586+ stubApiKeys,
587+ stubAutumn,
588+ Layer.succeed(AccountCaller)({ session: session(ADMIN) }),
589+ ),
590+ ),
591+ Layer.provideMerge(stores),
592+ Layer.provide(DbService.Live),
593+ );
594+ };
595+
596+ // TARGET as an existing member of `org`, seeded through the live mirror.
597+ const seedTarget = (org: string) =>
598+ Effect.gen(function* () {
599+ const users = yield* UserStoreService;
600+ const mirror = yield* WorkOsMirror;
601+ yield* users.use("upsertOrganization", (s) =>
602+ s.upsertOrganization({
603+ id: org,
604+ name: `Org ${org}`,
605+ updatedAt: new Date(T1),
606+ }),
607+ );
608+ yield* mirror.upsertMembership({
609+ id: `om_${TARGET}_${org}`,
610+ accountId: TARGET,
611+ organizationId: org,
612+ role: "member",
613+ status: "active",
614+ updatedAt: new Date(T1),
615+ });
616+ });
617+
618+ const membersOf = (org: string) =>
619+ Effect.flatMap(MemberDirectory.asEffect(), (directory) => directory.members(org));
620+
621+ it.effect("removeMember tombstones the mirror row after the WorkOS delete", () => {
622+ const org = freshId("org");
623+ const deleted: string[] = [];
624+ return Effect.gen(function* () {
625+ yield* seedTarget(org);
626+ const account = yield* AccountProvider;
627+
628+ const result = yield* account.removeMember(
629+ { [ORG_SELECTOR_HEADER]: org },
630+ `om_${TARGET}_${org}`,
631+ );
632+
633+ expect(result).toEqual({ success: true });
634+ expect(deleted, "WorkOS is the authority and is written first").toEqual([
635+ `om_${TARGET}_${org}`,
636+ ]);
637+ const members = yield* membersOf(org);
638+ expect(members.map((m) => m.accountId)).not.toContain(TARGET);
639+
640+ // A login or backfill that listed TARGET's membership BEFORE the
641+ // removal writes it afterwards: the tombstone refuses it.
642+ const mirror = yield* WorkOsMirror;
643+ const replayed = yield* mirror.upsertMembership({
644+ id: `om_${TARGET}_${org}`,
645+ accountId: TARGET,
646+ organizationId: org,
647+ role: "member",
648+ status: "active",
649+ updatedAt: new Date(T1),
650+ });
651+ expect(replayed, "the pre-removal payload is refused").toBe(false);
652+ expect((yield* membersOf(org)).map((m) => m.accountId)).not.toContain(TARGET);
653+
654+ // The tombstone carries the membership's last WorkOS stamp (T1), not
655+ // the wall clock at the delete: a replacement membership WorkOS
656+ // created for TARGET while the removal was in flight — stamped T2,
657+ // long before any clock this test runs under — is accepted.
658+ const replaced = yield* mirror.upsertMembership({
659+ id: `om_${TARGET}_${org}_2`,
660+ accountId: TARGET,
661+ organizationId: org,
662+ role: "member",
663+ status: "active",
664+ updatedAt: new Date(T2),
665+ });
666+ expect(replaced, "a replacement newer than the removed state is accepted").toBe(true);
667+ expect((yield* membersOf(org)).map((m) => m.accountId)).toContain(TARGET);
668+ }).pipe(Effect.provide(providerLayer(org, deleted)));
669+ });
670+
671+ it.effect("updateMemberRole writes the role WorkOS returned", () => {
672+ const org = freshId("org");
673+ return Effect.gen(function* () {
674+ yield* seedTarget(org);
675+ const account = yield* AccountProvider;
676+
677+ const result = yield* account.updateMemberRole(
678+ { [ORG_SELECTOR_HEADER]: org },
679+ `om_${TARGET}_${org}`,
680+ "admin",
681+ );
682+
683+ expect(result).toEqual({ success: true });
684+ const members = yield* membersOf(org);
685+ expect(members.find((m) => m.accountId === TARGET)?.role).toBe("admin");
686+ }).pipe(Effect.provide(providerLayer(org, [])));
687+ });
688+});
689+
690+describe("backfill", () => {
691+ /** A fake WorkOS holding `orgs` → members, counting `getUser` calls. */
692+ const source = (orgs: ReadonlyMap<string, readonly FakeMembership[]>, userCalls: string[]) => ({
693+ listOrganizationIds: () => Effect.succeed([...orgs.keys()]),
694+ listOrgMembers: (organizationId: string) => Effect.succeed(orgs.get(organizationId) ?? []),
695+ getUser: (userId: string) =>
696+ Effect.sync(() => {
697+ userCalls.push(userId);
698+ return workosUser(userId);
699+ }),
700+ });
701+
702+ const withMirror = <A, E>(body: (mirror: WorkOsMirrorShape) => Effect.Effect<A, E>) =>
703+ Effect.runPromise(
704+ Effect.flatMap(WorkOsMirror.asEffect(), body).pipe(
705+ Effect.provide(WorkOsMirror.Live.pipe(Layer.provide(DbService.Live))),
706+ Effect.scoped,
707+ ),
708+ );
709+
710+ const runBackfill = (
711+ orgs: ReadonlyMap<string, readonly FakeMembership[]>,
712+ dryRun: boolean,
713+ userCalls: string[] = [],
714+ ) =>
715+ withMirror((mirror) =>
716+ backfillWorkOsMirror(source(orgs, userCalls), mirror, {
717+ dryRun,
718+ log: () => undefined,
719+ }),
720+ );
721+
722+ /** The instance-wide replay boundary a completed run records. */
723+ const syncState = () => withMirror((mirror) => mirror.replayBoundary());
724+
725+ /** When a run first covered every organization, or null: the authorization gate's first half. */
726+ const completedAt = () => withMirror((mirror) => mirror.backfillCompletedAt());
727+
728+ /** Drop the instance-wide events row, so the run under test is the first ever. */
729+ const clearEventsRow = () =>
730+ Effect.runPromise(
731+ Effect.flatMap(DbService.asEffect(), ({ db }) =>
732+ Effect.promise(() => db.execute(sql`delete from workos_sync where id = 'events'`)),
733+ ).pipe(Effect.provide(DbService.Live), Effect.scoped),
734+ );
735+
736+ const backfilledAt = (org: string) =>
737+ withMirror((mirror) => mirror.organizationBackfilledAt(org));
738+
739+ it("records the replay boundary on first completion, marks and mirrors every organization's members, counts the writes, converges and repairs on a re-run", async () => {
740+ const orgA = freshId("org");
741+ const orgB = freshId("org");
742+ await seedOrganization(orgA);
743+ await seedOrganization(orgB);
744+ const shared = freshId("user");
745+ const leaving = freshId("user");
746+ const orgs = new Map([
747+ [orgA, [workosMembership(shared, orgA), workosMembership(leaving, orgA)]],
748+ [orgB, [workosMembership(shared, orgB, { status: "pending" })]],
749+ ]);
750+
751+ // The boundary is instance-wide (migration 0019 seeds it on the empty
752+ // test database, other tests may have written it): start as a database
753+ // that has never been backfilled.
754+ await clearEventsRow();
755+ const startedAt = Date.now();
756+
757+ const dry = await runBackfill(orgs, true);
758+ expect(dry).toEqual({
759+ organizations: 2,
760+ memberships: 3,
761+ usersWritten: 0,
762+ membershipsWritten: 0,
763+ membershipsTombstoned: 0,
764+ });
765+ expect(await readMembers(orgA), "a dry run writes nothing").toEqual([]);
766+ expect(await syncState(), "a dry run records no boundary").toBeNull();
767+ expect(await completedAt(), "nor a completion").toBeNull();
768+ expect(await backfilledAt(orgA), "and marks nothing").toBeNull();
769+
770+ const userCalls: string[] = [];
771+ const first = await runBackfill(orgs, false, userCalls);
772+ expect(first).toEqual({
773+ organizations: 2,
774+ memberships: 3,
775+ usersWritten: 3,
776+ membershipsWritten: 3,
777+ membershipsTombstoned: 0,
778+ });
779+ expect(userCalls, "one getUser per membership").toHaveLength(3);
780+ expect((await readMembers(orgA)).map((m) => m.status)).toEqual(["active", "active"]);
781+ expect((await readMembers(orgB)).map((m) => m.status)).toEqual(["pending"]);
782+ const after = await syncState();
783+ expect(after, "the run records where the events replay starts").not.toBeNull();
784+ const firstCompletion = await completedAt();
785+ expect(firstCompletion, "and that every organization is now covered").not.toBeNull();
786+ expect(firstCompletion!.getTime()).toBeGreaterThanOrEqual(after!.getTime());
787+ expect(after!.getTime()).toBeGreaterThanOrEqual(startedAt);
788+ for (const org of [orgA, orgB]) {
789+ const marked = await backfilledAt(org);
790+ expect(marked, "each scanned organization is marked as of its listing").not.toBeNull();
791+ expect(marked!.getTime()).toBeGreaterThanOrEqual(after!.getTime());
792+ }
793+
794+ // Same payloads again, minus one member WorkOS no longer lists: the
795+ // `updatedAt` guard lets equal payloads through (replays converge), and
796+ // the missing membership is tombstoned — a re-run repairs a stale row.
797+ orgs.set(orgA, [workosMembership(shared, orgA)]);
798+ const again = await runBackfill(orgs, false);
799+ expect(again).toMatchObject({ memberships: 2, membershipsTombstoned: 1 });
800+ expect(
801+ await syncState(),
802+ "the re-run keeps the first boundary: an org rename or user deletion between the two runs is only in the events stream",
803+ ).toEqual(after);
804+ expect(await completedAt(), "and the first completion").toEqual(firstCompletion);
805+ expect(new Map((await readMembers(orgA)).map((m) => [m.accountId, m.status]))).toEqual(
806+ new Map([
807+ [shared, "active"],
808+ [leaving, "inactive"],
809+ ]),
810+ );
811+ expect(
812+ (await readMembers(orgB)).map((m) => m.status),
813+ "the other org is untouched",
814+ ).toEqual(["pending"]);
815+ // The tombstone is keyed to the deleted membership id, so the member's
816+ // pre-removal payload (as a late login or event would carry) is refused.
817+ const replayed = await withMirror((mirror) =>
818+ mirror.upsertMembership({
819+ id: `om_${leaving}_${orgA}`,
820+ accountId: leaving,
821+ organizationId: orgA,
822+ role: "member",
823+ status: "active",
824+ updatedAt: new Date(T1),
825+ }),
826+ );
827+ expect(replayed).toBe(false);
828+ });
829+
830+ it("keeps a membership WorkOS merely deactivated as inactive, not tombstoned, so a later reactivation lands", async () => {
831+ const org = freshId("org");
832+ await seedOrganization(org);
833+ const paused = freshId("user");
834+ // Mirrored while active (a sign-in), then deactivated in WorkOS: the
835+ // listing carries the membership under the SAME id with its real status.
836+ await withMirror((mirror) =>
837+ mirror.upsertMembership({
838+ id: `om_${paused}_${org}`,
839+ accountId: paused,
840+ organizationId: org,
841+ role: "member",
842+ status: "active",
843+ updatedAt: new Date(T1),
844+ }),
845+ );
846+
847+ const counts = await runBackfill(
848+ new Map([[org, [workosMembership(paused, org, { status: "inactive", updatedAt: T2 })]]]),
849+ false,
850+ );
851+ expect(counts, "the deactivated membership is written, not tombstoned").toMatchObject({
852+ memberships: 1,
853+ membershipsWritten: 1,
854+ membershipsTombstoned: 0,
855+ });
856+ expect((await readMembers(org)).map((m) => [m.accountId, m.status])).toEqual([
857+ [paused, "inactive"],
858+ ]);
859+
860+ // WorkOS reactivates it under the same id AFTER the scan (so the payload
861+ // is stamped past the org's `backfilled_at`): an ordinary newer payload,
862+ // which a tombstone keyed to that id would have refused for good.
863+ const reactivated = await withMirror((mirror) =>
864+ mirror.upsertMembership({
865+ id: `om_${paused}_${org}`,
866+ accountId: paused,
867+ organizationId: org,
868+ role: "member",
869+ status: "active",
870+ updatedAt: new Date(Date.now() + 60 * 1000),
871+ }),
872+ );
873+ expect(reactivated).toBe(true);
874+ expect((await readMembers(org)).map((m) => m.status)).toEqual(["active"]);
875+ });
876+
877+ it("leaves a membership written after its listing alone when tombstoning what the listing lacks", async () => {
878+ const org = freshId("org");
879+ await seedOrganization(org);
880+ const listed = freshId("user");
881+ const stale = freshId("user");
882+ const joinedMeanwhile = freshId("user");
883+ // Both rows are absent from the listing below. `stale` is stamped before
884+ // the listing (a genuine leaver); `joinedMeanwhile` carries a stamp AFTER
885+ // any listing this run can take — the membership WorkOS created between
886+ // the listing and the cleanup, whose own event lands via the reconciler.
887+ const afterListing = new Date(Date.now() + 60 * 60 * 1000);
888+ await withMirror((mirror) =>
889+ Effect.all([
890+ mirror.upsertMembership({
891+ id: `om_${stale}_${org}`,
892+ accountId: stale,
893+ organizationId: org,
894+ role: "member",
895+ status: "active",
896+ updatedAt: new Date(T1),
897+ }),
898+ mirror.upsertMembership({
899+ id: `om_${joinedMeanwhile}_${org}`,
900+ accountId: joinedMeanwhile,
901+ organizationId: org,
902+ role: "member",
903+ status: "active",
904+ updatedAt: afterListing,
905+ }),
906+ ]),
907+ );
908+
909+ const counts = await runBackfill(new Map([[org, [workosMembership(listed, org)]]]), false);
910+
911+ expect(counts).toMatchObject({ memberships: 1, membershipsTombstoned: 1 });
912+ expect(new Map((await readMembers(org)).map((m) => [m.accountId, m.status]))).toEqual(
913+ new Map([
914+ [listed, "active"],
915+ [stale, "inactive"],
916+ [joinedMeanwhile, "active"],
917+ ]),
918+ );
919+ });
920+
921+ it("keeps the marks of the organizations it finished but records no replay boundary when a run fails part-way", async () => {
922+ const orgA = freshId("org");
923+ const orgB = freshId("org");
924+ await seedOrganization(orgA);
925+ await seedOrganization(orgB);
926+ const member = freshId("user");
927+ const orgs = new Map([
928+ [orgA, [workosMembership(member, orgA)]],
929+ [orgB, [workosMembership(member, orgB)]],
930+ ]);
931+ // A completed run first, so there IS a boundary to protect.
932+ await runBackfill(orgs, false);
933+ const completed = await syncState();
934+ expect(completed).not.toBeNull();
935+ const markedA = await backfilledAt(orgA);
936+
937+ // A re-run whose second organization fails on a WorkOS read: the first
938+ // org was written, but the run as a whole did not complete — and even a
939+ // completed one would keep the first boundary.
940+ const failing = {
941+ ...source(orgs, []),
942+ listOrgMembers: (organizationId: string) =>
943+ organizationId === orgB
944+ ? Effect.fail(new WorkOSError({ status: 503 }))
945+ : Effect.succeed(orgs.get(organizationId) ?? []),
946+ };
947+ const exit = await withMirror((mirror) =>
948+ Effect.exit(
949+ backfillWorkOsMirror(failing, mirror, {
950+ dryRun: false,
951+ log: () => undefined,
952+ }),
953+ ),
954+ );
955+ expect(Exit.isFailure(exit), "the run fails rather than skipping the org").toBe(true);
956+ expect(await syncState(), "the completed run's boundary stands").toEqual(completed);
957+ expect(await completedAt(), "and so does its completion mark").not.toBeNull();
958+ expect(
959+ (await backfilledAt(orgA))!.getTime(),
960+ "the org the failed run did finish is marked as of its new listing",
961+ ).toBeGreaterThanOrEqual(markedA!.getTime());
962+ });
963+
964+ it("scans one organization on demand and marks only that one", async () => {
965+ const org = freshId("org");
966+ const other = freshId("org");
967+ await seedOrganization(org);
968+ await seedOrganization(other);
969+ const member = freshId("user");
970+ const orgs = new Map([
971+ [org, [workosMembership(member, org)]],
972+ [other, [workosMembership(member, other)]],
973+ ]);
974+
975+ const counts = await withMirror((mirror) =>
976+ backfillOrganization(source(orgs, []), mirror, org, { dryRun: false }),
977+ );
978+
979+ expect(counts).toEqual({
980+ applied: true,
981+ memberships: 1,
982+ usersWritten: 1,
983+ membershipsWritten: 1,
984+ membershipsTombstoned: 0,
985+ });
986+ expect((await readMembers(org)).map((m) => m.accountId)).toEqual([member]);
987+ expect(await backfilledAt(org)).not.toBeNull();
988+ expect(await readMembers(other), "the other organization is not scanned").toEqual([]);
989+ expect(await backfilledAt(other), "nor marked").toBeNull();
990+ });
991+
992+ it("refuses a scan that stalled while a later scan found a membership gone, so the revoked member stays revoked", async () => {
993+ const org = freshId("org");
994+ await seedOrganization(org);
995+ const staying = freshId("user");
996+ const leaving = freshId("user");
997+ const result = await withMirror((mirror) =>
998+ Effect.gen(function* () {
999+ // Scan A lists the org while `leaving` is still a member, then
1000+ // stalls (its listing is held behind the latch)...
1001+ const listedByA = yield* Latch.make(false);
1002+ const stalled = {
1003+ ...source(new Map(), []),
1004+ listOrgMembers: () =>
1005+ listedByA.await.pipe(
1006+ Effect.as([workosMembership(staying, org), workosMembership(leaving, org)]),
1007+ ),
1008+ };
1009+ const scanA = yield* Effect.forkChild(
1010+ backfillOrganization(stalled, mirror, org, { dryRun: false }),
1011+ { startImmediately: true },
1012+ );
1013+ // ...WorkOS removes `leaving`, and scan B lists and applies the
1014+ // org without them — no tombstone, the row was never there...
1015+ const b = yield* backfillOrganization(
1016+ source(new Map([[org, [workosMembership(staying, org)]]]), []),
1017+ mirror,
1018+ org,
1019+ { dryRun: false },
1020+ );
1021+ // ...then A resumes with its older listing.
1022+ yield* listedByA.open;
1023+ const a = yield* Fiber.join(scanA);
1024+ return { a, b };
1025+ }),
1026+ );
1027+ expect(result.b).toMatchObject({ applied: true, membershipsWritten: 1 });
1028+ expect(result.a, "the older listing is refused whole").toMatchObject({
1029+ applied: false,
1030+ memberships: 2,
1031+ usersWritten: 0,
1032+ membershipsWritten: 0,
1033+ membershipsTombstoned: 0,
1034+ });
1035+ expect(
1036+ new Map((await readMembers(org)).map((m) => [m.accountId, m.status])),
1037+ "the member the later listing no longer had was never inserted",
1038+ ).toEqual(new Map([[staying, "active"]]));
1039+ });
1040+});
apps/cloud/src/auth/mirror-feeders.ts+80−0
@@ -0,0 +1,80 @@
1+// ---------------------------------------------------------------------------
2+// The membership mirror's request-path FEEDERS: the writes the login callback
3+// and the Executor-initiated membership changes make through `WorkOsMirror`.
4+//
5+// Each feeder takes the WorkOS payload the caller ALREADY holds (the
6+// authenticated user, the membership list the callback fetches to pick a
7+// landing org, the membership a write returned) so feeding the mirror never
8+// adds a WorkOS read. Mirror failures fail the request: the mirror is the
9+// membership read path, so a login that could not record its memberships is
10+// not a login that finished.
11+// ---------------------------------------------------------------------------
12+
13+import { Effect } from "effect";
14+
15+import { UserStoreService } from "./context";
16+import {
17+ WorkOsMirror,
18+ mirrorMembershipFromWorkOs,
19+ mirrorUserFromWorkOs,
20+ type WorkOsMembershipPayload,
21+ type WorkOsUserPayload,
22+} from "./workos-mirror";
23+
24+/**
25+ * A membership as WorkOS lists it for a user: carries the organization's name,
26+ * which is what lets the sign-in feeder mirror the org row without a
27+ * `getOrganization` call. `OrganizationMembership` from the SDK satisfies it.
28+ */
29+export interface WorkOsSignInMembership extends WorkOsMembershipPayload {
30+ readonly organizationName: string;
31+}
32+
33+/**
34+ * Record a sign-in: the user's profile, then every organization WorkOS lists
35+ * them in (the org row first, so the membership's foreign key holds) and the
36+ * membership itself. Replays converge: every write is guarded on WorkOS
37+ * `updatedAt`, so a second login with the same payload changes nothing.
38+ *
39+ * `fetchedAt` is the instant the membership list was requested — taken
40+ * BEFORE the WorkOS read, so nothing that changed after it can be mistaken
41+ * for older. A membership list names each organization but carries no
42+ * organization timestamp, so `fetchedAt` is the stamp its name is written
43+ * under: a list fetched before a rename (a login that stalled) cannot revert
44+ * the rename after it landed. A membership of an organization the mirror
45+ * holds as deleted is refused by the mirror, and the organization is neither
46+ * re-minted nor renamed: a list fetched before a deletion cannot restore the
47+ * organization after its purge. And a membership stamped before the
48+ * organization's last full scan (`organizations.backfilled_at`) is refused
49+ * too: a list fetched before a revocation and written after the scan that
50+ * found the membership gone cannot reinstate it.
51+ */
52+export const mirrorSignIn = Effect.fn("workos_mirror.signIn")(function* (
53+ user: WorkOsUserPayload,
54+ memberships: readonly WorkOsSignInMembership[],
55+ fetchedAt: Date,
56+) {
57+ const mirror = yield* WorkOsMirror;
58+ const users = yield* UserStoreService;
59+ yield* mirror.upsertUser(mirrorUserFromWorkOs(user));
60+ for (const membership of memberships) {
61+ yield* users.use("upsertOrganization", (s) =>
62+ s.upsertOrganization({
63+ id: membership.organizationId,
64+ name: membership.organizationName,
65+ updatedAt: fetchedAt,
66+ }),
67+ );
68+ yield* mirror.upsertMembership(mirrorMembershipFromWorkOs(membership));
69+ }
70+});
71+
72+/**
73+ * Record one membership WorkOS just returned to a write (create, role
74+ * change, invitation acceptance). The organization must already be mirrored;
75+ * every caller has just upserted it or resolved it through the mirror.
76+ */
77+export const mirrorMembership = (membership: WorkOsMembershipPayload) =>
78+ Effect.flatMap(WorkOsMirror.asEffect(), (mirror) =>
79+ mirror.upsertMembership(mirrorMembershipFromWorkOs(membership)),
80+ );
apps/cloud/src/auth/org-api-key-auth.node.test.ts+13−1
@@ -83,18 +83,27 @@ const stubUsers = Layer.succeed(UserStoreService)({ · 82 unmodified lines
8383 upsertOrganization: async (org: { id: string; name: string }) => ({
8484 ...org,
8585 slug: `org-slug-${org.id}`,
86+ backfilledAt: null,
87+ deletedAt: null,
88+ workosUpdatedAt: null,
8689 createdAt,
8790 }),
8891 getOrganization: async (id: string) => ({
8992 id,
9093 name: `Org ${id}`,
9194 slug: `org-slug-${id}`,
95+ backfilledAt: null,
96+ deletedAt: null,
97+ workosUpdatedAt: null,
9298 createdAt,
9399 }),
94100 getOrganizationBySlug: async (slug: string) => ({
95101 id: "org_by_slug",
96102 name: `Org ${slug}`,
97103 slug,
104+ backfilledAt: null,
105+ deletedAt: null,
106+ workosUpdatedAt: null,
98107 createdAt,
99108 }),
100109 deleteOrganizationCascade: async () => {},
@@ -133,7 +142,10 @@ describe("org-level API keys", () => { · 32 unmodified lines
133142 const auth = yield* resolveBearerAuth(bearer("valid_user_key")).pipe(Effect.provide(layers));
134143
135144 expect(isPlatformAuth(auth)).toBe(false);
136− expect(auth).toMatchObject({ accountId: "user_123", organizationId: "org_123" });
145+ expect(auth).toMatchObject({
146+ accountId: "user_123",
147+ organizationId: "org_123",
148+ });
137149 }),
138150 );
139151
apps/cloud/src/auth/org-selector-auth.node.test.ts+9−0
@@ -86,12 +86,18 @@ const stubUsers = Layer.succeed(UserStoreService)({ · 85 unmodified lines
8686 upsertOrganization: async (org: { id: string; name: string }) => ({
8787 ...org,
8888 slug: org.id,
89+ backfilledAt: null,
90+ deletedAt: null,
91+ workosUpdatedAt: null,
8992 createdAt,
9093 }),
9194 getOrganization: async (id: string) => ({
9295 id,
9396 name: `Org ${id}`,
9497 slug: id,
98+ backfilledAt: null,
99+ deletedAt: null,
100+ workosUpdatedAt: null,
95101 createdAt,
96102 }),
97103 // The URL slug maps to URL_ORG (the member's other org); any other slug
@@ -100,6 +106,9 @@ const stubUsers = Layer.succeed(UserStoreService)({ · 2 unmodified lines
100106 id: slug === URL_SLUG ? URL_ORG : "org_outsider",
101107 name: `Org ${slug}`,
102108 slug,
109+ backfilledAt: null,
110+ deletedAt: null,
111+ workosUpdatedAt: null,
103112 createdAt,
104113 }),
105114 deleteOrganizationCascade: async () => {},
apps/cloud/src/auth/organization.ts+5−1
@@ -44,7 +44,11 @@ export const resolveOrganization = (organizationId: string) => · 43 unmodified lines
4444 const workos = yield* WorkOSClient;
4545 const fresh = yield* workos.getOrganization(organizationId);
4646 return yield* users.use("upsertOrganization", (s) =>
47− s.upsertOrganization({ id: fresh.id, name: fresh.name }),
47+ s.upsertOrganization({
48+ id: fresh.id,
49+ name: fresh.name,
50+ updatedAt: new Date(fresh.updatedAt),
51+ }),
4852 );
4953 });
5054
apps/cloud/src/auth/user-store.ts+49−12
@@ -7,7 +7,7 @@ · 6 unmodified lines
77 // so domain tables can foreign-key against them and so we can resolve org
88 // metadata without an API call on every request.
99
10−import { eq } from "drizzle-orm";
10+import { and, eq, isNull, lte, or } from "drizzle-orm";
1111
1212 import { generateOrgSlug } from "@executor-js/api";
1313
@@ -18,6 +18,29 @@ import { purgeOrganizationData } from "../db/org-deletion"; · 4 unmodified lines
1818 export type Account = typeof accounts.$inferSelect;
1919 export type Organization = typeof organizations.$inferSelect;
2020
21+/**
22+ * An organization as a feeder hands it to the mirror. `updatedAt` is when
23+ * `name` is known to have been the organization's name in WorkOS: the WorkOS
24+ * `updatedAt` of an organization payload, or the instant a membership list
25+ * naming the organization was fetched (a list carries the name but no
26+ * organization timestamp).
27+ */
28+export interface OrganizationPayload {
29+ readonly id: string;
30+ readonly name: string;
31+ readonly updatedAt: Date;
32+}
33+
34+/**
35+ * Which stored organization rows a name stamped `updatedAt` may rename: a
36+ * row with no stamp (predating the stamp), or one stamped at or before
37+ * `updatedAt` — feeders replay the same payload and must converge. Every
38+ * writer of `organizations.name` applies this, so a name fetched before a
39+ * rename can never revert the rename after it landed.
40+ */
41+export const organizationAcceptsName = (updatedAt: Date) =>
42+ or(isNull(organizations.workosUpdatedAt), lte(organizations.workosUpdatedAt, updatedAt));
43+
2144 export const makeUserStore = (db: DrizzleDb) => {
2245 const getOrganization = async (id: string) => {
2346 const rows = await db.select().from(organizations).where(eq(organizations.id, id));
@@ -38,10 +61,15 @@ export const makeUserStore = (db: DrizzleDb) => { · 14 unmodified lines
3861 // candidate was claimed by a different org). Returns the inserted row, or
3962 // null when either conflict swallowed the insert — the caller decides whether
4063 // to re-read (id race) or retry with a new candidate (slug race).
41− const tryInsertOrg = async (id: string, name: string, slug: string) => {
64+ const tryInsertOrg = async (org: OrganizationPayload, slug: string) => {
4265 const [row] = await db
4366 .insert(organizations)
44− .values({ id, name, slug })
67+ .values({
68+ id: org.id,
69+ name: org.name,
70+ slug,
71+ workosUpdatedAt: org.updatedAt,
72+ })
4573 .onConflictDoNothing()
4674 .returning();
4775 return row ?? null;
@@ -49,20 +77,27 @@ export const makeUserStore = (db: DrizzleDb) => { · 1 unmodified line
4977
5078 // Every new org row is born with a slug — there is no nullable window and no
5179 // self-healing. Existing rows keep their slug (stable across renames, so org
52− // URLs survive) and only refresh their name.
53− const upsertOrganization = async (org: { id: string; name: string }) => {
80+ // URLs survive) and only refresh their name — and only from a payload at
81+ // least as new as the one that last named it (`organizationAcceptsName`):
82+ // a sign-in whose membership list was fetched before a rename would
83+ // otherwise revert the rename after it landed. A row marked deleted is
84+ // returned as it is: the organization is gone, and nothing a feeder still
85+ // holds about it (a name, a membership fetched before the deletion) is
86+ // written — never re-minted live, never renamed.
87+ const upsertOrganization = async (org: OrganizationPayload) => {
5488 const existing = await getOrganization(org.id);
5589 if (existing) {
90+ if (existing.deletedAt !== null) return existing;
5691 const [updated] = await db
5792 .update(organizations)
58− .set({ name: org.name })
59− .where(eq(organizations.id, org.id))
93+ .set({ name: org.name, workosUpdatedAt: org.updatedAt })
94+ .where(and(eq(organizations.id, org.id), organizationAcceptsName(org.updatedAt)))
6095 .returning();
6196 return updated ?? existing;
6297 }
6398 for (let attempt = 0; attempt < 4; attempt++) {
6499 const slug = await generateOrgSlug(org.name, slugTaken);
65− const inserted = await tryInsertOrg(org.id, org.name, slug);
100+ const inserted = await tryInsertOrg(org, slug);
66101 if (inserted) return inserted;
67102 // The insert was swallowed by a conflict. If the id now exists, a
68103 // concurrent request mirrored it — return that row. Otherwise the slug
@@ -98,9 +133,11 @@ export const makeUserStore = (db: DrizzleDb) => { · 29 unmodified lines
98133 return rows[0] ?? null;
99134 },
100135
101− // Permanently delete an org and everything it owns (tenant data, secrets,
102− // identity mirror + cascaded memberships) in a single transaction. Callers
103− // sequence the external WorkOS/Autumn deletions around this.
104− deleteOrganizationCascade: (id: string) => purgeOrganizationData(db, id),
136+ // Permanently delete everything an org owns (tenant data, secrets, its
137+ // memberships) in a single transaction, leaving the organization row as
138+ // a tombstone marked `deletedAt` (see `purgeOrganizationData` for why).
139+ // Callers sequence the external WorkOS/Autumn deletions around this.
140+ deleteOrganizationCascade: (id: string, deletedAt: Date) =>
141+ purgeOrganizationData(db, id, deletedAt),
105142 };
106143 };
apps/cloud/src/auth/workos-callback-state.node.test.ts+28−0
@@ -20,6 +20,7 @@ import { CloudAuthPublicHandlers } from "./handlers"; · 19 unmodified lines
2020 import { CloudAuthPublicApi } from "./api";
2121 import { UserStoreService } from "./context";
2222 import { WorkOSClient, type WorkOSClientService } from "./workos";
23+import { WorkOsMirror } from "./workos-mirror";
2324 import { encodeLoginState } from "./login-state";
2425 import { AutumnService } from "../extensions/billing/service";
2526
@@ -79,18 +80,27 @@ const stubUsers = Layer.succeed(UserStoreService)({ · 53 unmodified lines
7980 upsertOrganization: async (org: { id: string; name: string }) => ({
8081 ...org,
8182 slug: org.id,
83+ backfilledAt: null,
84+ deletedAt: null,
85+ workosUpdatedAt: null,
8286 createdAt: new Date(),
8387 }),
8488 getOrganization: async (id: string) => ({
8589 id,
8690 name: "Org " + id,
8791 slug: id,
92+ backfilledAt: null,
93+ deletedAt: null,
94+ workosUpdatedAt: null,
8895 createdAt: new Date(),
8996 }),
9097 getOrganizationBySlug: async (slug: string) => ({
9198 id: slug,
9299 name: slug,
93100 slug,
101+ backfilledAt: null,
102+ deletedAt: null,
103+ workosUpdatedAt: null,
94104 createdAt: new Date(),
95105 }),
96106 deleteOrganizationCascade: async () => {},
@@ -98,6 +108,23 @@ const stubUsers = Layer.succeed(UserStoreService)({ · 1 unmodified line
98108 ),
99109 });
100110
111+// The callback records the sign-in (user + memberships) in the membership
112+// mirror; every other mirror operation is out of this route's reach.
113+const stubMirror = Layer.succeed(WorkOsMirror)({
114+ upsertUser: () => Effect.succeed(true),
115+ upsertMembership: () => Effect.succeed(true),
116+ deleteMembership: () => Effect.die("the callback does not delete memberships"),
117+ deleteUser: () => Effect.die("the callback does not delete users"),
118+ getCursor: () => Effect.die("the callback does not read the events cursor"),
119+ setCursor: () => Effect.die("the callback does not move the events cursor"),
120+ applyOrganizationScan: () => Effect.die("the callback does not run the backfill"),
121+ replayBoundary: () => Effect.die("the callback does not run the reconciler"),
122+ setReplayBoundary: () => Effect.die("the callback does not run the backfill"),
123+ backfillCompletedAt: () => Effect.die("the callback does not check mirror readiness"),
124+ markBackfillCompleted: () => Effect.die("the callback does not run the backfill"),
125+ organizationBackfilledAt: () => Effect.die("the callback does not report seats"),
126+});
127+
101128 // Only the public group is under test; the session group (and its SessionAuth
102129 // middleware, which needs a live DB) is out of scope — the callback route lives
103130 // in CloudAuthPublicApi and requires no middleware.
@@ -107,6 +134,7 @@ const App = HttpApiBuilder.layer(PublicApi).pipe( · 3 unmodified lines
107134 Layer.provide(CloudAuthPublicHandlers),
108135 Layer.provide(stubWorkOS),
109136 Layer.provide(stubUsers),
137+ Layer.provide(stubMirror),
110138 Layer.provide(AutumnService.Default),
111139 Layer.provide(HttpServer.layerServices),
112140 );
apps/cloud/src/auth/workos-mirror-backfill.ts+274−0
@@ -0,0 +1,274 @@
1+// ---------------------------------------------------------------------------
2+// Backfill of the membership mirror from WorkOS, one organization at a time:
3+// list EVERY membership WorkOS holds for it — active, pending, and inactive
4+// alike — fetch each member's user, write both through the mirror's guarded
5+// upserts, tombstone whatever the mirror still holds that WorkOS no longer
6+// lists, and mark the organization BACKFILLED as of the listing. Inactive
7+// memberships are listed on purpose: the scan tombstones every mirrored
8+// membership its listing lacks, and a tombstone is keyed to the membership
9+// id for good (`membershipAcceptsPayload`), so a listing that skipped the
10+// inactive ones would tombstone a membership WorkOS merely deactivated and
11+// refuse its reactivation — under the same id — forever. Listed with its
12+// real status it stays an ordinary `inactive` row that a newer payload
13+// reactivates. The core is a pure function over a `source`
14+// (WorkOS reads) and the mirror store, so the one-off script
15+// (`scripts/backfill-workos-mirror.ts`) can wire real clients, the request
16+// path can wire `WorkOSClient` (an organization whose mark is missing is
17+// scanned on demand before its seats are counted), and the test can wire
18+// fakes against the test database.
19+//
20+// Idempotent and repairing: the upserts are guarded on WorkOS `updatedAt`,
21+// so a re-scan over unchanged data writes nothing new, and every membership
22+// the mirror holds for the org that WorkOS no longer lists is TOMBSTONED
23+// (`inactive`, `deleted_at` = the time the listing was taken) — so a re-scan
24+// repairs a stale row instead of leaving it granting access. `dryRun` reads
25+// everything and writes nothing, so the printed counts are the plan.
26+//
27+// One scan is applied in ONE transaction (`WorkOsMirror.applyOrganizationScan`)
28+// that first moves the organization's `backfilled_at` forward to the
29+// listing's instant and writes nothing if a LATER listing already did. Two
30+// scans of the same organization can overlap (the one-off script and an
31+// on-demand scan from a request, or a stalled script run and its retry), and
32+// the `updatedAt` guard alone cannot order them: a scan that listed a
33+// membership, stalled, and resumed after a later scan had found it gone
34+// would insert it live — the later scan tombstoned nothing, because the row
35+// was not there to tombstone. Refusing the older listing whole is what keeps
36+// a membership revoked between the two listings revoked.
37+//
38+// Completeness is tracked PER ORGANIZATION (`organizations.backfilled_at`),
39+// never database-wide: the mark is written only by a scan that listed that
40+// organization's memberships in full, so an organization mirrored after a
41+// backfill ran (lazily by a request, or by a sign-in that records only the
42+// caller's own membership) starts unmarked and is scanned before any count
43+// read from the mirror is trusted. A run over every organization
44+// (`backfillWorkOsMirror`) additionally records the Events API replay
45+// boundary — the instant it began reading WorkOS, taken BEFORE anything is
46+// listed so no change can fall between the boundary and a listing — once
47+// every organization was written, and only if no boundary is recorded yet.
48+// A run that fails part-way records nothing, and a later completed run
49+// keeps the first boundary: a scan refreshes memberships and tombstones,
50+// not organization names or deleted users' profiles, so an
51+// `organization.updated` or `user.deleted` between two runs is covered only
52+// by the events stream — advancing the boundary past it would skip it for
53+// good. The reconciler's own cursor takes over from the boundary after its
54+// first page, so the boundary's only job is to name where that first page
55+// starts.
56+//
57+// The mark also orders every OTHER membership write against the scan: the
58+// mirror refuses a membership payload stamped before the organization's
59+// `backfilled_at` (`upsertMembership`). A login whose membership list was
60+// fetched before a revocation and written after the scan would otherwise
61+// reinstate the revoked membership — and that revocation predates the
62+// events replay boundary, so no event would ever tombstone it again.
63+// ---------------------------------------------------------------------------
64+
65+import { Clock, Effect, Option } from "effect";
66+
67+import {
68+ mirrorMembershipFromWorkOs,
69+ mirrorUserFromWorkOs,
70+ type WorkOsMembershipPayload,
71+ type WorkOsMirrorShape,
72+ type WorkOsOrganizationScanWrites,
73+ type WorkOsUserPayload,
74+} from "./workos-mirror-store";
75+
76+/** The WorkOS reads one organization's scan performs, over whatever client the caller wires. */
77+export interface WorkOsOrganizationScanSource<E> {
78+ /**
79+ * EVERY membership of one organization, all pages and all statuses
80+ * (active, pending, inactive). A source that filtered by status would
81+ * have the scan tombstone what it filtered out — see the header.
82+ */
83+ readonly listOrgMembers: (
84+ organizationId: string,
85+ ) => Effect.Effect<readonly WorkOsMembershipPayload[], E>;
86+ readonly getUser: (userId: string) => Effect.Effect<WorkOsUserPayload, E>;
87+}
88+
89+/** The reads the full backfill performs: every organization, then each one's scan. */
90+export interface WorkOsMirrorBackfillSource<E> extends WorkOsOrganizationScanSource<E> {
91+ /** Every organization id the mirror knows (FK target of `memberships`). */
92+ readonly listOrganizationIds: () => Effect.Effect<readonly string[], E>;
93+}
94+
95+export interface WorkOsMirrorBackfillOptions {
96+ readonly dryRun: boolean;
97+ /** One line per organization and one summary line; never a user's data. */
98+ readonly log: (line: string) => void;
99+}
100+
101+/** What one organization's scan did. */
102+export interface WorkOsOrganizationScanCounts extends WorkOsOrganizationScanWrites {
103+ /** Memberships WorkOS reported for the organization. */
104+ readonly memberships: number;
105+ /**
106+ * Whether the listing was written to the mirror. `false` on a dry run, and
107+ * when the mirror refused the listing whole: a later listing of the
108+ * organization had already been applied (an overlapping scan finished
109+ * first), or the organization is marked deleted or not mirrored. Every
110+ * write count is 0 then.
111+ */
112+ readonly applied: boolean;
113+}
114+
115+const NOTHING_WRITTEN: WorkOsOrganizationScanWrites = {
116+ usersWritten: 0,
117+ membershipsWritten: 0,
118+ membershipsTombstoned: 0,
119+};
120+
121+export interface WorkOsMirrorBackfillCounts extends WorkOsOrganizationScanWrites {
122+ readonly organizations: number;
123+ /** Memberships WorkOS reported across every organization. */
124+ readonly memberships: number;
125+}
126+
127+// Bounded fan-out for the per-member `getUser` calls: enough to overlap the
128+// WorkOS round-trips, low enough to stay clear of its rate limit.
129+const USER_FETCH_CONCURRENCY = 5;
130+
131+const now = () => Effect.map(Clock.currentTimeMillis, (millis) => new Date(millis));
132+
133+/**
134+ * Scan one organization: list every membership WorkOS holds for it, fetch
135+ * each member's user, and apply the listing to the mirror in one transaction
136+ * — the listed users and memberships upserted, the rest tombstoned, the
137+ * organization marked backfilled as of the listing — unless a later listing
138+ * was applied first, in which case nothing is written (`applied: false`).
139+ * The organization row must already be mirrored. Fails on the first source
140+ * or mirror failure and writes nothing then — a failed scan is safe to
141+ * repeat, so surfacing the failure beats a silent skip. A dry run reads
142+ * everything, writes nothing, and marks nothing.
143+ */
144+export const backfillOrganization = <E>(
145+ source: WorkOsOrganizationScanSource<E>,
146+ mirror: WorkOsMirrorShape,
147+ organizationId: string,
148+ options: { readonly dryRun: boolean },
149+) =>
150+ Effect.gen(function* () {
151+ // The listing's own instant, taken BEFORE the read so no change can fall
152+ // between them: the tombstone time for whatever the listing no longer
153+ // contains, the cut-off for what may be tombstoned at all (a row stamped
154+ // at or after it was written after the listing and is not missing from
155+ // it), and the organization's new `backfilled_at`.
156+ const listedAt = yield* now();
157+ const listed = yield* source.listOrgMembers(organizationId);
158+ const members = yield* Effect.forEach(
159+ listed,
160+ (membership) =>
161+ Effect.map(source.getUser(membership.userId), (user) => ({
162+ user: mirrorUserFromWorkOs(user),
163+ membership: mirrorMembershipFromWorkOs(membership),
164+ })),
165+ { concurrency: USER_FETCH_CONCURRENCY },
166+ );
167+ if (options.dryRun) {
168+ const counts: WorkOsOrganizationScanCounts = {
169+ ...NOTHING_WRITTEN,
170+ memberships: members.length,
171+ applied: false,
172+ };
173+ return counts;
174+ }
175+ const written = yield* mirror.applyOrganizationScan({
176+ organizationId,
177+ listedAt,
178+ members,
179+ });
180+ if (Option.isNone(written)) {
181+ yield* Effect.logWarning(
182+ "workos_mirror: organization scan not applied — a later listing was already applied, or the organization is deleted or not mirrored",
183+ { organizationId, listedAt: listedAt.toISOString() },
184+ );
185+ }
186+ const counts: WorkOsOrganizationScanCounts = {
187+ ...Option.getOrElse(written, () => NOTHING_WRITTEN),
188+ memberships: members.length,
189+ applied: Option.isSome(written),
190+ };
191+ return counts;
192+ }).pipe(
193+ Effect.withSpan("workos_mirror.backfillOrganization", {
194+ attributes: { organizationId },
195+ }),
196+ );
197+
198+/**
199+ * Run the full backfill: scan every organization the mirror knows, then
200+ * record the Events API replay boundary if none is recorded yet. Fails on
201+ * the first source or mirror failure — the organizations scanned so far stay
202+ * marked (each was covered in full), the boundary is not recorded, and the
203+ * run is safe to repeat.
204+ */
205+export const backfillWorkOsMirror = <E>(
206+ source: WorkOsMirrorBackfillSource<E>,
207+ mirror: WorkOsMirrorShape,
208+ options: WorkOsMirrorBackfillOptions,
209+) =>
210+ Effect.gen(function* () {
211+ // Taken before the first listing; recorded only once the run completes.
212+ const boundary = yield* now();
213+
214+ const organizationIds = yield* source.listOrganizationIds();
215+ let memberships = 0;
216+ let usersWritten = 0;
217+ let membershipsWritten = 0;
218+ let membershipsTombstoned = 0;
219+
220+ for (const organizationId of organizationIds) {
221+ const scanned = yield* backfillOrganization(source, mirror, organizationId, options);
222+ memberships += scanned.memberships;
223+ usersWritten += scanned.usersWritten;
224+ membershipsWritten += scanned.membershipsWritten;
225+ membershipsTombstoned += scanned.membershipsTombstoned;
226+ options.log(
227+ `${organizationId} ${scanned.memberships} membership(s)` +
228+ (options.dryRun
229+ ? ""
230+ : scanned.applied
231+ ? ` wrote ${scanned.usersWritten} user(s), ${scanned.membershipsWritten} membership(s), tombstoned ${scanned.membershipsTombstoned}`
232+ : " not applied (a later listing was already applied, or the organization is deleted)"),
233+ );
234+ }
235+
236+ const counts: WorkOsMirrorBackfillCounts = {
237+ organizations: organizationIds.length,
238+ memberships,
239+ usersWritten,
240+ membershipsWritten,
241+ membershipsTombstoned,
242+ };
243+ options.log(
244+ options.dryRun
245+ ? `dry run — ${counts.organizations} organization(s), ${counts.memberships} membership(s) would be mirrored`
246+ : `${counts.organizations} organization(s), ${counts.memberships} membership(s): wrote ${counts.usersWritten} user(s), ${counts.membershipsWritten} membership(s), tombstoned ${counts.membershipsTombstoned}`,
247+ );
248+ if (!options.dryRun) {
249+ // Every organization was read and written without failure (a failure
250+ // above fails the whole run) — or refused in favour of a listing taken
251+ // later still — so everything before `boundary` is now covered: record
252+ // it for the reconciler — unless an earlier run already did, in which
253+ // case the events between the two runs are the reconciler's to replay
254+ // and the earlier boundary stands.
255+ const recorded = yield* mirror.setReplayBoundary(boundary);
256+ options.log(
257+ recorded
258+ ? `events replay boundary set to ${boundary.toISOString()}`
259+ : "events replay boundary already recorded by an earlier run; kept (the events reconciler replays every change since it)",
260+ );
261+ // And every live organization is now covered: the mirror is complete
262+ // enough to authorize from (once the reconciler has caught up too),
263+ // which the authorization path reads as the first half of readiness.
264+ // Once: a re-run keeps the first completion.
265+ const completedAt = yield* now();
266+ const marked = yield* mirror.markBackfillCompleted(completedAt);
267+ options.log(
268+ marked
269+ ? `backfill completion recorded at ${completedAt.toISOString()}`
270+ : "backfill completion already recorded by an earlier run; kept",
271+ );
272+ }
273+ return counts;
274+ });
apps/cloud/src/auth/workos-mirror-store.ts+999−0
@@ -0,0 +1,999 @@
1+// ---------------------------------------------------------------------------
2+// The membership mirror's WRITE store — the Drizzle queries behind
3+// `WorkOsMirror`, plus the converters from WorkOS SDK payloads to mirror rows.
4+//
5+// Kept free of `cloudflare:workers` (no `DbService`, no `env`) so the one-off
6+// backfill (`scripts/backfill-workos-mirror.ts`) can run the SAME upserts over
7+// a plain postgres.js connection under bun. The request-scoped service that
8+// wraps this store is `workos-mirror.ts`.
9+//
10+// Every write is idempotent and out-of-order safe. Both upserts carry the
11+// WorkOS `updatedAt` of their payload and refuse to overwrite a row whose
12+// stored `workos_updated_at` is newer, so a replayed or late-arriving event
13+// can never regress the mirror. A delete never drops the row: it TOMBSTONES
14+// it (`status = 'inactive'`, `deleted_at` set) — and INSERTS the tombstone
15+// when the row is not there yet — so a feeder that fetched the membership
16+// before the deletion and writes it after (a login, the backfill) finds the
17+// tombstone instead of reinstating access. The tombstone is protected by
18+// IDENTITY, not by time: WorkOS never reuses a deleted `om_…` id, so a
19+// payload naming the tombstoned id is refused whatever it is stamped — a
20+// role change issued before the removal and delivered after it is stamped
21+// NEWER than the row and would beat any timestamp guard — while a payload
22+// naming a DIFFERENT id for the same (account, organization) is the member
23+// re-added in WorkOS, a replacement, and takes the row over under the usual
24+// `updatedAt` rule (WorkOS creates it after the old one is deleted, so it is
25+// always the newer). A tombstone keeps the stamp the row holds, the deleted
26+// membership's last reported state — never a local clock: read after WorkOS
27+// answered, it could post-date a replacement created meanwhile and refuse
28+// it for good. `deleted_at` is the deletion instant when the caller holds
29+// one (an event's `createdAt`, a scan's listing time) and `now()` otherwise;
30+// it records WHEN, it orders nothing. A membership WorkOS merely
31+// deactivated (`status = 'inactive'`, no `deleted_at`) is not a tombstone:
32+// WorkOS can reactivate it under the same id, and the `updatedAt` rule
33+// orders that as any other update. The row tombstone can only speak for the
34+// id the row happens to hold, so every delete ALSO records the deleted id in
35+// `membership_tombstones`, a ledger keyed by the WorkOS id alone, and every
36+// membership write consults it first: that covers the case the row cannot —
37+// membership A replaced by B in WorkOS before the mirror saw either, B then
38+// deleted. The delete finds a row holding A (not B's to tombstone) and would
39+// otherwise leave nothing behind; a later scan that still lists B would then
40+// insert it live, stamped after A and under another id, exactly what the row
41+// guard lets through. With the ledger the delete is recorded whatever the
42+// row holds, and B's payload is refused by identity. A scan records the ids
43+// it tombstones the same way. A deleted USER is protected by identity
44+// too: `deleteUser` leaves the account row behind as a tombstone (profile
45+// cleared, stamped with the deletion), and no membership naming that account
46+// is written again, however the payload is stamped — WorkOS never reuses a
47+// user id, and a membership the mirror had not seen has no row of its own
48+// for a guard to refuse the insert against. The cursor advances only by
49+// compare-and-set, so two reconciler runs cannot both own the stream.
50+//
51+// A backfill SCAN of one organization (its full membership listing, taken
52+// at one instant) is applied in ONE transaction that first compare-and-sets
53+// the organization's `backfilled_at` to the listing's instant
54+// (`applyOrganizationScan`): the row stays locked until commit, so two
55+// overlapping scans serialize on it, and the one whose listing is older
56+// than the recorded one writes NOTHING. The `updatedAt` guard alone cannot
57+// order two scans: a scan that listed a membership, stalled, and resumed
58+// after a later scan had found it gone would insert it live — the later scan
59+// left no tombstone, because the row was not there to tombstone. For the
60+// same reason every OTHER membership write is ordered against the scan too:
61+// a payload stamped before the organization's `backfilled_at` is refused
62+// unless the row already holds it — judged inside the write's own
63+// transaction, reading the organization row `FOR SHARE`, so a write that
64+// races a scan waits for the scan's commit and sees the mark it set, and
65+// never inserts a membership the scan has just proved revoked. A completed
66+// scan is the full listing as
67+// of that instant, so a membership it did not contain but an older payload
68+// still names (a login whose list was fetched before the revocation and
69+// written after the scan) was revoked before the scan — and that revocation
70+// predates the events replay boundary, so nothing would ever tombstone the
71+// reinstated row. The scan's own writes are the one exception: they are the
72+// listing that sets the mark.
73+//
74+// The events replay boundary (`workos_sync.range_start`) is written ONCE, by
75+// the first completed backfill, and never advanced: a later backfill
76+// refreshes memberships only, so an organization rename or user deletion
77+// between two runs is covered by the events stream alone, and moving the
78+// boundary past it would skip it for good.
79+// ---------------------------------------------------------------------------
80+
81+import { and, eq, isNotNull, isNull, lt, ne, notInArray, or, sql } from "drizzle-orm";
82+import type { AnyPgColumn } from "drizzle-orm/pg-core";
83+import { Effect, Option } from "effect";
84+
85+import type { MemberStatus } from "@executor-js/api/server";
86+
87+import {
88+ accounts,
89+ membershipTombstones,
90+ memberships,
91+ organizations,
92+ workosSync,
93+} from "../db/schema";
94+import type { DrizzleDb } from "../db/db";
95+import {
96+ WorkOsMirrorError,
97+ tryPromiseService,
98+ userStoreReasonFromCause,
99+ withServiceLogging,
100+} from "./errors";
101+
102+/** A WorkOS user, as the mirror stores it. `updatedAt` is WorkOS's own. */
103+export interface WorkOsMirrorUser {
104+ readonly id: string;
105+ readonly email: string;
106+ readonly firstName: string | null;
107+ readonly lastName: string | null;
108+ readonly avatarUrl: string | null;
109+ readonly lastSignInAt: Date | null;
110+ readonly updatedAt: Date;
111+}
112+
113+/**
114+ * A WorkOS organization membership, as the mirror stores it. `id` is the
115+ * WorkOS `om_…`; `accountId` the WorkOS user id; `updatedAt` is WorkOS's own.
116+ * The organization row must already be mirrored (`upsertOrganization`) — a
117+ * membership of an unknown org is a `query` failure, not a silent skip.
118+ */
119+export interface WorkOsMirrorMembership {
120+ readonly id: string;
121+ readonly accountId: string;
122+ readonly organizationId: string;
123+ readonly role: string;
124+ readonly status: MemberStatus;
125+ readonly updatedAt: Date;
126+}
127+
128+/**
129+ * What identifies a membership to a delete: the WorkOS `om_…` id AND the
130+ * (account, organization) pair it belongs to. The pair is the row's key,
131+ * and a delete must be able to mint the row as a tombstone when the mirror
132+ * has not seen the membership yet — an id alone cannot.
133+ */
134+export type WorkOsMirrorMembershipRef = Pick<
135+ WorkOsMirrorMembership,
136+ "id" | "accountId" | "organizationId"
137+>;
138+
139+/** One member as a backfill scan lists it: the membership and its user. */
140+export interface WorkOsScannedMember {
141+ readonly user: WorkOsMirrorUser;
142+ readonly membership: WorkOsMirrorMembership;
143+}
144+
145+/**
146+ * One organization's full membership listing, as `applyOrganizationScan`
147+ * applies it. `listedAt` is the instant the listing was taken — BEFORE the
148+ * WorkOS read, so no change can fall between the instant and the listing —
149+ * and is the tombstone time for what the listing no longer contains, the
150+ * cut-off for what may be tombstoned at all, and the organization's new
151+ * `backfilled_at`.
152+ */
153+export interface WorkOsOrganizationScan {
154+ readonly organizationId: string;
155+ readonly listedAt: Date;
156+ readonly members: readonly WorkOsScannedMember[];
157+}
158+
159+/** What an applied scan wrote: the upserts the `updatedAt` guard let through, and the tombstones. */
160+export interface WorkOsOrganizationScanWrites {
161+ readonly usersWritten: number;
162+ readonly membershipsWritten: number;
163+ readonly membershipsTombstoned: number;
164+}
165+
166+export interface WorkOsMirrorShape {
167+ /**
168+ * Insert or refresh a user row. `false` when the payload was refused and
169+ * the row left untouched: the stored row is newer than `updatedAt`, or is
170+ * a deletion tombstone stamped at `updatedAt` or later.
171+ */
172+ readonly upsertUser: (user: WorkOsMirrorUser) => Effect.Effect<boolean, WorkOsMirrorError>;
173+ /**
174+ * Insert or refresh a membership row, minting the bare account row first so
175+ * the foreign key holds when the membership arrives before its user. `false`
176+ * when the payload was refused: the stored row is newer than `updatedAt`,
177+ * or is a deletion tombstone of THIS membership id, or THIS membership id
178+ * is in the deletion ledger (`membership_tombstones`, written by every
179+ * delete and every scan tombstone, whatever row it found) — a deleted
180+ * WorkOS id never returns, however the payload is stamped; a payload under
181+ * another id is a replacement and is ordered by `updatedAt` against the
182+ * row's stamp — or the account is a deletion tombstone (`deleteUser`), whatever
183+ * the payload is stamped: WorkOS never reuses a user id, so a deleted user
184+ * has no memberships to mirror — or the organization is marked deleted
185+ * (`organizations.deleted_at`), in which case nothing is written at all:
186+ * no membership of a deleted organization is ever mirrored — or the
187+ * payload is stamped BEFORE the organization's last full scan
188+ * (`organizations.backfilled_at`): the scan listed everything WorkOS held
189+ * at that instant, so a membership it wrote already carries a stamp at
190+ * least this new (the write would change nothing) and one it did not
191+ * write was gone by then and must not come back from a list fetched
192+ * earlier. Only the scan itself writes past that mark
193+ * (`applyOrganizationScan`). The organization checks and the write run in
194+ * ONE transaction that holds the organization row `FOR SHARE`, so a scan
195+ * claiming the row at the same time is waited for and its mark seen — and
196+ * the account row `FOR SHARE` too, so a `deleteUser` tombstoning it at the
197+ * same time is waited for and its tombstone seen.
198+ */
199+ readonly upsertMembership: (
200+ membership: WorkOsMirrorMembership,
201+ ) => Effect.Effect<boolean, WorkOsMirrorError>;
202+ /**
203+ * Tombstone the membership as deleted: the row stays (or is minted, when
204+ * the mirror has not seen the membership yet), `inactive`, carrying the
205+ * deleted WorkOS id with `deleted_at` set. The tombstone is protected by
206+ * that id: `upsertMembership` refuses every later payload naming it,
207+ * however stamped, and accepts only a REPLACEMENT under another id. The
208+ * row's `workos_updated_at` is left as it is — the deleted membership's
209+ * last reported state, which a replacement is always stamped after; a
210+ * local clock read after WorkOS answered could post-date a replacement
211+ * created meanwhile and must never become the row's stamp. `deletedAt`
212+ * is the deletion instant when the caller holds one (a deletion event's
213+ * `createdAt`, a scan's listing time) or `null` when it holds none
214+ * (WorkOS answers a delete with no time): `deleted_at` then takes the
215+ * current time. It records when the membership was deleted; it orders
216+ * nothing. Matches the row by IDENTITY: `false` when the row is already
217+ * tombstoned (a replayed delete), or holds ANOTHER membership id — the
218+ * member re-added in WorkOS under a new id, which stands whether the
219+ * replacement was mirrored before or after this delete. In EVERY case the
220+ * deleted id is recorded in the deletion ledger (`membership_tombstones`),
221+ * so a membership the mirror never held under its own id (replaced and
222+ * deleted in WorkOS before the mirror saw it) cannot be inserted live by a
223+ * scan that listed it before the deletion: `true` when the row was
224+ * tombstoned OR the id was newly recorded. The organization row must
225+ * already be mirrored, as for `upsertMembership`.
226+ */
227+ readonly deleteMembership: (
228+ membership: WorkOsMirrorMembershipRef,
229+ deletedAt: Date | null,
230+ ) => Effect.Effect<boolean, WorkOsMirrorError>;
231+ /**
232+ * Tombstone a deleted WorkOS user: every membership of the account is
233+ * tombstoned as by `deleteMembership` (as of `deletedAt`; a deleted user's
234+ * membership ids never return), and the account row is kept (it anchors
235+ * foreign keys) — or minted, when the mirror has not seen the user yet —
236+ * with its profile cleared and stamped `deletedAt`, so a stale user
237+ * payload cannot restore it. `false` when the account already carries
238+ * this tombstone or a later one (a replayed delete). ONE transaction that
239+ * locks the account row first (`FOR NO KEY UPDATE`) and holds it until
240+ * the memberships are tombstoned: `upsertMembership` reads that row `FOR
241+ * SHARE` before it inserts, so a membership write racing the deletion
242+ * waits for it and sees the tombstone, or committed first and is
243+ * tombstoned here — never a live membership left behind for a deleted
244+ * user.
245+ */
246+ readonly deleteUser: (
247+ accountId: string,
248+ deletedAt: Date,
249+ ) => Effect.Effect<boolean, WorkOsMirrorError>;
250+ /** The id of the last WorkOS event applied, or `null` before the first run. */
251+ readonly getCursor: () => Effect.Effect<string | null, WorkOsMirrorError>;
252+ /**
253+ * Compare-and-set the cursor: advance to `next` only if it still reads
254+ * `prev` (`null` = no cursor yet). `false` means another run moved it first
255+ * — the caller must stop, it no longer owns the stream.
256+ */
257+ readonly setCursor: (
258+ prev: string | null,
259+ next: string,
260+ ) => Effect.Effect<boolean, WorkOsMirrorError>;
261+ /**
262+ * Apply one organization's backfill scan — the memberships (with their
263+ * users) a WorkOS listing taken at `listedAt` contained — atomically: in a
264+ * single transaction, compare-and-set the organization's `backfilled_at`
265+ * forward to `listedAt`, and only if that succeeded upsert every listed
266+ * user and membership and tombstone (as `deleteMembership` does, at
267+ * `listedAt`) every membership of the organization the listing did NOT
268+ * contain: the rows whose WorkOS id is not among the listed ones AND whose
269+ * stamp is older than the listing. A row stamped at or after `listedAt`
270+ * was written after the listing was taken (it could not be in it) and is
271+ * left alone — its own event lands through the reconciler; a tombstone
272+ * here would beat that event. Rows already tombstoned are left alone.
273+ *
274+ * The organization row stays locked until commit, so two overlapping
275+ * scans serialize on it and the one whose listing is older than (or the
276+ * same instant as) the recorded one writes nothing: `None` — the mirror
277+ * already holds the organization as of a later listing, and a membership
278+ * that listing no longer contained must not be inserted from this one.
279+ * `None` too for an organization the mirror does not hold or has marked
280+ * deleted: there is nothing to scan into. `Some` carries what was written.
281+ */
282+ readonly applyOrganizationScan: (
283+ scan: WorkOsOrganizationScan,
284+ ) => Effect.Effect<Option.Option<WorkOsOrganizationScanWrites>, WorkOsMirrorError>;
285+ /**
286+ * The Events API replay boundary: the instant the FIRST completed one-off
287+ * backfill began reading WorkOS, or `null` if none has completed. The
288+ * reconciler's first run (no cursor yet) reads the stream from here — the
289+ * backfill covers everything before it — and without a boundary it must
290+ * not guess.
291+ */
292+ readonly replayBoundary: () => Effect.Effect<Date | null, WorkOsMirrorError>;
293+ /**
294+ * Record the replay boundary, ONCE: `at` is the instant a completed
295+ * backfill began reading WorkOS, and it is kept only when no boundary is
296+ * recorded yet — `true` when this call recorded it. A later completed run
297+ * never moves it: the backfill refreshes memberships and tombstones only,
298+ * not organization names or deleted users' profiles, so a change between
299+ * two runs is covered only by the events stream, which must still be read
300+ * from the first boundary. Written only after every organization has been
301+ * written, so a run that fails part-way records nothing. Never touches the
302+ * cursor: a stream already being followed keeps its position, and the
303+ * boundary is then unused.
304+ */
305+ readonly setReplayBoundary: (at: Date) => Effect.Effect<boolean, WorkOsMirrorError>;
306+ /**
307+ * When a backfill run first wrote EVERY live organization
308+ * (`workos_sync.backfill_completed_at`), or `null` while none has. The
309+ * first half of the mirror's READINESS for authorization: until a run has
310+ * covered every organization, the mirror may lack members who have not
311+ * signed in since it shipped, and a membership check read from it would
312+ * deny them. The other half is the events reconciler being caught up,
313+ * which its cursor row reports.
314+ */
315+ readonly backfillCompletedAt: () => Effect.Effect<Date | null, WorkOsMirrorError>;
316+ /**
317+ * Record that a backfill run has written every live organization, as of
318+ * `at` — ONCE: a later completed run keeps the first mark (`false`), so
319+ * readiness never flips back. Written only after the last organization
320+ * was applied (or refused in favour of a later listing), so a run that
321+ * fails part-way records nothing here. Mints the events row when absent,
322+ * as `setReplayBoundary` does, and touches neither the cursor nor the
323+ * boundary.
324+ */
325+ readonly markBackfillCompleted: (at: Date) => Effect.Effect<boolean, WorkOsMirrorError>;
326+ /**
327+ * When the organization's membership list was last FULLY scanned from
328+ * WorkOS (`backfillOrganization` in workos-mirror-backfill.ts), or `null`
329+ * if it never was — or the organization is not mirrored. Until it has been,
330+ * the mirror may hold only the members login and write-through happened to
331+ * record, so a member count read from it is PARTIAL: every seat gate reads
332+ * this first and scans the organization when it is `null`. Per
333+ * organization, never database-wide, so an organization mirrored after a
334+ * backfill ran (lazily, or by a sign-in) is never mistaken for a scanned
335+ * one.
336+ */
337+ readonly organizationBackfilledAt: (
338+ organizationId: string,
339+ ) => Effect.Effect<Date | null, WorkOsMirrorError>;
340+}
341+
342+// ---------------------------------------------------------------------------
343+// SDK payload → mirror row. The feeders (login callback, write-through, the
344+// backfill, the Events reconciler) all hand the mirror WorkOS objects; this is
345+// the one place their field names and ISO timestamps are translated. Typed
346+// structurally (the fields actually read) so the SDK's `User` /
347+// `OrganizationMembership`, an event payload, and a test fixture all fit.
348+// ---------------------------------------------------------------------------
349+
350+/** The WorkOS user fields the mirror reads. `User` from the SDK satisfies it. */
351+export interface WorkOsUserPayload {
352+ readonly id: string;
353+ readonly email: string;
354+ readonly firstName: string | null;
355+ readonly lastName: string | null;
356+ readonly profilePictureUrl: string | null;
357+ readonly lastSignInAt: string | null;
358+ readonly updatedAt: string;
359+}
360+
361+/**
362+ * The WorkOS membership fields the mirror reads. `OrganizationMembership` from
363+ * the SDK satisfies it; its `status` is exactly the mirror's `MemberStatus`.
364+ */
365+export interface WorkOsMembershipPayload {
366+ readonly id: string;
367+ readonly userId: string;
368+ readonly organizationId: string;
369+ readonly role: { readonly slug: string };
370+ readonly status: MemberStatus;
371+ readonly updatedAt: string;
372+}
373+
374+/** Translate a WorkOS user payload to the row `upsertUser` stores. */
375+export const mirrorUserFromWorkOs = (user: WorkOsUserPayload): WorkOsMirrorUser => ({
376+ id: user.id,
377+ email: user.email,
378+ firstName: user.firstName,
379+ lastName: user.lastName,
380+ avatarUrl: user.profilePictureUrl,
381+ lastSignInAt: user.lastSignInAt === null ? null : new Date(user.lastSignInAt),
382+ updatedAt: new Date(user.updatedAt),
383+});
384+
385+/** Translate a WorkOS membership payload to the row `upsertMembership` stores. */
386+export const mirrorMembershipFromWorkOs = (
387+ membership: WorkOsMembershipPayload,
388+): WorkOsMirrorMembership => ({
389+ id: membership.id,
390+ accountId: membership.userId,
391+ organizationId: membership.organizationId,
392+ role: membership.role.slug,
393+ status: membership.status,
394+ updatedAt: new Date(membership.updatedAt),
395+});
396+
397+/**
398+ * The `workos_sync` row of the one WorkOS events stream the reconciler
399+ * follows. A row id rather than a singleton table so a second stream
400+ * (another WorkOS environment, a replay) can be added without a schema
401+ * change; the same row carries the backfill's replay boundary and
402+ * completion mark. Not a `db/schema.ts` export: that module's exports are
403+ * enumerated as tables by the purge-coverage test.
404+ */
405+export const WORKOS_EVENTS_STREAM_ID = "events";
406+
407+// A raw `sql` fragment binds a Date without the column's driver mapping, so
408+// every instant below is passed as ISO text and cast.
409+const instant = (at: Date) => sql`${at.toISOString()}::timestamptz`;
410+
411+// An account tombstone moves the row's timestamp to the deletion time —
412+// never backwards, so a row that somehow carries a newer WorkOS timestamp
413+// keeps it and the upsert guard stays at least as strict.
414+const noEarlierThan = (column: AnyPgColumn, at: Date) => sql`greatest(${column}, ${instant(at)})`;
415+
416+// A membership tombstone keeps the row, marks it `inactive`, and records the
417+// deletion in `deleted_at`: the instant the caller holds, or `now()` when it
418+// holds none (see `deleteMembership`). `workos_updated_at` is left as it is
419+// — the tombstone is protected by the deleted id, not by its stamp.
420+const tombstone = (deletedAt: Date | null) => ({
421+ status: "inactive" as const,
422+ deletedAt: deletedAt === null ? sql`now()` : instant(deletedAt),
423+});
424+
425+// Which stored membership rows a payload for membership `id` stamped
426+// `updatedAt` may overwrite. A row WorkOS still holds (`deleted_at` null) is
427+// ordered by time: a row with no stamp (predating the mirror), any row
428+// stamped earlier, and a LIVE row stamped the same instant — feeders replay
429+// the same payload and must converge, not stall. An `inactive` row stamped
430+// the same instant is not overwritten: a deactivation at T beats an active
431+// payload at T. A deletion tombstone is ordered by IDENTITY: never
432+// overwritten by a payload naming the deleted id — a deleted WorkOS
433+// membership id never returns, however the payload is stamped — and taken
434+// over by a payload naming another id, a replacement membership WorkOS
435+// created after the deletion and so stamped after the row's last state,
436+// under the plain timestamp rule. A tombstone with no id at all (a
437+// pre-mirror row of a deleted user) is never taken over: the user is gone.
438+const membershipAcceptsPayload = (id: string, updatedAt: Date) =>
439+ or(
440+ and(
441+ isNull(memberships.deletedAt),
442+ or(
443+ isNull(memberships.workosUpdatedAt),
444+ lt(memberships.workosUpdatedAt, updatedAt),
445+ and(eq(memberships.workosUpdatedAt, updatedAt), ne(memberships.status, "inactive")),
446+ ),
447+ ),
448+ and(
449+ isNotNull(memberships.deletedAt),
450+ ne(memberships.membershipId, id),
451+ or(isNull(memberships.workosUpdatedAt), lt(memberships.workosUpdatedAt, updatedAt)),
452+ ),
453+ );
454+
455+// A deleted user's account row, as `deleteUser` leaves it: the profile is
456+// cleared (every WorkOS user payload carries an email, so a stamped row with
457+// none was written by `deleteUser`) and the stamp is the deletion time. A row
458+// minted bare by `ensureAccount` has no stamp either, and is not a tombstone.
459+// Judged in code, over a row read under a lock (`writeMembership`), not as a
460+// predicate in the read: a `SELECT ... FOR SHARE` locks only the rows it
461+// returns, so a read filtered to tombstones would lock nothing for a live
462+// row — the one case the lock exists for.
463+const isAccountTombstone = (account: {
464+ readonly email: string | null;
465+ readonly workosUpdatedAt: Date | null;
466+}): boolean => account.email === null && account.workosUpdatedAt !== null;
467+
468+// The same rule as for a membership row, for an account row. A tombstone
469+// takes no payload stamped at or before the deletion; a bare row takes any.
470+const accountAcceptsPayload = (updatedAt: Date) =>
471+ or(
472+ isNull(accounts.workosUpdatedAt),
473+ lt(accounts.workosUpdatedAt, updatedAt),
474+ and(eq(accounts.workosUpdatedAt, updatedAt), isNotNull(accounts.email)),
475+ );
476+
477+// A delete is applied only to a row not yet tombstoned: a replayed deletion
478+// changes nothing and reports so.
479+const notDeleted = isNull(memberships.deletedAt);
480+
481+// Whether membership `id` is in the deletion ledger: a WorkOS id a delete or
482+// a scan has named as gone, which never returns. Identity alone — the
483+// ledger records WHEN for the record, not for ordering.
484+const membershipIdDeleted = async (db: DrizzleDb, id: string): Promise<boolean> => {
485+ const rows = await db
486+ .select({ membershipId: membershipTombstones.membershipId })
487+ .from(membershipTombstones)
488+ .where(eq(membershipTombstones.membershipId, id));
489+ return rows.length > 0;
490+};
491+
492+// Which (account, organization) row a delete of membership `id` may
493+// tombstone: the row carrying THIS id — whatever its stamp, a deleted id is
494+// never reused — or a row with no id at all (written before the mirror
495+// recorded WorkOS ids; the delete fills the id in). Never a row under
496+// ANOTHER id: that is a different membership of the same account and
497+// organization, the member re-added in WorkOS after this one was removed,
498+// and it stands however the two are stamped. Identity orders them,
499+// timestamps do not. A row already tombstoned is left alone (`notDeleted`)
500+// so a replayed delete reports `false`.
501+const membershipDeletableBy = (id: string) =>
502+ and(or(isNull(memberships.membershipId), eq(memberships.membershipId, id)), notDeleted);
503+
504+// The write queries, over `db` or over a transaction handle (drizzle's is a
505+// `PgDatabase` too): the one `applyOrganizationScan` opens, or the one the
506+// store opens per `upsertMembership`. Each answers whether it wrote a row;
507+// the public shape translates that.
508+const makeWrites = (db: DrizzleDb) => {
509+ const ensureAccount = (id: string) =>
510+ db.insert(accounts).values({ id }).onConflictDoNothing({ target: accounts.id });
511+
512+ // The membership upsert under the row guard (`membershipAcceptsPayload`)
513+ // and the account tombstone, never the organization mark.
514+ const writeMembership = async (membership: WorkOsMirrorMembership): Promise<boolean> => {
515+ await ensureAccount(membership.accountId);
516+ // Never for a DELETED user. The row guard below orders a payload against
517+ // the membership row it would overwrite; a membership the mirror has not
518+ // seen yet has no row, so the guard cannot refuse the INSERT — and a
519+ // feeder that fetched the membership before the user was deleted and
520+ // writes it after (a stalled login list, the backfill's older listing)
521+ // would insert it live. The account tombstone is the one row a deleted
522+ // user always leaves behind, so it is consulted first, by identity:
523+ // WorkOS never reuses a user id, so no payload naming a tombstoned
524+ // account is ever current.
525+ //
526+ // Read FOR SHARE, held until the transaction `db` belongs to commits: a
527+ // `deleteUser` applying the deletion at this moment holds the row FOR NO
528+ // KEY UPDATE until ITS commit, so this read waits for it and sees the
529+ // tombstone — and a deletion that arrives after this read waits for this
530+ // transaction, then tombstones the membership it inserted. Unlocked, a
531+ // deletion could land between this read and the insert below and leave
532+ // a live membership for a deleted user.
533+ const locked = await db
534+ .select({ email: accounts.email, workosUpdatedAt: accounts.workosUpdatedAt })
535+ .from(accounts)
536+ .where(eq(accounts.id, membership.accountId))
537+ .for("share");
538+ const account = locked[0];
539+ // `ensureAccount` guarantees the row; accounts are tombstoned, never
540+ // deleted, so a missing one is refused like a tombstone, not written for.
541+ if (account === undefined || isAccountTombstone(account)) return false;
542+ // Never under a DELETED membership id. The row guard below can only
543+ // refuse against the id the row holds; a delete of THIS id that found
544+ // the row under another id (see the header) left only the ledger entry
545+ // behind, and that is what refuses the payload here.
546+ if (await membershipIdDeleted(db, membership.id)) return false;
547+ const written = await db
548+ .insert(memberships)
549+ .values({
550+ accountId: membership.accountId,
551+ organizationId: membership.organizationId,
552+ membershipId: membership.id,
553+ role: membership.role,
554+ status: membership.status,
555+ workosUpdatedAt: membership.updatedAt,
556+ })
557+ .onConflictDoUpdate({
558+ target: [memberships.accountId, memberships.organizationId],
559+ set: {
560+ membershipId: membership.id,
561+ role: membership.role,
562+ status: membership.status,
563+ workosUpdatedAt: membership.updatedAt,
564+ // A replacement taking over a tombstone is live again; a row that
565+ // was not tombstoned had nothing here.
566+ deletedAt: null,
567+ },
568+ setWhere: membershipAcceptsPayload(membership.id, membership.updatedAt),
569+ })
570+ .returning({ accountId: memberships.accountId });
571+ return written.length > 0;
572+ };
573+
574+ return {
575+ upsertUser: async (user: WorkOsMirrorUser): Promise<boolean> => {
576+ const written = await db
577+ .insert(accounts)
578+ .values({
579+ id: user.id,
580+ email: user.email,
581+ firstName: user.firstName,
582+ lastName: user.lastName,
583+ avatarUrl: user.avatarUrl,
584+ lastSignInAt: user.lastSignInAt,
585+ workosUpdatedAt: user.updatedAt,
586+ })
587+ .onConflictDoUpdate({
588+ target: accounts.id,
589+ set: {
590+ email: user.email,
591+ firstName: user.firstName,
592+ lastName: user.lastName,
593+ avatarUrl: user.avatarUrl,
594+ lastSignInAt: user.lastSignInAt,
595+ workosUpdatedAt: user.updatedAt,
596+ },
597+ setWhere: accountAcceptsPayload(user.updatedAt),
598+ })
599+ .returning({ id: accounts.id });
600+ return written.length > 0;
601+ },
602+
603+ upsertMembership: async (membership: WorkOsMirrorMembership): Promise<boolean> => {
604+ // Never into an organization the mirror holds as DELETED: its
605+ // memberships are gone from WorkOS and purged (or being purged) here,
606+ // so a payload that still names it was fetched before the deletion —
607+ // a login that stalled across the purge, a stale event — and writing
608+ // it would grant access to a deleted organization. The tombstone row
609+ // outlives the purge for exactly this check. And never from a payload
610+ // stamped before the organization's last full scan: the scan is the
611+ // complete listing as of `backfilled_at`, so an older payload either
612+ // repeats a row the scan wrote (nothing to change) or names a
613+ // membership the scan found gone — revoked before the mirror's events
614+ // replay begins, so no event would ever tombstone it again. The row
615+ // is read FOR SHARE, so a scan claiming it at this moment
616+ // (`claimOrganizationScan`, an UPDATE that holds the row until its
617+ // transaction commits) and the purge marking it deleted are waited
618+ // for, and the mark read is the committed one: a scan cannot slip
619+ // between this read and the write below and leave a membership it
620+ // proved revoked inserted after it. The lock is held until the
621+ // transaction `db` belongs to ends — `makeWorkOsMirrorStore` opens one
622+ // per call, `applyPage` runs this on the page's own — which is what
623+ // orders the write after the scan. An organization the mirror does
624+ // not hold at all is still a foreign-key failure in the write.
625+ const organization = await db
626+ .select({
627+ deletedAt: organizations.deletedAt,
628+ backfilledAt: organizations.backfilledAt,
629+ })
630+ .from(organizations)
631+ .where(eq(organizations.id, membership.organizationId))
632+ .for("share");
633+ const row = organization[0];
634+ if (row?.deletedAt != null) return false;
635+ if (row?.backfilledAt != null && membership.updatedAt < row.backfilledAt) return false;
636+ return writeMembership(membership);
637+ },
638+
639+ // A scan's own membership writes: the scan has just claimed the
640+ // organization as of its listing, so its payloads are exactly what the
641+ // mark stands for and are not ordered against it.
642+ writeScannedMembership: writeMembership,
643+
644+ // An upsert, like the membership write it guards against: a delete the
645+ // mirror sees before the membership itself (the reconciler ahead of the
646+ // backfill) must leave the tombstone behind, or the later, older payload
647+ // would insert the row live.
648+ deleteMembership: async (
649+ membership: WorkOsMirrorMembershipRef,
650+ deletedAt: Date | null,
651+ ): Promise<boolean> => {
652+ await ensureAccount(membership.accountId);
653+ // Lock the account row FOR NO KEY UPDATE, as `deleteUser` does: a
654+ // membership write of this user reads the row FOR SHARE before it
655+ // consults the ledger (`writeMembership`), so a write racing this
656+ // delete either waits here and then finds the ledger entry, or
657+ // committed first and is tombstoned by the row upsert below. Without
658+ // the lock a write could pass the ledger check before this entry
659+ // lands and insert the deleted membership live after it.
660+ await db
661+ .select({ id: accounts.id })
662+ .from(accounts)
663+ .where(eq(accounts.id, membership.accountId))
664+ .for("no key update");
665+ // The ledger entry FIRST, whatever the row holds: the one record of
666+ // the deletion that does not depend on the row carrying the deleted
667+ // id. A replayed delete finds it there and records nothing new.
668+ const recorded = await db
669+ .insert(membershipTombstones)
670+ .values({
671+ membershipId: membership.id,
672+ accountId: membership.accountId,
673+ organizationId: membership.organizationId,
674+ ...(deletedAt === null ? {} : { deletedAt }),
675+ })
676+ .onConflictDoNothing({ target: membershipTombstones.membershipId })
677+ .returning({ membershipId: membershipTombstones.membershipId });
678+ const tombstoned = await db
679+ .insert(memberships)
680+ .values({
681+ accountId: membership.accountId,
682+ organizationId: membership.organizationId,
683+ membershipId: membership.id,
684+ ...tombstone(deletedAt),
685+ })
686+ .onConflictDoUpdate({
687+ target: [memberships.accountId, memberships.organizationId],
688+ set: { membershipId: membership.id, ...tombstone(deletedAt) },
689+ setWhere: membershipDeletableBy(membership.id),
690+ })
691+ .returning({ accountId: memberships.accountId });
692+ return recorded.length > 0 || tombstoned.length > 0;
693+ },
694+
695+ // Only inside a transaction: the account row lock taken first is what
696+ // orders this against a membership write of the same user, and it lasts
697+ // exactly as long as the transaction `db` belongs to — the one the store
698+ // opens per call.
699+ deleteUser: async (accountId: string, deletedAt: Date): Promise<boolean> => {
700+ // Lock the account row FIRST — minted bare when absent, so there is a
701+ // row to lock (the tombstone is minted for the same reason as the
702+ // membership one: a later, older payload must find it) — and hold it
703+ // FOR NO KEY UPDATE until commit. A membership write reads the row FOR
704+ // SHARE before it inserts (`writeMembership`), so a write racing this
705+ // delete either waits here and then sees the tombstone, or committed
706+ // before this lock was granted and is caught by the membership
707+ // tombstoning below. Without the lock a write could read the row live
708+ // and insert its membership after the tombstoning had run.
709+ await ensureAccount(accountId);
710+ await db
711+ .select({ id: accounts.id })
712+ .from(accounts)
713+ .where(eq(accounts.id, accountId))
714+ .for("no key update");
715+ // The account tombstone: profile cleared, stamped with the deletion.
716+ // Applied unless the row already carries this tombstone or a later one
717+ // (a replayed delete).
718+ const cleared = await db
719+ .update(accounts)
720+ .set({
721+ email: null,
722+ firstName: null,
723+ lastName: null,
724+ avatarUrl: null,
725+ workosUpdatedAt: noEarlierThan(accounts.workosUpdatedAt, deletedAt),
726+ })
727+ .where(
728+ and(
729+ eq(accounts.id, accountId),
730+ or(
731+ isNotNull(accounts.email),
732+ isNull(accounts.workosUpdatedAt),
733+ lt(accounts.workosUpdatedAt, deletedAt),
734+ ),
735+ ),
736+ )
737+ .returning({ id: accounts.id });
738+ await db
739+ .update(memberships)
740+ .set(tombstone(deletedAt))
741+ .where(and(eq(memberships.accountId, accountId), notDeleted));
742+ return cleared.length > 0;
743+ },
744+
745+ // Tombstone (at `listedAt`) every membership of the organization that a
746+ // listing taken at `listedAt` did not contain. Only inside a scan's
747+ // transaction, after its `backfilled_at` CAS: on its own this could
748+ // tombstone what a LATER scan just wrote.
749+ tombstoneMembershipsExcept: async (
750+ organizationId: string,
751+ membershipIds: readonly string[],
752+ listedAt: Date,
753+ ): Promise<number> => {
754+ const tombstoned = await db
755+ .update(memberships)
756+ .set(tombstone(listedAt))
757+ .where(
758+ and(
759+ eq(memberships.organizationId, organizationId),
760+ // A row with no WorkOS id predates the mirror and is not in
761+ // any listing; if WorkOS still holds the membership, the
762+ // scan's upsert has just filled the id in.
763+ or(
764+ isNull(memberships.membershipId),
765+ notInArray(memberships.membershipId, [...membershipIds]),
766+ ),
767+ // Only rows the listing could have contained: stamped before
768+ // it was taken (or never stamped). Anything newer was written
769+ // after the listing and is not missing from it.
770+ or(isNull(memberships.workosUpdatedAt), lt(memberships.workosUpdatedAt, listedAt)),
771+ notDeleted,
772+ ),
773+ )
774+ .returning({
775+ accountId: memberships.accountId,
776+ membershipId: memberships.membershipId,
777+ });
778+ // The ids the listing proved gone go into the ledger too, as any other
779+ // delete's: they never return. A row with no id has none to record.
780+ const gone = tombstoned.flatMap((row) =>
781+ row.membershipId === null
782+ ? []
783+ : [
784+ {
785+ membershipId: row.membershipId,
786+ accountId: row.accountId,
787+ organizationId,
788+ deletedAt: listedAt,
789+ },
790+ ],
791+ );
792+ if (gone.length > 0) {
793+ await db
794+ .insert(membershipTombstones)
795+ .values(gone)
796+ .onConflictDoNothing({ target: membershipTombstones.membershipId });
797+ }
798+ return tombstoned.length;
799+ },
800+ };
801+};
802+
803+// Claim the organization for a scan listed at `listedAt`: move its
804+// `backfilled_at` forward to `listedAt` if the recorded mark is older (or
805+// missing). Run inside a transaction this also LOCKS the organization row
806+// until commit, so an overlapping scan's claim waits here, then reads the
807+// moved mark and matches nothing. An organization marked deleted is never
808+// claimed: its memberships are being (or have been) purged, and a scan
809+// that listed them before the deletion must not write them back.
810+const claimOrganizationScan = async (db: DrizzleDb, organizationId: string, listedAt: Date) => {
811+ const claimed = await db
812+ .update(organizations)
813+ .set({ backfilledAt: listedAt })
814+ .where(
815+ and(
816+ eq(organizations.id, organizationId),
817+ isNull(organizations.deletedAt),
818+ or(isNull(organizations.backfilledAt), lt(organizations.backfilledAt, listedAt)),
819+ ),
820+ )
821+ .returning({ id: organizations.id });
822+ return claimed.length > 0;
823+};
824+
825+/**
826+ * The mirror's write operations over `db`. Failures are `WorkOsMirrorError`
827+ * naming the operation and the classified driver reason; the full cause is
828+ * logged at the boundary.
829+ */
830+export const makeWorkOsMirrorStore = (db: DrizzleDb): WorkOsMirrorShape => {
831+ const run = <A>(op: string, fn: () => Promise<A>) =>
832+ withServiceLogging(
833+ `workos_mirror.${op}`,
834+ (failure) =>
835+ new WorkOsMirrorError({
836+ operation: op,
837+ reason: userStoreReasonFromCause(failure),
838+ }),
839+ tryPromiseService(fn),
840+ );
841+
842+ const writes = makeWrites(db);
843+
844+ return {
845+ upsertUser: (user) => run("upsertUser", () => writes.upsertUser(user)),
846+
847+ // One transaction per call: the organization guard inside locks the org
848+ // row until the write has landed (see `makeWrites`).
849+ upsertMembership: (membership) =>
850+ run("upsertMembership", () =>
851+ db.transaction((tx) => makeWrites(tx).upsertMembership(membership)),
852+ ),
853+
854+ // One transaction per call: the ledger entry and the row tombstone land
855+ // together or not at all.
856+ deleteMembership: (membership, deletedAt) =>
857+ run("deleteMembership", () =>
858+ db.transaction((tx) => makeWrites(tx).deleteMembership(membership, deletedAt)),
859+ ),
860+
861+ // One transaction per call: the account row lock inside is held until
862+ // the memberships are tombstoned (see `makeWrites`).
863+ deleteUser: (accountId, deletedAt) =>
864+ run("deleteUser", () =>
865+ db.transaction((tx) => makeWrites(tx).deleteUser(accountId, deletedAt)),
866+ ),
867+
868+ getCursor: () =>
869+ run("getCursor", async () => {
870+ const rows = await db
871+ .select({ cursor: workosSync.cursor })
872+ .from(workosSync)
873+ .where(eq(workosSync.id, WORKOS_EVENTS_STREAM_ID));
874+ // No row yet is the same state as a row with no cursor: nothing applied.
875+ return rows[0]?.cursor ?? null;
876+ }),
877+
878+ setCursor: (prev, next) =>
879+ run("setCursor", async () => {
880+ const now = new Date();
881+ if (prev === null) {
882+ // First advance: mint the row, or claim an existing row that still
883+ // has no cursor. A row that already carries one belongs to another
884+ // run and is left alone.
885+ const written = await db
886+ .insert(workosSync)
887+ .values({ id: WORKOS_EVENTS_STREAM_ID, cursor: next, updatedAt: now })
888+ .onConflictDoUpdate({
889+ target: workosSync.id,
890+ set: { cursor: next, updatedAt: now },
891+ setWhere: isNull(workosSync.cursor),
892+ })
893+ .returning({ id: workosSync.id });
894+ return written.length > 0;
895+ }
896+ const written = await db
897+ .update(workosSync)
898+ .set({ cursor: next, updatedAt: now })
899+ .where(and(eq(workosSync.id, WORKOS_EVENTS_STREAM_ID), eq(workosSync.cursor, prev)))
900+ .returning({ id: workosSync.id });
901+ return written.length > 0;
902+ }),
903+
904+ applyOrganizationScan: (scan) =>
905+ run("applyOrganizationScan", () =>
906+ db.transaction(async (tx) => {
907+ // The claim comes FIRST so the lock is held for every write below;
908+ // a scan that lost to a later listing commits an empty transaction.
909+ const claimed = await claimOrganizationScan(tx, scan.organizationId, scan.listedAt);
910+ if (!claimed) return Option.none();
911+ const txWrites = makeWrites(tx);
912+ let usersWritten = 0;
913+ let membershipsWritten = 0;
914+ for (const member of scan.members) {
915+ if (await txWrites.upsertUser(member.user)) usersWritten += 1;
916+ if (await txWrites.writeScannedMembership(member.membership)) membershipsWritten += 1;
917+ }
918+ const membershipsTombstoned = await txWrites.tombstoneMembershipsExcept(
919+ scan.organizationId,
920+ scan.members.map((member) => member.membership.id),
921+ scan.listedAt,
922+ );
923+ const written: WorkOsOrganizationScanWrites = {
924+ usersWritten,
925+ membershipsWritten,
926+ membershipsTombstoned,
927+ };
928+ return Option.some(written);
929+ }),
930+ ),
931+
932+ replayBoundary: () =>
933+ run("replayBoundary", async () => {
934+ const rows = await db
935+ .select({ rangeStart: workosSync.rangeStart })
936+ .from(workosSync)
937+ .where(eq(workosSync.id, WORKOS_EVENTS_STREAM_ID));
938+ return rows[0]?.rangeStart ?? null;
939+ }),
940+
941+ setReplayBoundary: (at) =>
942+ run("setReplayBoundary", async () => {
943+ const recorded = await db
944+ .insert(workosSync)
945+ .values({
946+ id: WORKOS_EVENTS_STREAM_ID,
947+ cursor: null,
948+ rangeStart: at,
949+ updatedAt: at,
950+ })
951+ .onConflictDoUpdate({
952+ target: workosSync.id,
953+ set: { rangeStart: at },
954+ // A boundary already recorded stands, whatever this run's is.
955+ setWhere: isNull(workosSync.rangeStart),
956+ })
957+ .returning({ id: workosSync.id });
958+ return recorded.length > 0;
959+ }),
960+
961+ backfillCompletedAt: () =>
962+ run("backfillCompletedAt", async () => {
963+ const rows = await db
964+ .select({ backfillCompletedAt: workosSync.backfillCompletedAt })
965+ .from(workosSync)
966+ .where(eq(workosSync.id, WORKOS_EVENTS_STREAM_ID));
967+ return rows[0]?.backfillCompletedAt ?? null;
968+ }),
969+
970+ markBackfillCompleted: (at) =>
971+ run("markBackfillCompleted", async () => {
972+ const recorded = await db
973+ .insert(workosSync)
974+ .values({
975+ id: WORKOS_EVENTS_STREAM_ID,
976+ cursor: null,
977+ backfillCompletedAt: at,
978+ updatedAt: at,
979+ })
980+ .onConflictDoUpdate({
981+ target: workosSync.id,
982+ set: { backfillCompletedAt: at },
983+ // The first completion stands, whatever this run's is.
984+ setWhere: isNull(workosSync.backfillCompletedAt),
985+ })
986+ .returning({ id: workosSync.id });
987+ return recorded.length > 0;
988+ }),
989+
990+ organizationBackfilledAt: (organizationId) =>
991+ run("organizationBackfilledAt", async () => {
992+ const rows = await db
993+ .select({ backfilledAt: organizations.backfilledAt })
994+ .from(organizations)
995+ .where(eq(organizations.id, organizationId));
996+ return rows[0]?.backfilledAt ?? null;
997+ }),
998+ };
999+};
apps/cloud/src/auth/workos-mirror.node.test.ts+543−19
@@ -6,10 +6,13 @@ · 5 unmodified lines
66 //
77 // What this pins:
88 // - an older WorkOS payload never overwrites a newer row (replay-safe)
9−// - a delete tombstones the row (inactive, stamped with the deletion time):
10−// a stale OR equal-timestamp upsert cannot resurrect it, a newer one
11−// reactivates it, and every default read treats the tombstone as no
12−// membership
9+// - a delete tombstones the row (inactive, `deleted_at` set) by IDENTITY:
10+// no payload naming the deleted membership id ever reactivates it,
11+// however it is stamped — not a stale one, not one stamped after the
12+// removal — while a replacement under a NEW id takes the row over live;
13+// every default read treats the tombstone as no membership
14+// - a membership WorkOS merely deactivated (inactive, no `deleted_at`)
15+// reactivates under the same id like any other update
1316 // - a delete of a membership or user the mirror has not seen yet leaves
1417 // the tombstone behind, so the backfill's older payload cannot insert
1518 // the row live afterwards
@@ -17,25 +20,40 @@ · 1 unmodified line
1720 // still carries the id it was replaced from) is recorded all the same,
1821 // so a later, newer payload of the deleted id cannot take the row over
1922 // - a deleted user takes no membership at all, however the payload is
20−// stamped: the account tombstone refuses the insert by identity
23+// stamped: the account tombstone refuses the insert by identity — and a
24+// membership write racing the deletion waits for its commit and sees
25+// the tombstone, never inserting a live membership for a deleted user
2126 // - a delete with no WorkOS instant keeps the row's own WorkOS stamp, so
2227 // a replacement membership WorkOS created meanwhile is not refused,
2328 // while the removed membership's own payload still is
2429 // - the cursor advances only by compare-and-set (one owner per stream)
30+// - a backfill scan is applied only if its listing is newer than the one
31+// already applied to the organization (one owner per listing instant),
32+// so an older listing cannot insert a membership the newer one lacked;
33+// a deleted organization takes no scan
34+// - a membership payload stamped before the organization's last scan is
35+// refused (a login list fetched before a revocation the scan already
36+// applied cannot reinstate it); one stamped at or after it is written —
37+// and a write racing a scan waits for the scan's commit and sees its
38+// mark, so it cannot insert a membership the scan just proved revoked
39+// - the events replay boundary is recorded once and never advanced
2540 // - `members` searches email AND name case-insensitively, pages stably
2641 // - `findByEmail` ignores the casing WorkOS stored
2742 // - a membership arriving before its user still holds (FK via ensureAccount)
2843 // ---------------------------------------------------------------------------
2944
3045 import { describe, expect, it } from "@effect/vitest";
31−import { Effect, Layer } from "effect";
46+import { eq, sql } from "drizzle-orm";
47+import { Context, Deferred, Effect, Fiber, Layer, Option } from "effect";
3248
3349 import { MemberDirectory } from "@executor-js/api/server";
3450
35−import { DbService } from "../db/db";
51+import { DbService, makeDbLayer } from "../db/db";
52+import { accounts, organizations } from "../db/schema";
3653 import { cloudMemberDirectoryLayer } from "./member-directory";
3754 import { UserStoreService } from "./context";
3855 import { WorkOsMirror, type WorkOsMirrorMembership, type WorkOsMirrorUser } from "./workos-mirror";
56+import { makeWorkOsMirrorStore } from "./workos-mirror-store";
3957
4058 const DbLive = DbService.Live;
4159 const Services = Layer.mergeAll(
@@ -44,8 +62,14 @@ const Services = Layer.mergeAll( · 2 unmodified lines
4462 UserStoreService.Live,
4563 ).pipe(Layer.provideMerge(DbLive));
4664
47−const run = <A, E>(body: Effect.Effect<A, E, WorkOsMirror | MemberDirectory | UserStoreService>) =>
48− Effect.runPromise(body.pipe(Effect.provide(Services), Effect.scoped));
65+const run = <A, E>(
66+ body: Effect.Effect<A, E, WorkOsMirror | MemberDirectory | UserStoreService | DbService>,
67+) => Effect.runPromise(body.pipe(Effect.provide(Services), Effect.scoped));
68+
69+/** The events row is instance-wide: drop it so a test starts as a never-backfilled database. */
70+const clearEventsRow = Effect.flatMap(DbService.asEffect(), ({ db }) =>
71+ Effect.promise(() => db.execute(sql`delete from workos_sync where id = 'events'`)),
72+);
4973
5074 const at = (iso: string) => new Date(iso);
5175 const T1 = at("2026-01-01T00:00:00.000Z");
@@ -59,7 +83,9 @@ const freshOrg = () => · 7 unmodified lines
5983 Effect.gen(function* () {
6084 const id = `org_${crypto.randomUUID().replaceAll("-", "")}`;
6185 const store = yield* UserStoreService;
62− yield* store.use("upsertOrganization", (s) => s.upsertOrganization({ id, name: "Mirror Org" }));
86+ yield* store.use("upsertOrganization", (s) =>
87+ s.upsertOrganization({ id, name: "Mirror Org", updatedAt: T1 }),
88+ );
6389 return id;
6490 });
6591
@@ -168,7 +194,7 @@ describe("WorkOsMirror upserts", () => { · 102 unmodified lines
168194 expect(result.filled?.name).toBe("Late");
169195 });
170196
171− it("tombstones a deleted membership so a stale upsert cannot resurrect it, and a newer one can", async () => {
197+ it("tombstones a deleted membership by identity: no payload of that id resurrects it, a replacement under a new id does", async () => {
172198 const result = await run(
173199 Effect.gen(function* () {
174200 const mirror = yield* WorkOsMirror;
@@ -198,6 +224,14 @@ describe("WorkOsMirror upserts", () => { · 23 unmodified lines
198224 membership(org, id, { id: membershipId, updatedAt: T2 }),
199225 );
200226 const afterEqual = yield* directory.membership(id, org, ["inactive"]);
227+ // A role change issued before the removal and delivered after it (a
228+ // stalled login list, a lagging feeder), stamped NEWER than anything
229+ // the row holds — the payload a timestamp guard would let through.
230+ // Same id: the membership is deleted, it never returns.
231+ const newerSameId = yield* mirror.upsertMembership(
232+ membership(org, id, { id: membershipId, role: "admin", updatedAt: T3 }),
233+ );
234+ const afterNewerSameId = yield* directory.membership(id, org, ["inactive"]);
201235 // The member re-added in WorkOS: a payload newer than the deletion,
202236 // under a new membership id (a deleted id is never reused).
203237 const readded = yield* mirror.upsertMembership(
@@ -224,6 +258,8 @@ describe("WorkOsMirror upserts", () => { · 20 unmodified lines
224258 afterStale,
225259 equal,
226260 afterEqual,
261+ newerSameId,
262+ afterNewerSameId,
227263 readded,
228264 afterReadd,
229265 lateDelete,
@@ -245,7 +281,12 @@ describe("WorkOsMirror upserts", () => { · 15 unmodified lines
245281 expect(result.afterStale?.status).toBe("inactive");
246282 expect(result.equal, "an upsert stamped AT the deletion is refused too").toBe(false);
247283 expect(result.afterEqual?.status).toBe("inactive");
248− expect(result.readded, "an upsert newer than the deletion reactivates").toBe(true);
284+ expect(
285+ result.newerSameId,
286+ "a payload of the deleted id stamped AFTER the removal is refused: identity, not time",
287+ ).toBe(false);
288+ expect(result.afterNewerSameId).toMatchObject({ status: "inactive", role: "member" });
289+ expect(result.readded, "a replacement under a new id reactivates").toBe(true);
249290 expect(result.afterReadd?.status).toBe("active");
250291 expect(result.lateDelete, "a replayed deletion of the OLD id is refused").toBe(false);
251292 expect(result.afterLateDelete?.status).toBe("active");
@@ -447,6 +488,59 @@ describe("WorkOsMirror upserts", () => { · 195 unmodified lines
447488 expect(result.afterReadd?.status).toBe("active");
448489 });
449490
491+ it("reactivates a membership WorkOS deactivated under the same id: a deactivation is not a deletion", async () => {
492+ const result = await run(
493+ Effect.gen(function* () {
494+ const mirror = yield* WorkOsMirror;
495+ const directory = yield* MemberDirectory;
496+ const org = yield* freshOrg();
497+ const id = `user_${crypto.randomUUID()}`;
498+ const membershipId = `om_${id}_${org}`;
499+ yield* mirror.upsertUser(user(id));
500+ yield* mirror.upsertMembership(membership(org, id, { id: membershipId, updatedAt: T1 }));
501+ // WorkOS deactivates the membership (an `organization_membership.updated`
502+ // with status inactive): the row is inactive but still WorkOS's, with
503+ // no `deleted_at` to protect it.
504+ const deactivated = yield* mirror.upsertMembership(
505+ membership(org, id, { id: membershipId, status: "inactive", updatedAt: T2 }),
506+ );
507+ const whileInactive = yield* directory.membership(id, org);
508+ // A payload older than the deactivation cannot undo it, nor one
509+ // stamped at the same instant.
510+ const older = yield* mirror.upsertMembership(
511+ membership(org, id, { id: membershipId, updatedAt: T1 }),
512+ );
513+ const equal = yield* mirror.upsertMembership(
514+ membership(org, id, { id: membershipId, updatedAt: T2 }),
515+ );
516+ // WorkOS reactivates it, same id, newer stamp: live again.
517+ const reactivated = yield* mirror.upsertMembership(
518+ membership(org, id, { id: membershipId, role: "admin", updatedAt: T3 }),
519+ );
520+ const afterReactivate = yield* directory.membership(id, org);
521+ return {
522+ membershipId,
523+ deactivated,
524+ whileInactive,
525+ older,
526+ equal,
527+ reactivated,
528+ afterReactivate,
529+ };
530+ }),
531+ );
532+ expect(result.deactivated).toBe(true);
533+ expect(result.whileInactive, "an inactive membership reads as no membership").toBeNull();
534+ expect(result.older, "an older payload cannot undo the deactivation").toBe(false);
535+ expect(result.equal, "nor one stamped at the deactivation").toBe(false);
536+ expect(result.reactivated, "a newer payload under the same id reactivates it").toBe(true);
537+ expect(result.afterReactivate).toMatchObject({
538+ status: "active",
539+ role: "admin",
540+ membershipId: result.membershipId,
541+ });
542+ });
543+
450544 it("tombstones every membership of a deleted user and clears the profile, keeping the account row", async () => {
451545 const result = await run(
452546 Effect.gen(function* () {
@@ -465,11 +559,13 @@ describe("WorkOsMirror upserts", () => { · 12 unmodified lines
465559 const inB = yield* directory.membership(id, orgB, ["inactive"]);
466560 // A stale user payload cannot restore the profile, nor can one
467561 // stamped at the deletion itself. No membership of the deleted user
468− // is written again — not one stamped after the deletion under a new
469− // id, the payload a timestamp guard would let through: the user is
470− // gone, and WorkOS never reuses the id.
562+ // is written again — not the deleted id (a deleted membership id
563+ // never returns), and not one stamped after the deletion under a
564+ // NEW id, the payload a timestamp guard would let through: the user
565+ // is gone, and WorkOS never reuses the id.
471566 const staleUser = yield* mirror.upsertUser(user(id, { firstName: "Back", updatedAt: T1 }));
472567 const equalUser = yield* mirror.upsertUser(user(id, { firstName: "Same", updatedAt: T2 }));
568+ const sameId = yield* mirror.upsertMembership(membership(orgA, id, { updatedAt: T3 }));
473569 const rejoined = yield* mirror.upsertMembership(
474570 membership(orgA, id, { id: `om_${id}_${orgA}_2`, updatedAt: T3 }),
475571 );
@@ -499,6 +595,7 @@ describe("WorkOsMirror upserts", () => { · 23 unmodified lines
499595 inB,
500596 staleUser,
501597 equalUser,
598+ sameId,
502599 rejoined,
503600 afterRejoin,
504601 unknown,
@@ -518,14 +615,12 @@ describe("WorkOsMirror upserts", () => { · 13 unmodified lines
518615 });
519616 expect(result.staleUser).toBe(false);
520617 expect(result.equalUser, "a profile stamped AT the deletion is refused").toBe(false);
618+ expect(result.sameId, "a deleted membership id never returns").toBe(false);
521619 expect(
522620 result.rejoined,
523621 "a membership of a deleted user is refused however it is stamped: identity, not time",
524622 ).toBe(false);
525− expect(result.afterRejoin).toMatchObject({
526− status: "inactive",
527− name: null,
528− });
623+ expect(result.afterRejoin).toMatchObject({ status: "inactive", name: null });
529624 expect(result.unknown, "deleting an unseen user leaves a tombstone").toBe(true);
530625 expect(result.unseenProfile, "which the older profile cannot fill").toBe(false);
531626 expect(
@@ -534,6 +629,150 @@ describe("WorkOsMirror upserts", () => { · 2 unmodified lines
534629 ).toBe(false);
535630 expect(result.unseenRow).toBeNull();
536631 });
632+
633+ it("waits for a user deletion holding the account row before judging a membership write against its tombstone", async () => {
634+ // A feeder (a stalled login list, the backfill's older listing) writes a
635+ // membership of a user whose `user.deleted` the reconciler is applying
636+ // at this moment. The deletion locks the account row for the length of
637+ // its transaction; the feeder's write must wait for that commit, see the
638+ // tombstone, and refuse the payload — never insert a live membership
639+ // for a deleted user. The deletion runs on the test's shared connection;
640+ // the feeder runs the same store over a second one, so the two
641+ // transactions are real peers on the server.
642+ const result = await run(
643+ Effect.scoped(
644+ Effect.gen(function* () {
645+ const org = yield* freshOrg();
646+ const gone = `user_${crypto.randomUUID()}`;
647+ const mirror = yield* WorkOsMirror;
648+ const { db: deleterDb } = yield* DbService;
649+ // The second connection is owned by this test body's scope.
650+ const feederDb = Context.get(yield* Layer.build(makeDbLayer()), DbService).db;
651+ const feeder = makeWorkOsMirrorStore(feederDb);
652+ const directory = yield* MemberDirectory;
653+ yield* mirror.upsertUser(user(gone, { firstName: "Gone" }));
654+
655+ // The deletion's transaction: the account row is locked as
656+ // `deleteUser` locks it, and the transaction is held open until
657+ // `release` — the window a feeder can race. The tombstone itself
658+ // is written after the feeder has started waiting.
659+ const locked = yield* Deferred.make<void>();
660+ let release: () => void = () => undefined;
661+ const held = new Promise<void>((resolve) => {
662+ release = resolve;
663+ });
664+ const deletion = yield* Effect.forkChild(
665+ Effect.promise(() =>
666+ deleterDb.transaction(async (tx) => {
667+ await tx
668+ .select({ id: accounts.id })
669+ .from(accounts)
670+ .where(eq(accounts.id, gone))
671+ .for("no key update");
672+ await Effect.runPromise(Deferred.succeed(locked, undefined));
673+ await held;
674+ await makeWorkOsMirrorStore(tx).deleteUser(gone, T2).pipe(Effect.runPromise);
675+ }),
676+ ),
677+ { startImmediately: true },
678+ );
679+ yield* Deferred.await(locked);
680+
681+ // The feeder's write starts while the deletion holds the row.
682+ const written = yield* Deferred.make<boolean>();
683+ yield* Effect.forkChild(
684+ feeder
685+ .upsertMembership(membership(org, gone, { updatedAt: T3 }))
686+ .pipe(Effect.flatMap((wrote) => Deferred.succeed(written, wrote))),
687+ { startImmediately: true },
688+ );
689+ const beforeCommit = yield* Effect.timeoutOption(Deferred.await(written), "250 millis");
690+ release();
691+ yield* Fiber.join(deletion);
692+ const afterCommit = yield* Deferred.await(written);
693+ const row = yield* directory.membership(gone, org, ["active", "pending", "inactive"]);
694+ return { beforeCommit, afterCommit, row };
695+ }),
696+ ),
697+ );
698+ expect(
699+ Option.isNone(result.beforeCommit),
700+ "the write waits while the deletion holds the account row",
701+ ).toBe(true);
702+ expect(
703+ result.afterCommit,
704+ "once the deletion has committed, its tombstone refuses the payload",
705+ ).toBe(false);
706+ expect(result.row, "so no membership was inserted for the deleted user").toBeNull();
707+ });
708+
709+ it("waits for a membership deletion holding the account row before judging a write of that id against the ledger", async () => {
710+ // Same race for a single membership: a feeder writes membership `om`
711+ // while the reconciler is applying its `organization_membership.deleted`.
712+ // The delete locks the account row for its transaction, so the feeder's
713+ // FOR SHARE read waits, then finds the ledger entry and refuses. Without
714+ // the lock the feeder could pass the ledger check first and insert the
715+ // deleted membership live after the delete committed.
716+ const result = await run(
717+ Effect.scoped(
718+ Effect.gen(function* () {
719+ const org = yield* freshOrg();
720+ const id = `user_${crypto.randomUUID()}`;
721+ const mirror = yield* WorkOsMirror;
722+ const { db: deleterDb } = yield* DbService;
723+ const feederDb = Context.get(yield* Layer.build(makeDbLayer()), DbService).db;
724+ const feeder = makeWorkOsMirrorStore(feederDb);
725+ const directory = yield* MemberDirectory;
726+ yield* mirror.upsertUser(user(id));
727+ const gone = membership(org, id, { updatedAt: T3 });
728+
729+ const locked = yield* Deferred.make<void>();
730+ let release: () => void = () => undefined;
731+ const held = new Promise<void>((resolve) => {
732+ release = resolve;
733+ });
734+ const deletion = yield* Effect.forkChild(
735+ Effect.promise(() =>
736+ deleterDb.transaction(async (tx) => {
737+ await tx
738+ .select({ id: accounts.id })
739+ .from(accounts)
740+ .where(eq(accounts.id, id))
741+ .for("no key update");
742+ await Effect.runPromise(Deferred.succeed(locked, undefined));
743+ await held;
744+ await makeWorkOsMirrorStore(tx).deleteMembership(gone, T2).pipe(Effect.runPromise);
745+ }),
746+ ),
747+ { startImmediately: true },
748+ );
749+ yield* Deferred.await(locked);
750+
751+ const written = yield* Deferred.make<boolean>();
752+ yield* Effect.forkChild(
753+ feeder
754+ .upsertMembership(gone)
755+ .pipe(Effect.flatMap((wrote) => Deferred.succeed(written, wrote))),
756+ { startImmediately: true },
757+ );
758+ const beforeCommit = yield* Effect.timeoutOption(Deferred.await(written), "250 millis");
759+ release();
760+ yield* Fiber.join(deletion);
761+ const afterCommit = yield* Deferred.await(written);
762+ const row = yield* directory.membership(id, org, ["active", "pending", "inactive"]);
763+ return { beforeCommit, afterCommit, status: row?.status ?? null };
764+ }),
765+ ),
766+ );
767+ expect(
768+ Option.isNone(result.beforeCommit),
769+ "the write waits while the deletion holds the account row",
770+ ).toBe(true);
771+ expect(result.afterCommit, "once the deletion has committed, the ledger refuses the id").toBe(
772+ false,
773+ );
774+ expect(result.status, "and the row is the tombstone, never live").toBe("inactive");
775+ });
537776 });
538777
539778 describe("WorkOsMirror cursor", () => {
@@ -560,6 +799,291 @@ describe("WorkOsMirror cursor", () => { · 20 unmodified lines
560799 });
561800 });
562801
802+describe("WorkOsMirror backfill sync state", () => {
803+ it("records the replay boundary and the backfill completion once each, without touching the cursor", async () => {
804+ const result = await run(
805+ Effect.gen(function* () {
806+ const mirror = yield* WorkOsMirror;
807+ // The events row is instance-wide (migration 0019 seeds it on the
808+ // empty test database, other tests may have written it): start from
809+ // no row, as a database that has never been backfilled has.
810+ yield* clearEventsRow;
811+ const first = yield* mirror.setReplayBoundary(T2);
812+ const boundary = yield* mirror.replayBoundary();
813+ const cursorAfterBoundary = yield* mirror.getCursor();
814+ // A later completed backfill: its boundary is not recorded.
815+ const again = yield* mirror.setReplayBoundary(T3);
816+ const afterAgain = yield* mirror.replayBoundary();
817+ // Nor once the stream is being followed.
818+ const cursorBefore = yield* mirror.getCursor();
819+ yield* mirror.setCursor(cursorBefore, "event_boundary");
820+ const afterCursor = yield* mirror.setReplayBoundary(T1);
821+ const boundaryWithCursor = yield* mirror.replayBoundary();
822+ const cursor = yield* mirror.getCursor();
823+ // The completion mark: absent until a run covers every organization,
824+ // then written once, beside the boundary and the cursor.
825+ const notCompleted = yield* mirror.backfillCompletedAt();
826+ const completed = yield* mirror.markBackfillCompleted(T3);
827+ const completedAgain = yield* mirror.markBackfillCompleted(T4);
828+ const completedAt = yield* mirror.backfillCompletedAt();
829+ const boundaryAfterCompletion = yield* mirror.replayBoundary();
830+ const cursorAfterCompletion = yield* mirror.getCursor();
831+ return {
832+ first,
833+ boundary,
834+ cursorAfterBoundary,
835+ again,
836+ afterAgain,
837+ afterCursor,
838+ boundaryWithCursor,
839+ cursor,
840+ notCompleted,
841+ completed,
842+ completedAgain,
843+ completedAt,
844+ boundaryAfterCompletion,
845+ cursorAfterCompletion,
846+ };
847+ }),
848+ );
849+ expect(result.first, "the first boundary is recorded").toBe(true);
850+ expect(result.boundary, "and reads back as written").toEqual(T2);
851+ expect(result.cursorAfterBoundary, "writing the boundary mints no cursor").toBeNull();
852+ expect(result.again, "a later run's boundary is refused").toBe(false);
853+ expect(result.afterAgain, "the first stands").toEqual(T2);
854+ expect(result.afterCursor).toBe(false);
855+ expect(result.boundaryWithCursor).toEqual(T2);
856+ expect(result.cursor, "and the cursor is untouched").toBe("event_boundary");
857+ expect(result.notCompleted, "no completion until a run covers every org").toBeNull();
858+ expect(result.completed, "the first completion is recorded").toBe(true);
859+ expect(result.completedAgain, "a later one is refused").toBe(false);
860+ expect(result.completedAt, "the first stands").toEqual(T3);
861+ expect(result.boundaryAfterCompletion, "the boundary is untouched").toEqual(T2);
862+ expect(result.cursorAfterCompletion, "and so is the cursor").toBe("event_boundary");
863+ });
864+
865+ it("refuses a membership payload stamped before the organization's last scan, and accepts one stamped at or after it", async () => {
866+ const result = await run(
867+ Effect.gen(function* () {
868+ const mirror = yield* WorkOsMirror;
869+ const directory = yield* MemberDirectory;
870+ const org = yield* freshOrg();
871+ const revoked = `user_${crypto.randomUUID()}`;
872+ const kept = `user_${crypto.randomUUID()}`;
873+ const joined = `user_${crypto.randomUUID()}`;
874+ // A login fetched its membership list at T1, while `revoked` was a
875+ // member, then stalled. WorkOS revoked them, and the backfill scanned
876+ // the org at T2 without them — nothing to tombstone, the row was
877+ // never there.
878+ yield* mirror.applyOrganizationScan({
879+ organizationId: org,
880+ listedAt: T2,
881+ members: [{ user: user(kept), membership: membership(org, kept, { updatedAt: T1 }) }],
882+ });
883+ // The stalled login resumes and writes what it holds: refused, the
884+ // revocation predates the scan and nothing would ever undo the row.
885+ const stale = yield* mirror.upsertMembership(membership(org, revoked, { updatedAt: T1 }));
886+ const staleRow = yield* directory.membership(revoked, org, [
887+ "active",
888+ "pending",
889+ "inactive",
890+ ]);
891+ // The same list's payload for a member the scan kept: refused too —
892+ // it changes nothing, the scan already wrote that state.
893+ const repeated = yield* mirror.upsertMembership(membership(org, kept, { updatedAt: T1 }));
894+ const keptRow = yield* directory.membership(kept, org);
895+ // A membership WorkOS created after the scan (its event, or a login
896+ // after it): stamped past the mark, written.
897+ const later = yield* mirror.upsertMembership(membership(org, joined, { updatedAt: T3 }));
898+ const atMark = yield* mirror.upsertMembership(
899+ membership(org, kept, { role: "admin", updatedAt: T2 }),
900+ );
901+ const keptAfter = yield* directory.membership(kept, org);
902+ return { stale, staleRow, repeated, keptRow, later, atMark, keptAfter };
903+ }),
904+ );
905+ expect(result.stale, "a payload older than the scan is refused").toBe(false);
906+ expect(result.staleRow, "and no row is minted for the revoked member").toBeNull();
907+ expect(result.repeated, "even for a member the scan kept").toBe(false);
908+ expect(result.keptRow?.status).toBe("active");
909+ expect(result.later, "a payload newer than the scan is written").toBe(true);
910+ expect(result.atMark, "as is one stamped at the scan's instant").toBe(true);
911+ expect(result.keptAfter?.role).toBe("admin");
912+ });
913+
914+ it("waits for a scan holding the organization row before judging a payload against the scan's mark", async () => {
915+ // A feeder (a login) writes a membership it fetched at T1 while a scan
916+ // listed at T2 — which no longer contains that membership — is being
917+ // applied. The scan claims the organization row for the length of its
918+ // transaction; the feeder's write must wait for that commit, observe
919+ // the mark, and refuse the payload — never insert the membership the
920+ // scan proved revoked. The scan holds the test's shared connection; the
921+ // feeder runs the same store over a second one, so the two transactions
922+ // are real peers on the server.
923+ const result = await run(
924+ Effect.scoped(
925+ Effect.gen(function* () {
926+ const org = yield* freshOrg();
927+ const revoked = `user_${crypto.randomUUID()}`;
928+ const { db: scanDb } = yield* DbService;
929+ // The second connection is owned by this test body's scope.
930+ const feederDb = Context.get(yield* Layer.build(makeDbLayer()), DbService).db;
931+ const feeder = makeWorkOsMirrorStore(feederDb);
932+ const directory = yield* MemberDirectory;
933+
934+ // The scan's transaction: its claim (the `backfilled_at` CAS, an
935+ // UPDATE that locks the row) is done, and the transaction is held
936+ // open until `release` — the window a feeder can race.
937+ const claimed = yield* Deferred.make<void>();
938+ let release: () => void = () => undefined;
939+ const held = new Promise<void>((resolve) => {
940+ release = resolve;
941+ });
942+ const scan = yield* Effect.forkChild(
943+ Effect.promise(() =>
944+ scanDb.transaction(async (tx) => {
945+ await tx
946+ .update(organizations)
947+ .set({ backfilledAt: T2 })
948+ .where(eq(organizations.id, org));
949+ await Effect.runPromise(Deferred.succeed(claimed, undefined));
950+ await held;
951+ }),
952+ ),
953+ { startImmediately: true },
954+ );
955+ yield* Deferred.await(claimed);
956+
957+ // The feeder's write starts while the scan holds the row.
958+ const written = yield* Deferred.make<boolean>();
959+ yield* Effect.forkChild(
960+ feeder
961+ .upsertMembership(membership(org, revoked, { updatedAt: T1 }))
962+ .pipe(Effect.flatMap((wrote) => Deferred.succeed(written, wrote))),
963+ { startImmediately: true },
964+ );
965+ const beforeCommit = yield* Effect.timeoutOption(Deferred.await(written), "250 millis");
966+ release();
967+ yield* Fiber.join(scan);
968+ const afterCommit = yield* Deferred.await(written);
969+ const row = yield* directory.membership(revoked, org, ["active", "pending", "inactive"]);
970+ return { beforeCommit, afterCommit, row };
971+ }),
972+ ),
973+ );
974+ expect(
975+ Option.isNone(result.beforeCommit),
976+ "the write waits while the scan holds the organization row",
977+ ).toBe(true);
978+ expect(result.afterCommit, "once the scan has committed, its mark refuses the payload").toBe(
979+ false,
980+ );
981+ expect(result.row, "so the revoked membership was never inserted").toBeNull();
982+ });
983+
984+ it("applies a scan only when its listing is newer than the one already applied, and never to a deleted or unknown organization", async () => {
985+ const result = await run(
986+ Effect.gen(function* () {
987+ const mirror = yield* WorkOsMirror;
988+ const directory = yield* MemberDirectory;
989+ const store = yield* UserStoreService;
990+ const scanned = yield* freshOrg();
991+ const untouched = yield* freshOrg();
992+ const staying = `user_${crypto.randomUUID()}`;
993+ const leaving = `user_${crypto.randomUUID()}`;
994+ const member = (id: string) => ({
995+ user: user(id),
996+ membership: membership(scanned, id),
997+ });
998+
999+ const before = yield* mirror.organizationBackfilledAt(scanned);
1000+ // The later listing (T2) no longer contains `leaving`; applied first.
1001+ const later = yield* mirror.applyOrganizationScan({
1002+ organizationId: scanned,
1003+ listedAt: T2,
1004+ members: [member(staying)],
1005+ });
1006+ const afterLater = yield* mirror.organizationBackfilledAt(scanned);
1007+ // The earlier listing (T1) still contains `leaving`: refused whole.
1008+ const earlier = yield* mirror.applyOrganizationScan({
1009+ organizationId: scanned,
1010+ listedAt: T1,
1011+ members: [member(staying), member(leaving)],
1012+ });
1013+ const afterEarlier = yield* mirror.organizationBackfilledAt(scanned);
1014+ const leavingRow = yield* directory.membership(leaving, scanned, [
1015+ "active",
1016+ "pending",
1017+ "inactive",
1018+ ]);
1019+ // The same instant is not newer either: a replayed listing writes nothing.
1020+ const same = yield* mirror.applyOrganizationScan({
1021+ organizationId: scanned,
1022+ listedAt: T2,
1023+ members: [],
1024+ });
1025+ const stayingRow = yield* directory.membership(staying, scanned);
1026+ const other = yield* mirror.organizationBackfilledAt(untouched);
1027+ const unknown = yield* mirror.applyOrganizationScan({
1028+ organizationId: "org_never_mirrored",
1029+ listedAt: T3,
1030+ members: [],
1031+ });
1032+ yield* store.use("deleteOrganizationCascade", (s) =>
1033+ s.deleteOrganizationCascade(untouched, T3),
1034+ );
1035+ const deleted = yield* mirror.applyOrganizationScan({
1036+ organizationId: untouched,
1037+ listedAt: T4,
1038+ members: [{ user: user(staying), membership: membership(untouched, staying) }],
1039+ });
1040+ const deletedRow = yield* directory.membership(staying, untouched);
1041+ return {
1042+ before,
1043+ later,
1044+ afterLater,
1045+ earlier,
1046+ afterEarlier,
1047+ leavingRow,
1048+ same,
1049+ stayingRow,
1050+ other,
1051+ unknown,
1052+ deleted,
1053+ deletedRow,
1054+ };
1055+ }),
1056+ );
1057+ expect(result.before, "a freshly mirrored organization is unscanned").toBeNull();
1058+ expect(result.later).toEqual(
1059+ Option.some({
1060+ usersWritten: 1,
1061+ membershipsWritten: 1,
1062+ membershipsTombstoned: 0,
1063+ }),
1064+ );
1065+ expect(result.afterLater, "the scan marks the organization as of its listing").toEqual(T2);
1066+ expect(result.earlier, "an older listing is refused whole").toEqual(Option.none());
1067+ expect(result.afterEarlier, "and the mark never moves backwards").toEqual(T2);
1068+ expect(
1069+ result.leavingRow,
1070+ "the membership only the older listing held was never written",
1071+ ).toBeNull();
1072+ expect(result.same, "a listing at the recorded instant is refused too").toEqual(Option.none());
1073+ expect(result.stayingRow?.status, "so it tombstones nothing the newer one wrote").toBe(
1074+ "active",
1075+ );
1076+ expect(result.other, "another organization's mark is its own").toBeNull();
1077+ expect(result.unknown, "an organization the mirror does not hold takes no scan").toEqual(
1078+ Option.none(),
1079+ );
1080+ expect(result.deleted, "nor does a deleted one, however new the listing").toEqual(
1081+ Option.none(),
1082+ );
1083+ expect(result.deletedRow).toBeNull();
1084+ });
1085+});
1086+
5631087 describe("cloud MemberDirectory", () => {
5641088 const seed = (org: string) =>
5651089 Effect.gen(function* () {
apps/cloud/src/auth/workos-mirror.ts+33−497
@@ -5,509 +5,43 @@ · 4 unmodified lines
55 // `accounts` / `memberships` rows (db/schema.ts) in step with it so the read
66 // side (`auth/member-directory.ts`, the cloud `MemberDirectory`) never has to
77 // ask WorkOS. Three feeders write through it: the login callback (user +
8−// memberships already in hand), Executor-initiated changes (write-through),
9−// and the WorkOS Events API reconciler (dashboard-side changes, replayed in
10−// order from a persisted cursor).
8+// memberships already in hand), Executor-initiated changes (write-through in
9+// `auth/handlers.ts` and `account/workos-account-service.ts`), and the WorkOS
10+// Events API reconciler (dashboard-side changes, replayed in order from a
11+// persisted cursor). The one-off backfill runs the same store out-of-band.
1112 //
12−// Every write is idempotent and out-of-order safe. Both upserts carry the
13−// WorkOS `updatedAt` of their payload and refuse to overwrite a row whose
14−// stored `workos_updated_at` is newer, so a replayed or late-arriving event
15−// can never regress the mirror. A delete never drops the row: it TOMBSTONES
16−// it (`status = 'inactive'`, `workos_updated_at` = the deletion time) — and
17−// INSERTS the tombstone when the row is not there yet — so a feeder that
18−// fetched the membership before the deletion and writes it after (a login,
19−// the backfill) is refused by that same guard instead of reinstating access;
20−// only a payload newer than the deletion (the member re-added in WorkOS)
21−// reactivates it. At an EQUAL timestamp the tombstone wins: a live row
22−// accepts a payload stamped the same instant (feeders replay the same
23−// payload and must converge), a tombstone does not, so a deletion at T is
24−// never undone by an active payload at T. Every stamp is on WorkOS's clock:
25−// a deletion replayed from the events stream is stamped with the event's
26−// time; one Executor made itself — WorkOS answers a delete with no time —
27−// is stamped with the `updatedAt` WorkOS last reported for the membership,
28−// or keeps the stamp the row already holds. Never a local clock: read after
29−// WorkOS answered, it can post-date a replacement membership WorkOS created
30−// for the same member meanwhile, and a tombstone stamped with it would
31−// refuse that replacement for good. The row tombstone can only speak for the
32−// membership id the row happens to hold, so every delete ALSO records the
33−// deleted WorkOS id in `membership_tombstones`, a ledger keyed by that id
34−// alone: WorkOS never reuses a deleted `om_…` id, so no membership write
35−// naming a recorded id is ever current, however it is stamped. That is what
36−// covers the case the row cannot: membership A replaced by B in WorkOS before
37−// the mirror saw either, B then deleted — the delete finds a row holding A
38−// (not B's to tombstone) and would otherwise leave nothing behind, and a
39−// later scan that still lists B would insert it live, stamped after A. With
40−// the ledger the delete is recorded whatever the row holds, and the scan's
41−// payload is refused by identity. A deleted USER is protected by identity
42−// the same way: `deleteUser` leaves the account row behind as a tombstone
43−// (profile cleared, stamped with the deletion), and no membership naming
44−// that account is ever written again, however the payload is stamped —
45−// WorkOS never reuses a user id, and a membership the mirror had not seen
46−// has no row of its own for a timestamp guard to refuse the insert against.
47−// The cursor advances only by compare-and-set, so two reconciler runs cannot
48−// both own the stream.
49−//
50−// Per-request layer shape, like `UserStoreService`: it holds the request's
51−// postgres socket, so it is rebuilt per request (`RequestScopedServicesLive`)
52−// and never shared across Workers requests.
13+// The queries live in `workos-mirror-store.ts`; this file binds them to the
14+// per-request `DbService`. Per-request layer shape, like `UserStoreService`:
15+// it holds the request's postgres socket, so it is rebuilt per request
16+// (`RequestScopedServicesLive`) and never shared across Workers requests.
5317 // ---------------------------------------------------------------------------
5418
55−import { and, eq, isNotNull, isNull, lt, ne, or, sql } from "drizzle-orm";
56−import type { AnyPgColumn } from "drizzle-orm/pg-core";
57−import { Context, Effect, Layer, Schema } from "effect";
58−
59−import { type MemberStatus } from "@executor-js/api/server";
60−
61−import { accounts, membershipTombstones, memberships, workosSync } from "../db/schema";
62−import { DbService, type DrizzleDb } from "../db/db";
63−import {
64− USER_STORE_FAILURE_REASONS,
65− tryPromiseService,
66− userStoreReasonFromCause,
67− withServiceLogging,
68−} from "./errors";
69−
70−/** A WorkOS user, as the mirror stores it. `updatedAt` is WorkOS's own. */
71−export interface WorkOsMirrorUser {
72− readonly id: string;
73− readonly email: string;
74− readonly firstName: string | null;
75− readonly lastName: string | null;
76− readonly avatarUrl: string | null;
77− readonly lastSignInAt: Date | null;
78− readonly updatedAt: Date;
79−}
80−
81−/**
82− * A WorkOS organization membership, as the mirror stores it. `id` is the
83− * WorkOS `om_…`; `accountId` the WorkOS user id; `updatedAt` is WorkOS's own.
84− * The organization row must already be mirrored (`upsertOrganization`) — a
85− * membership of an unknown org is a `query` failure, not a silent skip.
86− */
87−export interface WorkOsMirrorMembership {
88− readonly id: string;
89− readonly accountId: string;
90− readonly organizationId: string;
91− readonly role: string;
92− readonly status: MemberStatus;
93− readonly updatedAt: Date;
94−}
95−
96−/**
97− * What identifies a membership to a delete: the WorkOS `om_…` id AND the
98− * (account, organization) pair it belongs to. The pair is the row's key,
99− * and a delete must be able to mint the row as a tombstone when the mirror
100− * has not seen the membership yet — an id alone cannot.
101− */
102−export type WorkOsMirrorMembershipRef = Pick<
103− WorkOsMirrorMembership,
104− "id" | "accountId" | "organizationId"
105−>;
106−
107−/**
108− * The public failure of every mirror write: which call, and how it failed
109− * (classified from the driver cause the same way `UserStoreError` is).
110− */
111−export class WorkOsMirrorError extends Schema.TaggedErrorClass<WorkOsMirrorError>()(
112− "WorkOsMirrorError",
113− {
114− operation: Schema.String,
115− reason: Schema.Literals(USER_STORE_FAILURE_REASONS),
116− },
117− { httpApiStatus: 500 },
118−) {
119− override get message(): string {
120− return `workos mirror ${this.operation} failed: ${this.reason}`;
121− }
122−}
123−
124−export interface WorkOsMirrorShape {
125− /**
126− * Insert or refresh a user row. `false` when the payload was refused and
127− * the row left untouched: the stored row is newer than `updatedAt`, or is
128− * a deletion tombstone stamped at `updatedAt` or later.
129− */
130− readonly upsertUser: (user: WorkOsMirrorUser) => Effect.Effect<boolean, WorkOsMirrorError>;
131− /**
132− * Insert or refresh a membership row, minting the bare account row first so
133− * the foreign key holds when the membership arrives before its user. `false`
134− * when the payload was refused: the stored row is newer than `updatedAt`,
135− * or is a tombstone stamped at `updatedAt` or later — or the membership id
136− * is recorded in the deletion ledger (`membership_tombstones`, written by
137− * `deleteMembership`), whatever the payload is stamped: WorkOS never
138− * reuses a deleted `om_…` id, so no payload naming a recorded id is
139− * current — or the account is a deletion tombstone (`deleteUser`),
140− * whatever the payload is stamped: WorkOS never reuses a user id either,
141− * so a deleted user has no memberships to mirror.
142− */
143− readonly upsertMembership: (
144− membership: WorkOsMirrorMembership,
145− ) => Effect.Effect<boolean, WorkOsMirrorError>;
146− /**
147− * Tombstone the membership as deleted: the row stays (or is minted, when
148− * the mirror has not seen the membership yet), `inactive`, carrying the
149− * deleted WorkOS id and stamped so that only a payload newer than the
150− * stamp can reactivate the row. The stamp is on WorkOS's clock, so it
151− * orders against every payload the mirror is fed: `deletedAt` is an
152− * instant at or after the membership's last reported state — a deletion
153− * event's `createdAt`, or the `updatedAt` WorkOS last reported for the
154− * membership when Executor deletes it (WorkOS answers a delete with no
155− * time) — or `null` when the caller holds no WorkOS instant at all: the
156− * row then keeps the stamp it holds, the last state WorkOS reported for
157− * this membership. Never a local clock: read after WorkOS answered, it
158− * can post-date a replacement membership WorkOS created for the same
159− * member meanwhile, and a tombstone stamped with it would refuse that
160− * replacement for good. Every payload of the deleted membership a feeder
161− * could have fetched is at or before the stamp (a tombstone wins a tie),
162− * and every replacement is after it. A row the mirror does not hold and
163− * no instant to stamp it with takes the current time: there is no WorkOS
164− * stamp to keep. Matches the row by the membership's IDENTITY, never by
165− * timestamp: `false` when the row already carries this tombstone or a
166− * later one (a replayed delete), or holds ANOTHER membership id — the
167− * member re-added in WorkOS under a new id, which stands whether the
168− * replacement was mirrored before or after this delete was stamped. In
169− * EVERY case the deleted id is recorded in the deletion ledger
170− * (`membership_tombstones`) — a row holding another id is left alone, but
171− * the id this delete names is still refused to every later write, so a
172− * membership the mirror never held under its own id (replaced and deleted
173− * in WorkOS before the mirror saw it) cannot be inserted live by a scan
174− * that listed it before the deletion. `true` when the delete changed
175− * anything: the row was tombstoned, or the id was newly recorded. The
176− * organization row must already be mirrored, as for `upsertMembership`.
177− */
178− readonly deleteMembership: (
179− membership: WorkOsMirrorMembershipRef,
180− deletedAt: Date | null,
181− ) => Effect.Effect<boolean, WorkOsMirrorError>;
182− /**
183− * Tombstone a deleted WorkOS user: every membership of the account is
184− * tombstoned as by `deleteMembership`, and the account row is kept (it
185− * anchors foreign keys) — or minted, when the mirror has not seen the user
186− * yet — with its profile cleared and stamped `deletedAt`, so a stale user
187− * payload cannot restore it. `false` when the account already carries
188− * this tombstone or a later one (a replayed delete).
189− */
190− readonly deleteUser: (
191− accountId: string,
192− deletedAt: Date,
193− ) => Effect.Effect<boolean, WorkOsMirrorError>;
194− /** The id of the last WorkOS event applied, or `null` before the first run. */
195− readonly getCursor: () => Effect.Effect<string | null, WorkOsMirrorError>;
196− /**
197− * Compare-and-set the cursor: advance to `next` only if it still reads
198− * `prev` (`null` = no cursor yet). `false` means another run moved it first
199− * — the caller must stop, it no longer owns the stream.
200− */
201− readonly setCursor: (
202− prev: string | null,
203− next: string,
204− ) => Effect.Effect<boolean, WorkOsMirrorError>;
205−}
206−
207−// The one events stream the reconciler follows. A row id rather than a
208−// singleton table so a second stream (another WorkOS environment, a replay)
209−// can be added without a schema change.
210−const EVENTS_CURSOR_ID = "events";
211−
212−// A tombstone keeps the row, marks it `inactive`, and moves its timestamp to
213−// the deletion time — never backwards, so a row that somehow carries a newer
214−// WorkOS timestamp keeps it and the upsert guard stays at least as strict.
215−// With no instant at all (`deletedAt` null, see `deleteMembership`) the row
216−// keeps the stamp it holds, and a row that has none takes the current time
217−// — the only instant known. (A raw `sql`
218−// fragment binds the Date without the column's driver mapping, so it is
219−// passed as ISO text and cast.)
220−const noEarlierThan = (column: AnyPgColumn, at: Date | null) =>
221− at === null
222− ? sql`coalesce(${column}, now())`
223− : sql`greatest(${column}, ${at.toISOString()}::timestamptz)`;
224−
225−const tombstone = (deletedAt: Date | null) => ({
226− status: "inactive" as const,
227− workosUpdatedAt: noEarlierThan(memberships.workosUpdatedAt, deletedAt),
228−});
229−
230−// Which stored membership rows a payload stamped `updatedAt` may overwrite:
231−// a row with no stamp (predating the mirror), any row stamped earlier, and a
232−// LIVE row stamped the same instant — feeders replay the same payload and
233−// must converge, not stall. An `inactive` row stamped the same instant is
234−// not overwritten: a deletion (or deactivation) at T beats an active payload
235−// at T, so an equal-timestamp payload can never undo a tombstone.
236−const membershipAcceptsPayload = (updatedAt: Date) =>
237− or(
238− isNull(memberships.workosUpdatedAt),
239− lt(memberships.workosUpdatedAt, updatedAt),
240− and(eq(memberships.workosUpdatedAt, updatedAt), ne(memberships.status, "inactive")),
241− );
242−
243−// A deleted user's account row, as `deleteUser` leaves it: the profile is
244−// cleared (every WorkOS user payload carries an email, so a stamped row with
245−// none was written by `deleteUser`) and the stamp is the deletion time. A row
246−// minted bare by `ensureAccount` has no stamp either, and is not a tombstone.
247−const accountIsTombstone = and(isNull(accounts.email), isNotNull(accounts.workosUpdatedAt));
248−
249−// Whether membership `id` is in the deletion ledger: a WorkOS id a delete
250−// has named, which never returns. Judged by identity alone — the ledger
251−// records WHEN for the record, not for ordering.
252−const membershipIdDeleted = async (db: DrizzleDb, id: string): Promise<boolean> => {
253− const rows = await db
254− .select({ membershipId: membershipTombstones.membershipId })
255− .from(membershipTombstones)
256− .where(eq(membershipTombstones.membershipId, id));
257− return rows.length > 0;
258−};
259−
260−// The same rule as for a membership row, for an account row. A tombstone
261−// takes no payload stamped at or before the deletion; a bare row takes any.
262−const accountAcceptsPayload = (updatedAt: Date) =>
263− or(
264− isNull(accounts.workosUpdatedAt),
265− lt(accounts.workosUpdatedAt, updatedAt),
266− and(eq(accounts.workosUpdatedAt, updatedAt), isNotNull(accounts.email)),
267− );
268−
269−// A delete is applied unless the row already carries this tombstone or a
270−// later one: a replayed deletion changes nothing and reports so. With no
271−// instant given, any tombstone the row carries is this one or later.
272−const notTombstonedSince = (deletedAt: Date | null) =>
273− or(
274− ne(memberships.status, "inactive"),
275− isNull(memberships.workosUpdatedAt),
276− deletedAt === null ? undefined : lt(memberships.workosUpdatedAt, deletedAt),
277− );
278−
279−// Which (account, organization) row a delete of membership `id` at
280−// `deletedAt` may tombstone: the row carrying THIS id — whatever its stamp,
281−// a deleted id is never reused — or a row with no id at all (written before
282−// the mirror recorded WorkOS ids; the delete fills the id in). Never a row
283−// under ANOTHER id: that is a different membership of the same account and
284−// organization, the member re-added in WorkOS after this one was removed,
285−// and it stands however the two are stamped. Its stamp cannot be trusted to
286−// order them: a removal Executor makes is stamped by a local clock, and one
287−// that reads the clock after WorkOS answered can post-date a replacement
288−// WorkOS created while the request was in flight. Identity orders them,
289−// timestamps do not. A row already carrying this tombstone or a later one is
290−// left alone (`notTombstonedSince`) so a replayed delete reports `false`.
291−const membershipDeletableBy = (id: string, deletedAt: Date | null) =>
292− and(
293− or(isNull(memberships.membershipId), eq(memberships.membershipId, id)),
294− notTombstonedSince(deletedAt),
295− );
296−
297−const makeService = (db: DrizzleDb): WorkOsMirrorShape => {
298− const run = <A>(op: string, fn: () => Promise<A>) =>
299− withServiceLogging(
300− `workos_mirror.${op}`,
301− (failure) =>
302− new WorkOsMirrorError({
303− operation: op,
304− reason: userStoreReasonFromCause(failure),
305− }),
306− tryPromiseService(fn),
307− );
308−
309− // Over `db` or a transaction handle (drizzle's is a `PgDatabase` too).
310− const ensureAccount = (on: DrizzleDb, id: string) =>
311− on.insert(accounts).values({ id }).onConflictDoNothing({ target: accounts.id });
312−
313− return {
314− upsertUser: (user) =>
315− run("upsertUser", async () => {
316− const written = await db
317− .insert(accounts)
318− .values({
319− id: user.id,
320− email: user.email,
321− firstName: user.firstName,
322− lastName: user.lastName,
323− avatarUrl: user.avatarUrl,
324− lastSignInAt: user.lastSignInAt,
325− workosUpdatedAt: user.updatedAt,
326− })
327− .onConflictDoUpdate({
328− target: accounts.id,
329− set: {
330− email: user.email,
331− firstName: user.firstName,
332− lastName: user.lastName,
333− avatarUrl: user.avatarUrl,
334− lastSignInAt: user.lastSignInAt,
335− workosUpdatedAt: user.updatedAt,
336− },
337− setWhere: accountAcceptsPayload(user.updatedAt),
338− })
339− .returning({ id: accounts.id });
340− return written.length > 0;
341− }),
342−
343− upsertMembership: (membership) =>
344− run("upsertMembership", async () => {
345− await ensureAccount(db, membership.accountId);
346− // Never for a DELETED user. The row guard below orders a payload
347− // against the membership row it would overwrite; a membership the
348− // mirror has not seen yet has no row, so the guard cannot refuse
349− // the INSERT — and a feeder that fetched the membership before the
350− // user was deleted and writes it after (a stalled login list, the
351− // backfill's older listing) would insert it live. The account
352− // tombstone is the one row a deleted user always leaves behind, so
353− // it is consulted first, by identity: WorkOS never reuses a user
354− // id, so no payload naming a tombstoned account is ever current.
355− const deleted = await db
356− .select({ id: accounts.id })
357− .from(accounts)
358− .where(and(eq(accounts.id, membership.accountId), accountIsTombstone));
359− if (deleted.length > 0) return false;
360− // Never under a DELETED membership id. The row guard below can only
361− // refuse against the id the row holds; a delete of THIS id that
362− // found the row under another id (see the header) left only the
363− // ledger entry behind, and that is what refuses the payload here.
364− if (await membershipIdDeleted(db, membership.id)) return false;
365− const written = await db
366− .insert(memberships)
367− .values({
368− accountId: membership.accountId,
369− organizationId: membership.organizationId,
370− membershipId: membership.id,
371− role: membership.role,
372− status: membership.status,
373− workosUpdatedAt: membership.updatedAt,
374− })
375− .onConflictDoUpdate({
376− target: [memberships.accountId, memberships.organizationId],
377− set: {
378− membershipId: membership.id,
379− role: membership.role,
380− status: membership.status,
381− workosUpdatedAt: membership.updatedAt,
382− },
383− setWhere: membershipAcceptsPayload(membership.updatedAt),
384− })
385− .returning({ accountId: memberships.accountId });
386− return written.length > 0;
387− }),
388−
389− // An upsert, like the membership write it guards against: a delete the
390− // mirror sees before the membership itself (the reconciler ahead of the
391− // backfill) must leave the tombstone behind, or the later, older payload
392− // would insert the row live.
393− deleteMembership: (membership, deletedAt) =>
394− run("deleteMembership", () =>
395− db.transaction(async (tx) => {
396− await ensureAccount(tx, membership.accountId);
397− // The ledger entry FIRST, whatever the row holds: this is the one
398− // record of the deletion that does not depend on the row carrying
399− // the deleted id. A replayed delete finds it there (`DO NOTHING`)
400− // and records nothing new.
401− const recorded = await tx
402− .insert(membershipTombstones)
403− .values({
404− membershipId: membership.id,
405− accountId: membership.accountId,
406− organizationId: membership.organizationId,
407− ...(deletedAt === null ? {} : { deletedAt }),
408− })
409− .onConflictDoNothing({ target: membershipTombstones.membershipId })
410− .returning({ membershipId: membershipTombstones.membershipId });
411− const tombstoned = await tx
412− .insert(memberships)
413− .values({
414− accountId: membership.accountId,
415− organizationId: membership.organizationId,
416− membershipId: membership.id,
417− status: "inactive",
418− workosUpdatedAt: deletedAt ?? new Date(),
419− })
420− .onConflictDoUpdate({
421− target: [memberships.accountId, memberships.organizationId],
422− set: { membershipId: membership.id, ...tombstone(deletedAt) },
423− setWhere: membershipDeletableBy(membership.id, deletedAt),
424− })
425− .returning({ accountId: memberships.accountId });
426− return recorded.length > 0 || tombstoned.length > 0;
427− }),
428− ),
429−
430− deleteUser: (accountId, deletedAt) =>
431− run("deleteUser", async () => {
432− // The account tombstone FIRST, then the memberships. A membership
433− // write checks the account tombstone before it inserts
434− // (`upsertMembership`), so a write racing this delete either sees
435− // the tombstone and refuses, or has landed before it and is caught
436− // by the membership tombstoning below. In the other order a write
437− // between the two statements would slip through live.
438− //
439− // The account tombstone: profile cleared, stamped with the deletion.
440− // Minted when absent, for the same reason as the membership one.
441− const cleared = await db
442− .insert(accounts)
443− .values({ id: accountId, workosUpdatedAt: deletedAt })
444− .onConflictDoUpdate({
445− target: accounts.id,
446− set: {
447− email: null,
448− firstName: null,
449− lastName: null,
450− avatarUrl: null,
451− workosUpdatedAt: noEarlierThan(accounts.workosUpdatedAt, deletedAt),
452− },
453− // Applied unless the row already carries this tombstone or a
454− // later one (a replayed delete).
455− setWhere: or(
456− isNotNull(accounts.email),
457− isNull(accounts.workosUpdatedAt),
458− lt(accounts.workosUpdatedAt, deletedAt),
459− ),
460− })
461− .returning({ id: accounts.id });
462− await db
463− .update(memberships)
464− .set(tombstone(deletedAt))
465− .where(and(eq(memberships.accountId, accountId), notTombstonedSince(deletedAt)));
466− return cleared.length > 0;
467− }),
468−
469− getCursor: () =>
470− run("getCursor", async () => {
471− const rows = await db
472− .select({ cursor: workosSync.cursor })
473− .from(workosSync)
474− .where(eq(workosSync.id, EVENTS_CURSOR_ID));
475− // No row yet is the same state as a row with no cursor: nothing applied.
476− return rows[0]?.cursor ?? null;
477− }),
478−
479− setCursor: (prev, next) =>
480− run("setCursor", async () => {
481− const now = new Date();
482− if (prev === null) {
483− // First advance: mint the row, or claim an existing row that still
484− // has no cursor. A row that already carries one belongs to another
485− // run and is left alone.
486− const written = await db
487− .insert(workosSync)
488− .values({ id: EVENTS_CURSOR_ID, cursor: next, updatedAt: now })
489− .onConflictDoUpdate({
490− target: workosSync.id,
491− set: { cursor: next, updatedAt: now },
492− setWhere: isNull(workosSync.cursor),
493− })
494− .returning({ id: workosSync.id });
495− return written.length > 0;
496− }
497− const written = await db
498− .update(workosSync)
499− .set({ cursor: next, updatedAt: now })
500− .where(and(eq(workosSync.id, EVENTS_CURSOR_ID), eq(workosSync.cursor, prev)))
501− .returning({ id: workosSync.id });
502− return written.length > 0;
503− }),
504− };
505−};
19+import { Context, Effect, Layer } from "effect";
20+
21+import { DbService } from "../db/db";
22+import { makeWorkOsMirrorStore, type WorkOsMirrorShape } from "./workos-mirror-store";
23+
24+export { WorkOsMirrorError } from "./errors";
25+export {
26+ mirrorMembershipFromWorkOs,
27+ mirrorUserFromWorkOs,
28+ type WorkOsMembershipPayload,
29+ type WorkOsMirrorMembership,
30+ type WorkOsMirrorMembershipRef,
31+ type WorkOsMirrorShape,
32+ type WorkOsMirrorUser,
33+ type WorkOsOrganizationScan,
34+ type WorkOsOrganizationScanWrites,
35+ type WorkOsScannedMember,
36+ type WorkOsUserPayload,
37+} from "./workos-mirror-store";
50638
50739 export class WorkOsMirror extends Context.Service<WorkOsMirror, WorkOsMirrorShape>()(
50840 "@executor-js/cloud/WorkOsMirror",
50941 ) {
510− static Live = Layer.effect(this)(Effect.map(DbService.asEffect(), ({ db }) => makeService(db)));
42+ static Live = Layer.effect(this)(
43+ Effect.map(DbService.asEffect(), ({ db }) => makeWorkOsMirrorStore(db)),
44+ );
51145 }
51246
51347 /**
@@ -516,4 +50,6 @@ export class WorkOsMirror extends Context.Service<WorkOsMirror, WorkOsMirrorShap · 2 unmodified lines
51650 * `makeUserStoreLayer` exists. See [[makeDbLayer]].
51751 */
51852 export const makeWorkOsMirrorLayer = (): Layer.Layer<WorkOsMirror, never, DbService> =>
519− Layer.effect(WorkOsMirror)(Effect.map(DbService.asEffect(), ({ db }) => makeService(db)));
53+ Layer.effect(WorkOsMirror)(
54+ Effect.map(DbService.asEffect(), ({ db }) => makeWorkOsMirrorStore(db)),
55+ );
apps/cloud/src/db/db.test.ts+42−4
@@ -94,7 +94,11 @@ describe("DbService", () => { · 93 unmodified lines
9494 Effect.gen(function* () {
9595 const { db } = yield* DbService;
9696 yield* Effect.promise(() =>
97− makeUserStore(db).upsertOrganization({ id: organizationId, name: "Acme" }),
97+ makeUserStore(db).upsertOrganization({
98+ id: organizationId,
99+ name: "Acme",
100+ updatedAt: new Date(),
101+ }),
98102 );
99103 }),
100104 ),
@@ -121,16 +125,24 @@ describe("DbService", () => { · 20 unmodified lines
121125 });
122126
123127 describe("upsertOrganization · slug is minted at insert", () => {
124− const upsert = (org: { id: string; name: string }) =>
128+ const upsert = (org: { id: string; name: string; updatedAt?: Date }) =>
125129 program(
126130 Effect.gen(function* () {
127131 const { db } = yield* DbService;
128− return yield* Effect.promise(() => makeUserStore(db).upsertOrganization(org));
132+ return yield* Effect.promise(() =>
133+ makeUserStore(db).upsertOrganization({
134+ updatedAt: new Date(),
135+ ...org,
136+ }),
137+ );
129138 }),
130139 );
131140
132141 it("mints a valid slug on insert", async () => {
133− const org = await upsert({ id: `org_${crypto.randomUUID()}`, name: "Slug Mint Co" });
142+ const org = await upsert({
143+ id: `org_${crypto.randomUUID()}`,
144+ name: "Slug Mint Co",
145+ });
134146 expect(org.slug, "a new org row is born with a slug").toBeTruthy();
135147 expect(isValidOrgSlug(org.slug), "the minted slug fits the URL grammar").toBe(true);
136148 });
@@ -143,6 +155,32 @@ describe("upsertOrganization · slug is minted at insert", () => { · 6 unmodified lines
143155 expect(renamed.name, "the name is refreshed on conflict").toBe("Renamed Org");
144156 });
145157
158+ it("refuses a name stamped earlier than the one it holds, and never renames a deleted org", async () => {
159+ const id = `org_${crypto.randomUUID()}`;
160+ const t1 = new Date("2026-01-01T00:00:00.000Z");
161+ const t2 = new Date("2026-01-02T00:00:00.000Z");
162+ const t3 = new Date("2026-01-03T00:00:00.000Z");
163+ await upsert({ id, name: "Original Name", updatedAt: t2 });
164+ // A payload fetched before the rename landed (a login that stalled).
165+ const stale = await upsert({ id, name: "Stale Name", updatedAt: t1 });
166+ expect(stale.name, "an older name never reverts a newer one").toBe("Original Name");
167+ const replay = await upsert({ id, name: "Replayed Name", updatedAt: t2 });
168+ expect(replay.name, "the same instant is accepted, so replays converge").toBe("Replayed Name");
169+ await program(
170+ Effect.gen(function* () {
171+ const { db } = yield* DbService;
172+ yield* Effect.promise(() => makeUserStore(db).deleteOrganizationCascade(id, t3));
173+ }),
174+ );
175+ const afterDelete = await upsert({
176+ id,
177+ name: "Resurrected Name",
178+ updatedAt: t3,
179+ });
180+ expect(afterDelete.deletedAt, "a deleted org stays deleted").toEqual(t3);
181+ expect(afterDelete.name, "and keeps its last name").toBe("Replayed Name");
182+ });
183+
146184 it("discriminates same-name collisions into distinct slugs", async () => {
147185 // Same name → same slug base; the second insert collides on the unique
148186 // index and gets a discriminated slug.
apps/cloud/src/db/org-deletion.test.ts+28−13
@@ -7,7 +7,8 @@ · 6 unmodified lines
77 // two orgs across every tenant table + blob namespace, purges one, and asserts:
88 // - every executor tenant table row for the target org is gone
99 // - org- and user-scoped secret blobs for the target org are gone
10−// - the identity row is gone and its memberships cascade with it
10+// - the org's memberships are gone; the identity row stays as a tombstone
11+// marked deleted (so a delayed feeder cannot re-mint the org live)
1112 // - a second org's data is completely untouched
1213 // - the blob prefix match escapes LIKE wildcards (a `_` in the org id must
1314 // not widen the match to a look-alike namespace)
@@ -196,11 +197,12 @@ const NOT_ORG_OWNED: Record<string, string> = { · 182 unmodified lines
196197 blob: "org-scoped by namespace prefix, purged via the LIKE match",
197198 // Instance-wide, not owned by any org.
198199 private_executor_cloud_settings: "singleton instance settings, not org-scoped",
199− // Identity mirror. `organizations` is deleted directly and `memberships`
200− // cascades from its FK; `accounts` deliberately outlives the org.
201− organizations: "the identity row itself, deleted directly",
202− memberships: "cascades from the organizations FK",
203− membership_tombstones: "cascades from the organizations FK",
200+ // Identity mirror. `organizations` is kept as a tombstone marked deleted,
201+ // `memberships` are deleted by organization id; `accounts` deliberately
202+ // outlives the org.
203+ organizations: "the identity row itself, kept as a tombstone marked deleted",
204+ memberships: "deleted by organization id",
205+ membership_tombstones: "deleted by organization id",
204206 accounts: "shared across orgs — deliberately survives",
205207 workos_sync: "the WorkOS Events API cursor, instance-wide and not org-scoped",
206208 };
@@ -231,14 +233,24 @@ describe("purgeOrganizationData", () => { · 24 unmodified lines
231233 // A look-alike blob that only an UNescaped `_` wildcard would match:
232234 // `o:<orgA>/…` with the underscore replaced by another char.
233235 const trapNs = `o:${orgA.replace("_", "X")}/plugin`;
236+ const now = new Date();
237+ const deletedAt = new Date("2026-01-02T00:00:00.000Z");
234238
235239 await program(
236240 Effect.gen(function* () {
237241 const { db } = yield* DbService;
238242 yield* Effect.promise(async () => {
239243 const store = makeUserStore(db);
240− await store.upsertOrganization({ id: orgA, name: "Delete Me" });
241− await store.upsertOrganization({ id: orgB, name: "Keep Me" });
244+ await store.upsertOrganization({
245+ id: orgA,
246+ name: "Delete Me",
247+ updatedAt: now,
248+ });
249+ await store.upsertOrganization({
250+ id: orgB,
251+ name: "Keep Me",
252+ updatedAt: now,
253+ });
242254 await store.ensureAccount(accountId);
243255 await db.insert(memberships).values({ accountId, organizationId: orgA });
244256 await db.insert(memberships).values({ accountId, organizationId: orgB });
@@ -257,7 +269,7 @@ describe("purgeOrganizationData", () => { · 12 unmodified lines
257269 await program(
258270 Effect.gen(function* () {
259271 const { db } = yield* DbService;
260− yield* Effect.promise(() => makeUserStore(db).deleteOrganizationCascade(orgA));
272+ yield* Effect.promise(() => makeUserStore(db).deleteOrganizationCascade(orgA, deletedAt));
261273 }),
262274 );
263275
@@ -267,10 +279,13 @@ describe("purgeOrganizationData", () => { · 3 unmodified lines
267279 yield* Effect.promise(async () => {
268280 const store = makeUserStore(db);
269281
270− // Target org: every tenant row + blob gone, identity gone, membership
271− // cascaded, but the shared account survives (it may join other orgs).
282+ // Target org: every tenant row + blob gone, memberships gone, the
283+ // identity row kept as a tombstone, and the shared account survives
284+ // (it may join other orgs).
272285 expect(await countTenantRows(db, orgA)).toBe(0);
273− expect(await store.getOrganization(orgA)).toBeNull();
286+ const tombstone = await store.getOrganization(orgA);
287+ expect(tombstone?.deletedAt, "the org row stays, marked deleted").toEqual(deletedAt);
288+ expect(tombstone?.name, "as it was").toBe("Delete Me");
274289 const orgAMemberships = await db
275290 .select()
276291 .from(memberships)
@@ -285,7 +300,7 @@ describe("purgeOrganizationData", () => { · 8 unmodified lines
285300
286301 // Second org: fully intact.
287302 expect(await countTenantRows(db, orgB)).toBeGreaterThan(0);
288− expect(await store.getOrganization(orgB)).not.toBeNull();
303+ expect((await store.getOrganization(orgB))?.deletedAt).toBeNull();
289304 const orgBMemberships = await db
290305 .select()
291306 .from(memberships)
apps/cloud/src/db/org-deletion.ts+30−7
@@ -10,11 +10,21 @@ · 9 unmodified lines
1010 //
1111 // External side effects (the WorkOS org, the Autumn customer) are NOT touched
1212 // here — the caller (auth handler) sequences those around this purge.
13+//
14+// The `organizations` row itself is NOT deleted: it stays as a TOMBSTONE,
15+// marked `deleted_at`, with its memberships removed. The membership mirror's
16+// feeders write whatever WorkOS payload they hold — a login that fetched its
17+// membership list before the deletion can write it after this purge — and
18+// the tombstone is what makes those writes refuse: `upsertOrganization`
19+// never re-mints or renames a marked organization, and the mirror never
20+// inserts a membership of one. Without it the login would insert a fresh,
21+// live organization row plus an active membership, and the deleted
22+// organization would authorize again.
1323
1424 import { eq, or, sql } from "drizzle-orm";
1525
1626 import type { DrizzleDb } from "./db";
17−import { organizations } from "./schema";
27+import { memberships, organizations } from "./schema";
1828 import {
1929 artifact,
2030 blob,
@@ -36,10 +46,16 @@ const escapeLike = (value: string): string => value.replace(/[\\%_]/g, "\\$&"); · 15 unmodified lines
3646
3747 /**
3848 * Delete all rows owned by `organizationId`: every executor tenant table, the
39− * org's secret blobs (org- and user-scoped), and the identity mirror row (which
40− * cascades to local `memberships`). Idempotent — a second run deletes nothing.
49+ * org's secret blobs (org- and user-scoped), and its local `memberships` —
50+ * and mark the identity row deleted as of `deletedAt` (an earlier mark
51+ * stands), keeping it as a tombstone. Idempotent — a second run deletes
52+ * nothing and keeps the first mark.
4153 */
42−export const purgeOrganizationData = (db: DrizzleDb, organizationId: string): Promise<void> =>
54+export const purgeOrganizationData = (
55+ db: DrizzleDb,
56+ organizationId: string,
57+ deletedAt: Date,
58+): Promise<void> =>
4359 db.transaction(async (tx) => {
4460 // Executor tenant tables — every row is scoped by `tenant = organizationId`.
4561 await tx.delete(tool).where(eq(tool.tenant, organizationId));
@@ -66,7 +82,14 @@ export const purgeOrganizationData = (db: DrizzleDb, organizationId: string): Pr · 20 unmodified lines
6682 ),
6783 );
6884
69− // Identity mirror — FK `ON DELETE CASCADE` removes local memberships too.
70− // `accounts` are intentionally left: a user may belong to other orgs.
71− await tx.delete(organizations).where(eq(organizations.id, organizationId));
85+ // Identity mirror: the memberships go, the organization row stays as a
86+ // tombstone (see the header). `accounts` are intentionally left: a user
87+ // may belong to other orgs.
88+ await tx.delete(memberships).where(eq(memberships.organizationId, organizationId));
89+ await tx
90+ .update(organizations)
91+ .set({
92+ deletedAt: sql`coalesce(${organizations.deletedAt}, ${deletedAt.toISOString()}::timestamptz)`,
93+ })
94+ .where(eq(organizations.id, organizationId));
7295 });
apps/cloud/src/db/schema.ts+71−5
@@ -9,7 +9,9 @@ · 8 unmodified lines
99 // - `organizations` — billing entity, scoping root for all domain data
1010 // - `memberships` — which accounts belong to which organizations, with
1111 // the WorkOS role and status
12−// - `workos_sync` — the WorkOS Events API cursor the reconciler resumes from
12+// - `workos_sync` — the WorkOS Events API cursor the reconciler resumes
13+// from, and the replay boundary the one-off backfill
14+// records
1315 //
1416 // The mirror is fed by login (the callback has the user + memberships in
1517 // hand), write-through on every Executor-initiated change, and the WorkOS
@@ -65,6 +67,41 @@ export const organizations = pgTable( · 49 unmodified lines
6567 id: text("id").primaryKey(),
6668 name: text("name").notNull(),
6769 slug: text("slug").notNull(),
70+ /**
71+ * When this organization's membership list was last FULLY scanned from
72+ * WorkOS (the one-off backfill, or the on-demand scan a seat count
73+ * triggers), or null if it never was. Until then the mirror may hold only
74+ * the members login and write-through happened to record, so a count read
75+ * from it is partial; every seat gate checks this mark first. Per
76+ * organization, never database-wide: an org mirrored lazily after a
77+ * backfill ran starts unmarked and is scanned on its first count. The
78+ * mark also orders membership writes: a payload stamped before it is
79+ * refused, since the scan was the full listing at that instant and a
80+ * membership it did not contain was revoked before it — before the
81+ * events replay boundary, so nothing would tombstone it again.
82+ */
83+ backfilledAt: timestamp("backfilled_at", { withTimezone: true }),
84+ /**
85+ * When this organization was deleted, or null while it is live. Set by
86+ * cloud's own deletion flow and by the `organization.deleted` event, and
87+ * KEPT by the local purge (`db/org-deletion.ts`), which removes the
88+ * organization's memberships and tenant data but leaves this row as a
89+ * tombstone: a feeder that fetched a membership before the deletion and
90+ * writes it after (a login that stalled across the purge) finds the
91+ * tombstone and does not re-mint the organization live. A marked
92+ * organization is never renamed and authorizes nobody.
93+ */
94+ deletedAt: timestamp("deleted_at", { withTimezone: true }),
95+ /**
96+ * The instant the stored `name` is known to have been the organization's
97+ * name in WorkOS: the WorkOS `updatedAt` of the organization payload that
98+ * wrote it, or — for a name learned from a membership list at sign-in,
99+ * which carries no organization timestamp — the instant that list was
100+ * fetched. A name write stamped earlier than this is refused
101+ * (`upsertOrganization`), so a sign-in whose list predates a rename cannot
102+ * revert it. Null only on rows written before the stamp existed.
103+ */
104+ workosUpdatedAt: timestamp("workos_updated_at", { withTimezone: true }),
68105 createdAt: timestamp("created_at", { withTimezone: true }).notNull().defaultNow(),
69106 },
70107 (t) => ({
@@ -157,13 +194,42 @@ export const membershipTombstones = pgTable( · 86 unmodified lines
157194 );
158195
159196 /**
160− * The WorkOS Events API cursor. One row per stream (`id` names the stream;
161− * the reconciler uses `"events"`), holding the id of the last event applied.
162− * Advanced only by compare-and-set, so two concurrent reconciler runs cannot
163− * both believe they own the stream: the loser's CAS fails and it stops.
197+ * The WorkOS Events API sync state. One row per stream (`id` names the
198+ * stream; the reconciler uses `"events"`), holding the id of the last event
199+ * applied. Advanced only by compare-and-set, so two concurrent reconciler
200+ * runs cannot both believe they own the stream: the loser's CAS fails and it
201+ * stops.
202+ *
203+ * `range_start` on the `"events"` row is the REPLAY BOUNDARY: the instant the
204+ * FIRST completed one-off backfill (`scripts/backfill-workos-mirror.ts`) began
205+ * reading WorkOS. Everything before it is covered by that backfill; the
206+ * reconciler's first run (no cursor yet) reads the events stream from here,
207+ * so a revocation between the backfill and the first run is never skipped.
208+ * Written once: a backfill that fails part-way records nothing, and a later
209+ * completed one keeps it, because the backfill does not refresh everything
210+ * the events stream carries (organization renames, deleted users'
211+ * profiles) — those between two runs are replayed from the first boundary.
212+ * Without a cursor or a boundary the reconciler does not guess; it waits for
213+ * the backfill.
214+ *
215+ * `backfill_completed_at` is when a backfill run first wrote EVERY live
216+ * organization (`scripts/backfill-workos-mirror.ts` completing, or refusing
217+ * an organization only because a later listing was already applied). Until
218+ * it is set, the mirror may lack members who have not signed in since it
219+ * shipped, so a membership check read from it would deny them: it is the
220+ * first half of the mirror-readiness mark the authorization path consults
221+ * before trusting the mirror over WorkOS. Write-once — a later completed run
222+ * keeps the first instant, so readiness never flips back. Per-organization
223+ * completeness for the seat gates is tracked separately
224+ * (`organizations.backfilled_at`).
225+ *
226+ * Migration 0019 seeds the boundary and the completion mark on a database
227+ * with no organizations, where there is nothing to backfill.
164228 */
165229 export const workosSync = pgTable("workos_sync", {
166230 id: text("id").primaryKey(),
167231 cursor: text("cursor"),
232+ rangeStart: timestamp("range_start", { withTimezone: true }),
233+ backfillCompletedAt: timestamp("backfill_completed_at", { withTimezone: true }),
168234 updatedAt: timestamp("updated_at", { withTimezone: true }).notNull().defaultNow(),
169235 });
apps/cloud/src/extensions/billing/route.node.test.ts+9−0
@@ -55,18 +55,27 @@ const stubUsers = Layer.succeed(UserStoreService)({ · 54 unmodified lines
5555 upsertOrganization: async (org: { id: string; name: string }) => ({
5656 ...org,
5757 slug: org.id,
58+ backfilledAt: null,
59+ deletedAt: null,
60+ workosUpdatedAt: null,
5861 createdAt,
5962 }),
6063 getOrganization: async (id: string) => ({
6164 id,
6265 name: `Org ${id}`,
6366 slug: id,
67+ backfilledAt: null,
68+ deletedAt: null,
69+ workosUpdatedAt: null,
6470 createdAt,
6571 }),
6672 getOrganizationBySlug: async (slug: string) => ({
6773 id: slug === URL_SLUG ? URL_ORG : "org_outsider",
6874 name: `Org ${slug}`,
6975 slug,
76+ backfilledAt: null,
77+ deletedAt: null,
78+ workosUpdatedAt: null,
7079 createdAt,
7180 }),
7281 deleteOrganizationCascade: async () => {},
apps/cloud/src/extensions/routes.ts+4−1
@@ -28,6 +28,7 @@ import { AccountApi, AdminUsersApi } from "@executor-js/api"; · 27 unmodified lines
2828 import { requestScopedMiddleware } from "@executor-js/api/server";
2929
3030 import { UserStoreService } from "../auth/context";
31+import { WorkOsMirror } from "../auth/workos-mirror";
3132 import {
3233 CloudAuthPublicHandlers,
3334 CloudSessionAuthHandlers,
@@ -72,7 +73,9 @@ const spec = OpenApi.fromApi(CloudOpenApi); · 38 unmodified lines
7273 * read it — the few app-only billing touchpoints. It is NOT on the neutral boot
7374 * core.
7475 */
75−export const makeCloudExtensionRoutes = (rsLive: Layer.Layer<DbService | UserStoreService>) => {
76+export const makeCloudExtensionRoutes = (
77+ rsLive: Layer.Layer<DbService | UserStoreService | WorkOsMirror>,
78+) => {
7679 // Session routes (login / callback / me / switch-org / …). Handlers yield
7780 // `UserStoreService` directly; the per-request DB combine keeps the postgres
7881 // socket request-scoped.