History

Feed the cloud membership mirror from login and member writes (#2004)

Rhys Sullivan committed 1 month ago

Commit ddfe3f51 parent 89f424633 files · +5374 −606Browse files

Unified · Split

33 files
  1. apps/cloud/drizzle/0019_workos_mirror_sync_state.sql +21 −0
  2. apps/cloud/drizzle/meta/0019_snapshot.json +1754 −0
  3. apps/cloud/drizzle/meta/_journal.json +7 −0
  4. apps/cloud/package.json +2 −0
  5. apps/cloud/scripts/backfill-workos-mirror.ts +102 −0
  6. apps/cloud/scripts/test-globalsetup.ts +5 −1
  7. apps/cloud/src/account/account-api.ts +7 −3
  8. apps/cloud/src/account/org-api-key-revoke.node.test.ts +27 −0
  9. apps/cloud/src/account/workos-account-service.ts +61 −11
  10. apps/cloud/src/api/layers.ts +3 −1
  11. apps/cloud/src/api/protected-api-key-auth.node.test.ts +9 −0
  12. apps/cloud/src/api/protected-jwt-auth.node.test.ts +9 −0
  13. apps/cloud/src/api/router.ts +4 −1
  14. apps/cloud/src/auth/api.ts +4 −2
  15. apps/cloud/src/auth/errors.ts +21 −0
  16. apps/cloud/src/auth/handlers.ts +81 −27
  17. apps/cloud/src/auth/mirror-feeders.node.test.ts +1040 −0
  18. apps/cloud/src/auth/mirror-feeders.ts +80 −0
  19. apps/cloud/src/auth/org-api-key-auth.node.test.ts +13 −1
  20. apps/cloud/src/auth/org-selector-auth.node.test.ts +9 −0
  21. apps/cloud/src/auth/organization.ts +5 −1
  22. apps/cloud/src/auth/user-store.ts +49 −12
  23. apps/cloud/src/auth/workos-callback-state.node.test.ts +28 −0
  24. apps/cloud/src/auth/workos-mirror-backfill.ts +274 −0
  25. apps/cloud/src/auth/workos-mirror-store.ts +999 −0
  26. apps/cloud/src/auth/workos-mirror.node.test.ts +543 −19
  27. apps/cloud/src/auth/workos-mirror.ts +33 −497
  28. apps/cloud/src/db/db.test.ts +42 −4
  29. apps/cloud/src/db/org-deletion.test.ts +28 −13
  30. apps/cloud/src/db/org-deletion.ts +30 −7
  31. apps/cloud/src/db/schema.ts +71 −5
  32. apps/cloud/src/extensions/billing/route.node.test.ts +9 −0
  33. apps/cloud/src/extensions/routes.ts +4 −1
apps/cloud/drizzle/0019_workos_mirror_sync_state.sql+21−0
@@ -0,0 +1,21 @@
1+-- Backfill completeness per organization (`organizations.backfilled_at`: when
2+-- its membership list was last fully scanned from WorkOS), the organization
3+-- tombstone (`organizations.deleted_at`: kept by the local purge so a delayed
4+-- login cannot re-mint a deleted organization), the organization name stamp
5+-- (`organizations.workos_updated_at`: a name write stamped earlier is refused,
6+-- so a delayed login cannot revert a rename), and on the "events" row of
7+-- `workos_sync` the Events API replay boundary (`range_start`) the
8+-- reconciler's first run reads from plus the backfill completion mark
9+-- (`backfill_completed_at`) the authorization path checks before it trusts
10+-- the mirror over WorkOS. A database with no organizations has nothing to
11+-- backfill, so seed both there (fresh dev, test, and e2e databases); a
12+-- database that already holds organizations gets them from the backfill
13+-- script (scripts/backfill-workos-mirror.ts).
14+ALTER TABLE "organizations" ADD COLUMN "backfilled_at" timestamp with time zone;--> statement-breakpoint
15+ALTER TABLE "organizations" ADD COLUMN "deleted_at" timestamp with time zone;--> statement-breakpoint
16+ALTER TABLE "organizations" ADD COLUMN "workos_updated_at" timestamp with time zone;--> statement-breakpoint
17+ALTER TABLE "workos_sync" ADD COLUMN "range_start" timestamp with time zone;--> statement-breakpoint
18+ALTER TABLE "workos_sync" ADD COLUMN "backfill_completed_at" timestamp with time zone;--> statement-breakpoint
19+INSERT INTO "workos_sync" ("id", "cursor", "range_start", "backfill_completed_at", "updated_at")
20+SELECT 'events', NULL, now(), now(), now()
21+WHERE NOT EXISTS (SELECT 1 FROM "organizations");
apps/cloud/drizzle/meta/0019_snapshot.json+1754−0
@@ -0,0 +1,1754 @@
1+{
2+ "id": "26e5a445-9146-40bb-afaf-0f26bfb00818",
3+ "prevId": "fe5ddc71-31c5-4144-a879-11ea71a63735",
4+ "version": "7",
5+ "dialect": "postgresql",
6+ "tables": {
7+ "public.accounts": {
8+ "name": "accounts",
9+ "schema": "",
10+ "columns": {
11+ "id": {
12+ "name": "id",
13+ "type": "text",
14+ "primaryKey": true,
15+ "notNull": true
16+ },
17+ "email": {
18+ "name": "email",
19+ "type": "text",
20+ "primaryKey": false,
21+ "notNull": false
22+ },
23+ "first_name": {
24+ "name": "first_name",
25+ "type": "text",
26+ "primaryKey": false,
27+ "notNull": false
28+ },
29+ "last_name": {
30+ "name": "last_name",
31+ "type": "text",
32+ "primaryKey": false,
33+ "notNull": false
34+ },
35+ "avatar_url": {
36+ "name": "avatar_url",
37+ "type": "text",
38+ "primaryKey": false,
39+ "notNull": false
40+ },
41+ "workos_updated_at": {
42+ "name": "workos_updated_at",
43+ "type": "timestamp with time zone",
44+ "primaryKey": false,
45+ "notNull": false
46+ },
47+ "last_sign_in_at": {
48+ "name": "last_sign_in_at",
49+ "type": "timestamp with time zone",
50+ "primaryKey": false,
51+ "notNull": false
52+ },
53+ "created_at": {
54+ "name": "created_at",
55+ "type": "timestamp with time zone",
56+ "primaryKey": false,
57+ "notNull": true,
58+ "default": "now()"
59+ }
60+ },
61+ "indexes": {
62+ "accounts_email_lower_idx": {
63+ "name": "accounts_email_lower_idx",
64+ "columns": [
65+ {
66+ "expression": "lower(\"email\")",
67+ "asc": true,
68+ "isExpression": true,
69+ "nulls": "last"
70+ }
71+ ],
72+ "isUnique": false,
73+ "concurrently": false,
74+ "method": "btree",
75+ "with": {}
76+ }
77+ },
78+ "foreignKeys": {},
79+ "compositePrimaryKeys": {},
80+ "uniqueConstraints": {},
81+ "policies": {},
82+ "checkConstraints": {},
83+ "isRLSEnabled": false
84+ },
85+ "public.membership_tombstones": {
86+ "name": "membership_tombstones",
87+ "schema": "",
88+ "columns": {
89+ "membership_id": {
90+ "name": "membership_id",
91+ "type": "text",
92+ "primaryKey": true,
93+ "notNull": true
94+ },
95+ "account_id": {
96+ "name": "account_id",
97+ "type": "text",
98+ "primaryKey": false,
99+ "notNull": true
100+ },
101+ "organization_id": {
102+ "name": "organization_id",
103+ "type": "text",
104+ "primaryKey": false,
105+ "notNull": true
106+ },
107+ "deleted_at": {
108+ "name": "deleted_at",
109+ "type": "timestamp with time zone",
110+ "primaryKey": false,
111+ "notNull": true,
112+ "default": "now()"
113+ }
114+ },
115+ "indexes": {
116+ "membership_tombstones_organization_id_idx": {
117+ "name": "membership_tombstones_organization_id_idx",
118+ "columns": [
119+ {
120+ "expression": "organization_id",
121+ "isExpression": false,
122+ "asc": true,
123+ "nulls": "last"
124+ }
125+ ],
126+ "isUnique": false,
127+ "concurrently": false,
128+ "method": "btree",
129+ "with": {}
130+ }
131+ },
132+ "foreignKeys": {
133+ "membership_tombstones_account_id_accounts_id_fk": {
134+ "name": "membership_tombstones_account_id_accounts_id_fk",
135+ "tableFrom": "membership_tombstones",
136+ "tableTo": "accounts",
137+ "columnsFrom": ["account_id"],
138+ "columnsTo": ["id"],
139+ "onDelete": "cascade",
140+ "onUpdate": "no action"
141+ },
142+ "membership_tombstones_organization_id_organizations_id_fk": {
143+ "name": "membership_tombstones_organization_id_organizations_id_fk",
144+ "tableFrom": "membership_tombstones",
145+ "tableTo": "organizations",
146+ "columnsFrom": ["organization_id"],
147+ "columnsTo": ["id"],
148+ "onDelete": "cascade",
149+ "onUpdate": "no action"
150+ }
151+ },
152+ "compositePrimaryKeys": {},
153+ "uniqueConstraints": {},
154+ "policies": {},
155+ "checkConstraints": {},
156+ "isRLSEnabled": false
157+ },
158+ "public.memberships": {
159+ "name": "memberships",
160+ "schema": "",
161+ "columns": {
162+ "account_id": {
163+ "name": "account_id",
164+ "type": "text",
165+ "primaryKey": false,
166+ "notNull": true
167+ },
168+ "organization_id": {
169+ "name": "organization_id",
170+ "type": "text",
171+ "primaryKey": false,
172+ "notNull": true
173+ },
174+ "membership_id": {
175+ "name": "membership_id",
176+ "type": "text",
177+ "primaryKey": false,
178+ "notNull": false
179+ },
180+ "role": {
181+ "name": "role",
182+ "type": "text",
183+ "primaryKey": false,
184+ "notNull": true,
185+ "default": "'member'"
186+ },
187+ "status": {
188+ "name": "status",
189+ "type": "text",
190+ "primaryKey": false,
191+ "notNull": true,
192+ "default": "'active'"
193+ },
194+ "workos_updated_at": {
195+ "name": "workos_updated_at",
196+ "type": "timestamp with time zone",
197+ "primaryKey": false,
198+ "notNull": false
199+ },
200+ "deleted_at": {
201+ "name": "deleted_at",
202+ "type": "timestamp with time zone",
203+ "primaryKey": false,
204+ "notNull": false
205+ },
206+ "created_at": {
207+ "name": "created_at",
208+ "type": "timestamp with time zone",
209+ "primaryKey": false,
210+ "notNull": true,
211+ "default": "now()"
212+ }
213+ },
214+ "indexes": {
215+ "memberships_membership_id_unique": {
216+ "name": "memberships_membership_id_unique",
217+ "columns": [
218+ {
219+ "expression": "membership_id",
220+ "isExpression": false,
221+ "asc": true,
222+ "nulls": "last"
223+ }
224+ ],
225+ "isUnique": true,
226+ "concurrently": false,
227+ "method": "btree",
228+ "with": {}
229+ },
230+ "memberships_organization_id_idx": {
231+ "name": "memberships_organization_id_idx",
232+ "columns": [
233+ {
234+ "expression": "organization_id",
235+ "isExpression": false,
236+ "asc": true,
237+ "nulls": "last"
238+ }
239+ ],
240+ "isUnique": false,
241+ "concurrently": false,
242+ "method": "btree",
243+ "with": {}
244+ }
245+ },
246+ "foreignKeys": {
247+ "memberships_account_id_accounts_id_fk": {
248+ "name": "memberships_account_id_accounts_id_fk",
249+ "tableFrom": "memberships",
250+ "tableTo": "accounts",
251+ "columnsFrom": ["account_id"],
252+ "columnsTo": ["id"],
253+ "onDelete": "cascade",
254+ "onUpdate": "no action"
255+ },
256+ "memberships_organization_id_organizations_id_fk": {
257+ "name": "memberships_organization_id_organizations_id_fk",
258+ "tableFrom": "memberships",
259+ "tableTo": "organizations",
260+ "columnsFrom": ["organization_id"],
261+ "columnsTo": ["id"],
262+ "onDelete": "cascade",
263+ "onUpdate": "no action"
264+ }
265+ },
266+ "compositePrimaryKeys": {
267+ "memberships_account_id_organization_id_pk": {
268+ "name": "memberships_account_id_organization_id_pk",
269+ "columns": ["account_id", "organization_id"]
270+ }
271+ },
272+ "uniqueConstraints": {},
273+ "policies": {},
274+ "checkConstraints": {},
275+ "isRLSEnabled": false
276+ },
277+ "public.organizations": {
278+ "name": "organizations",
279+ "schema": "",
280+ "columns": {
281+ "id": {
282+ "name": "id",
283+ "type": "text",
284+ "primaryKey": true,
285+ "notNull": true
286+ },
287+ "name": {
288+ "name": "name",
289+ "type": "text",
290+ "primaryKey": false,
291+ "notNull": true
292+ },
293+ "slug": {
294+ "name": "slug",
295+ "type": "text",
296+ "primaryKey": false,
297+ "notNull": true
298+ },
299+ "backfilled_at": {
300+ "name": "backfilled_at",
301+ "type": "timestamp with time zone",
302+ "primaryKey": false,
303+ "notNull": false
304+ },
305+ "deleted_at": {
306+ "name": "deleted_at",
307+ "type": "timestamp with time zone",
308+ "primaryKey": false,
309+ "notNull": false
310+ },
311+ "workos_updated_at": {
312+ "name": "workos_updated_at",
313+ "type": "timestamp with time zone",
314+ "primaryKey": false,
315+ "notNull": false
316+ },
317+ "created_at": {
318+ "name": "created_at",
319+ "type": "timestamp with time zone",
320+ "primaryKey": false,
321+ "notNull": true,
322+ "default": "now()"
323+ }
324+ },
325+ "indexes": {
326+ "organizations_slug_unique": {
327+ "name": "organizations_slug_unique",
328+ "columns": [
329+ {
330+ "expression": "slug",
331+ "isExpression": false,
332+ "asc": true,
333+ "nulls": "last"
334+ }
335+ ],
336+ "isUnique": true,
337+ "concurrently": false,
338+ "method": "btree",
339+ "with": {}
340+ }
341+ },
342+ "foreignKeys": {},
343+ "compositePrimaryKeys": {},
344+ "uniqueConstraints": {},
345+ "policies": {},
346+ "checkConstraints": {},
347+ "isRLSEnabled": false
348+ },
349+ "public.workos_sync": {
350+ "name": "workos_sync",
351+ "schema": "",
352+ "columns": {
353+ "id": {
354+ "name": "id",
355+ "type": "text",
356+ "primaryKey": true,
357+ "notNull": true
358+ },
359+ "cursor": {
360+ "name": "cursor",
361+ "type": "text",
362+ "primaryKey": false,
363+ "notNull": false
364+ },
365+ "range_start": {
366+ "name": "range_start",
367+ "type": "timestamp with time zone",
368+ "primaryKey": false,
369+ "notNull": false
370+ },
371+ "backfill_completed_at": {
372+ "name": "backfill_completed_at",
373+ "type": "timestamp with time zone",
374+ "primaryKey": false,
375+ "notNull": false
376+ },
377+ "updated_at": {
378+ "name": "updated_at",
379+ "type": "timestamp with time zone",
380+ "primaryKey": false,
381+ "notNull": true,
382+ "default": "now()"
383+ }
384+ },
385+ "indexes": {},
386+ "foreignKeys": {},
387+ "compositePrimaryKeys": {},
388+ "uniqueConstraints": {},
389+ "policies": {},
390+ "checkConstraints": {},
391+ "isRLSEnabled": false
392+ },
393+ "public.artifact": {
394+ "name": "artifact",
395+ "schema": "",
396+ "columns": {
397+ "id": {
398+ "name": "id",
399+ "type": "varchar(255)",
400+ "primaryKey": false,
401+ "notNull": true
402+ },
403+ "title": {
404+ "name": "title",
405+ "type": "text",
406+ "primaryKey": false,
407+ "notNull": true
408+ },
409+ "description": {
410+ "name": "description",
411+ "type": "text",
412+ "primaryKey": false,
413+ "notNull": false
414+ },
415+ "code": {
416+ "name": "code",
417+ "type": "text",
418+ "primaryKey": false,
419+ "notNull": true
420+ },
421+ "bindings": {
422+ "name": "bindings",
423+ "type": "json",
424+ "primaryKey": false,
425+ "notNull": false
426+ },
427+ "preview": {
428+ "name": "preview",
429+ "type": "text",
430+ "primaryKey": false,
431+ "notNull": false
432+ },
433+ "created_at": {
434+ "name": "created_at",
435+ "type": "timestamp",
436+ "primaryKey": false,
437+ "notNull": true
438+ },
439+ "updated_at": {
440+ "name": "updated_at",
441+ "type": "timestamp",
442+ "primaryKey": false,
443+ "notNull": true
444+ },
445+ "row_id": {
446+ "name": "row_id",
447+ "type": "varchar(255)",
448+ "primaryKey": true,
449+ "notNull": true
450+ },
451+ "tenant": {
452+ "name": "tenant",
453+ "type": "varchar(255)",
454+ "primaryKey": false,
455+ "notNull": true
456+ },
457+ "owner": {
458+ "name": "owner",
459+ "type": "varchar(255)",
460+ "primaryKey": false,
461+ "notNull": true
462+ },
463+ "subject": {
464+ "name": "subject",
465+ "type": "varchar(255)",
466+ "primaryKey": false,
467+ "notNull": true
468+ }
469+ },
470+ "indexes": {
471+ "artifact_uidx": {
472+ "name": "artifact_uidx",
473+ "columns": [
474+ {
475+ "expression": "tenant",
476+ "isExpression": false,
477+ "asc": true,
478+ "nulls": "last"
479+ },
480+ {
481+ "expression": "owner",
482+ "isExpression": false,
483+ "asc": true,
484+ "nulls": "last"
485+ },
486+ {
487+ "expression": "subject",
488+ "isExpression": false,
489+ "asc": true,
490+ "nulls": "last"
491+ },
492+ {
493+ "expression": "id",
494+ "isExpression": false,
495+ "asc": true,
496+ "nulls": "last"
497+ }
498+ ],
499+ "isUnique": true,
500+ "concurrently": false,
501+ "method": "btree",
502+ "with": {}
503+ }
504+ },
505+ "foreignKeys": {},
506+ "compositePrimaryKeys": {},
507+ "uniqueConstraints": {},
508+ "policies": {},
509+ "checkConstraints": {},
510+ "isRLSEnabled": false
511+ },
512+ "public.blob": {
513+ "name": "blob",
514+ "schema": "",
515+ "columns": {
516+ "namespace": {
517+ "name": "namespace",
518+ "type": "varchar(255)",
519+ "primaryKey": false,
520+ "notNull": true
521+ },
522+ "key": {
523+ "name": "key",
524+ "type": "varchar(255)",
525+ "primaryKey": false,
526+ "notNull": true
527+ },
528+ "value": {
529+ "name": "value",
530+ "type": "text",
531+ "primaryKey": false,
532+ "notNull": true
533+ },
534+ "row_id": {
535+ "name": "row_id",
536+ "type": "varchar(255)",
537+ "primaryKey": true,
538+ "notNull": true
539+ },
540+ "id": {
541+ "name": "id",
542+ "type": "varchar(255)",
543+ "primaryKey": false,
544+ "notNull": true
545+ }
546+ },
547+ "indexes": {
548+ "blob_id_uidx": {
549+ "name": "blob_id_uidx",
550+ "columns": [
551+ {
552+ "expression": "id",
553+ "isExpression": false,
554+ "asc": true,
555+ "nulls": "last"
556+ }
557+ ],
558+ "isUnique": true,
559+ "concurrently": false,
560+ "method": "btree",
561+ "with": {}
562+ }
563+ },
564+ "foreignKeys": {},
565+ "compositePrimaryKeys": {},
566+ "uniqueConstraints": {},
567+ "policies": {},
568+ "checkConstraints": {},
569+ "isRLSEnabled": false
570+ },
571+ "public.connection": {
572+ "name": "connection",
573+ "schema": "",
574+ "columns": {
575+ "integration": {
576+ "name": "integration",
577+ "type": "varchar(255)",
578+ "primaryKey": false,
579+ "notNull": true
580+ },
581+ "name": {
582+ "name": "name",
583+ "type": "varchar(255)",
584+ "primaryKey": false,
585+ "notNull": true
586+ },
587+ "template": {
588+ "name": "template",
589+ "type": "text",
590+ "primaryKey": false,
591+ "notNull": true
592+ },
593+ "provider": {
594+ "name": "provider",
595+ "type": "text",
596+ "primaryKey": false,
597+ "notNull": true
598+ },
599+ "item_ids": {
600+ "name": "item_ids",
601+ "type": "json",
602+ "primaryKey": false,
603+ "notNull": true
604+ },
605+ "credential_write": {
606+ "name": "credential_write",
607+ "type": "json",
608+ "primaryKey": false,
609+ "notNull": false
610+ },
611+ "identity_label": {
612+ "name": "identity_label",
613+ "type": "text",
614+ "primaryKey": false,
615+ "notNull": false
616+ },
617+ "description": {
618+ "name": "description",
619+ "type": "text",
620+ "primaryKey": false,
621+ "notNull": false
622+ },
623+ "last_health": {
624+ "name": "last_health",
625+ "type": "json",
626+ "primaryKey": false,
627+ "notNull": false
628+ },
629+ "tools_synced_at": {
630+ "name": "tools_synced_at",
631+ "type": "bigint",
632+ "primaryKey": false,
633+ "notNull": false
634+ },
635+ "oauth_client": {
636+ "name": "oauth_client",
637+ "type": "text",
638+ "primaryKey": false,
639+ "notNull": false
640+ },
641+ "oauth_client_owner": {
642+ "name": "oauth_client_owner",
643+ "type": "text",
644+ "primaryKey": false,
645+ "notNull": false
646+ },
647+ "refresh_item_id": {
648+ "name": "refresh_item_id",
649+ "type": "text",
650+ "primaryKey": false,
651+ "notNull": false
652+ },
653+ "expires_at": {
654+ "name": "expires_at",
655+ "type": "bigint",
656+ "primaryKey": false,
657+ "notNull": false
658+ },
659+ "oauth_scope": {
660+ "name": "oauth_scope",
661+ "type": "text",
662+ "primaryKey": false,
663+ "notNull": false
664+ },
665+ "oauth_token_url": {
666+ "name": "oauth_token_url",
667+ "type": "text",
668+ "primaryKey": false,
669+ "notNull": false
670+ },
671+ "provider_state": {
672+ "name": "provider_state",
673+ "type": "json",
674+ "primaryKey": false,
675+ "notNull": false
676+ },
677+ "created_at": {
678+ "name": "created_at",
679+ "type": "timestamp",
680+ "primaryKey": false,
681+ "notNull": true
682+ },
683+ "updated_at": {
684+ "name": "updated_at",
685+ "type": "timestamp",
686+ "primaryKey": false,
687+ "notNull": true
688+ },
689+ "row_id": {
690+ "name": "row_id",
691+ "type": "varchar(255)",
692+ "primaryKey": true,
693+ "notNull": true
694+ },
695+ "tenant": {
696+ "name": "tenant",
697+ "type": "varchar(255)",
698+ "primaryKey": false,
699+ "notNull": true
700+ },
701+ "owner": {
702+ "name": "owner",
703+ "type": "varchar(255)",
704+ "primaryKey": false,
705+ "notNull": true
706+ },
707+ "subject": {
708+ "name": "subject",
709+ "type": "varchar(255)",
710+ "primaryKey": false,
711+ "notNull": true
712+ }
713+ },
714+ "indexes": {
715+ "connection_uidx": {
716+ "name": "connection_uidx",
717+ "columns": [
718+ {
719+ "expression": "tenant",
720+ "isExpression": false,
721+ "asc": true,
722+ "nulls": "last"
723+ },
724+ {
725+ "expression": "owner",
726+ "isExpression": false,
727+ "asc": true,
728+ "nulls": "last"
729+ },
730+ {
731+ "expression": "subject",
732+ "isExpression": false,
733+ "asc": true,
734+ "nulls": "last"
735+ },
736+ {
737+ "expression": "integration",
738+ "isExpression": false,
739+ "asc": true,
740+ "nulls": "last"
741+ },
742+ {
743+ "expression": "name",
744+ "isExpression": false,
745+ "asc": true,
746+ "nulls": "last"
747+ }
748+ ],
749+ "isUnique": true,
750+ "concurrently": false,
751+ "method": "btree",
752+ "with": {}
753+ }
754+ },
755+ "foreignKeys": {},
756+ "compositePrimaryKeys": {},
757+ "uniqueConstraints": {},
758+ "policies": {},
759+ "checkConstraints": {},
760+ "isRLSEnabled": false
761+ },
762+ "public.definition": {
763+ "name": "definition",
764+ "schema": "",
765+ "columns": {
766+ "integration": {
767+ "name": "integration",
768+ "type": "varchar(255)",
769+ "primaryKey": false,
770+ "notNull": true
771+ },
772+ "connection": {
773+ "name": "connection",
774+ "type": "varchar(255)",
775+ "primaryKey": false,
776+ "notNull": true
777+ },
778+ "plugin_id": {
779+ "name": "plugin_id",
780+ "type": "text",
781+ "primaryKey": false,
782+ "notNull": true
783+ },
784+ "name": {
785+ "name": "name",
786+ "type": "text",
787+ "primaryKey": false,
788+ "notNull": true
789+ },
790+ "schema": {
791+ "name": "schema",
792+ "type": "json",
793+ "primaryKey": false,
794+ "notNull": true
795+ },
796+ "created_at": {
797+ "name": "created_at",
798+ "type": "timestamp",
799+ "primaryKey": false,
800+ "notNull": true
801+ },
802+ "row_id": {
803+ "name": "row_id",
804+ "type": "varchar(255)",
805+ "primaryKey": true,
806+ "notNull": true
807+ },
808+ "tenant": {
809+ "name": "tenant",
810+ "type": "varchar(255)",
811+ "primaryKey": false,
812+ "notNull": true
813+ },
814+ "owner": {
815+ "name": "owner",
816+ "type": "varchar(255)",
817+ "primaryKey": false,
818+ "notNull": true
819+ },
820+ "subject": {
821+ "name": "subject",
822+ "type": "varchar(255)",
823+ "primaryKey": false,
824+ "notNull": true
825+ }
826+ },
827+ "indexes": {
828+ "definition_uidx": {
829+ "name": "definition_uidx",
830+ "columns": [
831+ {
832+ "expression": "tenant",
833+ "isExpression": false,
834+ "asc": true,
835+ "nulls": "last"
836+ },
837+ {
838+ "expression": "owner",
839+ "isExpression": false,
840+ "asc": true,
841+ "nulls": "last"
842+ },
843+ {
844+ "expression": "subject",
845+ "isExpression": false,
846+ "asc": true,
847+ "nulls": "last"
848+ },
849+ {
850+ "expression": "integration",
851+ "isExpression": false,
852+ "asc": true,
853+ "nulls": "last"
854+ },
855+ {
856+ "expression": "connection",
857+ "isExpression": false,
858+ "asc": true,
859+ "nulls": "last"
860+ },
861+ {
862+ "expression": "name",
863+ "isExpression": false,
864+ "asc": true,
865+ "nulls": "last"
866+ }
867+ ],
868+ "isUnique": true,
869+ "concurrently": false,
870+ "method": "btree",
871+ "with": {}
872+ }
873+ },
874+ "foreignKeys": {},
875+ "compositePrimaryKeys": {},
876+ "uniqueConstraints": {},
877+ "policies": {},
878+ "checkConstraints": {},
879+ "isRLSEnabled": false
880+ },
881+ "public.integration": {
882+ "name": "integration",
883+ "schema": "",
884+ "columns": {
885+ "slug": {
886+ "name": "slug",
887+ "type": "varchar(255)",
888+ "primaryKey": false,
889+ "notNull": true
890+ },
891+ "plugin_id": {
892+ "name": "plugin_id",
893+ "type": "text",
894+ "primaryKey": false,
895+ "notNull": true
896+ },
897+ "name": {
898+ "name": "name",
899+ "type": "text",
900+ "primaryKey": false,
901+ "notNull": false
902+ },
903+ "description": {
904+ "name": "description",
905+ "type": "text",
906+ "primaryKey": false,
907+ "notNull": false
908+ },
909+ "config": {
910+ "name": "config",
911+ "type": "json",
912+ "primaryKey": false,
913+ "notNull": false
914+ },
915+ "health_check": {
916+ "name": "health_check",
917+ "type": "json",
918+ "primaryKey": false,
919+ "notNull": false
920+ },
921+ "config_revised_at": {
922+ "name": "config_revised_at",
923+ "type": "bigint",
924+ "primaryKey": false,
925+ "notNull": false
926+ },
927+ "can_remove": {
928+ "name": "can_remove",
929+ "type": "boolean",
930+ "primaryKey": false,
931+ "notNull": true,
932+ "default": true
933+ },
934+ "can_refresh": {
935+ "name": "can_refresh",
936+ "type": "boolean",
937+ "primaryKey": false,
938+ "notNull": true,
939+ "default": false
940+ },
941+ "created_at": {
942+ "name": "created_at",
943+ "type": "timestamp",
944+ "primaryKey": false,
945+ "notNull": true
946+ },
947+ "updated_at": {
948+ "name": "updated_at",
949+ "type": "timestamp",
950+ "primaryKey": false,
951+ "notNull": true
952+ },
953+ "row_id": {
954+ "name": "row_id",
955+ "type": "varchar(255)",
956+ "primaryKey": true,
957+ "notNull": true
958+ },
959+ "tenant": {
960+ "name": "tenant",
961+ "type": "varchar(255)",
962+ "primaryKey": false,
963+ "notNull": true
964+ }
965+ },
966+ "indexes": {
967+ "integration_uidx": {
968+ "name": "integration_uidx",
969+ "columns": [
970+ {
971+ "expression": "tenant",
972+ "isExpression": false,
973+ "asc": true,
974+ "nulls": "last"
975+ },
976+ {
977+ "expression": "slug",
978+ "isExpression": false,
979+ "asc": true,
980+ "nulls": "last"
981+ }
982+ ],
983+ "isUnique": true,
984+ "concurrently": false,
985+ "method": "btree",
986+ "with": {}
987+ }
988+ },
989+ "foreignKeys": {},
990+ "compositePrimaryKeys": {},
991+ "uniqueConstraints": {},
992+ "policies": {},
993+ "checkConstraints": {},
994+ "isRLSEnabled": false
995+ },
996+ "public.oauth_client": {
997+ "name": "oauth_client",
998+ "schema": "",
999+ "columns": {
1000+ "slug": {
1001+ "name": "slug",
1002+ "type": "varchar(255)",
1003+ "primaryKey": false,
1004+ "notNull": true
1005+ },
1006+ "authorization_url": {
1007+ "name": "authorization_url",
1008+ "type": "text",
1009+ "primaryKey": false,
1010+ "notNull": true
1011+ },
1012+ "token_url": {
1013+ "name": "token_url",
1014+ "type": "text",
1015+ "primaryKey": false,
1016+ "notNull": true
1017+ },
1018+ "grant": {
1019+ "name": "grant",
1020+ "type": "text",
1021+ "primaryKey": false,
1022+ "notNull": true
1023+ },
1024+ "client_id": {
1025+ "name": "client_id",
1026+ "type": "text",
1027+ "primaryKey": false,
1028+ "notNull": true
1029+ },
1030+ "client_secret_item_id": {
1031+ "name": "client_secret_item_id",
1032+ "type": "text",
1033+ "primaryKey": false,
1034+ "notNull": false
1035+ },
1036+ "credential_write": {
1037+ "name": "credential_write",
1038+ "type": "json",
1039+ "primaryKey": false,
1040+ "notNull": false
1041+ },
1042+ "token_endpoint_auth_method": {
1043+ "name": "token_endpoint_auth_method",
1044+ "type": "text",
1045+ "primaryKey": false,
1046+ "notNull": false
1047+ },
1048+ "resource": {
1049+ "name": "resource",
1050+ "type": "text",
1051+ "primaryKey": false,
1052+ "notNull": false
1053+ },
1054+ "origin_kind": {
1055+ "name": "origin_kind",
1056+ "type": "text",
1057+ "primaryKey": false,
1058+ "notNull": false
1059+ },
1060+ "origin_integration": {
1061+ "name": "origin_integration",
1062+ "type": "text",
1063+ "primaryKey": false,
1064+ "notNull": false
1065+ },
1066+ "origin_issuer": {
1067+ "name": "origin_issuer",
1068+ "type": "text",
1069+ "primaryKey": false,
1070+ "notNull": false
1071+ },
1072+ "origin_redirect_uri": {
1073+ "name": "origin_redirect_uri",
1074+ "type": "text",
1075+ "primaryKey": false,
1076+ "notNull": false
1077+ },
1078+ "created_at": {
1079+ "name": "created_at",
1080+ "type": "timestamp",
1081+ "primaryKey": false,
1082+ "notNull": true
1083+ },
1084+ "row_id": {
1085+ "name": "row_id",
1086+ "type": "varchar(255)",
1087+ "primaryKey": true,
1088+ "notNull": true
1089+ },
1090+ "tenant": {
1091+ "name": "tenant",
1092+ "type": "varchar(255)",
1093+ "primaryKey": false,
1094+ "notNull": true
1095+ },
1096+ "owner": {
1097+ "name": "owner",
1098+ "type": "varchar(255)",
1099+ "primaryKey": false,
1100+ "notNull": true
1101+ },
1102+ "subject": {
1103+ "name": "subject",
1104+ "type": "varchar(255)",
1105+ "primaryKey": false,
1106+ "notNull": true
1107+ }
1108+ },
1109+ "indexes": {
1110+ "oauth_client_uidx": {
1111+ "name": "oauth_client_uidx",
1112+ "columns": [
1113+ {
1114+ "expression": "tenant",
1115+ "isExpression": false,
1116+ "asc": true,
1117+ "nulls": "last"
1118+ },
1119+ {
1120+ "expression": "owner",
1121+ "isExpression": false,
1122+ "asc": true,
1123+ "nulls": "last"
1124+ },
1125+ {
1126+ "expression": "subject",
1127+ "isExpression": false,
1128+ "asc": true,
1129+ "nulls": "last"
1130+ },
1131+ {
1132+ "expression": "slug",
1133+ "isExpression": false,
1134+ "asc": true,
1135+ "nulls": "last"
1136+ }
1137+ ],
1138+ "isUnique": true,
1139+ "concurrently": false,
1140+ "method": "btree",
1141+ "with": {}
1142+ }
1143+ },
1144+ "foreignKeys": {},
1145+ "compositePrimaryKeys": {},
1146+ "uniqueConstraints": {},
1147+ "policies": {},
1148+ "checkConstraints": {},
1149+ "isRLSEnabled": false
1150+ },
1151+ "public.oauth_session": {
1152+ "name": "oauth_session",
1153+ "schema": "",
1154+ "columns": {
1155+ "state": {
1156+ "name": "state",
1157+ "type": "varchar(255)",
1158+ "primaryKey": false,
1159+ "notNull": true
1160+ },
1161+ "client_slug": {
1162+ "name": "client_slug",
1163+ "type": "text",
1164+ "primaryKey": false,
1165+ "notNull": true
1166+ },
1167+ "integration": {
1168+ "name": "integration",
1169+ "type": "text",
1170+ "primaryKey": false,
1171+ "notNull": true
1172+ },
1173+ "name": {
1174+ "name": "name",
1175+ "type": "text",
1176+ "primaryKey": false,
1177+ "notNull": true
1178+ },
1179+ "template": {
1180+ "name": "template",
1181+ "type": "text",
1182+ "primaryKey": false,
1183+ "notNull": true
1184+ },
1185+ "redirect_url": {
1186+ "name": "redirect_url",
1187+ "type": "text",
1188+ "primaryKey": false,
1189+ "notNull": true
1190+ },
1191+ "pkce_verifier": {
1192+ "name": "pkce_verifier",
1193+ "type": "text",
1194+ "primaryKey": false,
1195+ "notNull": false
1196+ },
1197+ "identity_label": {
1198+ "name": "identity_label",
1199+ "type": "text",
1200+ "primaryKey": false,
1201+ "notNull": false
1202+ },
1203+ "payload": {
1204+ "name": "payload",
1205+ "type": "json",
1206+ "primaryKey": false,
1207+ "notNull": true
1208+ },
1209+ "expires_at": {
1210+ "name": "expires_at",
1211+ "type": "bigint",
1212+ "primaryKey": false,
1213+ "notNull": true
1214+ },
1215+ "created_at": {
1216+ "name": "created_at",
1217+ "type": "timestamp",
1218+ "primaryKey": false,
1219+ "notNull": true
1220+ },
1221+ "row_id": {
1222+ "name": "row_id",
1223+ "type": "varchar(255)",
1224+ "primaryKey": true,
1225+ "notNull": true
1226+ },
1227+ "tenant": {
1228+ "name": "tenant",
1229+ "type": "varchar(255)",
1230+ "primaryKey": false,
1231+ "notNull": true
1232+ },
1233+ "owner": {
1234+ "name": "owner",
1235+ "type": "varchar(255)",
1236+ "primaryKey": false,
1237+ "notNull": true
1238+ },
1239+ "subject": {
1240+ "name": "subject",
1241+ "type": "varchar(255)",
1242+ "primaryKey": false,
1243+ "notNull": true
1244+ }
1245+ },
1246+ "indexes": {
1247+ "oauth_session_uidx": {
1248+ "name": "oauth_session_uidx",
1249+ "columns": [
1250+ {
1251+ "expression": "tenant",
1252+ "isExpression": false,
1253+ "asc": true,
1254+ "nulls": "last"
1255+ },
1256+ {
1257+ "expression": "state",
1258+ "isExpression": false,
1259+ "asc": true,
1260+ "nulls": "last"
1261+ }
1262+ ],
1263+ "isUnique": true,
1264+ "concurrently": false,
1265+ "method": "btree",
1266+ "with": {}
1267+ }
1268+ },
1269+ "foreignKeys": {},
1270+ "compositePrimaryKeys": {},
1271+ "uniqueConstraints": {},
1272+ "policies": {},
1273+ "checkConstraints": {},
1274+ "isRLSEnabled": false
1275+ },
1276+ "public.plugin_storage": {
1277+ "name": "plugin_storage",
1278+ "schema": "",
1279+ "columns": {
1280+ "plugin_id": {
1281+ "name": "plugin_id",
1282+ "type": "varchar(255)",
1283+ "primaryKey": false,
1284+ "notNull": true
1285+ },
1286+ "collection": {
1287+ "name": "collection",
1288+ "type": "varchar(255)",
1289+ "primaryKey": false,
1290+ "notNull": true
1291+ },
1292+ "key": {
1293+ "name": "key",
1294+ "type": "varchar(255)",
1295+ "primaryKey": false,
1296+ "notNull": true
1297+ },
1298+ "data": {
1299+ "name": "data",
1300+ "type": "json",
1301+ "primaryKey": false,
1302+ "notNull": true
1303+ },
1304+ "created_at": {
1305+ "name": "created_at",
1306+ "type": "timestamp",
1307+ "primaryKey": false,
1308+ "notNull": true
1309+ },
1310+ "updated_at": {
1311+ "name": "updated_at",
1312+ "type": "timestamp",
1313+ "primaryKey": false,
1314+ "notNull": true
1315+ },
1316+ "row_id": {
1317+ "name": "row_id",
1318+ "type": "varchar(255)",
1319+ "primaryKey": true,
1320+ "notNull": true
1321+ },
1322+ "tenant": {
1323+ "name": "tenant",
1324+ "type": "varchar(255)",
1325+ "primaryKey": false,
1326+ "notNull": true
1327+ },
1328+ "owner": {
1329+ "name": "owner",
1330+ "type": "varchar(255)",
1331+ "primaryKey": false,
1332+ "notNull": true
1333+ },
1334+ "subject": {
1335+ "name": "subject",
1336+ "type": "varchar(255)",
1337+ "primaryKey": false,
1338+ "notNull": true
1339+ }
1340+ },
1341+ "indexes": {
1342+ "plugin_storage_uidx": {
1343+ "name": "plugin_storage_uidx",
1344+ "columns": [
1345+ {
1346+ "expression": "tenant",
1347+ "isExpression": false,
1348+ "asc": true,
1349+ "nulls": "last"
1350+ },
1351+ {
1352+ "expression": "owner",
1353+ "isExpression": false,
1354+ "asc": true,
1355+ "nulls": "last"
1356+ },
1357+ {
1358+ "expression": "subject",
1359+ "isExpression": false,
1360+ "asc": true,
1361+ "nulls": "last"
1362+ },
1363+ {
1364+ "expression": "plugin_id",
1365+ "isExpression": false,
1366+ "asc": true,
1367+ "nulls": "last"
1368+ },
1369+ {
1370+ "expression": "collection",
1371+ "isExpression": false,
1372+ "asc": true,
1373+ "nulls": "last"
1374+ },
1375+ {
1376+ "expression": "key",
1377+ "isExpression": false,
1378+ "asc": true,
1379+ "nulls": "last"
1380+ }
1381+ ],
1382+ "isUnique": true,
1383+ "concurrently": false,
1384+ "method": "btree",
1385+ "with": {}
1386+ }
1387+ },
1388+ "foreignKeys": {},
1389+ "compositePrimaryKeys": {},
1390+ "uniqueConstraints": {},
1391+ "policies": {},
1392+ "checkConstraints": {},
1393+ "isRLSEnabled": false
1394+ },
1395+ "public.private_executor_cloud_settings": {
1396+ "name": "private_executor_cloud_settings",
1397+ "schema": "",
1398+ "columns": {
1399+ "id": {
1400+ "name": "id",
1401+ "type": "varchar(255)",
1402+ "primaryKey": true,
1403+ "notNull": true
1404+ },
1405+ "version": {
1406+ "name": "version",
1407+ "type": "varchar(255)",
1408+ "primaryKey": false,
1409+ "notNull": true,
1410+ "default": "'1.0.0'"
1411+ }
1412+ },
1413+ "indexes": {},
1414+ "foreignKeys": {},
1415+ "compositePrimaryKeys": {},
1416+ "uniqueConstraints": {},
1417+ "policies": {},
1418+ "checkConstraints": {},
1419+ "isRLSEnabled": false
1420+ },
1421+ "public.subject": {
1422+ "name": "subject",
1423+ "schema": "",
1424+ "columns": {
1425+ "external_id": {
1426+ "name": "external_id",
1427+ "type": "varchar(255)",
1428+ "primaryKey": false,
1429+ "notNull": true
1430+ },
1431+ "created_at": {
1432+ "name": "created_at",
1433+ "type": "timestamp",
1434+ "primaryKey": false,
1435+ "notNull": true
1436+ },
1437+ "last_seen_at": {
1438+ "name": "last_seen_at",
1439+ "type": "bigint",
1440+ "primaryKey": false,
1441+ "notNull": false
1442+ },
1443+ "status": {
1444+ "name": "status",
1445+ "type": "text",
1446+ "primaryKey": false,
1447+ "notNull": false
1448+ },
1449+ "row_id": {
1450+ "name": "row_id",
1451+ "type": "varchar(255)",
1452+ "primaryKey": true,
1453+ "notNull": true
1454+ },
1455+ "tenant": {
1456+ "name": "tenant",
1457+ "type": "varchar(255)",
1458+ "primaryKey": false,
1459+ "notNull": true
1460+ }
1461+ },
1462+ "indexes": {
1463+ "subject_uidx": {
1464+ "name": "subject_uidx",
1465+ "columns": [
1466+ {
1467+ "expression": "tenant",
1468+ "isExpression": false,
1469+ "asc": true,
1470+ "nulls": "last"
1471+ },
1472+ {
1473+ "expression": "external_id",
1474+ "isExpression": false,
1475+ "asc": true,
1476+ "nulls": "last"
1477+ }
1478+ ],
1479+ "isUnique": true,
1480+ "concurrently": false,
1481+ "method": "btree",
1482+ "with": {}
1483+ }
1484+ },
1485+ "foreignKeys": {},
1486+ "compositePrimaryKeys": {},
1487+ "uniqueConstraints": {},
1488+ "policies": {},
1489+ "checkConstraints": {},
1490+ "isRLSEnabled": false
1491+ },
1492+ "public.tool": {
1493+ "name": "tool",
1494+ "schema": "",
1495+ "columns": {
1496+ "integration": {
1497+ "name": "integration",
1498+ "type": "varchar(255)",
1499+ "primaryKey": false,
1500+ "notNull": true
1501+ },
1502+ "connection": {
1503+ "name": "connection",
1504+ "type": "varchar(255)",
1505+ "primaryKey": false,
1506+ "notNull": true
1507+ },
1508+ "plugin_id": {
1509+ "name": "plugin_id",
1510+ "type": "text",
1511+ "primaryKey": false,
1512+ "notNull": true
1513+ },
1514+ "name": {
1515+ "name": "name",
1516+ "type": "varchar(255)",
1517+ "primaryKey": false,
1518+ "notNull": true
1519+ },
1520+ "description": {
1521+ "name": "description",
1522+ "type": "text",
1523+ "primaryKey": false,
1524+ "notNull": true
1525+ },
1526+ "input_schema": {
1527+ "name": "input_schema",
1528+ "type": "json",
1529+ "primaryKey": false,
1530+ "notNull": false
1531+ },
1532+ "output_schema": {
1533+ "name": "output_schema",
1534+ "type": "json",
1535+ "primaryKey": false,
1536+ "notNull": false
1537+ },
1538+ "annotations": {
1539+ "name": "annotations",
1540+ "type": "json",
1541+ "primaryKey": false,
1542+ "notNull": false
1543+ },
1544+ "created_at": {
1545+ "name": "created_at",
1546+ "type": "timestamp",
1547+ "primaryKey": false,
1548+ "notNull": true
1549+ },
1550+ "updated_at": {
1551+ "name": "updated_at",
1552+ "type": "timestamp",
1553+ "primaryKey": false,
1554+ "notNull": true
1555+ },
1556+ "row_id": {
1557+ "name": "row_id",
1558+ "type": "varchar(255)",
1559+ "primaryKey": true,
1560+ "notNull": true
1561+ },
1562+ "tenant": {
1563+ "name": "tenant",
1564+ "type": "varchar(255)",
1565+ "primaryKey": false,
1566+ "notNull": true
1567+ },
1568+ "owner": {
1569+ "name": "owner",
1570+ "type": "varchar(255)",
1571+ "primaryKey": false,
1572+ "notNull": true
1573+ },
1574+ "subject": {
1575+ "name": "subject",
1576+ "type": "varchar(255)",
1577+ "primaryKey": false,
1578+ "notNull": true
1579+ }
1580+ },
1581+ "indexes": {
1582+ "tool_uidx": {
1583+ "name": "tool_uidx",
1584+ "columns": [
1585+ {
1586+ "expression": "tenant",
1587+ "isExpression": false,
1588+ "asc": true,
1589+ "nulls": "last"
1590+ },
1591+ {
1592+ "expression": "owner",
1593+ "isExpression": false,
1594+ "asc": true,
1595+ "nulls": "last"
1596+ },
1597+ {
1598+ "expression": "subject",
1599+ "isExpression": false,
1600+ "asc": true,
1601+ "nulls": "last"
1602+ },
1603+ {
1604+ "expression": "integration",
1605+ "isExpression": false,
1606+ "asc": true,
1607+ "nulls": "last"
1608+ },
1609+ {
1610+ "expression": "connection",
1611+ "isExpression": false,
1612+ "asc": true,
1613+ "nulls": "last"
1614+ },
1615+ {
1616+ "expression": "name",
1617+ "isExpression": false,
1618+ "asc": true,
1619+ "nulls": "last"
1620+ }
1621+ ],
1622+ "isUnique": true,
1623+ "concurrently": false,
1624+ "method": "btree",
1625+ "with": {}
1626+ }
1627+ },
1628+ "foreignKeys": {},
1629+ "compositePrimaryKeys": {},
1630+ "uniqueConstraints": {},
1631+ "policies": {},
1632+ "checkConstraints": {},
1633+ "isRLSEnabled": false
1634+ },
1635+ "public.tool_policy": {
1636+ "name": "tool_policy",
1637+ "schema": "",
1638+ "columns": {
1639+ "id": {
1640+ "name": "id",
1641+ "type": "varchar(255)",
1642+ "primaryKey": false,
1643+ "notNull": true
1644+ },
1645+ "pattern": {
1646+ "name": "pattern",
1647+ "type": "text",
1648+ "primaryKey": false,
1649+ "notNull": true
1650+ },
1651+ "action": {
1652+ "name": "action",
1653+ "type": "text",
1654+ "primaryKey": false,
1655+ "notNull": true
1656+ },
1657+ "position": {
1658+ "name": "position",
1659+ "type": "text",
1660+ "primaryKey": false,
1661+ "notNull": true
1662+ },
1663+ "created_at": {
1664+ "name": "created_at",
1665+ "type": "timestamp",
1666+ "primaryKey": false,
1667+ "notNull": true
1668+ },
1669+ "updated_at": {
1670+ "name": "updated_at",
1671+ "type": "timestamp",
1672+ "primaryKey": false,
1673+ "notNull": true
1674+ },
1675+ "row_id": {
1676+ "name": "row_id",
1677+ "type": "varchar(255)",
1678+ "primaryKey": true,
1679+ "notNull": true
1680+ },
1681+ "tenant": {
1682+ "name": "tenant",
1683+ "type": "varchar(255)",
1684+ "primaryKey": false,
1685+ "notNull": true
1686+ },
1687+ "owner": {
1688+ "name": "owner",
1689+ "type": "varchar(255)",
1690+ "primaryKey": false,
1691+ "notNull": true
1692+ },
1693+ "subject": {
1694+ "name": "subject",
1695+ "type": "varchar(255)",
1696+ "primaryKey": false,
1697+ "notNull": true
1698+ }
1699+ },
1700+ "indexes": {
1701+ "tool_policy_uidx": {
1702+ "name": "tool_policy_uidx",
1703+ "columns": [
1704+ {
1705+ "expression": "tenant",
1706+ "isExpression": false,
1707+ "asc": true,
1708+ "nulls": "last"
1709+ },
1710+ {
1711+ "expression": "owner",
1712+ "isExpression": false,
1713+ "asc": true,
1714+ "nulls": "last"
1715+ },
1716+ {
1717+ "expression": "subject",
1718+ "isExpression": false,
1719+ "asc": true,
1720+ "nulls": "last"
1721+ },
1722+ {
1723+ "expression": "id",
1724+ "isExpression": false,
1725+ "asc": true,
1726+ "nulls": "last"
1727+ }
1728+ ],
1729+ "isUnique": true,
1730+ "concurrently": false,
1731+ "method": "btree",
1732+ "with": {}
1733+ }
1734+ },
1735+ "foreignKeys": {},
1736+ "compositePrimaryKeys": {},
1737+ "uniqueConstraints": {},
1738+ "policies": {},
1739+ "checkConstraints": {},
1740+ "isRLSEnabled": false
1741+ }
1742+ },
1743+ "enums": {},
1744+ "schemas": {},
1745+ "sequences": {},
1746+ "roles": {},
1747+ "policies": {},
1748+ "views": {},
1749+ "_meta": {
1750+ "columns": {},
1751+ "schemas": {},
1752+ "tables": {}
1753+ }
1754+}
apps/cloud/drizzle/meta/_journal.json+7−0
@@ -134,6 +134,13 @@ · 133 unmodified lines
134134 "when": 1789570778982,
135135 "tag": "0018_member_directory_mirror",
136136 "breakpoints": true
137+ },
138+ {
139+ "idx": 19,
140+ "version": "7",
141+ "when": 1789571259533,
142+ "tag": "0019_workos_mirror_sync_state",
143+ "breakpoints": true
137144 }
138145 ]
139146 }
apps/cloud/package.json+2−0
@@ -32,6 +32,8 @@ · 31 unmodified lines
3232 "db:backfill-org-slugs:dev": "op run --env-file=.env.op -- bun run scripts/backfill-org-slugs.ts",
3333 "db:backfill-subjects:prod": "op run --env-file=.env.production -- bun run scripts/backfill-subjects.ts",
3434 "db:backfill-subjects:dev": "op run --env-file=.env.op -- bun run scripts/backfill-subjects.ts",
35+ "db:backfill-workos-mirror:prod": "op run --env-file=.env.production -- bun run scripts/backfill-workos-mirror.ts",
36+ "db:backfill-workos-mirror:dev": "op run --env-file=.env.op -- bun run scripts/backfill-workos-mirror.ts",
3537 "routes:gen": "bun scripts/gen-routes.ts",
3638 "vendor-wasm": "bun run scripts/vendor-quickjs-wasm.ts"
3739 },
apps/cloud/scripts/backfill-workos-mirror.ts+102−0
@@ -0,0 +1,102 @@
1+// ---------------------------------------------------------------------------
2+// One-off data backfill: fill the membership mirror (`accounts` profile
3+// columns + `memberships` rows, migration 0018) from WorkOS for every
4+// organization the mirror already knows.
5+//
6+// bun run db:backfill-workos-mirror:prod # op run --env-file=.env.production
7+// bun run db:backfill-workos-mirror:dev # against the local PGlite dev db
8+//
9+// For each live row in `organizations`: list EVERY membership WorkOS holds
10+// for it (active, pending, and inactive — a listing that skipped inactive
11+// ones would have the scan tombstone them, see `auth/workos-mirror-backfill.ts`),
12+// fetch each member's user (concurrency 5), and apply the listing in one
13+// transaction through the same guarded store the request path
14+// uses (`auth/workos-mirror-store.ts`): upsert user + membership, tombstone
15+// any mirrored membership of that org WorkOS no longer lists, and mark the
16+// org backfilled (`organizations.backfilled_at`) — the per-org mark the seat
17+// gates check before trusting a count from the mirror; an org left unmarked
18+// is scanned on demand the first time its seats are counted. Idempotent —
19+// the upserts refuse anything older than the stored WorkOS `updatedAt`, so
20+// re-running is safe, never rewinds a fresher row, and repairs a stale one;
21+// and a listing older than one already applied (two runs overlapping) is
22+// refused whole, so it cannot resurrect a membership the later listing found
23+// gone. Pass --dry-run to read and count without writing.
24+//
25+// DEPLOY ORDER: run this against production BEFORE deploying the builds that
26+// reconcile from the Events API and read seat counts from the mirror, so no
27+// request pays for an on-demand scan. The FIRST completed run records the
28+// events replay boundary (the reconciler's first run reads from it; without
29+// one it waits); later runs keep it, since only the events stream covers the
30+// org renames and user deletions between two runs. A run that fails part-way
31+// keeps the marks of the orgs it finished, records no boundary, and is safe
32+// to repeat. Verify the printed membership count against the WorkOS
33+// dashboard.
34+// ---------------------------------------------------------------------------
35+
36+import { asc, isNull } from "drizzle-orm";
37+import { drizzle } from "drizzle-orm/postgres-js";
38+import { Effect } from "effect";
39+import postgres from "postgres";
40+import { WorkOS } from "@workos-inc/node";
41+
42+import { backfillWorkOsMirror } from "../src/auth/workos-mirror-backfill";
43+import { makeWorkOsMirrorStore } from "../src/auth/workos-mirror-store";
44+import { organizations } from "../src/db/schema";
45+
46+const dryRun = process.argv.includes("--dry-run");
47+
48+const connectionString = process.env.DATABASE_URL;
49+if (!connectionString) {
50+ console.error("DATABASE_URL is not set");
51+ process.exit(1);
52+}
53+const apiKey = process.env.WORKOS_API_KEY;
54+if (!apiKey) {
55+ console.error("WORKOS_API_KEY is not set");
56+ process.exit(1);
57+}
58+
59+const usesLocalDatabase =
60+ connectionString.includes("127.0.0.1") || connectionString.includes("localhost");
61+
62+const sql = postgres(connectionString, {
63+ max: 1,
64+ prepare: false,
65+ ...(usesLocalDatabase ? {} : { ssl: "require" as const }),
66+});
67+const db = drizzle(sql);
68+const workos = new WorkOS(apiKey);
69+
70+// The script boundary: raw SDK / driver promises lifted once, here.
71+const fromPromise = <A>(fn: () => Promise<A>) =>
72+ Effect.tryPromise({ try: fn, catch: (cause) => cause });
73+
74+await Effect.runPromise(
75+ backfillWorkOsMirror(
76+ {
77+ listOrganizationIds: () =>
78+ fromPromise(async () => {
79+ // Never a deleted organization: its row is a tombstone (its
80+ // memberships are purged, WorkOS no longer has it) and the mirror
81+ // refuses a scan of it anyway.
82+ const rows = await db
83+ .select({ id: organizations.id })
84+ .from(organizations)
85+ .where(isNull(organizations.deletedAt))
86+ .orderBy(asc(organizations.createdAt));
87+ return rows.map((row) => row.id);
88+ }),
89+ listOrgMembers: (organizationId) =>
90+ fromPromise(async () => {
91+ const page = await workos.userManagement.listOrganizationMemberships({
92+ organizationId,
93+ statuses: ["active", "pending", "inactive"],
94+ });
95+ return page.listMetadata.after ? page.autoPagination() : page.data;
96+ }),
97+ getUser: (userId) => fromPromise(() => workos.userManagement.getUser(userId)),
98+ },
99+ makeWorkOsMirrorStore(db),
100+ { dryRun, log: (line) => console.log(line) },
101+ ).pipe(Effect.ensuring(Effect.promise(() => sql.end({ timeout: 5 })))),
102+);
apps/cloud/scripts/test-globalsetup.ts+5−1
@@ -42,7 +42,11 @@ export default async function setup() { · 41 unmodified lines
4242 db = await PGlite.create();
4343 await migrate(drizzle(db), { migrationsFolder: MIGRATIONS_FOLDER });
4444
45− server = new PGLiteSocketServer({ db, port: PORT, host: "127.0.0.1" });
45+ // PGlite is single-session; pglite-socket multiplexes connections onto it
46+ // by queueing whole transactions. Two connections let a test open two
47+ // transactions and interleave them (the mirror's scan-vs-feeder race in
48+ // auth/workos-mirror.node.test.ts); a third would be refused at once.
49+ server = new PGLiteSocketServer({ db, port: PORT, host: "127.0.0.1", maxConnections: 2 });
4650 await server.start();
4751
4852 // eslint-disable-next-line no-console
apps/cloud/src/account/account-api.ts+7−3
@@ -9,6 +9,7 @@ import { · 8 unmodified lines
99
1010 import { ApiKeyService } from "../auth/api-keys";
1111 import { UserStoreService } from "../auth/context";
12+import { WorkOsMirror } from "../auth/workos-mirror";
1213 import { sessionFromSealed, type Session } from "../auth/middleware";
1314 import { WorkOSClient } from "../auth/workos";
1415 import { AutumnService } from "../extensions/billing/service";
@@ -95,10 +96,13 @@ const AccountProviderMiddleware = HttpRouter.middleware<{ provides: AccountProvi · 80 unmodified lines
9596 * account service closes over the per-request postgres socket). `AutumnService`
9697 * (the seat-gate) stays a residual requirement, satisfied by the app `boot`.
9798 */
98−export const workosAccountMiddleware = (rsLive: Layer.Layer<DbService | UserStoreService>) =>
99− AccountProviderMiddleware.combine(requestScopedMiddleware(rsLive)).layer;
99+export const workosAccountMiddleware = (
100+ rsLive: Layer.Layer<DbService | UserStoreService | WorkOsMirror>,
101+) => AccountProviderMiddleware.combine(requestScopedMiddleware(rsLive)).layer;
100102
101−export const makeAccountApiLive = (rsLive: Layer.Layer<DbService | UserStoreService>) => {
103+export const makeAccountApiLive = (
104+ rsLive: Layer.Layer<DbService | UserStoreService | WorkOsMirror>,
105+) => {
102106 // Cloud builds the WorkOS `AccountProvider` INSIDE the request body (so it
103107 // closes over the per-request postgres socket), so it can't be a self-
104108 // contained `Layer<AccountProvider>` — it combines its own middleware with
apps/cloud/src/account/org-api-key-revoke.node.test.ts+27−0
@@ -8,6 +8,7 @@ import { ApiKeyService, OrgApiKeyNotFound } from "../auth/api-keys"; · 7 unmodified lines
88 import { UserStoreService } from "../auth/context";
99 import { ORG_SELECTOR_HEADER } from "../auth/organization";
1010 import { WorkOSClient, type WorkOSClientService } from "../auth/workos";
11+import { WorkOsMirror } from "../auth/workos-mirror";
1112 import { AutumnService } from "../extensions/billing/service";
1213 import { AccountCaller, workosAccountProvider } from "./workos-account-service";
1314
@@ -100,18 +101,27 @@ const stubUsers = Layer.succeed(UserStoreService)({ · 86 unmodified lines
100101 upsertOrganization: async (org: { id: string; name: string }) => ({
101102 ...org,
102103 slug: org.id,
104+ backfilledAt: null,
105+ deletedAt: null,
106+ workosUpdatedAt: null,
103107 createdAt,
104108 }),
105109 getOrganization: async (id: string) => ({
106110 id,
107111 name: `Org ${id}`,
108112 slug: id,
113+ backfilledAt: null,
114+ deletedAt: null,
115+ workosUpdatedAt: null,
109116 createdAt,
110117 }),
111118 getOrganizationBySlug: async (slug: string) => ({
112119 id: slug,
113120 name: `Org ${slug}`,
114121 slug,
122+ backfilledAt: null,
123+ deletedAt: null,
124+ workosUpdatedAt: null,
115125 createdAt,
116126 }),
117127 deleteOrganizationCascade: async () => {},
@@ -119,6 +129,22 @@ const stubUsers = Layer.succeed(UserStoreService)({ · 1 unmodified line
119129 ),
120130 });
121131
132+// Revoke changes no membership, so the mirror is never written.
133+const stubMirror = Layer.succeed(WorkOsMirror)({
134+ upsertUser: () => Effect.die("revoke does not write the membership mirror"),
135+ upsertMembership: () => Effect.die("revoke does not write the membership mirror"),
136+ deleteMembership: () => Effect.die("revoke does not write the membership mirror"),
137+ deleteUser: () => Effect.die("revoke does not write the membership mirror"),
138+ getCursor: () => Effect.die("revoke does not read the events cursor"),
139+ setCursor: () => Effect.die("revoke does not move the events cursor"),
140+ applyOrganizationScan: () => Effect.die("revoke does not run the backfill"),
141+ replayBoundary: () => Effect.die("revoke does not run the reconciler"),
142+ setReplayBoundary: () => Effect.die("revoke does not run the backfill"),
143+ backfillCompletedAt: () => Effect.die("revoke does not check mirror readiness"),
144+ markBackfillCompleted: () => Effect.die("revoke does not run the backfill"),
145+ organizationBackfilledAt: () => Effect.die("revoke does not report seats"),
146+});
147+
122148 const stubAutumn = Layer.succeed(AutumnService)({
123149 use: () => Effect.die("revoke does not touch billing"),
124150 ensureCustomer: () => Effect.die("revoke does not touch billing"),
@@ -156,6 +182,7 @@ const providerWith = (accountId: string) => { · 31 unmodified lines
156182 Layer.mergeAll(
157183 stubWorkOS,
158184 stubUsers,
185+ stubMirror,
159186 stubApiKeys,
160187 stubAutumn,
161188 Layer.succeed(AccountCaller)({ session: session(accountId) }),
apps/cloud/src/account/workos-account-service.ts+61−11
@@ -12,6 +12,7 @@ import { ApiKeyService } from "../auth/api-keys"; · 11 unmodified lines
1212 import { UserStoreService } from "../auth/context";
1313 import type { Session } from "../auth/middleware";
1414 import { WorkOSClient } from "../auth/workos";
15+import { WorkOsMirror, mirrorMembershipFromWorkOs } from "../auth/workos-mirror";
1516 import { ORG_SELECTOR_HEADER, authorizeOrganizationSelector } from "../auth/organization";
1617 import { AutumnService } from "../extensions/billing/service";
1718 import { forkReportMemberSeats } from "../extensions/billing/member-seats";
@@ -50,7 +51,7 @@ export class AccountCaller extends Context.Service< · 32 unmodified lines
5051 // (me / API keys) and `org/handlers.ts` (members / roles / invite / role /
5152 // name). Native WorkOS / store failures are mapped at this boundary onto the
5253 // neutral account errors so the shared UI sees one shape:
53−// WorkOSError | UserStoreError | ApiKeyManagementError → AccountError
54+// WorkOSError | UserStoreError | ApiKeyManagementError | WorkOsMirrorError → AccountError
5455 // no organization in session → AccountNoOrganization
5556 // not-an-admin / over-seat-limit / not-allowed → AccountForbidden
5657 // ---------------------------------------------------------------------------
@@ -65,13 +66,17 @@ const toAccountError = () => Effect.fail(new AccountError({ message: "Account re · 8 unmodified lines
6566 export const workosAccountProvider: Layer.Layer<
6667 AccountProvider,
6768 never,
68− WorkOSClient | UserStoreService | ApiKeyService | AutumnService | AccountCaller
69+ WorkOSClient | UserStoreService | WorkOsMirror | ApiKeyService | AutumnService | AccountCaller
6970 > = Layer.effect(AccountProvider)(
7071 Effect.gen(function* () {
7172 const workos = yield* WorkOSClient;
7273 const apiKeys = yield* ApiKeyService;
7374 const autumn = yield* AutumnService;
7475 const users = yield* UserStoreService;
76+ // Membership writes below go to WorkOS FIRST (the authority), then are
77+ // written through to the local mirror so the member list and the seat
78+ // count read the change without waiting for the Events reconciler.
79+ const mirror = yield* WorkOsMirror;
7580
7681 // The caller, resolved once per request by the cookie-only session
7782 // middleware (account-api.ts) — the same credential `SessionAuthLive`
@@ -137,6 +142,7 @@ export const workosAccountProvider: Layer.Layer< · 59 unmodified lines
137142 if (!membership || membership.organizationId !== organizationId) {
138143 return yield* new AccountForbidden();
139144 }
145+ return membership;
140146 });
141147
142148 // Mirror of org/handlers `getMemberSeats` — live seat usage from WorkOS.
@@ -215,7 +221,10 @@ export const workosAccountProvider: Layer.Layer< · 72 unmodified lines
215221 Effect.gen(function* () {
216222 const { session, org } = yield* requireOrganization(headers);
217223 const keys = yield* apiKeys
218− .listUserKeys({ accountId: session.accountId, organizationId: org.id })
224+ .listUserKeys({
225+ accountId: session.accountId,
226+ organizationId: org.id,
227+ })
219228 .pipe(Effect.catchTag("ApiKeyManagementError", toAccountError));
220229 return { apiKeys: keys };
221230 }),
@@ -225,10 +234,16 @@ export const workosAccountProvider: Layer.Layer< · 3 unmodified lines
225234 const { session, org } = yield* requireOrganization(headers);
226235 const trimmed = name.trim().slice(0, MAX_API_KEY_NAME_LENGTH);
227236 if (!trimmed) {
228− return yield* new AccountError({ message: "API key name is required" });
237+ return yield* new AccountError({
238+ message: "API key name is required",
239+ });
229240 }
230241 return yield* apiKeys
231− .createUserKey({ accountId: session.accountId, organizationId: org.id, name: trimmed })
242+ .createUserKey({
243+ accountId: session.accountId,
244+ organizationId: org.id,
245+ name: trimmed,
246+ })
232247 .pipe(Effect.catchTag("ApiKeyManagementError", toAccountError));
233248 }),
234249
@@ -236,7 +251,10 @@ export const workosAccountProvider: Layer.Layer< · 1 unmodified line
236251 Effect.gen(function* () {
237252 const { session, org } = yield* requireOrganization(headers);
238253 const ownedKeys = yield* apiKeys
239− .listUserKeys({ accountId: session.accountId, organizationId: org.id })
254+ .listUserKeys({
255+ accountId: session.accountId,
256+ organizationId: org.id,
257+ })
240258 .pipe(Effect.catchTag("ApiKeyManagementError", toAccountError));
241259 if (!ownedKeys.some((key) => key.id === apiKeyId)) {
242260 return yield* new AccountError({ message: "API key not found" });
@@ -266,7 +284,9 @@ export const workosAccountProvider: Layer.Layer< · 23 unmodified lines
266284 yield* requireAdmin(session.accountId, org.id);
267285 const trimmed = name.trim().slice(0, MAX_API_KEY_NAME_LENGTH);
268286 if (!trimmed) {
269− return yield* new AccountError({ message: "API key name is required" });
287+ return yield* new AccountError({
288+ message: "API key name is required",
289+ });
270290 }
271291 return yield* apiKeys
272292 .createOrgKey({ organizationId: org.id, name: trimmed })
@@ -287,7 +307,11 @@ export const workosAccountProvider: Layer.Layer< · 14 unmodified lines
287307 yield* apiKeys.revokeOrgKey({ organizationId: org.id, keyId: apiKeyId }).pipe(
288308 Effect.catchTag("ApiKeyManagementError", toAccountError),
289309 Effect.catchTag("OrgApiKeyNotFound", () =>
290− Effect.fail(new AccountError({ message: "Organization API key not found" })),
310+ Effect.fail(
311+ new AccountError({
312+ message: "Organization API key not found",
313+ }),
314+ ),
291315 ),
292316 );
293317 return { success: true };
@@ -361,10 +385,29 @@ export const workosAccountProvider: Layer.Layer< · 67 unmodified lines
361385 Effect.gen(function* () {
362386 const { session, org } = yield* requireOrganization(headers);
363387 yield* requireAdmin(session.accountId, org.id);
364− yield* assertMembershipInOrg(org.id, membershipId);
388+ const membership = yield* assertMembershipInOrg(org.id, membershipId);
365389 yield* workos
366390 .deleteOrgMembership(membershipId)
367391 .pipe(Effect.catchTag("WorkOSError", toAccountError));
392+ // Tombstoned by identity: the deleted WorkOS id never returns, so
393+ // a login or backfill that fetched this membership before the
394+ // delete — or a role change issued before it and delivered after
395+ // — is refused however it is stamped, while a replacement
396+ // membership WorkOS creates for the same member (a new id) is
397+ // not. No WorkOS instant is in hand (WorkOS answers a delete with
398+ // no time): the row keeps its own stamp, never the local clock,
399+ // which read after WorkOS answered could post-date that
400+ // replacement.
401+ yield* mirror
402+ .deleteMembership(
403+ {
404+ id: membershipId,
405+ accountId: membership.userId,
406+ organizationId: membership.organizationId,
407+ },
408+ new Date(membership.updatedAt),
409+ )
410+ .pipe(Effect.catchTag("WorkOsMirrorError", toAccountError));
368411 yield* forkReportMemberSeats(org.id).pipe(Effect.provideContext(ctx));
369412 return { success: true };
370413 }),
@@ -374,9 +417,12 @@ export const workosAccountProvider: Layer.Layer< · 3 unmodified lines
374417 const { session, org } = yield* requireOrganization(headers);
375418 yield* requireAdmin(session.accountId, org.id);
376419 yield* assertMembershipInOrg(org.id, membershipId);
377− yield* workos
420+ const updated = yield* workos
378421 .updateOrgMembershipRole(membershipId, roleSlug)
379422 .pipe(Effect.catchTag("WorkOSError", toAccountError));
423+ yield* mirror
424+ .upsertMembership(mirrorMembershipFromWorkOs(updated))
425+ .pipe(Effect.catchTag("WorkOsMirrorError", toAccountError));
380426 return { success: true };
381427 }),
382428
@@ -389,7 +435,11 @@ export const workosAccountProvider: Layer.Layer< · 6 unmodified lines
389435 .pipe(Effect.catchTag("WorkOSError", toAccountError));
390436 yield* users
391437 .use("upsertOrganization", (s) =>
392− s.upsertOrganization({ id: updated.id, name: updated.name }),
438+ s.upsertOrganization({
439+ id: updated.id,
440+ name: updated.name,
441+ updatedAt: new Date(updated.updatedAt),
442+ }),
393443 )
394444 .pipe(Effect.catchTag("UserStoreError", toAccountError));
395445 return { name: updated.name };
apps/cloud/src/api/layers.ts+3−1
@@ -56,7 +56,9 @@ export const BootSharedServices = Layer.mergeAll( · 55 unmodified lines
5656 // `AutumnService.Default` is provided HERE because the `createOrganization`
5757 // handler reads it for the free-organizations-per-user limit gate — one of the
5858 // few app-only billing touchpoints. (It is NOT on the neutral boot core.)
59−export const makeNonProtectedApiLive = (rsLive: Layer.Layer<DbService | UserStoreService>) =>
59+export const makeNonProtectedApiLive = (
60+ rsLive: Layer.Layer<DbService | UserStoreService | WorkOsMirror>,
61+) =>
6062 HttpApiBuilder.layer(NonProtectedApi).pipe(
6163 Layer.provide(Layer.mergeAll(CloudAuthPublicHandlers, CloudSessionAuthHandlers)),
6264 Layer.provide(requestScopedMiddleware(rsLive).layer),
apps/cloud/src/api/protected-api-key-auth.node.test.ts+9−0
@@ -68,18 +68,27 @@ const stubUsers = Layer.succeed(UserStoreService)({ · 67 unmodified lines
6868 upsertOrganization: async (org: { id: string; name: string }) => ({
6969 ...org,
7070 slug: `org-slug-${org.id}`,
71+ backfilledAt: null,
72+ deletedAt: null,
73+ workosUpdatedAt: null,
7174 createdAt,
7275 }),
7376 getOrganization: async (id: string) => ({
7477 id,
7578 name: `Org ${id}`,
7679 slug: `org-slug-${id}`,
80+ backfilledAt: null,
81+ deletedAt: null,
82+ workosUpdatedAt: null,
7783 createdAt,
7884 }),
7985 getOrganizationBySlug: async (slug: string) => ({
8086 id: "org_by_slug",
8187 name: `Org ${slug}`,
8288 slug,
89+ backfilledAt: null,
90+ deletedAt: null,
91+ workosUpdatedAt: null,
8392 createdAt,
8493 }),
8594 deleteOrganizationCascade: async () => {},
apps/cloud/src/api/protected-jwt-auth.node.test.ts+9−0
@@ -85,18 +85,27 @@ const stubUsers = Layer.succeed(UserStoreService)({ · 84 unmodified lines
8585 upsertOrganization: async (org: { id: string; name: string }) => ({
8686 ...org,
8787 slug: `org-slug-${org.id}`,
88+ backfilledAt: null,
89+ deletedAt: null,
90+ workosUpdatedAt: null,
8891 createdAt,
8992 }),
9093 getOrganization: async (id: string) => ({
9194 id,
9295 name: `Org ${id}`,
9396 slug: `org-slug-${id}`,
97+ backfilledAt: null,
98+ deletedAt: null,
99+ workosUpdatedAt: null,
94100 createdAt,
95101 }),
96102 getOrganizationBySlug: async (slug: string) => ({
97103 id: "org_by_slug",
98104 name: `Org ${slug}`,
99105 slug,
106+ backfilledAt: null,
107+ deletedAt: null,
108+ workosUpdatedAt: null,
100109 createdAt,
101110 }),
102111 deleteOrganizationCascade: async () => {},
apps/cloud/src/api/router.ts+4−1
@@ -4,6 +4,7 @@ import { HttpRouter } from "effect/unstable/http"; · 3 unmodified lines
44 import { RouterConfigLive, requestScopedMiddleware } from "@executor-js/api/server";
55
66 import { UserStoreService } from "../auth/context";
7+import { WorkOsMirror } from "../auth/workos-mirror";
78 import { DbService } from "../db/db";
89 import { makeAccountApiLive } from "../account/account-api";
910
@@ -29,7 +30,9 @@ import { makeProtectedApiLive } from "./protected"; · 19 unmodified lines
2930 // so tests can substitute a counting fake for `DbService.Live` and
3031 // assert per-request semantics — see
3132 // `apps/cloud/src/api.request-scope.node.test.ts`.
32−export const makeApiLive = (requestScopedLive: Layer.Layer<DbService | UserStoreService>) => {
33+export const makeApiLive = (
34+ requestScopedLive: Layer.Layer<DbService | UserStoreService | WorkOsMirror>,
35+) => {
3336 const BillingRoutesLive = AutumnRoutesLive.pipe(
3437 Layer.provide(requestScopedMiddleware(requestScopedLive).layer),
3538 );
apps/cloud/src/auth/api.ts+4−2
@@ -1,6 +1,6 @@
11 import { HttpApiEndpoint, HttpApiGroup } from "effect/unstable/httpapi";
22 import { Schema } from "effect";
3−import { UserStoreError, WorkOSError } from "./errors";
3+import { UserStoreError, WorkOSError, WorkOsMirrorError } from "./errors";
44 import { NoOrganization } from "@executor-js/api/server";
55 import { SessionAuth } from "./middleware";
66
@@ -172,7 +172,9 @@ export const AUTH_PATHS = { · 165 unmodified lines
172172 callback: "/api/auth/callback",
173173 } as const;
174174
175−const AuthErrors = [UserStoreError, WorkOSError] as const;
175+// The login callback and the org handlers feed the membership mirror, so a
176+// mirror write failure is one of their wire errors (same 500 as a store failure).
177+const AuthErrors = [UserStoreError, WorkOSError, WorkOsMirrorError] as const;
176178 const McpApprovalErrors = [
177179 NoOrganization,
178180 McpExecutionNotFoundError,
apps/cloud/src/auth/errors.ts+21−0
@@ -38,6 +38,27 @@ export class UserStoreError extends Schema.TaggedErrorClass<UserStoreError>()( · 37 unmodified lines
3838 }
3939 }
4040
41+/**
42+ * The public failure of every cloud membership-mirror write (`WorkOsMirror`).
43+ * Same two diagnosable fields as `UserStoreError` — which mirror call failed,
44+ * and how — classified from the driver cause the same way. Declared here,
45+ * beside `UserStoreError`, because the auth API (`auth/api.ts`, in the SPA
46+ * bundle) names it on the wire for the login and org handlers that feed the
47+ * mirror; the service itself lives in `workos-mirror.ts`.
48+ */
49+export class WorkOsMirrorError extends Schema.TaggedErrorClass<WorkOsMirrorError>()(
50+ "WorkOsMirrorError",
51+ {
52+ operation: Schema.String,
53+ reason: Schema.Literals(USER_STORE_FAILURE_REASONS),
54+ },
55+ { httpApiStatus: 500 },
56+) {
57+ override get message(): string {
58+ return `workos mirror ${this.operation} failed: ${this.reason}`;
59+ }
60+}
61+
4162 /** Reasons a retry can plausibly clear: the query never reached a healthy
4263 * server. A `query` failure is deterministic and must not be retried. */
4364 export const isTransientUserStoreReason = (reason: UserStoreFailureReason): boolean =>
apps/cloud/src/auth/handlers.ts+81−27
@@ -1,6 +1,6 @@
11 import { HttpApi, HttpApiBuilder } from "effect/unstable/httpapi";
22 import { HttpServerRequest, HttpServerResponse } from "effect/unstable/http";
3−import { Duration, Effect, Predicate } from "effect";
3+import { Clock, Duration, Effect, Predicate } from "effect";
44 import { isValidOrgSlug } from "@executor-js/api";
55
66 import {
@@ -18,6 +18,7 @@ import { SessionContext, SessionCookies } from "./middleware"; · 11 unmodified lines
1818 import { encodeLoginState, decodeLoginState } from "./login-state";
1919 import { safeReturnTo } from "./return-to";
2020 import { UserStoreService } from "./context";
21+import { mirrorMembership, mirrorSignIn } from "./mirror-feeders";
2122 import { env } from "cloudflare:workers";
2223 import { WorkOSError } from "./errors";
2324 import { WorkOSClient } from "./workos";
@@ -209,8 +210,16 @@ export const CloudAuthPublicHandlers = HttpApiBuilder.group( · 185 unmodified lines
209210
210211 const result = yield* workos.authenticateWithCode(query.code);
211212
212− // Mirror the account locally
213− yield* users.use("ensureAccount", (s) => s.ensureAccount(result.user.id));
213+ // ONE membership list for the whole callback. It feeds the mirror
214+ // (the user + every org they hold a membership in, all already in
215+ // hand) and it is the membership check for every landing-org
216+ // candidate below, so the callback makes no per-candidate WorkOS
217+ // call. The user's account row is minted by the mirror's user
218+ // upsert. The list's fetch instant, taken before the read, stamps
219+ // the organization names it carries (see `mirrorSignIn`).
220+ const fetchedAt = new Date(yield* Clock.currentTimeMillis);
221+ const memberships = yield* workos.listUserMemberships(result.user.id);
222+ yield* mirrorSignIn(result.user, memberships.data, fetchedAt);
214223
215224 let sealedSession = result.sealedSession;
216225
@@ -220,34 +229,43 @@ export const CloudAuthPublicHandlers = HttpApiBuilder.group( · 3 unmodified lines
220229 // any other untrusted path.
221230 const returnTo = safeReturnTo(decodeLoginState(query.state)?.returnTo) ?? "/";
222231 const requestedOrgSelector = requestedOrgSelectorFromReturnTo(returnTo);
223− const requestedOrg = requestedOrgSelector
224− ? yield* authorizeOrganizationSelector(result.user.id, requestedOrgSelector).pipe(
232+
233+ // An org SLUG (both candidate sources below are slug-validated, so
234+ // an `org_…` id never reaches here) resolves to its id only when the
235+ // list above holds an ACTIVE membership in it. Pending memberships
236+ // are skipped because refreshing into one 400s and would bypass
237+ // invite consent. A slug that fails to resolve (unknown, or a store
238+ // hiccup) is not a candidate, the same as an org the user is not in.
239+ const activeOrganizationIds = new Set(
240+ memberships.data.filter((m) => m.status === "active").map((m) => m.organizationId),
241+ );
242+ const activeOrganizationFor = (slug: string) =>
243+ users
244+ .use("getOrganizationBySlug", (s) => s.getOrganizationBySlug(slug))
245+ .pipe(
246+ Effect.map((org) => (org && activeOrganizationIds.has(org.id) ? org.id : null)),
225247 Effect.orElseSucceed(() => null),
226− )
227− : null;
248+ );
228249
229250 // Prefer the org in the URL that sent the user to login. If the URL
230251 // is bare, or not an org route, prefer the org this browser last
231252 // worked in (the last-org cookie — it outlives the session precisely
232253 // so a fresh login lands where the user left off), then WorkOS's
233254 // org, then the first active membership for org-less sessions.
234− // Pending memberships are skipped because refreshing into one 400s
235− // and would bypass invite consent. The cookie is membership-checked
236− // like any selector, so a stale one just falls through.
237− let targetOrganizationId = requestedOrg?.id ?? null;
255+ // The cookie is membership-checked like any selector, so a stale
256+ // one just falls through.
257+ let targetOrganizationId = requestedOrgSelector
258+ ? yield* activeOrganizationFor(requestedOrgSelector)
259+ : null;
238260 if (!targetOrganizationId && !requestedOrgSelector) {
239261 const lastOrgSlug = request.cookies[LAST_ORG_COOKIE];
240− const lastOrg =
262+ targetOrganizationId =
241263 lastOrgSlug && isValidOrgSlug(lastOrgSlug)
242− ? yield* authorizeOrganizationSelector(result.user.id, lastOrgSlug).pipe(
243− Effect.orElseSucceed(() => null),
244− )
264+ ? yield* activeOrganizationFor(lastOrgSlug)
245265 : null;
246− targetOrganizationId = lastOrg?.id ?? null;
247266 }
248267 targetOrganizationId ??= result.organizationId ?? null;
249268 if (!targetOrganizationId && !requestedOrgSelector) {
250− const memberships = yield* workos.listUserMemberships(result.user.id);
251269 const existingActive = memberships.data.find((m) => m.status === "active");
252270 targetOrganizationId = existingActive?.organizationId ?? null;
253271 }
@@ -311,7 +329,9 @@ export const CloudAuthPublicHandlers = HttpApiBuilder.group( · 57 unmodified lines
311329 ? yield* workos.logoutUrl(sealedSession, origin ? `${origin}/` : undefined)
312330 : null;
313331
314− const response = HttpServerResponse.redirect(logoutUrl ?? "/", { status: 302 });
332+ const response = HttpServerResponse.redirect(logoutUrl ?? "/", {
333+ status: 302,
334+ });
315335
316336 // Drop only what this browser actually presented. Both cookies are
317337 // SameSite=Lax, so a cross-site form POST carries neither — it gets
@@ -447,11 +467,18 @@ export const CloudSessionAuthHandlers = HttpApiBuilder.group( · 129 unmodified lines
447467 }
448468
449469 const org = yield* workos.createOrganization(name);
450− yield* workos.createMembership(org.id, session.accountId, "admin");
470+ const membership = yield* workos.createMembership(org.id, session.accountId, "admin");
451471 // `upsertOrganization` mints the slug at insert — no separate heal step.
452472 const mirrored = yield* users.use("upsertOrganization", (s) =>
453− s.upsertOrganization({ id: org.id, name: org.name }),
473+ s.upsertOrganization({
474+ id: org.id,
475+ name: org.name,
476+ updatedAt: new Date(org.updatedAt),
477+ }),
454478 );
479+ // Write-through: the creator's admin membership, from the create
480+ // response, lands in the mirror before anything reads it.
481+ yield* mirrorMembership(membership);
455482
456483 // Provision the org's billing customer while we're the ones creating
457484 // the org. Without this the first billing call an org ever makes is a
@@ -535,13 +562,18 @@ export const CloudSessionAuthHandlers = HttpApiBuilder.group( · 77 unmodified lines
535562 // empty workspace when a later request re-mirrors it with a new slug.
536563 yield* workos.deleteOrganization(organizationId);
537564
538− // Purge all local tenant data, secrets, and the identity mirror
539− // (cascades local memberships) in one transaction. If this fails
540− // after the WorkOS delete already succeeded, the org is gone for
541− // everyone (unreachable) but its secrets/tenant rows linger orphaned —
542− // alert loudly so that window gets swept, then surface the failure.
565+ // Purge all local tenant data, secrets, and the org's memberships in
566+ // one transaction, leaving the org row as a tombstone marked deleted
567+ // (so a login that fetched its membership list before the deletion
568+ // cannot re-mint the org afterwards). If this fails after the WorkOS
569+ // delete already succeeded, the org is gone for everyone
570+ // (unreachable) but its secrets/tenant rows linger orphaned — alert
571+ // loudly so that window gets swept, then surface the failure.
572+ const deletedAt = new Date(yield* Clock.currentTimeMillis);
543573 yield* users
544− .use("deleteOrganizationCascade", (s) => s.deleteOrganizationCascade(organizationId))
574+ .use("deleteOrganizationCascade", (s) =>
575+ s.deleteOrganizationCascade(organizationId, deletedAt),
576+ )
545577 .pipe(
546578 Effect.tapError((error) =>
547579 Effect.logError(
@@ -645,9 +677,31 @@ export const CloudSessionAuthHandlers = HttpApiBuilder.group( · 97 unmodified lines
645677 // upsert mints the slug at insert — no separate heal step.
646678 const org = yield* workos.getOrganization(invitation.organizationId);
647679 const mirrored = yield* users.use("upsertOrganization", (s) =>
648− s.upsertOrganization({ id: org.id, name: org.name }),
680+ s.upsertOrganization({
681+ id: org.id,
682+ name: org.name,
683+ updatedAt: new Date(org.updatedAt),
684+ }),
649685 );
650686
687+ // Write-through: acceptance returns the invitation, not the
688+ // membership it activated, so this is the one feeder that reads the
689+ // membership back (a rare path; one extra call). WorkOS activates it
690+ // as part of acceptance, so its absence is worth a warning — the
691+ // Events reconciler will still land it.
692+ const membership = yield* workos.getUserOrgMembership(org.id, session.accountId);
693+ if (membership) {
694+ yield* mirrorMembership(membership);
695+ } else {
696+ yield* Effect.logWarning(
697+ "acceptInvitation: accepted invitation has no membership yet",
698+ {
699+ userId: session.accountId,
700+ organizationId: org.id,
701+ },
702+ );
703+ }
704+
651705 // The membership is active in WorkOS from this point even if
652706 // attaching the session below fails, so reconcile the org's billed
653707 // seat count now.
apps/cloud/src/auth/mirror-feeders.node.test.ts+1040−0
@@ -0,0 +1,1040 @@
1+// ---------------------------------------------------------------------------
2+// The membership mirror's FEEDERS, end to end through the code that runs in
3+// production, against the real PGlite Postgres every cloud unit test runs on
4+// (scripts/test-globalsetup.ts). WorkOS is a fake `WorkOSClient` (the
5+// emulator has no list-users / events routes); the mirror, the user store,
6+// and the directory read are the live layers over `DbService.Live`.
7+//
8+// What this pins:
9+// - the login callback records the signed-in user and EVERY membership
10+// WorkOS lists (active and pending), with the org row minted so the FK
11+// holds — from the one membership list it already fetches
12+// - the callback picks the landing org from that same list: a returnTo
13+// slug or last-org cookie lands only in an ACTIVE membership, an unknown
14+// or pending one falls through
15+// - `removeMember` tombstones the mirror row after the WorkOS delete,
16+// stamped with the membership's last WorkOS state (never a local clock),
17+// so a replay of the membership as it was before the delete cannot
18+// restore it while a replacement WorkOS created meanwhile is accepted
19+// - `updateMemberRole` writes the role WorkOS returned
20+// - the backfill mirrors every org's members and counts what it wrote,
21+// writes nothing on a dry run, converges on a re-run, tombstones a
22+// membership WorkOS no longer lists — but never one written after its
23+// listing was taken — marks each org backfilled as of its listing, and
24+// records the events replay boundary only when it completes and only
25+// ONCE: a run that fails part-way keeps the marks of the orgs it
26+// finished and records no boundary, and a later completed run keeps the
27+// first boundary (the org renames and user deletions between two runs
28+// are the events stream's to replay)
29+// - two scans of one org that overlap cannot resurrect a membership: a scan
30+// that listed it, stalled, and resumed after a later listing (which no
31+// longer had it) was applied is refused whole
32+// - a login whose membership list was fetched BEFORE the org was purged and
33+// written after cannot re-mint the org or its membership, one fetched
34+// before a rename cannot revert the rename, and one fetched before a
35+// revocation the backfill has since scanned cannot reinstate the
36+// membership
37+// ---------------------------------------------------------------------------
38+
39+import { describe, expect, it } from "@effect/vitest";
40+import { sql } from "drizzle-orm";
41+import { Effect, Exit, Fiber, Latch, Layer } from "effect";
42+import { HttpRouter, HttpServer } from "effect/unstable/http";
43+import { HttpApiBuilder } from "effect/unstable/httpapi";
44+
45+import {
46+ AccountProvider,
47+ MemberDirectory,
48+ RouterConfigLive,
49+ requestScopedMiddleware,
50+} from "@executor-js/api/server";
51+
52+import { AccountCaller, workosAccountProvider } from "../account/workos-account-service";
53+import { RequestScopedServicesLive } from "../api/layers";
54+import { DbService } from "../db/db";
55+import { AutumnService } from "../extensions/billing/service";
56+import { ApiKeyService } from "./api-keys";
57+import { UserStoreService } from "./context";
58+import { WorkOSError } from "./errors";
59+import { CloudAuthPublicHandlers, CloudSessionAuthHandlers, NonProtectedApi } from "./handlers";
60+import { LAST_ORG_COOKIE } from "./last-org-cookie";
61+import { encodeLoginState } from "./login-state";
62+import { cloudMemberDirectoryLayer } from "./member-directory";
63+import { SessionAuthLive } from "./middleware-live";
64+import { mirrorSignIn } from "./mirror-feeders";
65+import { ORG_SELECTOR_HEADER } from "./organization";
66+import { WorkOSClient, type WorkOSClientService } from "./workos";
67+import { WorkOsMirror, type WorkOsMirrorShape } from "./workos-mirror";
68+import { backfillOrganization, backfillWorkOsMirror } from "./workos-mirror-backfill";
69+import type { WorkOsMembershipPayload, WorkOsUserPayload } from "./workos-mirror-store";
70+
71+const T1 = "2026-01-01T00:00:00.000Z";
72+const T2 = "2026-01-02T00:00:00.000Z";
73+
74+// Synthetic identities only. Every test mints its own org ids so the shared
75+// test database never couples two tests.
76+const freshId = (prefix: string) => `${prefix}_${crypto.randomUUID().replaceAll("-", "")}`;
77+
78+const workosUser = (id: string, overrides: Partial<WorkOsUserPayload> = {}) => ({
79+ object: "user" as const,
80+ id,
81+ email: `${id}@placeholder.test`,
82+ emailVerified: true,
83+ firstName: "Ada",
84+ lastName: "Placeholder",
85+ profilePictureUrl: null,
86+ lastSignInAt: T1,
87+ locale: null,
88+ createdAt: T1,
89+ updatedAt: T1,
90+ externalId: null,
91+ metadata: {},
92+ ...overrides,
93+});
94+
95+interface FakeMembership extends WorkOsMembershipPayload {
96+ readonly organizationName: string;
97+}
98+
99+const workosMembership = (
100+ userId: string,
101+ organizationId: string,
102+ overrides: Partial<FakeMembership> = {},
103+): FakeMembership => ({
104+ id: `om_${userId}_${organizationId}`,
105+ userId,
106+ organizationId,
107+ organizationName: `Org ${organizationId}`,
108+ role: { slug: "member" },
109+ status: "active",
110+ updatedAt: T1,
111+ ...overrides,
112+});
113+
114+/** Mirrored rows for one org, read through the live cloud `MemberDirectory`. */
115+const readMembers = (organizationId: string) =>
116+ Effect.runPromise(
117+ Effect.flatMap(MemberDirectory.asEffect(), (directory) =>
118+ directory.members(organizationId, {
119+ statuses: ["active", "pending", "inactive"],
120+ }),
121+ ).pipe(
122+ Effect.provide(cloudMemberDirectoryLayer.pipe(Layer.provide(DbService.Live))),
123+ Effect.scoped,
124+ ),
125+ );
126+
127+/** Mirror an org row (named as of T1) and return the URL slug the store minted for it. */
128+const seedOrganization = (id: string) =>
129+ Effect.runPromise(
130+ Effect.flatMap(UserStoreService.asEffect(), (users) =>
131+ users.use("upsertOrganization", (s) =>
132+ s.upsertOrganization({
133+ id,
134+ name: `Org ${id}`,
135+ updatedAt: new Date(T1),
136+ }),
137+ ),
138+ ).pipe(
139+ Effect.map((org) => org.slug),
140+ Effect.provide(UserStoreService.Live.pipe(Layer.provide(DbService.Live))),
141+ Effect.scoped,
142+ ),
143+ );
144+
145+const stubAutumn = Layer.succeed(AutumnService)({
146+ use: () => Effect.die("feeders do not read billing"),
147+ ensureCustomer: () => Effect.void,
148+ checkExecutionBalance: () => Effect.die("feeders do not check balances"),
149+ trackExecution: () => Effect.void,
150+ setMemberSeats: () => Effect.void,
151+});
152+
153+/**
154+ * A `WorkOSClient` whose every method is one of `methods`; anything else is
155+ * an unexpected call and dies, so a feeder that silently adds a WorkOS read
156+ * fails the test instead of passing on a fake.
157+ */
158+const stubWorkOS = (methods: Partial<WorkOSClientService>) =>
159+ Layer.succeed(
160+ WorkOSClient,
161+ new Proxy({} as WorkOSClientService, {
162+ get: (_target, prop) =>
163+ (methods as Record<PropertyKey, unknown>)[prop] ??
164+ (() => Effect.die(`unexpected WorkOSClient.${String(prop)} call`)),
165+ }),
166+ );
167+
168+describe("login callback", () => {
169+ const callbackHandler = (workos: Layer.Layer<WorkOSClient>) =>
170+ HttpRouter.toWebHandler(
171+ HttpApiBuilder.layer(NonProtectedApi).pipe(
172+ Layer.provide(Layer.mergeAll(CloudAuthPublicHandlers, CloudSessionAuthHandlers)),
173+ Layer.provide(requestScopedMiddleware(RequestScopedServicesLive).layer),
174+ Layer.provideMerge(SessionAuthLive),
175+ Layer.provideMerge(stubAutumn),
176+ Layer.provideMerge(workos),
177+ Layer.provideMerge(HttpServer.layerServices),
178+ Layer.provideMerge(RouterConfigLive),
179+ ),
180+ { disableLogger: true },
181+ ).handler;
182+
183+ const STATE_COOKIE = "wos-login-state";
184+
185+ /**
186+ * A callback handler over a fake WorkOS that authenticates `user` with the
187+ * memberships `listed`, recording every WorkOS read (`calls`) and every
188+ * session refresh (`refreshedInto`, the org ids) so the landing-org choice
189+ * is assertable from the outside.
190+ */
191+ const signIn = (user: ReturnType<typeof workosUser>, listed: readonly FakeMembership[]) => {
192+ const calls: string[] = [];
193+ const refreshedInto: (string | undefined)[] = [];
194+ const handler = callbackHandler(
195+ stubWorkOS({
196+ authenticateWithCode: () =>
197+ Effect.succeed({
198+ user,
199+ organizationId: undefined,
200+ accessToken: "access",
201+ refreshToken: "refresh",
202+ sealedSession: "sealed",
203+ }),
204+ listUserMemberships: (id) => {
205+ calls.push(`listUserMemberships:${id}`);
206+ return Effect.succeed({
207+ object: "list" as const,
208+ data: listed as never[],
209+ listMetadata: { before: null, after: null },
210+ });
211+ },
212+ // The forked seat recount after login.
213+ listOrgMembers: () =>
214+ Effect.succeed({
215+ object: "list" as const,
216+ data: [] as never[],
217+ listMetadata: { before: null, after: null },
218+ }),
219+ refreshSession: (_sealed, organizationId) => {
220+ refreshedInto.push(organizationId);
221+ return Effect.succeed("sealed-refreshed");
222+ },
223+ }),
224+ );
225+ return { handler, calls, refreshedInto };
226+ };
227+
228+ /**
229+ * `GET /auth/callback` with the CSRF-matched login `state` (the callback
230+ * refuses any request without one) and any extra cookies; `returnTo`
231+ * rides inside the state as /login mints it.
232+ */
233+ const callbackRequest = (options: { returnTo?: string; cookies?: Record<string, string> }) => {
234+ const url = new URL("http://test.local/auth/callback");
235+ url.searchParams.set("code", "code_1");
236+ const state = encodeLoginState({
237+ nonce: "nonce",
238+ ...(options.returnTo === undefined ? {} : { returnTo: options.returnTo }),
239+ });
240+ url.searchParams.set("state", state);
241+ const cookies = { ...options.cookies, [STATE_COOKIE]: state };
242+ const cookie = Object.entries(cookies)
243+ .map(([name, value]) => `${name}=${value}`)
244+ .join("; ");
245+ return new Request(url, { headers: cookie ? { cookie } : {} });
246+ };
247+
248+ it("records the user and every listed membership from the one list it already fetches", async () => {
249+ const userId = freshId("user");
250+ const activeOrg = freshId("org");
251+ const pendingOrg = freshId("org");
252+ const { handler, calls } = signIn(
253+ workosUser(userId, {
254+ firstName: "Grace",
255+ lastName: "Hopper",
256+ updatedAt: T2,
257+ }),
258+ [
259+ workosMembership(userId, activeOrg, {
260+ role: { slug: "admin" },
261+ updatedAt: T2,
262+ }),
263+ workosMembership(userId, pendingOrg, { status: "pending" }),
264+ ],
265+ );
266+
267+ const response = await handler(callbackRequest({}));
268+
269+ expect(response.status).toBe(302);
270+ expect(calls, "one membership list for the whole callback").toEqual([
271+ `listUserMemberships:${userId}`,
272+ ]);
273+
274+ const active = await readMembers(activeOrg);
275+ expect(active).toHaveLength(1);
276+ expect(active[0]).toMatchObject({
277+ accountId: userId,
278+ membershipId: `om_${userId}_${activeOrg}`,
279+ email: `${userId}@placeholder.test`,
280+ name: "Grace Hopper",
281+ role: "admin",
282+ status: "active",
283+ lastActiveAt: new Date(T1).getTime(),
284+ });
285+ const pending = await readMembers(pendingOrg);
286+ expect(
287+ pending.map((m) => m.status),
288+ "pending memberships are mirrored too",
289+ ).toEqual(["pending"]);
290+ });
291+
292+ describe("lands in the org the returnTo slug names", () => {
293+ it("when the user holds an active membership there", async () => {
294+ const userId = freshId("user");
295+ const requested = freshId("org");
296+ const other = freshId("org");
297+ const slug = await seedOrganization(requested);
298+ const { handler, refreshedInto } = signIn(workosUser(userId), [
299+ workosMembership(userId, other),
300+ workosMembership(userId, requested),
301+ ]);
302+
303+ const response = await handler(callbackRequest({ returnTo: `/${slug}/settings` }));
304+
305+ expect(response.status).toBe(302);
306+ expect(response.headers.get("location")).toBe(`/${slug}/settings`);
307+ expect(refreshedInto, "the session is switched into the requested org").toEqual([requested]);
308+ });
309+
310+ it("never when the membership there is only pending", async () => {
311+ const userId = freshId("user");
312+ const requested = freshId("org");
313+ const other = freshId("org");
314+ const slug = await seedOrganization(requested);
315+ const { handler, refreshedInto } = signIn(workosUser(userId), [
316+ workosMembership(userId, other),
317+ workosMembership(userId, requested, { status: "pending" }),
318+ ]);
319+
320+ const response = await handler(callbackRequest({ returnTo: `/${slug}` }));
321+
322+ expect(response.status).toBe(302);
323+ expect(response.headers.get("location")).toBe(`/${slug}`);
324+ expect(
325+ refreshedInto,
326+ "a pending membership is not a landing candidate, and an explicit slug does not fall back to another org",
327+ ).toEqual([]);
328+ });
329+ });
330+
331+ describe("without a returnTo org", () => {
332+ it("lands in the last-org cookie's org when the user is active there", async () => {
333+ const userId = freshId("user");
334+ const last = freshId("org");
335+ const other = freshId("org");
336+ const slug = await seedOrganization(last);
337+ const { handler, refreshedInto } = signIn(workosUser(userId), [
338+ workosMembership(userId, other),
339+ workosMembership(userId, last),
340+ ]);
341+
342+ const response = await handler(callbackRequest({ cookies: { [LAST_ORG_COOKIE]: slug } }));
343+
344+ expect(response.status).toBe(302);
345+ expect(response.headers.get("location")).toBe("/");
346+ expect(refreshedInto).toEqual([last]);
347+ });
348+
349+ it("falls through an unknown last-org slug to the first active membership", async () => {
350+ const userId = freshId("user");
351+ const pendingOrg = freshId("org");
352+ const activeOrg = freshId("org");
353+ const { handler, refreshedInto } = signIn(workosUser(userId), [
354+ workosMembership(userId, pendingOrg, { status: "pending" }),
355+ workosMembership(userId, activeOrg),
356+ ]);
357+
358+ const response = await handler(
359+ // Valid slug grammar, never minted: the store finds no org for it.
360+ callbackRequest({ cookies: { [LAST_ORG_COOKIE]: "no-such-org-slug" } }),
361+ );
362+
363+ expect(response.status).toBe(302);
364+ expect(refreshedInto).toEqual([activeOrg]);
365+ });
366+ });
367+});
368+
369+describe("a delayed sign-in feeder", () => {
370+ /** The live mirror, user store, and directory over one test-db socket. */
371+ const Services = Layer.mergeAll(
372+ UserStoreService.Live,
373+ WorkOsMirror.Live,
374+ cloudMemberDirectoryLayer,
375+ ).pipe(Layer.provideMerge(DbService.Live));
376+
377+ const run = <A, E>(
378+ body: Effect.Effect<A, E, UserStoreService | WorkOsMirror | MemberDirectory | DbService>,
379+ ) => Effect.runPromise(body.pipe(Effect.provide(Services), Effect.scoped));
380+
381+ const readOrganization = (org: string) =>
382+ Effect.flatMap(UserStoreService.asEffect(), (users) =>
383+ users.use("getOrganization", (s) => s.getOrganization(org)),
384+ );
385+
386+ const readMembership = (userId: string, org: string) =>
387+ Effect.flatMap(MemberDirectory.asEffect(), (directory) =>
388+ directory.membership(userId, org, ["active", "pending", "inactive"]),
389+ );
390+
391+ it("cannot re-mint a purged organization or its membership from a list fetched before the purge", async () => {
392+ const userId = freshId("user");
393+ const org = freshId("org");
394+ await seedOrganization(org);
395+ const result = await run(
396+ Effect.gen(function* () {
397+ const users = yield* UserStoreService;
398+ // The login fetched its membership list at T1, while the org lived...
399+ const fetchedAt = new Date(T1);
400+ const listed = [workosMembership(userId, org)];
401+ // ...then stalled while cloud's deletion flow purged the org at T2.
402+ yield* users.use("deleteOrganizationCascade", (s) =>
403+ s.deleteOrganizationCascade(org, new Date(T2)),
404+ );
405+ // The stalled login resumes and writes what it holds.
406+ yield* mirrorSignIn(workosUser(userId), listed, fetchedAt);
407+ return {
408+ organization: yield* readOrganization(org),
409+ membership: yield* readMembership(userId, org),
410+ };
411+ }),
412+ );
413+ expect(result.organization?.deletedAt, "the org stays a deleted tombstone").toEqual(
414+ new Date(T2),
415+ );
416+ expect(result.membership, "and holds no membership: nothing to authorize").toBeNull();
417+ });
418+
419+ it("cannot reinstate a membership from a list fetched before a revocation the backfill has since scanned", async () => {
420+ const userId = freshId("user");
421+ const org = freshId("org");
422+ await seedOrganization(org);
423+ const result = await run(
424+ Effect.gen(function* () {
425+ const mirror = yield* WorkOsMirror;
426+ // The login fetched its membership list at T1, while the user was
427+ // a member, then stalled...
428+ const fetchedAt = new Date(T1);
429+ const listed = [workosMembership(userId, org)];
430+ // ...WorkOS revoked the membership before the mirror was ever
431+ // backfilled, and the backfill then scanned the org at T2 without
432+ // it: no tombstone, the row was never there — and the revocation
433+ // predates the events replay boundary, so no event will land it.
434+ yield* mirror.applyOrganizationScan({
435+ organizationId: org,
436+ listedAt: new Date(T2),
437+ members: [],
438+ });
439+ // The stalled login resumes and writes what it holds.
440+ yield* mirrorSignIn(workosUser(userId), listed, fetchedAt);
441+ const membership = yield* readMembership(userId, org);
442+ // A login after the scan, carrying a membership WorkOS created
443+ // since (stamped past the scan), is recorded.
444+ const rejoinedAt = "2026-01-03T00:00:00.000Z";
445+ yield* mirrorSignIn(
446+ workosUser(userId),
447+ [workosMembership(userId, org, { id: `om_${userId}_${org}_2`, updatedAt: rejoinedAt })],
448+ new Date(rejoinedAt),
449+ );
450+ return { membership, rejoined: yield* readMembership(userId, org) };
451+ }),
452+ );
453+ expect(result.membership, "the pre-scan list reinstates nothing").toBeNull();
454+ expect(result.rejoined?.membershipId, "a membership newer than the scan is recorded").toBe(
455+ `om_${userId}_${org}_2`,
456+ );
457+ });
458+
459+ it("cannot revert a rename from a list fetched before it, and applies a newer name", async () => {
460+ const userId = freshId("user");
461+ const org = freshId("org");
462+ await seedOrganization(org);
463+ const result = await run(
464+ Effect.gen(function* () {
465+ const users = yield* UserStoreService;
466+ // The org is renamed through Executor (write-through of the WorkOS
467+ // organization payload, stamped T2)...
468+ yield* users.use("upsertOrganization", (s) =>
469+ s.upsertOrganization({
470+ id: org,
471+ name: "Renamed Org",
472+ updatedAt: new Date(T2),
473+ }),
474+ );
475+ // ...after a login had fetched a list still carrying the old name at T1.
476+ yield* mirrorSignIn(
477+ workosUser(userId),
478+ [workosMembership(userId, org, { organizationName: `Org ${org}` })],
479+ new Date(T1),
480+ );
481+ const afterStale = yield* readOrganization(org);
482+ // A login whose list was fetched after the rename carries the new name.
483+ yield* mirrorSignIn(
484+ workosUser(userId),
485+ [
486+ workosMembership(userId, org, {
487+ organizationName: "Renamed Again",
488+ }),
489+ ],
490+ new Date("2026-01-03T00:00:00.000Z"),
491+ );
492+ const afterNewer = yield* readOrganization(org);
493+ return {
494+ afterStale,
495+ afterNewer,
496+ membership: yield* readMembership(userId, org),
497+ };
498+ }),
499+ );
500+ expect(result.afterStale?.name, "the stale list does not revert the rename").toBe(
501+ "Renamed Org",
502+ );
503+ expect(result.afterStale?.slug, "and the slug is untouched").toBe(result.afterNewer?.slug);
504+ expect(result.afterNewer?.name, "a list fetched after the rename is applied").toBe(
505+ "Renamed Again",
506+ );
507+ expect(result.membership?.status, "the membership itself is recorded either way").toBe(
508+ "active",
509+ );
510+ });
511+});
512+
513+describe("account service writes through to the mirror", () => {
514+ const ADMIN = freshId("user");
515+ const TARGET = freshId("user");
516+
517+ const session = (accountId: string) => ({
518+ accountId,
519+ email: `${accountId}@placeholder.test`,
520+ name: null,
521+ avatarUrl: null,
522+ organizationId: null,
523+ sealedSession: "sealed",
524+ refreshedSession: null,
525+ });
526+
527+ const stubApiKeys = Layer.succeed(ApiKeyService)({
528+ validate: () => Effect.die("membership writes do not validate keys"),
529+ listUserKeys: () => Effect.die("membership writes do not list keys"),
530+ createUserKey: () => Effect.die("membership writes do not create keys"),
531+ revokeUserKey: () => Effect.die("membership writes do not revoke keys"),
532+ listOrgKeys: () => Effect.die("membership writes do not list keys"),
533+ createOrgKey: () => Effect.die("membership writes do not create keys"),
534+ revokeOrgKey: () => Effect.die("membership writes do not revoke keys"),
535+ });
536+
537+ /**
538+ * The provider layer over the LIVE mirror + user store (test db) and a fake
539+ * WorkOS in which ADMIN administers `org` and TARGET is a plain member.
540+ * `deleted` records the WorkOS-side deletes so "WorkOS first" is assertable.
541+ * Provided around the WHOLE test body so the postgres socket outlives the
542+ * provider call under test.
543+ */
544+ const providerLayer = (org: string, deleted: string[]) => {
545+ const list = (data: readonly unknown[]) =>
546+ Effect.succeed({
547+ object: "list" as const,
548+ data: data as never[],
549+ listMetadata: { before: null, after: null },
550+ });
551+ const workos = stubWorkOS({
552+ listUserMemberships: (userId) => list([workosMembership(userId, org)]),
553+ getUserOrgMembership: (organizationId, userId) =>
554+ Effect.succeed(
555+ workosMembership(userId, organizationId, {
556+ role: { slug: userId === ADMIN ? "admin" : "member" },
557+ }) as never,
558+ ),
559+ getOrgMembership: (membershipId) =>
560+ Effect.succeed(workosMembership(TARGET, org, { id: membershipId }) as never),
561+ deleteOrgMembership: (membershipId) =>
562+ Effect.sync(() => {
563+ deleted.push(membershipId);
564+ }),
565+ updateOrgMembershipRole: (membershipId, roleSlug) =>
566+ Effect.succeed(
567+ workosMembership(TARGET, org, {
568+ id: membershipId,
569+ role: { slug: roleSlug },
570+ updatedAt: T2,
571+ }) as never,
572+ ),
573+ listOrgMembers: () => list([]),
574+ });
575+ // The test database serves ONE connection at a time, so the seed, the
576+ // provider, and the directory read all share this layer's socket.
577+ const stores = Layer.mergeAll(
578+ UserStoreService.Live,
579+ WorkOsMirror.Live,
580+ cloudMemberDirectoryLayer,
581+ );
582+ return workosAccountProvider.pipe(
583+ Layer.provide(
584+ Layer.mergeAll(
585+ workos,
586+ stubApiKeys,
587+ stubAutumn,
588+ Layer.succeed(AccountCaller)({ session: session(ADMIN) }),
589+ ),
590+ ),
591+ Layer.provideMerge(stores),
592+ Layer.provide(DbService.Live),
593+ );
594+ };
595+
596+ // TARGET as an existing member of `org`, seeded through the live mirror.
597+ const seedTarget = (org: string) =>
598+ Effect.gen(function* () {
599+ const users = yield* UserStoreService;
600+ const mirror = yield* WorkOsMirror;
601+ yield* users.use("upsertOrganization", (s) =>
602+ s.upsertOrganization({
603+ id: org,
604+ name: `Org ${org}`,
605+ updatedAt: new Date(T1),
606+ }),
607+ );
608+ yield* mirror.upsertMembership({
609+ id: `om_${TARGET}_${org}`,
610+ accountId: TARGET,
611+ organizationId: org,
612+ role: "member",
613+ status: "active",
614+ updatedAt: new Date(T1),
615+ });
616+ });
617+
618+ const membersOf = (org: string) =>
619+ Effect.flatMap(MemberDirectory.asEffect(), (directory) => directory.members(org));
620+
621+ it.effect("removeMember tombstones the mirror row after the WorkOS delete", () => {
622+ const org = freshId("org");
623+ const deleted: string[] = [];
624+ return Effect.gen(function* () {
625+ yield* seedTarget(org);
626+ const account = yield* AccountProvider;
627+
628+ const result = yield* account.removeMember(
629+ { [ORG_SELECTOR_HEADER]: org },
630+ `om_${TARGET}_${org}`,
631+ );
632+
633+ expect(result).toEqual({ success: true });
634+ expect(deleted, "WorkOS is the authority and is written first").toEqual([
635+ `om_${TARGET}_${org}`,
636+ ]);
637+ const members = yield* membersOf(org);
638+ expect(members.map((m) => m.accountId)).not.toContain(TARGET);
639+
640+ // A login or backfill that listed TARGET's membership BEFORE the
641+ // removal writes it afterwards: the tombstone refuses it.
642+ const mirror = yield* WorkOsMirror;
643+ const replayed = yield* mirror.upsertMembership({
644+ id: `om_${TARGET}_${org}`,
645+ accountId: TARGET,
646+ organizationId: org,
647+ role: "member",
648+ status: "active",
649+ updatedAt: new Date(T1),
650+ });
651+ expect(replayed, "the pre-removal payload is refused").toBe(false);
652+ expect((yield* membersOf(org)).map((m) => m.accountId)).not.toContain(TARGET);
653+
654+ // The tombstone carries the membership's last WorkOS stamp (T1), not
655+ // the wall clock at the delete: a replacement membership WorkOS
656+ // created for TARGET while the removal was in flight — stamped T2,
657+ // long before any clock this test runs under — is accepted.
658+ const replaced = yield* mirror.upsertMembership({
659+ id: `om_${TARGET}_${org}_2`,
660+ accountId: TARGET,
661+ organizationId: org,
662+ role: "member",
663+ status: "active",
664+ updatedAt: new Date(T2),
665+ });
666+ expect(replaced, "a replacement newer than the removed state is accepted").toBe(true);
667+ expect((yield* membersOf(org)).map((m) => m.accountId)).toContain(TARGET);
668+ }).pipe(Effect.provide(providerLayer(org, deleted)));
669+ });
670+
671+ it.effect("updateMemberRole writes the role WorkOS returned", () => {
672+ const org = freshId("org");
673+ return Effect.gen(function* () {
674+ yield* seedTarget(org);
675+ const account = yield* AccountProvider;
676+
677+ const result = yield* account.updateMemberRole(
678+ { [ORG_SELECTOR_HEADER]: org },
679+ `om_${TARGET}_${org}`,
680+ "admin",
681+ );
682+
683+ expect(result).toEqual({ success: true });
684+ const members = yield* membersOf(org);
685+ expect(members.find((m) => m.accountId === TARGET)?.role).toBe("admin");
686+ }).pipe(Effect.provide(providerLayer(org, [])));
687+ });
688+});
689+
690+describe("backfill", () => {
691+ /** A fake WorkOS holding `orgs` → members, counting `getUser` calls. */
692+ const source = (orgs: ReadonlyMap<string, readonly FakeMembership[]>, userCalls: string[]) => ({
693+ listOrganizationIds: () => Effect.succeed([...orgs.keys()]),
694+ listOrgMembers: (organizationId: string) => Effect.succeed(orgs.get(organizationId) ?? []),
695+ getUser: (userId: string) =>
696+ Effect.sync(() => {
697+ userCalls.push(userId);
698+ return workosUser(userId);
699+ }),
700+ });
701+
702+ const withMirror = <A, E>(body: (mirror: WorkOsMirrorShape) => Effect.Effect<A, E>) =>
703+ Effect.runPromise(
704+ Effect.flatMap(WorkOsMirror.asEffect(), body).pipe(
705+ Effect.provide(WorkOsMirror.Live.pipe(Layer.provide(DbService.Live))),
706+ Effect.scoped,
707+ ),
708+ );
709+
710+ const runBackfill = (
711+ orgs: ReadonlyMap<string, readonly FakeMembership[]>,
712+ dryRun: boolean,
713+ userCalls: string[] = [],
714+ ) =>
715+ withMirror((mirror) =>
716+ backfillWorkOsMirror(source(orgs, userCalls), mirror, {
717+ dryRun,
718+ log: () => undefined,
719+ }),
720+ );
721+
722+ /** The instance-wide replay boundary a completed run records. */
723+ const syncState = () => withMirror((mirror) => mirror.replayBoundary());
724+
725+ /** When a run first covered every organization, or null: the authorization gate's first half. */
726+ const completedAt = () => withMirror((mirror) => mirror.backfillCompletedAt());
727+
728+ /** Drop the instance-wide events row, so the run under test is the first ever. */
729+ const clearEventsRow = () =>
730+ Effect.runPromise(
731+ Effect.flatMap(DbService.asEffect(), ({ db }) =>
732+ Effect.promise(() => db.execute(sql`delete from workos_sync where id = 'events'`)),
733+ ).pipe(Effect.provide(DbService.Live), Effect.scoped),
734+ );
735+
736+ const backfilledAt = (org: string) =>
737+ withMirror((mirror) => mirror.organizationBackfilledAt(org));
738+
739+ it("records the replay boundary on first completion, marks and mirrors every organization's members, counts the writes, converges and repairs on a re-run", async () => {
740+ const orgA = freshId("org");
741+ const orgB = freshId("org");
742+ await seedOrganization(orgA);
743+ await seedOrganization(orgB);
744+ const shared = freshId("user");
745+ const leaving = freshId("user");
746+ const orgs = new Map([
747+ [orgA, [workosMembership(shared, orgA), workosMembership(leaving, orgA)]],
748+ [orgB, [workosMembership(shared, orgB, { status: "pending" })]],
749+ ]);
750+
751+ // The boundary is instance-wide (migration 0019 seeds it on the empty
752+ // test database, other tests may have written it): start as a database
753+ // that has never been backfilled.
754+ await clearEventsRow();
755+ const startedAt = Date.now();
756+
757+ const dry = await runBackfill(orgs, true);
758+ expect(dry).toEqual({
759+ organizations: 2,
760+ memberships: 3,
761+ usersWritten: 0,
762+ membershipsWritten: 0,
763+ membershipsTombstoned: 0,
764+ });
765+ expect(await readMembers(orgA), "a dry run writes nothing").toEqual([]);
766+ expect(await syncState(), "a dry run records no boundary").toBeNull();
767+ expect(await completedAt(), "nor a completion").toBeNull();
768+ expect(await backfilledAt(orgA), "and marks nothing").toBeNull();
769+
770+ const userCalls: string[] = [];
771+ const first = await runBackfill(orgs, false, userCalls);
772+ expect(first).toEqual({
773+ organizations: 2,
774+ memberships: 3,
775+ usersWritten: 3,
776+ membershipsWritten: 3,
777+ membershipsTombstoned: 0,
778+ });
779+ expect(userCalls, "one getUser per membership").toHaveLength(3);
780+ expect((await readMembers(orgA)).map((m) => m.status)).toEqual(["active", "active"]);
781+ expect((await readMembers(orgB)).map((m) => m.status)).toEqual(["pending"]);
782+ const after = await syncState();
783+ expect(after, "the run records where the events replay starts").not.toBeNull();
784+ const firstCompletion = await completedAt();
785+ expect(firstCompletion, "and that every organization is now covered").not.toBeNull();
786+ expect(firstCompletion!.getTime()).toBeGreaterThanOrEqual(after!.getTime());
787+ expect(after!.getTime()).toBeGreaterThanOrEqual(startedAt);
788+ for (const org of [orgA, orgB]) {
789+ const marked = await backfilledAt(org);
790+ expect(marked, "each scanned organization is marked as of its listing").not.toBeNull();
791+ expect(marked!.getTime()).toBeGreaterThanOrEqual(after!.getTime());
792+ }
793+
794+ // Same payloads again, minus one member WorkOS no longer lists: the
795+ // `updatedAt` guard lets equal payloads through (replays converge), and
796+ // the missing membership is tombstoned — a re-run repairs a stale row.
797+ orgs.set(orgA, [workosMembership(shared, orgA)]);
798+ const again = await runBackfill(orgs, false);
799+ expect(again).toMatchObject({ memberships: 2, membershipsTombstoned: 1 });
800+ expect(
801+ await syncState(),
802+ "the re-run keeps the first boundary: an org rename or user deletion between the two runs is only in the events stream",
803+ ).toEqual(after);
804+ expect(await completedAt(), "and the first completion").toEqual(firstCompletion);
805+ expect(new Map((await readMembers(orgA)).map((m) => [m.accountId, m.status]))).toEqual(
806+ new Map([
807+ [shared, "active"],
808+ [leaving, "inactive"],
809+ ]),
810+ );
811+ expect(
812+ (await readMembers(orgB)).map((m) => m.status),
813+ "the other org is untouched",
814+ ).toEqual(["pending"]);
815+ // The tombstone is keyed to the deleted membership id, so the member's
816+ // pre-removal payload (as a late login or event would carry) is refused.
817+ const replayed = await withMirror((mirror) =>
818+ mirror.upsertMembership({
819+ id: `om_${leaving}_${orgA}`,
820+ accountId: leaving,
821+ organizationId: orgA,
822+ role: "member",
823+ status: "active",
824+ updatedAt: new Date(T1),
825+ }),
826+ );
827+ expect(replayed).toBe(false);
828+ });
829+
830+ it("keeps a membership WorkOS merely deactivated as inactive, not tombstoned, so a later reactivation lands", async () => {
831+ const org = freshId("org");
832+ await seedOrganization(org);
833+ const paused = freshId("user");
834+ // Mirrored while active (a sign-in), then deactivated in WorkOS: the
835+ // listing carries the membership under the SAME id with its real status.
836+ await withMirror((mirror) =>
837+ mirror.upsertMembership({
838+ id: `om_${paused}_${org}`,
839+ accountId: paused,
840+ organizationId: org,
841+ role: "member",
842+ status: "active",
843+ updatedAt: new Date(T1),
844+ }),
845+ );
846+
847+ const counts = await runBackfill(
848+ new Map([[org, [workosMembership(paused, org, { status: "inactive", updatedAt: T2 })]]]),
849+ false,
850+ );
851+ expect(counts, "the deactivated membership is written, not tombstoned").toMatchObject({
852+ memberships: 1,
853+ membershipsWritten: 1,
854+ membershipsTombstoned: 0,
855+ });
856+ expect((await readMembers(org)).map((m) => [m.accountId, m.status])).toEqual([
857+ [paused, "inactive"],
858+ ]);
859+
860+ // WorkOS reactivates it under the same id AFTER the scan (so the payload
861+ // is stamped past the org's `backfilled_at`): an ordinary newer payload,
862+ // which a tombstone keyed to that id would have refused for good.
863+ const reactivated = await withMirror((mirror) =>
864+ mirror.upsertMembership({
865+ id: `om_${paused}_${org}`,
866+ accountId: paused,
867+ organizationId: org,
868+ role: "member",
869+ status: "active",
870+ updatedAt: new Date(Date.now() + 60 * 1000),
871+ }),
872+ );
873+ expect(reactivated).toBe(true);
874+ expect((await readMembers(org)).map((m) => m.status)).toEqual(["active"]);
875+ });
876+
877+ it("leaves a membership written after its listing alone when tombstoning what the listing lacks", async () => {
878+ const org = freshId("org");
879+ await seedOrganization(org);
880+ const listed = freshId("user");
881+ const stale = freshId("user");
882+ const joinedMeanwhile = freshId("user");
883+ // Both rows are absent from the listing below. `stale` is stamped before
884+ // the listing (a genuine leaver); `joinedMeanwhile` carries a stamp AFTER
885+ // any listing this run can take — the membership WorkOS created between
886+ // the listing and the cleanup, whose own event lands via the reconciler.
887+ const afterListing = new Date(Date.now() + 60 * 60 * 1000);
888+ await withMirror((mirror) =>
889+ Effect.all([
890+ mirror.upsertMembership({
891+ id: `om_${stale}_${org}`,
892+ accountId: stale,
893+ organizationId: org,
894+ role: "member",
895+ status: "active",
896+ updatedAt: new Date(T1),
897+ }),
898+ mirror.upsertMembership({
899+ id: `om_${joinedMeanwhile}_${org}`,
900+ accountId: joinedMeanwhile,
901+ organizationId: org,
902+ role: "member",
903+ status: "active",
904+ updatedAt: afterListing,
905+ }),
906+ ]),
907+ );
908+
909+ const counts = await runBackfill(new Map([[org, [workosMembership(listed, org)]]]), false);
910+
911+ expect(counts).toMatchObject({ memberships: 1, membershipsTombstoned: 1 });
912+ expect(new Map((await readMembers(org)).map((m) => [m.accountId, m.status]))).toEqual(
913+ new Map([
914+ [listed, "active"],
915+ [stale, "inactive"],
916+ [joinedMeanwhile, "active"],
917+ ]),
918+ );
919+ });
920+
921+ it("keeps the marks of the organizations it finished but records no replay boundary when a run fails part-way", async () => {
922+ const orgA = freshId("org");
923+ const orgB = freshId("org");
924+ await seedOrganization(orgA);
925+ await seedOrganization(orgB);
926+ const member = freshId("user");
927+ const orgs = new Map([
928+ [orgA, [workosMembership(member, orgA)]],
929+ [orgB, [workosMembership(member, orgB)]],
930+ ]);
931+ // A completed run first, so there IS a boundary to protect.
932+ await runBackfill(orgs, false);
933+ const completed = await syncState();
934+ expect(completed).not.toBeNull();
935+ const markedA = await backfilledAt(orgA);
936+
937+ // A re-run whose second organization fails on a WorkOS read: the first
938+ // org was written, but the run as a whole did not complete — and even a
939+ // completed one would keep the first boundary.
940+ const failing = {
941+ ...source(orgs, []),
942+ listOrgMembers: (organizationId: string) =>
943+ organizationId === orgB
944+ ? Effect.fail(new WorkOSError({ status: 503 }))
945+ : Effect.succeed(orgs.get(organizationId) ?? []),
946+ };
947+ const exit = await withMirror((mirror) =>
948+ Effect.exit(
949+ backfillWorkOsMirror(failing, mirror, {
950+ dryRun: false,
951+ log: () => undefined,
952+ }),
953+ ),
954+ );
955+ expect(Exit.isFailure(exit), "the run fails rather than skipping the org").toBe(true);
956+ expect(await syncState(), "the completed run's boundary stands").toEqual(completed);
957+ expect(await completedAt(), "and so does its completion mark").not.toBeNull();
958+ expect(
959+ (await backfilledAt(orgA))!.getTime(),
960+ "the org the failed run did finish is marked as of its new listing",
961+ ).toBeGreaterThanOrEqual(markedA!.getTime());
962+ });
963+
964+ it("scans one organization on demand and marks only that one", async () => {
965+ const org = freshId("org");
966+ const other = freshId("org");
967+ await seedOrganization(org);
968+ await seedOrganization(other);
969+ const member = freshId("user");
970+ const orgs = new Map([
971+ [org, [workosMembership(member, org)]],
972+ [other, [workosMembership(member, other)]],
973+ ]);
974+
975+ const counts = await withMirror((mirror) =>
976+ backfillOrganization(source(orgs, []), mirror, org, { dryRun: false }),
977+ );
978+
979+ expect(counts).toEqual({
980+ applied: true,
981+ memberships: 1,
982+ usersWritten: 1,
983+ membershipsWritten: 1,
984+ membershipsTombstoned: 0,
985+ });
986+ expect((await readMembers(org)).map((m) => m.accountId)).toEqual([member]);
987+ expect(await backfilledAt(org)).not.toBeNull();
988+ expect(await readMembers(other), "the other organization is not scanned").toEqual([]);
989+ expect(await backfilledAt(other), "nor marked").toBeNull();
990+ });
991+
992+ it("refuses a scan that stalled while a later scan found a membership gone, so the revoked member stays revoked", async () => {
993+ const org = freshId("org");
994+ await seedOrganization(org);
995+ const staying = freshId("user");
996+ const leaving = freshId("user");
997+ const result = await withMirror((mirror) =>
998+ Effect.gen(function* () {
999+ // Scan A lists the org while `leaving` is still a member, then
1000+ // stalls (its listing is held behind the latch)...
1001+ const listedByA = yield* Latch.make(false);
1002+ const stalled = {
1003+ ...source(new Map(), []),
1004+ listOrgMembers: () =>
1005+ listedByA.await.pipe(
1006+ Effect.as([workosMembership(staying, org), workosMembership(leaving, org)]),
1007+ ),
1008+ };
1009+ const scanA = yield* Effect.forkChild(
1010+ backfillOrganization(stalled, mirror, org, { dryRun: false }),
1011+ { startImmediately: true },
1012+ );
1013+ // ...WorkOS removes `leaving`, and scan B lists and applies the
1014+ // org without them — no tombstone, the row was never there...
1015+ const b = yield* backfillOrganization(
1016+ source(new Map([[org, [workosMembership(staying, org)]]]), []),
1017+ mirror,
1018+ org,
1019+ { dryRun: false },
1020+ );
1021+ // ...then A resumes with its older listing.
1022+ yield* listedByA.open;
1023+ const a = yield* Fiber.join(scanA);
1024+ return { a, b };
1025+ }),
1026+ );
1027+ expect(result.b).toMatchObject({ applied: true, membershipsWritten: 1 });
1028+ expect(result.a, "the older listing is refused whole").toMatchObject({
1029+ applied: false,
1030+ memberships: 2,
1031+ usersWritten: 0,
1032+ membershipsWritten: 0,
1033+ membershipsTombstoned: 0,
1034+ });
1035+ expect(
1036+ new Map((await readMembers(org)).map((m) => [m.accountId, m.status])),
1037+ "the member the later listing no longer had was never inserted",
1038+ ).toEqual(new Map([[staying, "active"]]));
1039+ });
1040+});
apps/cloud/src/auth/mirror-feeders.ts+80−0
@@ -0,0 +1,80 @@
1+// ---------------------------------------------------------------------------
2+// The membership mirror's request-path FEEDERS: the writes the login callback
3+// and the Executor-initiated membership changes make through `WorkOsMirror`.
4+//
5+// Each feeder takes the WorkOS payload the caller ALREADY holds (the
6+// authenticated user, the membership list the callback fetches to pick a
7+// landing org, the membership a write returned) so feeding the mirror never
8+// adds a WorkOS read. Mirror failures fail the request: the mirror is the
9+// membership read path, so a login that could not record its memberships is
10+// not a login that finished.
11+// ---------------------------------------------------------------------------
12+
13+import { Effect } from "effect";
14+
15+import { UserStoreService } from "./context";
16+import {
17+ WorkOsMirror,
18+ mirrorMembershipFromWorkOs,
19+ mirrorUserFromWorkOs,
20+ type WorkOsMembershipPayload,
21+ type WorkOsUserPayload,
22+} from "./workos-mirror";
23+
24+/**
25+ * A membership as WorkOS lists it for a user: carries the organization's name,
26+ * which is what lets the sign-in feeder mirror the org row without a
27+ * `getOrganization` call. `OrganizationMembership` from the SDK satisfies it.
28+ */
29+export interface WorkOsSignInMembership extends WorkOsMembershipPayload {
30+ readonly organizationName: string;
31+}
32+
33+/**
34+ * Record a sign-in: the user's profile, then every organization WorkOS lists
35+ * them in (the org row first, so the membership's foreign key holds) and the
36+ * membership itself. Replays converge: every write is guarded on WorkOS
37+ * `updatedAt`, so a second login with the same payload changes nothing.
38+ *
39+ * `fetchedAt` is the instant the membership list was requested — taken
40+ * BEFORE the WorkOS read, so nothing that changed after it can be mistaken
41+ * for older. A membership list names each organization but carries no
42+ * organization timestamp, so `fetchedAt` is the stamp its name is written
43+ * under: a list fetched before a rename (a login that stalled) cannot revert
44+ * the rename after it landed. A membership of an organization the mirror
45+ * holds as deleted is refused by the mirror, and the organization is neither
46+ * re-minted nor renamed: a list fetched before a deletion cannot restore the
47+ * organization after its purge. And a membership stamped before the
48+ * organization's last full scan (`organizations.backfilled_at`) is refused
49+ * too: a list fetched before a revocation and written after the scan that
50+ * found the membership gone cannot reinstate it.
51+ */
52+export const mirrorSignIn = Effect.fn("workos_mirror.signIn")(function* (
53+ user: WorkOsUserPayload,
54+ memberships: readonly WorkOsSignInMembership[],
55+ fetchedAt: Date,
56+) {
57+ const mirror = yield* WorkOsMirror;
58+ const users = yield* UserStoreService;
59+ yield* mirror.upsertUser(mirrorUserFromWorkOs(user));
60+ for (const membership of memberships) {
61+ yield* users.use("upsertOrganization", (s) =>
62+ s.upsertOrganization({
63+ id: membership.organizationId,
64+ name: membership.organizationName,
65+ updatedAt: fetchedAt,
66+ }),
67+ );
68+ yield* mirror.upsertMembership(mirrorMembershipFromWorkOs(membership));
69+ }
70+});
71+
72+/**
73+ * Record one membership WorkOS just returned to a write (create, role
74+ * change, invitation acceptance). The organization must already be mirrored;
75+ * every caller has just upserted it or resolved it through the mirror.
76+ */
77+export const mirrorMembership = (membership: WorkOsMembershipPayload) =>
78+ Effect.flatMap(WorkOsMirror.asEffect(), (mirror) =>
79+ mirror.upsertMembership(mirrorMembershipFromWorkOs(membership)),
80+ );
apps/cloud/src/auth/org-api-key-auth.node.test.ts+13−1
@@ -83,18 +83,27 @@ const stubUsers = Layer.succeed(UserStoreService)({ · 82 unmodified lines
8383 upsertOrganization: async (org: { id: string; name: string }) => ({
8484 ...org,
8585 slug: `org-slug-${org.id}`,
86+ backfilledAt: null,
87+ deletedAt: null,
88+ workosUpdatedAt: null,
8689 createdAt,
8790 }),
8891 getOrganization: async (id: string) => ({
8992 id,
9093 name: `Org ${id}`,
9194 slug: `org-slug-${id}`,
95+ backfilledAt: null,
96+ deletedAt: null,
97+ workosUpdatedAt: null,
9298 createdAt,
9399 }),
94100 getOrganizationBySlug: async (slug: string) => ({
95101 id: "org_by_slug",
96102 name: `Org ${slug}`,
97103 slug,
104+ backfilledAt: null,
105+ deletedAt: null,
106+ workosUpdatedAt: null,
98107 createdAt,
99108 }),
100109 deleteOrganizationCascade: async () => {},
@@ -133,7 +142,10 @@ describe("org-level API keys", () => { · 32 unmodified lines
133142 const auth = yield* resolveBearerAuth(bearer("valid_user_key")).pipe(Effect.provide(layers));
134143
135144 expect(isPlatformAuth(auth)).toBe(false);
136− expect(auth).toMatchObject({ accountId: "user_123", organizationId: "org_123" });
145+ expect(auth).toMatchObject({
146+ accountId: "user_123",
147+ organizationId: "org_123",
148+ });
137149 }),
138150 );
139151
apps/cloud/src/auth/org-selector-auth.node.test.ts+9−0
@@ -86,12 +86,18 @@ const stubUsers = Layer.succeed(UserStoreService)({ · 85 unmodified lines
8686 upsertOrganization: async (org: { id: string; name: string }) => ({
8787 ...org,
8888 slug: org.id,
89+ backfilledAt: null,
90+ deletedAt: null,
91+ workosUpdatedAt: null,
8992 createdAt,
9093 }),
9194 getOrganization: async (id: string) => ({
9295 id,
9396 name: `Org ${id}`,
9497 slug: id,
98+ backfilledAt: null,
99+ deletedAt: null,
100+ workosUpdatedAt: null,
95101 createdAt,
96102 }),
97103 // The URL slug maps to URL_ORG (the member's other org); any other slug
@@ -100,6 +106,9 @@ const stubUsers = Layer.succeed(UserStoreService)({ · 2 unmodified lines
100106 id: slug === URL_SLUG ? URL_ORG : "org_outsider",
101107 name: `Org ${slug}`,
102108 slug,
109+ backfilledAt: null,
110+ deletedAt: null,
111+ workosUpdatedAt: null,
103112 createdAt,
104113 }),
105114 deleteOrganizationCascade: async () => {},
apps/cloud/src/auth/organization.ts+5−1
@@ -44,7 +44,11 @@ export const resolveOrganization = (organizationId: string) => · 43 unmodified lines
4444 const workos = yield* WorkOSClient;
4545 const fresh = yield* workos.getOrganization(organizationId);
4646 return yield* users.use("upsertOrganization", (s) =>
47− s.upsertOrganization({ id: fresh.id, name: fresh.name }),
47+ s.upsertOrganization({
48+ id: fresh.id,
49+ name: fresh.name,
50+ updatedAt: new Date(fresh.updatedAt),
51+ }),
4852 );
4953 });
5054
apps/cloud/src/auth/user-store.ts+49−12
@@ -7,7 +7,7 @@ · 6 unmodified lines
77 // so domain tables can foreign-key against them and so we can resolve org
88 // metadata without an API call on every request.
99
10−import { eq } from "drizzle-orm";
10+import { and, eq, isNull, lte, or } from "drizzle-orm";
1111
1212 import { generateOrgSlug } from "@executor-js/api";
1313
@@ -18,6 +18,29 @@ import { purgeOrganizationData } from "../db/org-deletion"; · 4 unmodified lines
1818 export type Account = typeof accounts.$inferSelect;
1919 export type Organization = typeof organizations.$inferSelect;
2020
21+/**
22+ * An organization as a feeder hands it to the mirror. `updatedAt` is when
23+ * `name` is known to have been the organization's name in WorkOS: the WorkOS
24+ * `updatedAt` of an organization payload, or the instant a membership list
25+ * naming the organization was fetched (a list carries the name but no
26+ * organization timestamp).
27+ */
28+export interface OrganizationPayload {
29+ readonly id: string;
30+ readonly name: string;
31+ readonly updatedAt: Date;
32+}
33+
34+/**
35+ * Which stored organization rows a name stamped `updatedAt` may rename: a
36+ * row with no stamp (predating the stamp), or one stamped at or before
37+ * `updatedAt` — feeders replay the same payload and must converge. Every
38+ * writer of `organizations.name` applies this, so a name fetched before a
39+ * rename can never revert the rename after it landed.
40+ */
41+export const organizationAcceptsName = (updatedAt: Date) =>
42+ or(isNull(organizations.workosUpdatedAt), lte(organizations.workosUpdatedAt, updatedAt));
43+
2144 export const makeUserStore = (db: DrizzleDb) => {
2245 const getOrganization = async (id: string) => {
2346 const rows = await db.select().from(organizations).where(eq(organizations.id, id));
@@ -38,10 +61,15 @@ export const makeUserStore = (db: DrizzleDb) => { · 14 unmodified lines
3861 // candidate was claimed by a different org). Returns the inserted row, or
3962 // null when either conflict swallowed the insert — the caller decides whether
4063 // to re-read (id race) or retry with a new candidate (slug race).
41− const tryInsertOrg = async (id: string, name: string, slug: string) => {
64+ const tryInsertOrg = async (org: OrganizationPayload, slug: string) => {
4265 const [row] = await db
4366 .insert(organizations)
44− .values({ id, name, slug })
67+ .values({
68+ id: org.id,
69+ name: org.name,
70+ slug,
71+ workosUpdatedAt: org.updatedAt,
72+ })
4573 .onConflictDoNothing()
4674 .returning();
4775 return row ?? null;
@@ -49,20 +77,27 @@ export const makeUserStore = (db: DrizzleDb) => { · 1 unmodified line
4977
5078 // Every new org row is born with a slug — there is no nullable window and no
5179 // self-healing. Existing rows keep their slug (stable across renames, so org
52− // URLs survive) and only refresh their name.
53− const upsertOrganization = async (org: { id: string; name: string }) => {
80+ // URLs survive) and only refresh their name — and only from a payload at
81+ // least as new as the one that last named it (`organizationAcceptsName`):
82+ // a sign-in whose membership list was fetched before a rename would
83+ // otherwise revert the rename after it landed. A row marked deleted is
84+ // returned as it is: the organization is gone, and nothing a feeder still
85+ // holds about it (a name, a membership fetched before the deletion) is
86+ // written — never re-minted live, never renamed.
87+ const upsertOrganization = async (org: OrganizationPayload) => {
5488 const existing = await getOrganization(org.id);
5589 if (existing) {
90+ if (existing.deletedAt !== null) return existing;
5691 const [updated] = await db
5792 .update(organizations)
58− .set({ name: org.name })
59− .where(eq(organizations.id, org.id))
93+ .set({ name: org.name, workosUpdatedAt: org.updatedAt })
94+ .where(and(eq(organizations.id, org.id), organizationAcceptsName(org.updatedAt)))
6095 .returning();
6196 return updated ?? existing;
6297 }
6398 for (let attempt = 0; attempt < 4; attempt++) {
6499 const slug = await generateOrgSlug(org.name, slugTaken);
65− const inserted = await tryInsertOrg(org.id, org.name, slug);
100+ const inserted = await tryInsertOrg(org, slug);
66101 if (inserted) return inserted;
67102 // The insert was swallowed by a conflict. If the id now exists, a
68103 // concurrent request mirrored it — return that row. Otherwise the slug
@@ -98,9 +133,11 @@ export const makeUserStore = (db: DrizzleDb) => { · 29 unmodified lines
98133 return rows[0] ?? null;
99134 },
100135
101− // Permanently delete an org and everything it owns (tenant data, secrets,
102− // identity mirror + cascaded memberships) in a single transaction. Callers
103− // sequence the external WorkOS/Autumn deletions around this.
104− deleteOrganizationCascade: (id: string) => purgeOrganizationData(db, id),
136+ // Permanently delete everything an org owns (tenant data, secrets, its
137+ // memberships) in a single transaction, leaving the organization row as
138+ // a tombstone marked `deletedAt` (see `purgeOrganizationData` for why).
139+ // Callers sequence the external WorkOS/Autumn deletions around this.
140+ deleteOrganizationCascade: (id: string, deletedAt: Date) =>
141+ purgeOrganizationData(db, id, deletedAt),
105142 };
106143 };
apps/cloud/src/auth/workos-callback-state.node.test.ts+28−0
@@ -20,6 +20,7 @@ import { CloudAuthPublicHandlers } from "./handlers"; · 19 unmodified lines
2020 import { CloudAuthPublicApi } from "./api";
2121 import { UserStoreService } from "./context";
2222 import { WorkOSClient, type WorkOSClientService } from "./workos";
23+import { WorkOsMirror } from "./workos-mirror";
2324 import { encodeLoginState } from "./login-state";
2425 import { AutumnService } from "../extensions/billing/service";
2526
@@ -79,18 +80,27 @@ const stubUsers = Layer.succeed(UserStoreService)({ · 53 unmodified lines
7980 upsertOrganization: async (org: { id: string; name: string }) => ({
8081 ...org,
8182 slug: org.id,
83+ backfilledAt: null,
84+ deletedAt: null,
85+ workosUpdatedAt: null,
8286 createdAt: new Date(),
8387 }),
8488 getOrganization: async (id: string) => ({
8589 id,
8690 name: "Org " + id,
8791 slug: id,
92+ backfilledAt: null,
93+ deletedAt: null,
94+ workosUpdatedAt: null,
8895 createdAt: new Date(),
8996 }),
9097 getOrganizationBySlug: async (slug: string) => ({
9198 id: slug,
9299 name: slug,
93100 slug,
101+ backfilledAt: null,
102+ deletedAt: null,
103+ workosUpdatedAt: null,
94104 createdAt: new Date(),
95105 }),
96106 deleteOrganizationCascade: async () => {},
@@ -98,6 +108,23 @@ const stubUsers = Layer.succeed(UserStoreService)({ · 1 unmodified line
98108 ),
99109 });
100110
111+// The callback records the sign-in (user + memberships) in the membership
112+// mirror; every other mirror operation is out of this route's reach.
113+const stubMirror = Layer.succeed(WorkOsMirror)({
114+ upsertUser: () => Effect.succeed(true),
115+ upsertMembership: () => Effect.succeed(true),
116+ deleteMembership: () => Effect.die("the callback does not delete memberships"),
117+ deleteUser: () => Effect.die("the callback does not delete users"),
118+ getCursor: () => Effect.die("the callback does not read the events cursor"),
119+ setCursor: () => Effect.die("the callback does not move the events cursor"),
120+ applyOrganizationScan: () => Effect.die("the callback does not run the backfill"),
121+ replayBoundary: () => Effect.die("the callback does not run the reconciler"),
122+ setReplayBoundary: () => Effect.die("the callback does not run the backfill"),
123+ backfillCompletedAt: () => Effect.die("the callback does not check mirror readiness"),
124+ markBackfillCompleted: () => Effect.die("the callback does not run the backfill"),
125+ organizationBackfilledAt: () => Effect.die("the callback does not report seats"),
126+});
127+
101128 // Only the public group is under test; the session group (and its SessionAuth
102129 // middleware, which needs a live DB) is out of scope — the callback route lives
103130 // in CloudAuthPublicApi and requires no middleware.
@@ -107,6 +134,7 @@ const App = HttpApiBuilder.layer(PublicApi).pipe( · 3 unmodified lines
107134 Layer.provide(CloudAuthPublicHandlers),
108135 Layer.provide(stubWorkOS),
109136 Layer.provide(stubUsers),
137+ Layer.provide(stubMirror),
110138 Layer.provide(AutumnService.Default),
111139 Layer.provide(HttpServer.layerServices),
112140 );
apps/cloud/src/auth/workos-mirror-backfill.ts+274−0
@@ -0,0 +1,274 @@
1+// ---------------------------------------------------------------------------
2+// Backfill of the membership mirror from WorkOS, one organization at a time:
3+// list EVERY membership WorkOS holds for it — active, pending, and inactive
4+// alike — fetch each member's user, write both through the mirror's guarded
5+// upserts, tombstone whatever the mirror still holds that WorkOS no longer
6+// lists, and mark the organization BACKFILLED as of the listing. Inactive
7+// memberships are listed on purpose: the scan tombstones every mirrored
8+// membership its listing lacks, and a tombstone is keyed to the membership
9+// id for good (`membershipAcceptsPayload`), so a listing that skipped the
10+// inactive ones would tombstone a membership WorkOS merely deactivated and
11+// refuse its reactivation — under the same id — forever. Listed with its
12+// real status it stays an ordinary `inactive` row that a newer payload
13+// reactivates. The core is a pure function over a `source`
14+// (WorkOS reads) and the mirror store, so the one-off script
15+// (`scripts/backfill-workos-mirror.ts`) can wire real clients, the request
16+// path can wire `WorkOSClient` (an organization whose mark is missing is
17+// scanned on demand before its seats are counted), and the test can wire
18+// fakes against the test database.
19+//
20+// Idempotent and repairing: the upserts are guarded on WorkOS `updatedAt`,
21+// so a re-scan over unchanged data writes nothing new, and every membership
22+// the mirror holds for the org that WorkOS no longer lists is TOMBSTONED
23+// (`inactive`, `deleted_at` = the time the listing was taken) — so a re-scan
24+// repairs a stale row instead of leaving it granting access. `dryRun` reads
25+// everything and writes nothing, so the printed counts are the plan.
26+//
27+// One scan is applied in ONE transaction (`WorkOsMirror.applyOrganizationScan`)
28+// that first moves the organization's `backfilled_at` forward to the
29+// listing's instant and writes nothing if a LATER listing already did. Two
30+// scans of the same organization can overlap (the one-off script and an
31+// on-demand scan from a request, or a stalled script run and its retry), and
32+// the `updatedAt` guard alone cannot order them: a scan that listed a
33+// membership, stalled, and resumed after a later scan had found it gone
34+// would insert it live — the later scan tombstoned nothing, because the row
35+// was not there to tombstone. Refusing the older listing whole is what keeps
36+// a membership revoked between the two listings revoked.
37+//
38+// Completeness is tracked PER ORGANIZATION (`organizations.backfilled_at`),
39+// never database-wide: the mark is written only by a scan that listed that
40+// organization's memberships in full, so an organization mirrored after a
41+// backfill ran (lazily by a request, or by a sign-in that records only the
42+// caller's own membership) starts unmarked and is scanned before any count
43+// read from the mirror is trusted. A run over every organization
44+// (`backfillWorkOsMirror`) additionally records the Events API replay
45+// boundary — the instant it began reading WorkOS, taken BEFORE anything is
46+// listed so no change can fall between the boundary and a listing — once
47+// every organization was written, and only if no boundary is recorded yet.
48+// A run that fails part-way records nothing, and a later completed run
49+// keeps the first boundary: a scan refreshes memberships and tombstones,
50+// not organization names or deleted users' profiles, so an
51+// `organization.updated` or `user.deleted` between two runs is covered only
52+// by the events stream — advancing the boundary past it would skip it for
53+// good. The reconciler's own cursor takes over from the boundary after its
54+// first page, so the boundary's only job is to name where that first page
55+// starts.
56+//
57+// The mark also orders every OTHER membership write against the scan: the
58+// mirror refuses a membership payload stamped before the organization's
59+// `backfilled_at` (`upsertMembership`). A login whose membership list was
60+// fetched before a revocation and written after the scan would otherwise
61+// reinstate the revoked membership — and that revocation predates the
62+// events replay boundary, so no event would ever tombstone it again.
63+// ---------------------------------------------------------------------------
64+
65+import { Clock, Effect, Option } from "effect";
66+
67+import {
68+ mirrorMembershipFromWorkOs,
69+ mirrorUserFromWorkOs,
70+ type WorkOsMembershipPayload,
71+ type WorkOsMirrorShape,
72+ type WorkOsOrganizationScanWrites,
73+ type WorkOsUserPayload,
74+} from "./workos-mirror-store";
75+
76+/** The WorkOS reads one organization's scan performs, over whatever client the caller wires. */
77+export interface WorkOsOrganizationScanSource<E> {
78+ /**
79+ * EVERY membership of one organization, all pages and all statuses
80+ * (active, pending, inactive). A source that filtered by status would
81+ * have the scan tombstone what it filtered out — see the header.
82+ */
83+ readonly listOrgMembers: (
84+ organizationId: string,
85+ ) => Effect.Effect<readonly WorkOsMembershipPayload[], E>;
86+ readonly getUser: (userId: string) => Effect.Effect<WorkOsUserPayload, E>;
87+}
88+
89+/** The reads the full backfill performs: every organization, then each one's scan. */
90+export interface WorkOsMirrorBackfillSource<E> extends WorkOsOrganizationScanSource<E> {
91+ /** Every organization id the mirror knows (FK target of `memberships`). */
92+ readonly listOrganizationIds: () => Effect.Effect<readonly string[], E>;
93+}
94+
95+export interface WorkOsMirrorBackfillOptions {
96+ readonly dryRun: boolean;
97+ /** One line per organization and one summary line; never a user's data. */
98+ readonly log: (line: string) => void;
99+}
100+
101+/** What one organization's scan did. */
102+export interface WorkOsOrganizationScanCounts extends WorkOsOrganizationScanWrites {
103+ /** Memberships WorkOS reported for the organization. */
104+ readonly memberships: number;
105+ /**
106+ * Whether the listing was written to the mirror. `false` on a dry run, and
107+ * when the mirror refused the listing whole: a later listing of the
108+ * organization had already been applied (an overlapping scan finished
109+ * first), or the organization is marked deleted or not mirrored. Every
110+ * write count is 0 then.
111+ */
112+ readonly applied: boolean;
113+}
114+
115+const NOTHING_WRITTEN: WorkOsOrganizationScanWrites = {
116+ usersWritten: 0,
117+ membershipsWritten: 0,
118+ membershipsTombstoned: 0,
119+};
120+
121+export interface WorkOsMirrorBackfillCounts extends WorkOsOrganizationScanWrites {
122+ readonly organizations: number;
123+ /** Memberships WorkOS reported across every organization. */
124+ readonly memberships: number;
125+}
126+
127+// Bounded fan-out for the per-member `getUser` calls: enough to overlap the
128+// WorkOS round-trips, low enough to stay clear of its rate limit.
129+const USER_FETCH_CONCURRENCY = 5;
130+
131+const now = () => Effect.map(Clock.currentTimeMillis, (millis) => new Date(millis));
132+
133+/**
134+ * Scan one organization: list every membership WorkOS holds for it, fetch
135+ * each member's user, and apply the listing to the mirror in one transaction
136+ * — the listed users and memberships upserted, the rest tombstoned, the
137+ * organization marked backfilled as of the listing — unless a later listing
138+ * was applied first, in which case nothing is written (`applied: false`).
139+ * The organization row must already be mirrored. Fails on the first source
140+ * or mirror failure and writes nothing then — a failed scan is safe to
141+ * repeat, so surfacing the failure beats a silent skip. A dry run reads
142+ * everything, writes nothing, and marks nothing.
143+ */
144+export const backfillOrganization = <E>(
145+ source: WorkOsOrganizationScanSource<E>,
146+ mirror: WorkOsMirrorShape,
147+ organizationId: string,
148+ options: { readonly dryRun: boolean },
149+) =>
150+ Effect.gen(function* () {
151+ // The listing's own instant, taken BEFORE the read so no change can fall
152+ // between them: the tombstone time for whatever the listing no longer
153+ // contains, the cut-off for what may be tombstoned at all (a row stamped
154+ // at or after it was written after the listing and is not missing from
155+ // it), and the organization's new `backfilled_at`.
156+ const listedAt = yield* now();
157+ const listed = yield* source.listOrgMembers(organizationId);
158+ const members = yield* Effect.forEach(
159+ listed,
160+ (membership) =>
161+ Effect.map(source.getUser(membership.userId), (user) => ({
162+ user: mirrorUserFromWorkOs(user),
163+ membership: mirrorMembershipFromWorkOs(membership),
164+ })),
165+ { concurrency: USER_FETCH_CONCURRENCY },
166+ );
167+ if (options.dryRun) {
168+ const counts: WorkOsOrganizationScanCounts = {
169+ ...NOTHING_WRITTEN,
170+ memberships: members.length,
171+ applied: false,
172+ };
173+ return counts;
174+ }
175+ const written = yield* mirror.applyOrganizationScan({
176+ organizationId,
177+ listedAt,
178+ members,
179+ });
180+ if (Option.isNone(written)) {
181+ yield* Effect.logWarning(
182+ "workos_mirror: organization scan not applied — a later listing was already applied, or the organization is deleted or not mirrored",
183+ { organizationId, listedAt: listedAt.toISOString() },
184+ );
185+ }
186+ const counts: WorkOsOrganizationScanCounts = {
187+ ...Option.getOrElse(written, () => NOTHING_WRITTEN),
188+ memberships: members.length,
189+ applied: Option.isSome(written),
190+ };
191+ return counts;
192+ }).pipe(
193+ Effect.withSpan("workos_mirror.backfillOrganization", {
194+ attributes: { organizationId },
195+ }),
196+ );
197+
198+/**
199+ * Run the full backfill: scan every organization the mirror knows, then
200+ * record the Events API replay boundary if none is recorded yet. Fails on
201+ * the first source or mirror failure — the organizations scanned so far stay
202+ * marked (each was covered in full), the boundary is not recorded, and the
203+ * run is safe to repeat.
204+ */
205+export const backfillWorkOsMirror = <E>(
206+ source: WorkOsMirrorBackfillSource<E>,
207+ mirror: WorkOsMirrorShape,
208+ options: WorkOsMirrorBackfillOptions,
209+) =>
210+ Effect.gen(function* () {
211+ // Taken before the first listing; recorded only once the run completes.
212+ const boundary = yield* now();
213+
214+ const organizationIds = yield* source.listOrganizationIds();
215+ let memberships = 0;
216+ let usersWritten = 0;
217+ let membershipsWritten = 0;
218+ let membershipsTombstoned = 0;
219+
220+ for (const organizationId of organizationIds) {
221+ const scanned = yield* backfillOrganization(source, mirror, organizationId, options);
222+ memberships += scanned.memberships;
223+ usersWritten += scanned.usersWritten;
224+ membershipsWritten += scanned.membershipsWritten;
225+ membershipsTombstoned += scanned.membershipsTombstoned;
226+ options.log(
227+ `${organizationId} ${scanned.memberships} membership(s)` +
228+ (options.dryRun
229+ ? ""
230+ : scanned.applied
231+ ? ` wrote ${scanned.usersWritten} user(s), ${scanned.membershipsWritten} membership(s), tombstoned ${scanned.membershipsTombstoned}`
232+ : " not applied (a later listing was already applied, or the organization is deleted)"),
233+ );
234+ }
235+
236+ const counts: WorkOsMirrorBackfillCounts = {
237+ organizations: organizationIds.length,
238+ memberships,
239+ usersWritten,
240+ membershipsWritten,
241+ membershipsTombstoned,
242+ };
243+ options.log(
244+ options.dryRun
245+ ? `dry run — ${counts.organizations} organization(s), ${counts.memberships} membership(s) would be mirrored`
246+ : `${counts.organizations} organization(s), ${counts.memberships} membership(s): wrote ${counts.usersWritten} user(s), ${counts.membershipsWritten} membership(s), tombstoned ${counts.membershipsTombstoned}`,
247+ );
248+ if (!options.dryRun) {
249+ // Every organization was read and written without failure (a failure
250+ // above fails the whole run) — or refused in favour of a listing taken
251+ // later still — so everything before `boundary` is now covered: record
252+ // it for the reconciler — unless an earlier run already did, in which
253+ // case the events between the two runs are the reconciler's to replay
254+ // and the earlier boundary stands.
255+ const recorded = yield* mirror.setReplayBoundary(boundary);
256+ options.log(
257+ recorded
258+ ? `events replay boundary set to ${boundary.toISOString()}`
259+ : "events replay boundary already recorded by an earlier run; kept (the events reconciler replays every change since it)",
260+ );
261+ // And every live organization is now covered: the mirror is complete
262+ // enough to authorize from (once the reconciler has caught up too),
263+ // which the authorization path reads as the first half of readiness.
264+ // Once: a re-run keeps the first completion.
265+ const completedAt = yield* now();
266+ const marked = yield* mirror.markBackfillCompleted(completedAt);
267+ options.log(
268+ marked
269+ ? `backfill completion recorded at ${completedAt.toISOString()}`
270+ : "backfill completion already recorded by an earlier run; kept",
271+ );
272+ }
273+ return counts;
274+ });
apps/cloud/src/auth/workos-mirror-store.ts+999−0
@@ -0,0 +1,999 @@
1+// ---------------------------------------------------------------------------
2+// The membership mirror's WRITE store — the Drizzle queries behind
3+// `WorkOsMirror`, plus the converters from WorkOS SDK payloads to mirror rows.
4+//
5+// Kept free of `cloudflare:workers` (no `DbService`, no `env`) so the one-off
6+// backfill (`scripts/backfill-workos-mirror.ts`) can run the SAME upserts over
7+// a plain postgres.js connection under bun. The request-scoped service that
8+// wraps this store is `workos-mirror.ts`.
9+//
10+// Every write is idempotent and out-of-order safe. Both upserts carry the
11+// WorkOS `updatedAt` of their payload and refuse to overwrite a row whose
12+// stored `workos_updated_at` is newer, so a replayed or late-arriving event
13+// can never regress the mirror. A delete never drops the row: it TOMBSTONES
14+// it (`status = 'inactive'`, `deleted_at` set) — and INSERTS the tombstone
15+// when the row is not there yet — so a feeder that fetched the membership
16+// before the deletion and writes it after (a login, the backfill) finds the
17+// tombstone instead of reinstating access. The tombstone is protected by
18+// IDENTITY, not by time: WorkOS never reuses a deleted `om_…` id, so a
19+// payload naming the tombstoned id is refused whatever it is stamped — a
20+// role change issued before the removal and delivered after it is stamped
21+// NEWER than the row and would beat any timestamp guard — while a payload
22+// naming a DIFFERENT id for the same (account, organization) is the member
23+// re-added in WorkOS, a replacement, and takes the row over under the usual
24+// `updatedAt` rule (WorkOS creates it after the old one is deleted, so it is
25+// always the newer). A tombstone keeps the stamp the row holds, the deleted
26+// membership's last reported state — never a local clock: read after WorkOS
27+// answered, it could post-date a replacement created meanwhile and refuse
28+// it for good. `deleted_at` is the deletion instant when the caller holds
29+// one (an event's `createdAt`, a scan's listing time) and `now()` otherwise;
30+// it records WHEN, it orders nothing. A membership WorkOS merely
31+// deactivated (`status = 'inactive'`, no `deleted_at`) is not a tombstone:
32+// WorkOS can reactivate it under the same id, and the `updatedAt` rule
33+// orders that as any other update. The row tombstone can only speak for the
34+// id the row happens to hold, so every delete ALSO records the deleted id in
35+// `membership_tombstones`, a ledger keyed by the WorkOS id alone, and every
36+// membership write consults it first: that covers the case the row cannot —
37+// membership A replaced by B in WorkOS before the mirror saw either, B then
38+// deleted. The delete finds a row holding A (not B's to tombstone) and would
39+// otherwise leave nothing behind; a later scan that still lists B would then
40+// insert it live, stamped after A and under another id, exactly what the row
41+// guard lets through. With the ledger the delete is recorded whatever the
42+// row holds, and B's payload is refused by identity. A scan records the ids
43+// it tombstones the same way. A deleted USER is protected by identity
44+// too: `deleteUser` leaves the account row behind as a tombstone (profile
45+// cleared, stamped with the deletion), and no membership naming that account
46+// is written again, however the payload is stamped — WorkOS never reuses a
47+// user id, and a membership the mirror had not seen has no row of its own
48+// for a guard to refuse the insert against. The cursor advances only by
49+// compare-and-set, so two reconciler runs cannot both own the stream.
50+//
51+// A backfill SCAN of one organization (its full membership listing, taken
52+// at one instant) is applied in ONE transaction that first compare-and-sets
53+// the organization's `backfilled_at` to the listing's instant
54+// (`applyOrganizationScan`): the row stays locked until commit, so two
55+// overlapping scans serialize on it, and the one whose listing is older
56+// than the recorded one writes NOTHING. The `updatedAt` guard alone cannot
57+// order two scans: a scan that listed a membership, stalled, and resumed
58+// after a later scan had found it gone would insert it live — the later scan
59+// left no tombstone, because the row was not there to tombstone. For the
60+// same reason every OTHER membership write is ordered against the scan too:
61+// a payload stamped before the organization's `backfilled_at` is refused
62+// unless the row already holds it — judged inside the write's own
63+// transaction, reading the organization row `FOR SHARE`, so a write that
64+// races a scan waits for the scan's commit and sees the mark it set, and
65+// never inserts a membership the scan has just proved revoked. A completed
66+// scan is the full listing as
67+// of that instant, so a membership it did not contain but an older payload
68+// still names (a login whose list was fetched before the revocation and
69+// written after the scan) was revoked before the scan — and that revocation
70+// predates the events replay boundary, so nothing would ever tombstone the
71+// reinstated row. The scan's own writes are the one exception: they are the
72+// listing that sets the mark.
73+//
74+// The events replay boundary (`workos_sync.range_start`) is written ONCE, by
75+// the first completed backfill, and never advanced: a later backfill
76+// refreshes memberships only, so an organization rename or user deletion
77+// between two runs is covered by the events stream alone, and moving the
78+// boundary past it would skip it for good.
79+// ---------------------------------------------------------------------------
80+
81+import { and, eq, isNotNull, isNull, lt, ne, notInArray, or, sql } from "drizzle-orm";
82+import type { AnyPgColumn } from "drizzle-orm/pg-core";
83+import { Effect, Option } from "effect";
84+
85+import type { MemberStatus } from "@executor-js/api/server";
86+
87+import {
88+ accounts,
89+ membershipTombstones,
90+ memberships,
91+ organizations,
92+ workosSync,
93+} from "../db/schema";
94+import type { DrizzleDb } from "../db/db";
95+import {
96+ WorkOsMirrorError,
97+ tryPromiseService,
98+ userStoreReasonFromCause,
99+ withServiceLogging,
100+} from "./errors";
101+
102+/** A WorkOS user, as the mirror stores it. `updatedAt` is WorkOS's own. */
103+export interface WorkOsMirrorUser {
104+ readonly id: string;
105+ readonly email: string;
106+ readonly firstName: string | null;
107+ readonly lastName: string | null;
108+ readonly avatarUrl: string | null;
109+ readonly lastSignInAt: Date | null;
110+ readonly updatedAt: Date;
111+}
112+
113+/**
114+ * A WorkOS organization membership, as the mirror stores it. `id` is the
115+ * WorkOS `om_…`; `accountId` the WorkOS user id; `updatedAt` is WorkOS's own.
116+ * The organization row must already be mirrored (`upsertOrganization`) — a
117+ * membership of an unknown org is a `query` failure, not a silent skip.
118+ */
119+export interface WorkOsMirrorMembership {
120+ readonly id: string;
121+ readonly accountId: string;
122+ readonly organizationId: string;
123+ readonly role: string;
124+ readonly status: MemberStatus;
125+ readonly updatedAt: Date;
126+}
127+
128+/**
129+ * What identifies a membership to a delete: the WorkOS `om_…` id AND the
130+ * (account, organization) pair it belongs to. The pair is the row's key,
131+ * and a delete must be able to mint the row as a tombstone when the mirror
132+ * has not seen the membership yet — an id alone cannot.
133+ */
134+export type WorkOsMirrorMembershipRef = Pick<
135+ WorkOsMirrorMembership,
136+ "id" | "accountId" | "organizationId"
137+>;
138+
139+/** One member as a backfill scan lists it: the membership and its user. */
140+export interface WorkOsScannedMember {
141+ readonly user: WorkOsMirrorUser;
142+ readonly membership: WorkOsMirrorMembership;
143+}
144+
145+/**
146+ * One organization's full membership listing, as `applyOrganizationScan`
147+ * applies it. `listedAt` is the instant the listing was taken — BEFORE the
148+ * WorkOS read, so no change can fall between the instant and the listing —
149+ * and is the tombstone time for what the listing no longer contains, the
150+ * cut-off for what may be tombstoned at all, and the organization's new
151+ * `backfilled_at`.
152+ */
153+export interface WorkOsOrganizationScan {
154+ readonly organizationId: string;
155+ readonly listedAt: Date;
156+ readonly members: readonly WorkOsScannedMember[];
157+}
158+
159+/** What an applied scan wrote: the upserts the `updatedAt` guard let through, and the tombstones. */
160+export interface WorkOsOrganizationScanWrites {
161+ readonly usersWritten: number;
162+ readonly membershipsWritten: number;
163+ readonly membershipsTombstoned: number;
164+}
165+
166+export interface WorkOsMirrorShape {
167+ /**
168+ * Insert or refresh a user row. `false` when the payload was refused and
169+ * the row left untouched: the stored row is newer than `updatedAt`, or is
170+ * a deletion tombstone stamped at `updatedAt` or later.
171+ */
172+ readonly upsertUser: (user: WorkOsMirrorUser) => Effect.Effect<boolean, WorkOsMirrorError>;
173+ /**
174+ * Insert or refresh a membership row, minting the bare account row first so
175+ * the foreign key holds when the membership arrives before its user. `false`
176+ * when the payload was refused: the stored row is newer than `updatedAt`,
177+ * or is a deletion tombstone of THIS membership id, or THIS membership id
178+ * is in the deletion ledger (`membership_tombstones`, written by every
179+ * delete and every scan tombstone, whatever row it found) — a deleted
180+ * WorkOS id never returns, however the payload is stamped; a payload under
181+ * another id is a replacement and is ordered by `updatedAt` against the
182+ * row's stamp — or the account is a deletion tombstone (`deleteUser`), whatever
183+ * the payload is stamped: WorkOS never reuses a user id, so a deleted user
184+ * has no memberships to mirror — or the organization is marked deleted
185+ * (`organizations.deleted_at`), in which case nothing is written at all:
186+ * no membership of a deleted organization is ever mirrored — or the
187+ * payload is stamped BEFORE the organization's last full scan
188+ * (`organizations.backfilled_at`): the scan listed everything WorkOS held
189+ * at that instant, so a membership it wrote already carries a stamp at
190+ * least this new (the write would change nothing) and one it did not
191+ * write was gone by then and must not come back from a list fetched
192+ * earlier. Only the scan itself writes past that mark
193+ * (`applyOrganizationScan`). The organization checks and the write run in
194+ * ONE transaction that holds the organization row `FOR SHARE`, so a scan
195+ * claiming the row at the same time is waited for and its mark seen — and
196+ * the account row `FOR SHARE` too, so a `deleteUser` tombstoning it at the
197+ * same time is waited for and its tombstone seen.
198+ */
199+ readonly upsertMembership: (
200+ membership: WorkOsMirrorMembership,
201+ ) => Effect.Effect<boolean, WorkOsMirrorError>;
202+ /**
203+ * Tombstone the membership as deleted: the row stays (or is minted, when
204+ * the mirror has not seen the membership yet), `inactive`, carrying the
205+ * deleted WorkOS id with `deleted_at` set. The tombstone is protected by
206+ * that id: `upsertMembership` refuses every later payload naming it,
207+ * however stamped, and accepts only a REPLACEMENT under another id. The
208+ * row's `workos_updated_at` is left as it is — the deleted membership's
209+ * last reported state, which a replacement is always stamped after; a
210+ * local clock read after WorkOS answered could post-date a replacement
211+ * created meanwhile and must never become the row's stamp. `deletedAt`
212+ * is the deletion instant when the caller holds one (a deletion event's
213+ * `createdAt`, a scan's listing time) or `null` when it holds none
214+ * (WorkOS answers a delete with no time): `deleted_at` then takes the
215+ * current time. It records when the membership was deleted; it orders
216+ * nothing. Matches the row by IDENTITY: `false` when the row is already
217+ * tombstoned (a replayed delete), or holds ANOTHER membership id — the
218+ * member re-added in WorkOS under a new id, which stands whether the
219+ * replacement was mirrored before or after this delete. In EVERY case the
220+ * deleted id is recorded in the deletion ledger (`membership_tombstones`),
221+ * so a membership the mirror never held under its own id (replaced and
222+ * deleted in WorkOS before the mirror saw it) cannot be inserted live by a
223+ * scan that listed it before the deletion: `true` when the row was
224+ * tombstoned OR the id was newly recorded. The organization row must
225+ * already be mirrored, as for `upsertMembership`.
226+ */
227+ readonly deleteMembership: (
228+ membership: WorkOsMirrorMembershipRef,
229+ deletedAt: Date | null,
230+ ) => Effect.Effect<boolean, WorkOsMirrorError>;
231+ /**
232+ * Tombstone a deleted WorkOS user: every membership of the account is
233+ * tombstoned as by `deleteMembership` (as of `deletedAt`; a deleted user's
234+ * membership ids never return), and the account row is kept (it anchors
235+ * foreign keys) — or minted, when the mirror has not seen the user yet —
236+ * with its profile cleared and stamped `deletedAt`, so a stale user
237+ * payload cannot restore it. `false` when the account already carries
238+ * this tombstone or a later one (a replayed delete). ONE transaction that
239+ * locks the account row first (`FOR NO KEY UPDATE`) and holds it until
240+ * the memberships are tombstoned: `upsertMembership` reads that row `FOR
241+ * SHARE` before it inserts, so a membership write racing the deletion
242+ * waits for it and sees the tombstone, or committed first and is
243+ * tombstoned here — never a live membership left behind for a deleted
244+ * user.
245+ */
246+ readonly deleteUser: (
247+ accountId: string,
248+ deletedAt: Date,
249+ ) => Effect.Effect<boolean, WorkOsMirrorError>;
250+ /** The id of the last WorkOS event applied, or `null` before the first run. */
251+ readonly getCursor: () => Effect.Effect<string | null, WorkOsMirrorError>;
252+ /**
253+ * Compare-and-set the cursor: advance to `next` only if it still reads
254+ * `prev` (`null` = no cursor yet). `false` means another run moved it first
255+ * — the caller must stop, it no longer owns the stream.
256+ */
257+ readonly setCursor: (
258+ prev: string | null,
259+ next: string,
260+ ) => Effect.Effect<boolean, WorkOsMirrorError>;
261+ /**
262+ * Apply one organization's backfill scan — the memberships (with their
263+ * users) a WorkOS listing taken at `listedAt` contained — atomically: in a
264+ * single transaction, compare-and-set the organization's `backfilled_at`
265+ * forward to `listedAt`, and only if that succeeded upsert every listed
266+ * user and membership and tombstone (as `deleteMembership` does, at
267+ * `listedAt`) every membership of the organization the listing did NOT
268+ * contain: the rows whose WorkOS id is not among the listed ones AND whose
269+ * stamp is older than the listing. A row stamped at or after `listedAt`
270+ * was written after the listing was taken (it could not be in it) and is
271+ * left alone — its own event lands through the reconciler; a tombstone
272+ * here would beat that event. Rows already tombstoned are left alone.
273+ *
274+ * The organization row stays locked until commit, so two overlapping
275+ * scans serialize on it and the one whose listing is older than (or the
276+ * same instant as) the recorded one writes nothing: `None` — the mirror
277+ * already holds the organization as of a later listing, and a membership
278+ * that listing no longer contained must not be inserted from this one.
279+ * `None` too for an organization the mirror does not hold or has marked
280+ * deleted: there is nothing to scan into. `Some` carries what was written.
281+ */
282+ readonly applyOrganizationScan: (
283+ scan: WorkOsOrganizationScan,
284+ ) => Effect.Effect<Option.Option<WorkOsOrganizationScanWrites>, WorkOsMirrorError>;
285+ /**
286+ * The Events API replay boundary: the instant the FIRST completed one-off
287+ * backfill began reading WorkOS, or `null` if none has completed. The
288+ * reconciler's first run (no cursor yet) reads the stream from here — the
289+ * backfill covers everything before it — and without a boundary it must
290+ * not guess.
291+ */
292+ readonly replayBoundary: () => Effect.Effect<Date | null, WorkOsMirrorError>;
293+ /**
294+ * Record the replay boundary, ONCE: `at` is the instant a completed
295+ * backfill began reading WorkOS, and it is kept only when no boundary is
296+ * recorded yet — `true` when this call recorded it. A later completed run
297+ * never moves it: the backfill refreshes memberships and tombstones only,
298+ * not organization names or deleted users' profiles, so a change between
299+ * two runs is covered only by the events stream, which must still be read
300+ * from the first boundary. Written only after every organization has been
301+ * written, so a run that fails part-way records nothing. Never touches the
302+ * cursor: a stream already being followed keeps its position, and the
303+ * boundary is then unused.
304+ */
305+ readonly setReplayBoundary: (at: Date) => Effect.Effect<boolean, WorkOsMirrorError>;
306+ /**
307+ * When a backfill run first wrote EVERY live organization
308+ * (`workos_sync.backfill_completed_at`), or `null` while none has. The
309+ * first half of the mirror's READINESS for authorization: until a run has
310+ * covered every organization, the mirror may lack members who have not
311+ * signed in since it shipped, and a membership check read from it would
312+ * deny them. The other half is the events reconciler being caught up,
313+ * which its cursor row reports.
314+ */
315+ readonly backfillCompletedAt: () => Effect.Effect<Date | null, WorkOsMirrorError>;
316+ /**
317+ * Record that a backfill run has written every live organization, as of
318+ * `at` — ONCE: a later completed run keeps the first mark (`false`), so
319+ * readiness never flips back. Written only after the last organization
320+ * was applied (or refused in favour of a later listing), so a run that
321+ * fails part-way records nothing here. Mints the events row when absent,
322+ * as `setReplayBoundary` does, and touches neither the cursor nor the
323+ * boundary.
324+ */
325+ readonly markBackfillCompleted: (at: Date) => Effect.Effect<boolean, WorkOsMirrorError>;
326+ /**
327+ * When the organization's membership list was last FULLY scanned from
328+ * WorkOS (`backfillOrganization` in workos-mirror-backfill.ts), or `null`
329+ * if it never was — or the organization is not mirrored. Until it has been,
330+ * the mirror may hold only the members login and write-through happened to
331+ * record, so a member count read from it is PARTIAL: every seat gate reads
332+ * this first and scans the organization when it is `null`. Per
333+ * organization, never database-wide, so an organization mirrored after a
334+ * backfill ran (lazily, or by a sign-in) is never mistaken for a scanned
335+ * one.
336+ */
337+ readonly organizationBackfilledAt: (
338+ organizationId: string,
339+ ) => Effect.Effect<Date | null, WorkOsMirrorError>;
340+}
341+
342+// ---------------------------------------------------------------------------
343+// SDK payload → mirror row. The feeders (login callback, write-through, the
344+// backfill, the Events reconciler) all hand the mirror WorkOS objects; this is
345+// the one place their field names and ISO timestamps are translated. Typed
346+// structurally (the fields actually read) so the SDK's `User` /
347+// `OrganizationMembership`, an event payload, and a test fixture all fit.
348+// ---------------------------------------------------------------------------
349+
350+/** The WorkOS user fields the mirror reads. `User` from the SDK satisfies it. */
351+export interface WorkOsUserPayload {
352+ readonly id: string;
353+ readonly email: string;
354+ readonly firstName: string | null;
355+ readonly lastName: string | null;
356+ readonly profilePictureUrl: string | null;
357+ readonly lastSignInAt: string | null;
358+ readonly updatedAt: string;
359+}
360+
361+/**
362+ * The WorkOS membership fields the mirror reads. `OrganizationMembership` from
363+ * the SDK satisfies it; its `status` is exactly the mirror's `MemberStatus`.
364+ */
365+export interface WorkOsMembershipPayload {
366+ readonly id: string;
367+ readonly userId: string;
368+ readonly organizationId: string;
369+ readonly role: { readonly slug: string };
370+ readonly status: MemberStatus;
371+ readonly updatedAt: string;
372+}
373+
374+/** Translate a WorkOS user payload to the row `upsertUser` stores. */
375+export const mirrorUserFromWorkOs = (user: WorkOsUserPayload): WorkOsMirrorUser => ({
376+ id: user.id,
377+ email: user.email,
378+ firstName: user.firstName,
379+ lastName: user.lastName,
380+ avatarUrl: user.profilePictureUrl,
381+ lastSignInAt: user.lastSignInAt === null ? null : new Date(user.lastSignInAt),
382+ updatedAt: new Date(user.updatedAt),
383+});
384+
385+/** Translate a WorkOS membership payload to the row `upsertMembership` stores. */
386+export const mirrorMembershipFromWorkOs = (
387+ membership: WorkOsMembershipPayload,
388+): WorkOsMirrorMembership => ({
389+ id: membership.id,
390+ accountId: membership.userId,
391+ organizationId: membership.organizationId,
392+ role: membership.role.slug,
393+ status: membership.status,
394+ updatedAt: new Date(membership.updatedAt),
395+});
396+
397+/**
398+ * The `workos_sync` row of the one WorkOS events stream the reconciler
399+ * follows. A row id rather than a singleton table so a second stream
400+ * (another WorkOS environment, a replay) can be added without a schema
401+ * change; the same row carries the backfill's replay boundary and
402+ * completion mark. Not a `db/schema.ts` export: that module's exports are
403+ * enumerated as tables by the purge-coverage test.
404+ */
405+export const WORKOS_EVENTS_STREAM_ID = "events";
406+
407+// A raw `sql` fragment binds a Date without the column's driver mapping, so
408+// every instant below is passed as ISO text and cast.
409+const instant = (at: Date) => sql`${at.toISOString()}::timestamptz`;
410+
411+// An account tombstone moves the row's timestamp to the deletion time —
412+// never backwards, so a row that somehow carries a newer WorkOS timestamp
413+// keeps it and the upsert guard stays at least as strict.
414+const noEarlierThan = (column: AnyPgColumn, at: Date) => sql`greatest(${column}, ${instant(at)})`;
415+
416+// A membership tombstone keeps the row, marks it `inactive`, and records the
417+// deletion in `deleted_at`: the instant the caller holds, or `now()` when it
418+// holds none (see `deleteMembership`). `workos_updated_at` is left as it is
419+// — the tombstone is protected by the deleted id, not by its stamp.
420+const tombstone = (deletedAt: Date | null) => ({
421+ status: "inactive" as const,
422+ deletedAt: deletedAt === null ? sql`now()` : instant(deletedAt),
423+});
424+
425+// Which stored membership rows a payload for membership `id` stamped
426+// `updatedAt` may overwrite. A row WorkOS still holds (`deleted_at` null) is
427+// ordered by time: a row with no stamp (predating the mirror), any row
428+// stamped earlier, and a LIVE row stamped the same instant — feeders replay
429+// the same payload and must converge, not stall. An `inactive` row stamped
430+// the same instant is not overwritten: a deactivation at T beats an active
431+// payload at T. A deletion tombstone is ordered by IDENTITY: never
432+// overwritten by a payload naming the deleted id — a deleted WorkOS
433+// membership id never returns, however the payload is stamped — and taken
434+// over by a payload naming another id, a replacement membership WorkOS
435+// created after the deletion and so stamped after the row's last state,
436+// under the plain timestamp rule. A tombstone with no id at all (a
437+// pre-mirror row of a deleted user) is never taken over: the user is gone.
438+const membershipAcceptsPayload = (id: string, updatedAt: Date) =>
439+ or(
440+ and(
441+ isNull(memberships.deletedAt),
442+ or(
443+ isNull(memberships.workosUpdatedAt),
444+ lt(memberships.workosUpdatedAt, updatedAt),
445+ and(eq(memberships.workosUpdatedAt, updatedAt), ne(memberships.status, "inactive")),
446+ ),
447+ ),
448+ and(
449+ isNotNull(memberships.deletedAt),
450+ ne(memberships.membershipId, id),
451+ or(isNull(memberships.workosUpdatedAt), lt(memberships.workosUpdatedAt, updatedAt)),
452+ ),
453+ );
454+
455+// A deleted user's account row, as `deleteUser` leaves it: the profile is
456+// cleared (every WorkOS user payload carries an email, so a stamped row with
457+// none was written by `deleteUser`) and the stamp is the deletion time. A row
458+// minted bare by `ensureAccount` has no stamp either, and is not a tombstone.
459+// Judged in code, over a row read under a lock (`writeMembership`), not as a
460+// predicate in the read: a `SELECT ... FOR SHARE` locks only the rows it
461+// returns, so a read filtered to tombstones would lock nothing for a live
462+// row — the one case the lock exists for.
463+const isAccountTombstone = (account: {
464+ readonly email: string | null;
465+ readonly workosUpdatedAt: Date | null;
466+}): boolean => account.email === null && account.workosUpdatedAt !== null;
467+
468+// The same rule as for a membership row, for an account row. A tombstone
469+// takes no payload stamped at or before the deletion; a bare row takes any.
470+const accountAcceptsPayload = (updatedAt: Date) =>
471+ or(
472+ isNull(accounts.workosUpdatedAt),
473+ lt(accounts.workosUpdatedAt, updatedAt),
474+ and(eq(accounts.workosUpdatedAt, updatedAt), isNotNull(accounts.email)),
475+ );
476+
477+// A delete is applied only to a row not yet tombstoned: a replayed deletion
478+// changes nothing and reports so.
479+const notDeleted = isNull(memberships.deletedAt);
480+
481+// Whether membership `id` is in the deletion ledger: a WorkOS id a delete or
482+// a scan has named as gone, which never returns. Identity alone — the
483+// ledger records WHEN for the record, not for ordering.
484+const membershipIdDeleted = async (db: DrizzleDb, id: string): Promise<boolean> => {
485+ const rows = await db
486+ .select({ membershipId: membershipTombstones.membershipId })
487+ .from(membershipTombstones)
488+ .where(eq(membershipTombstones.membershipId, id));
489+ return rows.length > 0;
490+};
491+
492+// Which (account, organization) row a delete of membership `id` may
493+// tombstone: the row carrying THIS id — whatever its stamp, a deleted id is
494+// never reused — or a row with no id at all (written before the mirror
495+// recorded WorkOS ids; the delete fills the id in). Never a row under
496+// ANOTHER id: that is a different membership of the same account and
497+// organization, the member re-added in WorkOS after this one was removed,
498+// and it stands however the two are stamped. Identity orders them,
499+// timestamps do not. A row already tombstoned is left alone (`notDeleted`)
500+// so a replayed delete reports `false`.
501+const membershipDeletableBy = (id: string) =>
502+ and(or(isNull(memberships.membershipId), eq(memberships.membershipId, id)), notDeleted);
503+
504+// The write queries, over `db` or over a transaction handle (drizzle's is a
505+// `PgDatabase` too): the one `applyOrganizationScan` opens, or the one the
506+// store opens per `upsertMembership`. Each answers whether it wrote a row;
507+// the public shape translates that.
508+const makeWrites = (db: DrizzleDb) => {
509+ const ensureAccount = (id: string) =>
510+ db.insert(accounts).values({ id }).onConflictDoNothing({ target: accounts.id });
511+
512+ // The membership upsert under the row guard (`membershipAcceptsPayload`)
513+ // and the account tombstone, never the organization mark.
514+ const writeMembership = async (membership: WorkOsMirrorMembership): Promise<boolean> => {
515+ await ensureAccount(membership.accountId);
516+ // Never for a DELETED user. The row guard below orders a payload against
517+ // the membership row it would overwrite; a membership the mirror has not
518+ // seen yet has no row, so the guard cannot refuse the INSERT — and a
519+ // feeder that fetched the membership before the user was deleted and
520+ // writes it after (a stalled login list, the backfill's older listing)
521+ // would insert it live. The account tombstone is the one row a deleted
522+ // user always leaves behind, so it is consulted first, by identity:
523+ // WorkOS never reuses a user id, so no payload naming a tombstoned
524+ // account is ever current.
525+ //
526+ // Read FOR SHARE, held until the transaction `db` belongs to commits: a
527+ // `deleteUser` applying the deletion at this moment holds the row FOR NO
528+ // KEY UPDATE until ITS commit, so this read waits for it and sees the
529+ // tombstone — and a deletion that arrives after this read waits for this
530+ // transaction, then tombstones the membership it inserted. Unlocked, a
531+ // deletion could land between this read and the insert below and leave
532+ // a live membership for a deleted user.
533+ const locked = await db
534+ .select({ email: accounts.email, workosUpdatedAt: accounts.workosUpdatedAt })
535+ .from(accounts)
536+ .where(eq(accounts.id, membership.accountId))
537+ .for("share");
538+ const account = locked[0];
539+ // `ensureAccount` guarantees the row; accounts are tombstoned, never
540+ // deleted, so a missing one is refused like a tombstone, not written for.
541+ if (account === undefined || isAccountTombstone(account)) return false;
542+ // Never under a DELETED membership id. The row guard below can only
543+ // refuse against the id the row holds; a delete of THIS id that found
544+ // the row under another id (see the header) left only the ledger entry
545+ // behind, and that is what refuses the payload here.
546+ if (await membershipIdDeleted(db, membership.id)) return false;
547+ const written = await db
548+ .insert(memberships)
549+ .values({
550+ accountId: membership.accountId,
551+ organizationId: membership.organizationId,
552+ membershipId: membership.id,
553+ role: membership.role,
554+ status: membership.status,
555+ workosUpdatedAt: membership.updatedAt,
556+ })
557+ .onConflictDoUpdate({
558+ target: [memberships.accountId, memberships.organizationId],
559+ set: {
560+ membershipId: membership.id,
561+ role: membership.role,
562+ status: membership.status,
563+ workosUpdatedAt: membership.updatedAt,
564+ // A replacement taking over a tombstone is live again; a row that
565+ // was not tombstoned had nothing here.
566+ deletedAt: null,
567+ },
568+ setWhere: membershipAcceptsPayload(membership.id, membership.updatedAt),
569+ })
570+ .returning({ accountId: memberships.accountId });
571+ return written.length > 0;
572+ };
573+
574+ return {
575+ upsertUser: async (user: WorkOsMirrorUser): Promise<boolean> => {
576+ const written = await db
577+ .insert(accounts)
578+ .values({
579+ id: user.id,
580+ email: user.email,
581+ firstName: user.firstName,
582+ lastName: user.lastName,
583+ avatarUrl: user.avatarUrl,
584+ lastSignInAt: user.lastSignInAt,
585+ workosUpdatedAt: user.updatedAt,
586+ })
587+ .onConflictDoUpdate({
588+ target: accounts.id,
589+ set: {
590+ email: user.email,
591+ firstName: user.firstName,
592+ lastName: user.lastName,
593+ avatarUrl: user.avatarUrl,
594+ lastSignInAt: user.lastSignInAt,
595+ workosUpdatedAt: user.updatedAt,
596+ },
597+ setWhere: accountAcceptsPayload(user.updatedAt),
598+ })
599+ .returning({ id: accounts.id });
600+ return written.length > 0;
601+ },
602+
603+ upsertMembership: async (membership: WorkOsMirrorMembership): Promise<boolean> => {
604+ // Never into an organization the mirror holds as DELETED: its
605+ // memberships are gone from WorkOS and purged (or being purged) here,
606+ // so a payload that still names it was fetched before the deletion —
607+ // a login that stalled across the purge, a stale event — and writing
608+ // it would grant access to a deleted organization. The tombstone row
609+ // outlives the purge for exactly this check. And never from a payload
610+ // stamped before the organization's last full scan: the scan is the
611+ // complete listing as of `backfilled_at`, so an older payload either
612+ // repeats a row the scan wrote (nothing to change) or names a
613+ // membership the scan found gone — revoked before the mirror's events
614+ // replay begins, so no event would ever tombstone it again. The row
615+ // is read FOR SHARE, so a scan claiming it at this moment
616+ // (`claimOrganizationScan`, an UPDATE that holds the row until its
617+ // transaction commits) and the purge marking it deleted are waited
618+ // for, and the mark read is the committed one: a scan cannot slip
619+ // between this read and the write below and leave a membership it
620+ // proved revoked inserted after it. The lock is held until the
621+ // transaction `db` belongs to ends — `makeWorkOsMirrorStore` opens one
622+ // per call, `applyPage` runs this on the page's own — which is what
623+ // orders the write after the scan. An organization the mirror does
624+ // not hold at all is still a foreign-key failure in the write.
625+ const organization = await db
626+ .select({
627+ deletedAt: organizations.deletedAt,
628+ backfilledAt: organizations.backfilledAt,
629+ })
630+ .from(organizations)
631+ .where(eq(organizations.id, membership.organizationId))
632+ .for("share");
633+ const row = organization[0];
634+ if (row?.deletedAt != null) return false;
635+ if (row?.backfilledAt != null && membership.updatedAt < row.backfilledAt) return false;
636+ return writeMembership(membership);
637+ },
638+
639+ // A scan's own membership writes: the scan has just claimed the
640+ // organization as of its listing, so its payloads are exactly what the
641+ // mark stands for and are not ordered against it.
642+ writeScannedMembership: writeMembership,
643+
644+ // An upsert, like the membership write it guards against: a delete the
645+ // mirror sees before the membership itself (the reconciler ahead of the
646+ // backfill) must leave the tombstone behind, or the later, older payload
647+ // would insert the row live.
648+ deleteMembership: async (
649+ membership: WorkOsMirrorMembershipRef,
650+ deletedAt: Date | null,
651+ ): Promise<boolean> => {
652+ await ensureAccount(membership.accountId);
653+ // Lock the account row FOR NO KEY UPDATE, as `deleteUser` does: a
654+ // membership write of this user reads the row FOR SHARE before it
655+ // consults the ledger (`writeMembership`), so a write racing this
656+ // delete either waits here and then finds the ledger entry, or
657+ // committed first and is tombstoned by the row upsert below. Without
658+ // the lock a write could pass the ledger check before this entry
659+ // lands and insert the deleted membership live after it.
660+ await db
661+ .select({ id: accounts.id })
662+ .from(accounts)
663+ .where(eq(accounts.id, membership.accountId))
664+ .for("no key update");
665+ // The ledger entry FIRST, whatever the row holds: the one record of
666+ // the deletion that does not depend on the row carrying the deleted
667+ // id. A replayed delete finds it there and records nothing new.
668+ const recorded = await db
669+ .insert(membershipTombstones)
670+ .values({
671+ membershipId: membership.id,
672+ accountId: membership.accountId,
673+ organizationId: membership.organizationId,
674+ ...(deletedAt === null ? {} : { deletedAt }),
675+ })
676+ .onConflictDoNothing({ target: membershipTombstones.membershipId })
677+ .returning({ membershipId: membershipTombstones.membershipId });
678+ const tombstoned = await db
679+ .insert(memberships)
680+ .values({
681+ accountId: membership.accountId,
682+ organizationId: membership.organizationId,
683+ membershipId: membership.id,
684+ ...tombstone(deletedAt),
685+ })
686+ .onConflictDoUpdate({
687+ target: [memberships.accountId, memberships.organizationId],
688+ set: { membershipId: membership.id, ...tombstone(deletedAt) },
689+ setWhere: membershipDeletableBy(membership.id),
690+ })
691+ .returning({ accountId: memberships.accountId });
692+ return recorded.length > 0 || tombstoned.length > 0;
693+ },
694+
695+ // Only inside a transaction: the account row lock taken first is what
696+ // orders this against a membership write of the same user, and it lasts
697+ // exactly as long as the transaction `db` belongs to — the one the store
698+ // opens per call.
699+ deleteUser: async (accountId: string, deletedAt: Date): Promise<boolean> => {
700+ // Lock the account row FIRST — minted bare when absent, so there is a
701+ // row to lock (the tombstone is minted for the same reason as the
702+ // membership one: a later, older payload must find it) — and hold it
703+ // FOR NO KEY UPDATE until commit. A membership write reads the row FOR
704+ // SHARE before it inserts (`writeMembership`), so a write racing this
705+ // delete either waits here and then sees the tombstone, or committed
706+ // before this lock was granted and is caught by the membership
707+ // tombstoning below. Without the lock a write could read the row live
708+ // and insert its membership after the tombstoning had run.
709+ await ensureAccount(accountId);
710+ await db
711+ .select({ id: accounts.id })
712+ .from(accounts)
713+ .where(eq(accounts.id, accountId))
714+ .for("no key update");
715+ // The account tombstone: profile cleared, stamped with the deletion.
716+ // Applied unless the row already carries this tombstone or a later one
717+ // (a replayed delete).
718+ const cleared = await db
719+ .update(accounts)
720+ .set({
721+ email: null,
722+ firstName: null,
723+ lastName: null,
724+ avatarUrl: null,
725+ workosUpdatedAt: noEarlierThan(accounts.workosUpdatedAt, deletedAt),
726+ })
727+ .where(
728+ and(
729+ eq(accounts.id, accountId),
730+ or(
731+ isNotNull(accounts.email),
732+ isNull(accounts.workosUpdatedAt),
733+ lt(accounts.workosUpdatedAt, deletedAt),
734+ ),
735+ ),
736+ )
737+ .returning({ id: accounts.id });
738+ await db
739+ .update(memberships)
740+ .set(tombstone(deletedAt))
741+ .where(and(eq(memberships.accountId, accountId), notDeleted));
742+ return cleared.length > 0;
743+ },
744+
745+ // Tombstone (at `listedAt`) every membership of the organization that a
746+ // listing taken at `listedAt` did not contain. Only inside a scan's
747+ // transaction, after its `backfilled_at` CAS: on its own this could
748+ // tombstone what a LATER scan just wrote.
749+ tombstoneMembershipsExcept: async (
750+ organizationId: string,
751+ membershipIds: readonly string[],
752+ listedAt: Date,
753+ ): Promise<number> => {
754+ const tombstoned = await db
755+ .update(memberships)
756+ .set(tombstone(listedAt))
757+ .where(
758+ and(
759+ eq(memberships.organizationId, organizationId),
760+ // A row with no WorkOS id predates the mirror and is not in
761+ // any listing; if WorkOS still holds the membership, the
762+ // scan's upsert has just filled the id in.
763+ or(
764+ isNull(memberships.membershipId),
765+ notInArray(memberships.membershipId, [...membershipIds]),
766+ ),
767+ // Only rows the listing could have contained: stamped before
768+ // it was taken (or never stamped). Anything newer was written
769+ // after the listing and is not missing from it.
770+ or(isNull(memberships.workosUpdatedAt), lt(memberships.workosUpdatedAt, listedAt)),
771+ notDeleted,
772+ ),
773+ )
774+ .returning({
775+ accountId: memberships.accountId,
776+ membershipId: memberships.membershipId,
777+ });
778+ // The ids the listing proved gone go into the ledger too, as any other
779+ // delete's: they never return. A row with no id has none to record.
780+ const gone = tombstoned.flatMap((row) =>
781+ row.membershipId === null
782+ ? []
783+ : [
784+ {
785+ membershipId: row.membershipId,
786+ accountId: row.accountId,
787+ organizationId,
788+ deletedAt: listedAt,
789+ },
790+ ],
791+ );
792+ if (gone.length > 0) {
793+ await db
794+ .insert(membershipTombstones)
795+ .values(gone)
796+ .onConflictDoNothing({ target: membershipTombstones.membershipId });
797+ }
798+ return tombstoned.length;
799+ },
800+ };
801+};
802+
803+// Claim the organization for a scan listed at `listedAt`: move its
804+// `backfilled_at` forward to `listedAt` if the recorded mark is older (or
805+// missing). Run inside a transaction this also LOCKS the organization row
806+// until commit, so an overlapping scan's claim waits here, then reads the
807+// moved mark and matches nothing. An organization marked deleted is never
808+// claimed: its memberships are being (or have been) purged, and a scan
809+// that listed them before the deletion must not write them back.
810+const claimOrganizationScan = async (db: DrizzleDb, organizationId: string, listedAt: Date) => {
811+ const claimed = await db
812+ .update(organizations)
813+ .set({ backfilledAt: listedAt })
814+ .where(
815+ and(
816+ eq(organizations.id, organizationId),
817+ isNull(organizations.deletedAt),
818+ or(isNull(organizations.backfilledAt), lt(organizations.backfilledAt, listedAt)),
819+ ),
820+ )
821+ .returning({ id: organizations.id });
822+ return claimed.length > 0;
823+};
824+
825+/**
826+ * The mirror's write operations over `db`. Failures are `WorkOsMirrorError`
827+ * naming the operation and the classified driver reason; the full cause is
828+ * logged at the boundary.
829+ */
830+export const makeWorkOsMirrorStore = (db: DrizzleDb): WorkOsMirrorShape => {
831+ const run = <A>(op: string, fn: () => Promise<A>) =>
832+ withServiceLogging(
833+ `workos_mirror.${op}`,
834+ (failure) =>
835+ new WorkOsMirrorError({
836+ operation: op,
837+ reason: userStoreReasonFromCause(failure),
838+ }),
839+ tryPromiseService(fn),
840+ );
841+
842+ const writes = makeWrites(db);
843+
844+ return {
845+ upsertUser: (user) => run("upsertUser", () => writes.upsertUser(user)),
846+
847+ // One transaction per call: the organization guard inside locks the org
848+ // row until the write has landed (see `makeWrites`).
849+ upsertMembership: (membership) =>
850+ run("upsertMembership", () =>
851+ db.transaction((tx) => makeWrites(tx).upsertMembership(membership)),
852+ ),
853+
854+ // One transaction per call: the ledger entry and the row tombstone land
855+ // together or not at all.
856+ deleteMembership: (membership, deletedAt) =>
857+ run("deleteMembership", () =>
858+ db.transaction((tx) => makeWrites(tx).deleteMembership(membership, deletedAt)),
859+ ),
860+
861+ // One transaction per call: the account row lock inside is held until
862+ // the memberships are tombstoned (see `makeWrites`).
863+ deleteUser: (accountId, deletedAt) =>
864+ run("deleteUser", () =>
865+ db.transaction((tx) => makeWrites(tx).deleteUser(accountId, deletedAt)),
866+ ),
867+
868+ getCursor: () =>
869+ run("getCursor", async () => {
870+ const rows = await db
871+ .select({ cursor: workosSync.cursor })
872+ .from(workosSync)
873+ .where(eq(workosSync.id, WORKOS_EVENTS_STREAM_ID));
874+ // No row yet is the same state as a row with no cursor: nothing applied.
875+ return rows[0]?.cursor ?? null;
876+ }),
877+
878+ setCursor: (prev, next) =>
879+ run("setCursor", async () => {
880+ const now = new Date();
881+ if (prev === null) {
882+ // First advance: mint the row, or claim an existing row that still
883+ // has no cursor. A row that already carries one belongs to another
884+ // run and is left alone.
885+ const written = await db
886+ .insert(workosSync)
887+ .values({ id: WORKOS_EVENTS_STREAM_ID, cursor: next, updatedAt: now })
888+ .onConflictDoUpdate({
889+ target: workosSync.id,
890+ set: { cursor: next, updatedAt: now },
891+ setWhere: isNull(workosSync.cursor),
892+ })
893+ .returning({ id: workosSync.id });
894+ return written.length > 0;
895+ }
896+ const written = await db
897+ .update(workosSync)
898+ .set({ cursor: next, updatedAt: now })
899+ .where(and(eq(workosSync.id, WORKOS_EVENTS_STREAM_ID), eq(workosSync.cursor, prev)))
900+ .returning({ id: workosSync.id });
901+ return written.length > 0;
902+ }),
903+
904+ applyOrganizationScan: (scan) =>
905+ run("applyOrganizationScan", () =>
906+ db.transaction(async (tx) => {
907+ // The claim comes FIRST so the lock is held for every write below;
908+ // a scan that lost to a later listing commits an empty transaction.
909+ const claimed = await claimOrganizationScan(tx, scan.organizationId, scan.listedAt);
910+ if (!claimed) return Option.none();
911+ const txWrites = makeWrites(tx);
912+ let usersWritten = 0;
913+ let membershipsWritten = 0;
914+ for (const member of scan.members) {
915+ if (await txWrites.upsertUser(member.user)) usersWritten += 1;
916+ if (await txWrites.writeScannedMembership(member.membership)) membershipsWritten += 1;
917+ }
918+ const membershipsTombstoned = await txWrites.tombstoneMembershipsExcept(
919+ scan.organizationId,
920+ scan.members.map((member) => member.membership.id),
921+ scan.listedAt,
922+ );
923+ const written: WorkOsOrganizationScanWrites = {
924+ usersWritten,
925+ membershipsWritten,
926+ membershipsTombstoned,
927+ };
928+ return Option.some(written);
929+ }),
930+ ),
931+
932+ replayBoundary: () =>
933+ run("replayBoundary", async () => {
934+ const rows = await db
935+ .select({ rangeStart: workosSync.rangeStart })
936+ .from(workosSync)
937+ .where(eq(workosSync.id, WORKOS_EVENTS_STREAM_ID));
938+ return rows[0]?.rangeStart ?? null;
939+ }),
940+
941+ setReplayBoundary: (at) =>
942+ run("setReplayBoundary", async () => {
943+ const recorded = await db
944+ .insert(workosSync)
945+ .values({
946+ id: WORKOS_EVENTS_STREAM_ID,
947+ cursor: null,
948+ rangeStart: at,
949+ updatedAt: at,
950+ })
951+ .onConflictDoUpdate({
952+ target: workosSync.id,
953+ set: { rangeStart: at },
954+ // A boundary already recorded stands, whatever this run's is.
955+ setWhere: isNull(workosSync.rangeStart),
956+ })
957+ .returning({ id: workosSync.id });
958+ return recorded.length > 0;
959+ }),
960+
961+ backfillCompletedAt: () =>
962+ run("backfillCompletedAt", async () => {
963+ const rows = await db
964+ .select({ backfillCompletedAt: workosSync.backfillCompletedAt })
965+ .from(workosSync)
966+ .where(eq(workosSync.id, WORKOS_EVENTS_STREAM_ID));
967+ return rows[0]?.backfillCompletedAt ?? null;
968+ }),
969+
970+ markBackfillCompleted: (at) =>
971+ run("markBackfillCompleted", async () => {
972+ const recorded = await db
973+ .insert(workosSync)
974+ .values({
975+ id: WORKOS_EVENTS_STREAM_ID,
976+ cursor: null,
977+ backfillCompletedAt: at,
978+ updatedAt: at,
979+ })
980+ .onConflictDoUpdate({
981+ target: workosSync.id,
982+ set: { backfillCompletedAt: at },
983+ // The first completion stands, whatever this run's is.
984+ setWhere: isNull(workosSync.backfillCompletedAt),
985+ })
986+ .returning({ id: workosSync.id });
987+ return recorded.length > 0;
988+ }),
989+
990+ organizationBackfilledAt: (organizationId) =>
991+ run("organizationBackfilledAt", async () => {
992+ const rows = await db
993+ .select({ backfilledAt: organizations.backfilledAt })
994+ .from(organizations)
995+ .where(eq(organizations.id, organizationId));
996+ return rows[0]?.backfilledAt ?? null;
997+ }),
998+ };
999+};
apps/cloud/src/auth/workos-mirror.node.test.ts+543−19
@@ -6,10 +6,13 @@ · 5 unmodified lines
66 //
77 // What this pins:
88 // - an older WorkOS payload never overwrites a newer row (replay-safe)
9−// - a delete tombstones the row (inactive, stamped with the deletion time):
10−// a stale OR equal-timestamp upsert cannot resurrect it, a newer one
11−// reactivates it, and every default read treats the tombstone as no
12−// membership
9+// - a delete tombstones the row (inactive, `deleted_at` set) by IDENTITY:
10+// no payload naming the deleted membership id ever reactivates it,
11+// however it is stamped — not a stale one, not one stamped after the
12+// removal — while a replacement under a NEW id takes the row over live;
13+// every default read treats the tombstone as no membership
14+// - a membership WorkOS merely deactivated (inactive, no `deleted_at`)
15+// reactivates under the same id like any other update
1316 // - a delete of a membership or user the mirror has not seen yet leaves
1417 // the tombstone behind, so the backfill's older payload cannot insert
1518 // the row live afterwards
@@ -17,25 +20,40 @@ · 1 unmodified line
1720 // still carries the id it was replaced from) is recorded all the same,
1821 // so a later, newer payload of the deleted id cannot take the row over
1922 // - a deleted user takes no membership at all, however the payload is
20−// stamped: the account tombstone refuses the insert by identity
23+// stamped: the account tombstone refuses the insert by identity — and a
24+// membership write racing the deletion waits for its commit and sees
25+// the tombstone, never inserting a live membership for a deleted user
2126 // - a delete with no WorkOS instant keeps the row's own WorkOS stamp, so
2227 // a replacement membership WorkOS created meanwhile is not refused,
2328 // while the removed membership's own payload still is
2429 // - the cursor advances only by compare-and-set (one owner per stream)
30+// - a backfill scan is applied only if its listing is newer than the one
31+// already applied to the organization (one owner per listing instant),
32+// so an older listing cannot insert a membership the newer one lacked;
33+// a deleted organization takes no scan
34+// - a membership payload stamped before the organization's last scan is
35+// refused (a login list fetched before a revocation the scan already
36+// applied cannot reinstate it); one stamped at or after it is written —
37+// and a write racing a scan waits for the scan's commit and sees its
38+// mark, so it cannot insert a membership the scan just proved revoked
39+// - the events replay boundary is recorded once and never advanced
2540 // - `members` searches email AND name case-insensitively, pages stably
2641 // - `findByEmail` ignores the casing WorkOS stored
2742 // - a membership arriving before its user still holds (FK via ensureAccount)
2843 // ---------------------------------------------------------------------------
2944
3045 import { describe, expect, it } from "@effect/vitest";
31−import { Effect, Layer } from "effect";
46+import { eq, sql } from "drizzle-orm";
47+import { Context, Deferred, Effect, Fiber, Layer, Option } from "effect";
3248
3349 import { MemberDirectory } from "@executor-js/api/server";
3450
35−import { DbService } from "../db/db";
51+import { DbService, makeDbLayer } from "../db/db";
52+import { accounts, organizations } from "../db/schema";
3653 import { cloudMemberDirectoryLayer } from "./member-directory";
3754 import { UserStoreService } from "./context";
3855 import { WorkOsMirror, type WorkOsMirrorMembership, type WorkOsMirrorUser } from "./workos-mirror";
56+import { makeWorkOsMirrorStore } from "./workos-mirror-store";
3957
4058 const DbLive = DbService.Live;
4159 const Services = Layer.mergeAll(
@@ -44,8 +62,14 @@ const Services = Layer.mergeAll( · 2 unmodified lines
4462 UserStoreService.Live,
4563 ).pipe(Layer.provideMerge(DbLive));
4664
47−const run = <A, E>(body: Effect.Effect<A, E, WorkOsMirror | MemberDirectory | UserStoreService>) =>
48− Effect.runPromise(body.pipe(Effect.provide(Services), Effect.scoped));
65+const run = <A, E>(
66+ body: Effect.Effect<A, E, WorkOsMirror | MemberDirectory | UserStoreService | DbService>,
67+) => Effect.runPromise(body.pipe(Effect.provide(Services), Effect.scoped));
68+
69+/** The events row is instance-wide: drop it so a test starts as a never-backfilled database. */
70+const clearEventsRow = Effect.flatMap(DbService.asEffect(), ({ db }) =>
71+ Effect.promise(() => db.execute(sql`delete from workos_sync where id = 'events'`)),
72+);
4973
5074 const at = (iso: string) => new Date(iso);
5175 const T1 = at("2026-01-01T00:00:00.000Z");
@@ -59,7 +83,9 @@ const freshOrg = () => · 7 unmodified lines
5983 Effect.gen(function* () {
6084 const id = `org_${crypto.randomUUID().replaceAll("-", "")}`;
6185 const store = yield* UserStoreService;
62− yield* store.use("upsertOrganization", (s) => s.upsertOrganization({ id, name: "Mirror Org" }));
86+ yield* store.use("upsertOrganization", (s) =>
87+ s.upsertOrganization({ id, name: "Mirror Org", updatedAt: T1 }),
88+ );
6389 return id;
6490 });
6591
@@ -168,7 +194,7 @@ describe("WorkOsMirror upserts", () => { · 102 unmodified lines
168194 expect(result.filled?.name).toBe("Late");
169195 });
170196
171− it("tombstones a deleted membership so a stale upsert cannot resurrect it, and a newer one can", async () => {
197+ it("tombstones a deleted membership by identity: no payload of that id resurrects it, a replacement under a new id does", async () => {
172198 const result = await run(
173199 Effect.gen(function* () {
174200 const mirror = yield* WorkOsMirror;
@@ -198,6 +224,14 @@ describe("WorkOsMirror upserts", () => { · 23 unmodified lines
198224 membership(org, id, { id: membershipId, updatedAt: T2 }),
199225 );
200226 const afterEqual = yield* directory.membership(id, org, ["inactive"]);
227+ // A role change issued before the removal and delivered after it (a
228+ // stalled login list, a lagging feeder), stamped NEWER than anything
229+ // the row holds — the payload a timestamp guard would let through.
230+ // Same id: the membership is deleted, it never returns.
231+ const newerSameId = yield* mirror.upsertMembership(
232+ membership(org, id, { id: membershipId, role: "admin", updatedAt: T3 }),
233+ );
234+ const afterNewerSameId = yield* directory.membership(id, org, ["inactive"]);
201235 // The member re-added in WorkOS: a payload newer than the deletion,
202236 // under a new membership id (a deleted id is never reused).
203237 const readded = yield* mirror.upsertMembership(
@@ -224,6 +258,8 @@ describe("WorkOsMirror upserts", () => { · 20 unmodified lines
224258 afterStale,
225259 equal,
226260 afterEqual,
261+ newerSameId,
262+ afterNewerSameId,
227263 readded,
228264 afterReadd,
229265 lateDelete,
@@ -245,7 +281,12 @@ describe("WorkOsMirror upserts", () => { · 15 unmodified lines
245281 expect(result.afterStale?.status).toBe("inactive");
246282 expect(result.equal, "an upsert stamped AT the deletion is refused too").toBe(false);
247283 expect(result.afterEqual?.status).toBe("inactive");
248− expect(result.readded, "an upsert newer than the deletion reactivates").toBe(true);
284+ expect(
285+ result.newerSameId,
286+ "a payload of the deleted id stamped AFTER the removal is refused: identity, not time",
287+ ).toBe(false);
288+ expect(result.afterNewerSameId).toMatchObject({ status: "inactive", role: "member" });
289+ expect(result.readded, "a replacement under a new id reactivates").toBe(true);
249290 expect(result.afterReadd?.status).toBe("active");
250291 expect(result.lateDelete, "a replayed deletion of the OLD id is refused").toBe(false);
251292 expect(result.afterLateDelete?.status).toBe("active");
@@ -447,6 +488,59 @@ describe("WorkOsMirror upserts", () => { · 195 unmodified lines
447488 expect(result.afterReadd?.status).toBe("active");
448489 });
449490
491+ it("reactivates a membership WorkOS deactivated under the same id: a deactivation is not a deletion", async () => {
492+ const result = await run(
493+ Effect.gen(function* () {
494+ const mirror = yield* WorkOsMirror;
495+ const directory = yield* MemberDirectory;
496+ const org = yield* freshOrg();
497+ const id = `user_${crypto.randomUUID()}`;
498+ const membershipId = `om_${id}_${org}`;
499+ yield* mirror.upsertUser(user(id));
500+ yield* mirror.upsertMembership(membership(org, id, { id: membershipId, updatedAt: T1 }));
501+ // WorkOS deactivates the membership (an `organization_membership.updated`
502+ // with status inactive): the row is inactive but still WorkOS's, with
503+ // no `deleted_at` to protect it.
504+ const deactivated = yield* mirror.upsertMembership(
505+ membership(org, id, { id: membershipId, status: "inactive", updatedAt: T2 }),
506+ );
507+ const whileInactive = yield* directory.membership(id, org);
508+ // A payload older than the deactivation cannot undo it, nor one
509+ // stamped at the same instant.
510+ const older = yield* mirror.upsertMembership(
511+ membership(org, id, { id: membershipId, updatedAt: T1 }),
512+ );
513+ const equal = yield* mirror.upsertMembership(
514+ membership(org, id, { id: membershipId, updatedAt: T2 }),
515+ );
516+ // WorkOS reactivates it, same id, newer stamp: live again.
517+ const reactivated = yield* mirror.upsertMembership(
518+ membership(org, id, { id: membershipId, role: "admin", updatedAt: T3 }),
519+ );
520+ const afterReactivate = yield* directory.membership(id, org);
521+ return {
522+ membershipId,
523+ deactivated,
524+ whileInactive,
525+ older,
526+ equal,
527+ reactivated,
528+ afterReactivate,
529+ };
530+ }),
531+ );
532+ expect(result.deactivated).toBe(true);
533+ expect(result.whileInactive, "an inactive membership reads as no membership").toBeNull();
534+ expect(result.older, "an older payload cannot undo the deactivation").toBe(false);
535+ expect(result.equal, "nor one stamped at the deactivation").toBe(false);
536+ expect(result.reactivated, "a newer payload under the same id reactivates it").toBe(true);
537+ expect(result.afterReactivate).toMatchObject({
538+ status: "active",
539+ role: "admin",
540+ membershipId: result.membershipId,
541+ });
542+ });
543+
450544 it("tombstones every membership of a deleted user and clears the profile, keeping the account row", async () => {
451545 const result = await run(
452546 Effect.gen(function* () {
@@ -465,11 +559,13 @@ describe("WorkOsMirror upserts", () => { · 12 unmodified lines
465559 const inB = yield* directory.membership(id, orgB, ["inactive"]);
466560 // A stale user payload cannot restore the profile, nor can one
467561 // stamped at the deletion itself. No membership of the deleted user
468− // is written again — not one stamped after the deletion under a new
469− // id, the payload a timestamp guard would let through: the user is
470− // gone, and WorkOS never reuses the id.
562+ // is written again — not the deleted id (a deleted membership id
563+ // never returns), and not one stamped after the deletion under a
564+ // NEW id, the payload a timestamp guard would let through: the user
565+ // is gone, and WorkOS never reuses the id.
471566 const staleUser = yield* mirror.upsertUser(user(id, { firstName: "Back", updatedAt: T1 }));
472567 const equalUser = yield* mirror.upsertUser(user(id, { firstName: "Same", updatedAt: T2 }));
568+ const sameId = yield* mirror.upsertMembership(membership(orgA, id, { updatedAt: T3 }));
473569 const rejoined = yield* mirror.upsertMembership(
474570 membership(orgA, id, { id: `om_${id}_${orgA}_2`, updatedAt: T3 }),
475571 );
@@ -499,6 +595,7 @@ describe("WorkOsMirror upserts", () => { · 23 unmodified lines
499595 inB,
500596 staleUser,
501597 equalUser,
598+ sameId,
502599 rejoined,
503600 afterRejoin,
504601 unknown,
@@ -518,14 +615,12 @@ describe("WorkOsMirror upserts", () => { · 13 unmodified lines
518615 });
519616 expect(result.staleUser).toBe(false);
520617 expect(result.equalUser, "a profile stamped AT the deletion is refused").toBe(false);
618+ expect(result.sameId, "a deleted membership id never returns").toBe(false);
521619 expect(
522620 result.rejoined,
523621 "a membership of a deleted user is refused however it is stamped: identity, not time",
524622 ).toBe(false);
525− expect(result.afterRejoin).toMatchObject({
526− status: "inactive",
527− name: null,
528− });
623+ expect(result.afterRejoin).toMatchObject({ status: "inactive", name: null });
529624 expect(result.unknown, "deleting an unseen user leaves a tombstone").toBe(true);
530625 expect(result.unseenProfile, "which the older profile cannot fill").toBe(false);
531626 expect(
@@ -534,6 +629,150 @@ describe("WorkOsMirror upserts", () => { · 2 unmodified lines
534629 ).toBe(false);
535630 expect(result.unseenRow).toBeNull();
536631 });
632+
633+ it("waits for a user deletion holding the account row before judging a membership write against its tombstone", async () => {
634+ // A feeder (a stalled login list, the backfill's older listing) writes a
635+ // membership of a user whose `user.deleted` the reconciler is applying
636+ // at this moment. The deletion locks the account row for the length of
637+ // its transaction; the feeder's write must wait for that commit, see the
638+ // tombstone, and refuse the payload — never insert a live membership
639+ // for a deleted user. The deletion runs on the test's shared connection;
640+ // the feeder runs the same store over a second one, so the two
641+ // transactions are real peers on the server.
642+ const result = await run(
643+ Effect.scoped(
644+ Effect.gen(function* () {
645+ const org = yield* freshOrg();
646+ const gone = `user_${crypto.randomUUID()}`;
647+ const mirror = yield* WorkOsMirror;
648+ const { db: deleterDb } = yield* DbService;
649+ // The second connection is owned by this test body's scope.
650+ const feederDb = Context.get(yield* Layer.build(makeDbLayer()), DbService).db;
651+ const feeder = makeWorkOsMirrorStore(feederDb);
652+ const directory = yield* MemberDirectory;
653+ yield* mirror.upsertUser(user(gone, { firstName: "Gone" }));
654+
655+ // The deletion's transaction: the account row is locked as
656+ // `deleteUser` locks it, and the transaction is held open until
657+ // `release` — the window a feeder can race. The tombstone itself
658+ // is written after the feeder has started waiting.
659+ const locked = yield* Deferred.make<void>();
660+ let release: () => void = () => undefined;
661+ const held = new Promise<void>((resolve) => {
662+ release = resolve;
663+ });
664+ const deletion = yield* Effect.forkChild(
665+ Effect.promise(() =>
666+ deleterDb.transaction(async (tx) => {
667+ await tx
668+ .select({ id: accounts.id })
669+ .from(accounts)
670+ .where(eq(accounts.id, gone))
671+ .for("no key update");
672+ await Effect.runPromise(Deferred.succeed(locked, undefined));
673+ await held;
674+ await makeWorkOsMirrorStore(tx).deleteUser(gone, T2).pipe(Effect.runPromise);
675+ }),
676+ ),
677+ { startImmediately: true },
678+ );
679+ yield* Deferred.await(locked);
680+
681+ // The feeder's write starts while the deletion holds the row.
682+ const written = yield* Deferred.make<boolean>();
683+ yield* Effect.forkChild(
684+ feeder
685+ .upsertMembership(membership(org, gone, { updatedAt: T3 }))
686+ .pipe(Effect.flatMap((wrote) => Deferred.succeed(written, wrote))),
687+ { startImmediately: true },
688+ );
689+ const beforeCommit = yield* Effect.timeoutOption(Deferred.await(written), "250 millis");
690+ release();
691+ yield* Fiber.join(deletion);
692+ const afterCommit = yield* Deferred.await(written);
693+ const row = yield* directory.membership(gone, org, ["active", "pending", "inactive"]);
694+ return { beforeCommit, afterCommit, row };
695+ }),
696+ ),
697+ );
698+ expect(
699+ Option.isNone(result.beforeCommit),
700+ "the write waits while the deletion holds the account row",
701+ ).toBe(true);
702+ expect(
703+ result.afterCommit,
704+ "once the deletion has committed, its tombstone refuses the payload",
705+ ).toBe(false);
706+ expect(result.row, "so no membership was inserted for the deleted user").toBeNull();
707+ });
708+
709+ it("waits for a membership deletion holding the account row before judging a write of that id against the ledger", async () => {
710+ // Same race for a single membership: a feeder writes membership `om`
711+ // while the reconciler is applying its `organization_membership.deleted`.
712+ // The delete locks the account row for its transaction, so the feeder's
713+ // FOR SHARE read waits, then finds the ledger entry and refuses. Without
714+ // the lock the feeder could pass the ledger check first and insert the
715+ // deleted membership live after the delete committed.
716+ const result = await run(
717+ Effect.scoped(
718+ Effect.gen(function* () {
719+ const org = yield* freshOrg();
720+ const id = `user_${crypto.randomUUID()}`;
721+ const mirror = yield* WorkOsMirror;
722+ const { db: deleterDb } = yield* DbService;
723+ const feederDb = Context.get(yield* Layer.build(makeDbLayer()), DbService).db;
724+ const feeder = makeWorkOsMirrorStore(feederDb);
725+ const directory = yield* MemberDirectory;
726+ yield* mirror.upsertUser(user(id));
727+ const gone = membership(org, id, { updatedAt: T3 });
728+
729+ const locked = yield* Deferred.make<void>();
730+ let release: () => void = () => undefined;
731+ const held = new Promise<void>((resolve) => {
732+ release = resolve;
733+ });
734+ const deletion = yield* Effect.forkChild(
735+ Effect.promise(() =>
736+ deleterDb.transaction(async (tx) => {
737+ await tx
738+ .select({ id: accounts.id })
739+ .from(accounts)
740+ .where(eq(accounts.id, id))
741+ .for("no key update");
742+ await Effect.runPromise(Deferred.succeed(locked, undefined));
743+ await held;
744+ await makeWorkOsMirrorStore(tx).deleteMembership(gone, T2).pipe(Effect.runPromise);
745+ }),
746+ ),
747+ { startImmediately: true },
748+ );
749+ yield* Deferred.await(locked);
750+
751+ const written = yield* Deferred.make<boolean>();
752+ yield* Effect.forkChild(
753+ feeder
754+ .upsertMembership(gone)
755+ .pipe(Effect.flatMap((wrote) => Deferred.succeed(written, wrote))),
756+ { startImmediately: true },
757+ );
758+ const beforeCommit = yield* Effect.timeoutOption(Deferred.await(written), "250 millis");
759+ release();
760+ yield* Fiber.join(deletion);
761+ const afterCommit = yield* Deferred.await(written);
762+ const row = yield* directory.membership(id, org, ["active", "pending", "inactive"]);
763+ return { beforeCommit, afterCommit, status: row?.status ?? null };
764+ }),
765+ ),
766+ );
767+ expect(
768+ Option.isNone(result.beforeCommit),
769+ "the write waits while the deletion holds the account row",
770+ ).toBe(true);
771+ expect(result.afterCommit, "once the deletion has committed, the ledger refuses the id").toBe(
772+ false,
773+ );
774+ expect(result.status, "and the row is the tombstone, never live").toBe("inactive");
775+ });
537776 });
538777
539778 describe("WorkOsMirror cursor", () => {
@@ -560,6 +799,291 @@ describe("WorkOsMirror cursor", () => { · 20 unmodified lines
560799 });
561800 });
562801
802+describe("WorkOsMirror backfill sync state", () => {
803+ it("records the replay boundary and the backfill completion once each, without touching the cursor", async () => {
804+ const result = await run(
805+ Effect.gen(function* () {
806+ const mirror = yield* WorkOsMirror;
807+ // The events row is instance-wide (migration 0019 seeds it on the
808+ // empty test database, other tests may have written it): start from
809+ // no row, as a database that has never been backfilled has.
810+ yield* clearEventsRow;
811+ const first = yield* mirror.setReplayBoundary(T2);
812+ const boundary = yield* mirror.replayBoundary();
813+ const cursorAfterBoundary = yield* mirror.getCursor();
814+ // A later completed backfill: its boundary is not recorded.
815+ const again = yield* mirror.setReplayBoundary(T3);
816+ const afterAgain = yield* mirror.replayBoundary();
817+ // Nor once the stream is being followed.
818+ const cursorBefore = yield* mirror.getCursor();
819+ yield* mirror.setCursor(cursorBefore, "event_boundary");
820+ const afterCursor = yield* mirror.setReplayBoundary(T1);
821+ const boundaryWithCursor = yield* mirror.replayBoundary();
822+ const cursor = yield* mirror.getCursor();
823+ // The completion mark: absent until a run covers every organization,
824+ // then written once, beside the boundary and the cursor.
825+ const notCompleted = yield* mirror.backfillCompletedAt();
826+ const completed = yield* mirror.markBackfillCompleted(T3);
827+ const completedAgain = yield* mirror.markBackfillCompleted(T4);
828+ const completedAt = yield* mirror.backfillCompletedAt();
829+ const boundaryAfterCompletion = yield* mirror.replayBoundary();
830+ const cursorAfterCompletion = yield* mirror.getCursor();
831+ return {
832+ first,
833+ boundary,
834+ cursorAfterBoundary,
835+ again,
836+ afterAgain,
837+ afterCursor,
838+ boundaryWithCursor,
839+ cursor,
840+ notCompleted,
841+ completed,
842+ completedAgain,
843+ completedAt,
844+ boundaryAfterCompletion,
845+ cursorAfterCompletion,
846+ };
847+ }),
848+ );
849+ expect(result.first, "the first boundary is recorded").toBe(true);
850+ expect(result.boundary, "and reads back as written").toEqual(T2);
851+ expect(result.cursorAfterBoundary, "writing the boundary mints no cursor").toBeNull();
852+ expect(result.again, "a later run's boundary is refused").toBe(false);
853+ expect(result.afterAgain, "the first stands").toEqual(T2);
854+ expect(result.afterCursor).toBe(false);
855+ expect(result.boundaryWithCursor).toEqual(T2);
856+ expect(result.cursor, "and the cursor is untouched").toBe("event_boundary");
857+ expect(result.notCompleted, "no completion until a run covers every org").toBeNull();
858+ expect(result.completed, "the first completion is recorded").toBe(true);
859+ expect(result.completedAgain, "a later one is refused").toBe(false);
860+ expect(result.completedAt, "the first stands").toEqual(T3);
861+ expect(result.boundaryAfterCompletion, "the boundary is untouched").toEqual(T2);
862+ expect(result.cursorAfterCompletion, "and so is the cursor").toBe("event_boundary");
863+ });
864+
865+ it("refuses a membership payload stamped before the organization's last scan, and accepts one stamped at or after it", async () => {
866+ const result = await run(
867+ Effect.gen(function* () {
868+ const mirror = yield* WorkOsMirror;
869+ const directory = yield* MemberDirectory;
870+ const org = yield* freshOrg();
871+ const revoked = `user_${crypto.randomUUID()}`;
872+ const kept = `user_${crypto.randomUUID()}`;
873+ const joined = `user_${crypto.randomUUID()}`;
874+ // A login fetched its membership list at T1, while `revoked` was a
875+ // member, then stalled. WorkOS revoked them, and the backfill scanned
876+ // the org at T2 without them — nothing to tombstone, the row was
877+ // never there.
878+ yield* mirror.applyOrganizationScan({
879+ organizationId: org,
880+ listedAt: T2,
881+ members: [{ user: user(kept), membership: membership(org, kept, { updatedAt: T1 }) }],
882+ });
883+ // The stalled login resumes and writes what it holds: refused, the
884+ // revocation predates the scan and nothing would ever undo the row.
885+ const stale = yield* mirror.upsertMembership(membership(org, revoked, { updatedAt: T1 }));
886+ const staleRow = yield* directory.membership(revoked, org, [
887+ "active",
888+ "pending",
889+ "inactive",
890+ ]);
891+ // The same list's payload for a member the scan kept: refused too —
892+ // it changes nothing, the scan already wrote that state.
893+ const repeated = yield* mirror.upsertMembership(membership(org, kept, { updatedAt: T1 }));
894+ const keptRow = yield* directory.membership(kept, org);
895+ // A membership WorkOS created after the scan (its event, or a login
896+ // after it): stamped past the mark, written.
897+ const later = yield* mirror.upsertMembership(membership(org, joined, { updatedAt: T3 }));
898+ const atMark = yield* mirror.upsertMembership(
899+ membership(org, kept, { role: "admin", updatedAt: T2 }),
900+ );
901+ const keptAfter = yield* directory.membership(kept, org);
902+ return { stale, staleRow, repeated, keptRow, later, atMark, keptAfter };
903+ }),
904+ );
905+ expect(result.stale, "a payload older than the scan is refused").toBe(false);
906+ expect(result.staleRow, "and no row is minted for the revoked member").toBeNull();
907+ expect(result.repeated, "even for a member the scan kept").toBe(false);
908+ expect(result.keptRow?.status).toBe("active");
909+ expect(result.later, "a payload newer than the scan is written").toBe(true);
910+ expect(result.atMark, "as is one stamped at the scan's instant").toBe(true);
911+ expect(result.keptAfter?.role).toBe("admin");
912+ });
913+
914+ it("waits for a scan holding the organization row before judging a payload against the scan's mark", async () => {
915+ // A feeder (a login) writes a membership it fetched at T1 while a scan
916+ // listed at T2 — which no longer contains that membership — is being
917+ // applied. The scan claims the organization row for the length of its
918+ // transaction; the feeder's write must wait for that commit, observe
919+ // the mark, and refuse the payload — never insert the membership the
920+ // scan proved revoked. The scan holds the test's shared connection; the
921+ // feeder runs the same store over a second one, so the two transactions
922+ // are real peers on the server.
923+ const result = await run(
924+ Effect.scoped(
925+ Effect.gen(function* () {
926+ const org = yield* freshOrg();
927+ const revoked = `user_${crypto.randomUUID()}`;
928+ const { db: scanDb } = yield* DbService;
929+ // The second connection is owned by this test body's scope.
930+ const feederDb = Context.get(yield* Layer.build(makeDbLayer()), DbService).db;
931+ const feeder = makeWorkOsMirrorStore(feederDb);
932+ const directory = yield* MemberDirectory;
933+
934+ // The scan's transaction: its claim (the `backfilled_at` CAS, an
935+ // UPDATE that locks the row) is done, and the transaction is held
936+ // open until `release` — the window a feeder can race.
937+ const claimed = yield* Deferred.make<void>();
938+ let release: () => void = () => undefined;
939+ const held = new Promise<void>((resolve) => {
940+ release = resolve;
941+ });
942+ const scan = yield* Effect.forkChild(
943+ Effect.promise(() =>
944+ scanDb.transaction(async (tx) => {
945+ await tx
946+ .update(organizations)
947+ .set({ backfilledAt: T2 })
948+ .where(eq(organizations.id, org));
949+ await Effect.runPromise(Deferred.succeed(claimed, undefined));
950+ await held;
951+ }),
952+ ),
953+ { startImmediately: true },
954+ );
955+ yield* Deferred.await(claimed);
956+
957+ // The feeder's write starts while the scan holds the row.
958+ const written = yield* Deferred.make<boolean>();
959+ yield* Effect.forkChild(
960+ feeder
961+ .upsertMembership(membership(org, revoked, { updatedAt: T1 }))
962+ .pipe(Effect.flatMap((wrote) => Deferred.succeed(written, wrote))),
963+ { startImmediately: true },
964+ );
965+ const beforeCommit = yield* Effect.timeoutOption(Deferred.await(written), "250 millis");
966+ release();
967+ yield* Fiber.join(scan);
968+ const afterCommit = yield* Deferred.await(written);
969+ const row = yield* directory.membership(revoked, org, ["active", "pending", "inactive"]);
970+ return { beforeCommit, afterCommit, row };
971+ }),
972+ ),
973+ );
974+ expect(
975+ Option.isNone(result.beforeCommit),
976+ "the write waits while the scan holds the organization row",
977+ ).toBe(true);
978+ expect(result.afterCommit, "once the scan has committed, its mark refuses the payload").toBe(
979+ false,
980+ );
981+ expect(result.row, "so the revoked membership was never inserted").toBeNull();
982+ });
983+
984+ it("applies a scan only when its listing is newer than the one already applied, and never to a deleted or unknown organization", async () => {
985+ const result = await run(
986+ Effect.gen(function* () {
987+ const mirror = yield* WorkOsMirror;
988+ const directory = yield* MemberDirectory;
989+ const store = yield* UserStoreService;
990+ const scanned = yield* freshOrg();
991+ const untouched = yield* freshOrg();
992+ const staying = `user_${crypto.randomUUID()}`;
993+ const leaving = `user_${crypto.randomUUID()}`;
994+ const member = (id: string) => ({
995+ user: user(id),
996+ membership: membership(scanned, id),
997+ });
998+
999+ const before = yield* mirror.organizationBackfilledAt(scanned);
1000+ // The later listing (T2) no longer contains `leaving`; applied first.
1001+ const later = yield* mirror.applyOrganizationScan({
1002+ organizationId: scanned,
1003+ listedAt: T2,
1004+ members: [member(staying)],
1005+ });
1006+ const afterLater = yield* mirror.organizationBackfilledAt(scanned);
1007+ // The earlier listing (T1) still contains `leaving`: refused whole.
1008+ const earlier = yield* mirror.applyOrganizationScan({
1009+ organizationId: scanned,
1010+ listedAt: T1,
1011+ members: [member(staying), member(leaving)],
1012+ });
1013+ const afterEarlier = yield* mirror.organizationBackfilledAt(scanned);
1014+ const leavingRow = yield* directory.membership(leaving, scanned, [
1015+ "active",
1016+ "pending",
1017+ "inactive",
1018+ ]);
1019+ // The same instant is not newer either: a replayed listing writes nothing.
1020+ const same = yield* mirror.applyOrganizationScan({
1021+ organizationId: scanned,
1022+ listedAt: T2,
1023+ members: [],
1024+ });
1025+ const stayingRow = yield* directory.membership(staying, scanned);
1026+ const other = yield* mirror.organizationBackfilledAt(untouched);
1027+ const unknown = yield* mirror.applyOrganizationScan({
1028+ organizationId: "org_never_mirrored",
1029+ listedAt: T3,
1030+ members: [],
1031+ });
1032+ yield* store.use("deleteOrganizationCascade", (s) =>
1033+ s.deleteOrganizationCascade(untouched, T3),
1034+ );
1035+ const deleted = yield* mirror.applyOrganizationScan({
1036+ organizationId: untouched,
1037+ listedAt: T4,
1038+ members: [{ user: user(staying), membership: membership(untouched, staying) }],
1039+ });
1040+ const deletedRow = yield* directory.membership(staying, untouched);
1041+ return {
1042+ before,
1043+ later,
1044+ afterLater,
1045+ earlier,
1046+ afterEarlier,
1047+ leavingRow,
1048+ same,
1049+ stayingRow,
1050+ other,
1051+ unknown,
1052+ deleted,
1053+ deletedRow,
1054+ };
1055+ }),
1056+ );
1057+ expect(result.before, "a freshly mirrored organization is unscanned").toBeNull();
1058+ expect(result.later).toEqual(
1059+ Option.some({
1060+ usersWritten: 1,
1061+ membershipsWritten: 1,
1062+ membershipsTombstoned: 0,
1063+ }),
1064+ );
1065+ expect(result.afterLater, "the scan marks the organization as of its listing").toEqual(T2);
1066+ expect(result.earlier, "an older listing is refused whole").toEqual(Option.none());
1067+ expect(result.afterEarlier, "and the mark never moves backwards").toEqual(T2);
1068+ expect(
1069+ result.leavingRow,
1070+ "the membership only the older listing held was never written",
1071+ ).toBeNull();
1072+ expect(result.same, "a listing at the recorded instant is refused too").toEqual(Option.none());
1073+ expect(result.stayingRow?.status, "so it tombstones nothing the newer one wrote").toBe(
1074+ "active",
1075+ );
1076+ expect(result.other, "another organization's mark is its own").toBeNull();
1077+ expect(result.unknown, "an organization the mirror does not hold takes no scan").toEqual(
1078+ Option.none(),
1079+ );
1080+ expect(result.deleted, "nor does a deleted one, however new the listing").toEqual(
1081+ Option.none(),
1082+ );
1083+ expect(result.deletedRow).toBeNull();
1084+ });
1085+});
1086+
5631087 describe("cloud MemberDirectory", () => {
5641088 const seed = (org: string) =>
5651089 Effect.gen(function* () {
apps/cloud/src/auth/workos-mirror.ts+33−497
@@ -5,509 +5,43 @@ · 4 unmodified lines
55 // `accounts` / `memberships` rows (db/schema.ts) in step with it so the read
66 // side (`auth/member-directory.ts`, the cloud `MemberDirectory`) never has to
77 // ask WorkOS. Three feeders write through it: the login callback (user +
8−// memberships already in hand), Executor-initiated changes (write-through),
9−// and the WorkOS Events API reconciler (dashboard-side changes, replayed in
10−// order from a persisted cursor).
8+// memberships already in hand), Executor-initiated changes (write-through in
9+// `auth/handlers.ts` and `account/workos-account-service.ts`), and the WorkOS
10+// Events API reconciler (dashboard-side changes, replayed in order from a
11+// persisted cursor). The one-off backfill runs the same store out-of-band.
1112 //
12−// Every write is idempotent and out-of-order safe. Both upserts carry the
13−// WorkOS `updatedAt` of their payload and refuse to overwrite a row whose
14−// stored `workos_updated_at` is newer, so a replayed or late-arriving event
15−// can never regress the mirror. A delete never drops the row: it TOMBSTONES
16−// it (`status = 'inactive'`, `workos_updated_at` = the deletion time) — and
17−// INSERTS the tombstone when the row is not there yet — so a feeder that
18−// fetched the membership before the deletion and writes it after (a login,
19−// the backfill) is refused by that same guard instead of reinstating access;
20−// only a payload newer than the deletion (the member re-added in WorkOS)
21−// reactivates it. At an EQUAL timestamp the tombstone wins: a live row
22−// accepts a payload stamped the same instant (feeders replay the same
23−// payload and must converge), a tombstone does not, so a deletion at T is
24−// never undone by an active payload at T. Every stamp is on WorkOS's clock:
25−// a deletion replayed from the events stream is stamped with the event's
26−// time; one Executor made itself — WorkOS answers a delete with no time —
27−// is stamped with the `updatedAt` WorkOS last reported for the membership,
28−// or keeps the stamp the row already holds. Never a local clock: read after
29−// WorkOS answered, it can post-date a replacement membership WorkOS created
30−// for the same member meanwhile, and a tombstone stamped with it would
31−// refuse that replacement for good. The row tombstone can only speak for the
32−// membership id the row happens to hold, so every delete ALSO records the
33−// deleted WorkOS id in `membership_tombstones`, a ledger keyed by that id
34−// alone: WorkOS never reuses a deleted `om_…` id, so no membership write
35−// naming a recorded id is ever current, however it is stamped. That is what
36−// covers the case the row cannot: membership A replaced by B in WorkOS before
37−// the mirror saw either, B then deleted — the delete finds a row holding A
38−// (not B's to tombstone) and would otherwise leave nothing behind, and a
39−// later scan that still lists B would insert it live, stamped after A. With
40−// the ledger the delete is recorded whatever the row holds, and the scan's
41−// payload is refused by identity. A deleted USER is protected by identity
42−// the same way: `deleteUser` leaves the account row behind as a tombstone
43−// (profile cleared, stamped with the deletion), and no membership naming
44−// that account is ever written again, however the payload is stamped —
45−// WorkOS never reuses a user id, and a membership the mirror had not seen
46−// has no row of its own for a timestamp guard to refuse the insert against.
47−// The cursor advances only by compare-and-set, so two reconciler runs cannot
48−// both own the stream.
49−//
50−// Per-request layer shape, like `UserStoreService`: it holds the request's
51−// postgres socket, so it is rebuilt per request (`RequestScopedServicesLive`)
52−// and never shared across Workers requests.
13+// The queries live in `workos-mirror-store.ts`; this file binds them to the
14+// per-request `DbService`. Per-request layer shape, like `UserStoreService`:
15+// it holds the request's postgres socket, so it is rebuilt per request
16+// (`RequestScopedServicesLive`) and never shared across Workers requests.
5317 // ---------------------------------------------------------------------------
5418
55−import { and, eq, isNotNull, isNull, lt, ne, or, sql } from "drizzle-orm";
56−import type { AnyPgColumn } from "drizzle-orm/pg-core";
57−import { Context, Effect, Layer, Schema } from "effect";
58−
59−import { type MemberStatus } from "@executor-js/api/server";
60−
61−import { accounts, membershipTombstones, memberships, workosSync } from "../db/schema";
62−import { DbService, type DrizzleDb } from "../db/db";
63−import {
64− USER_STORE_FAILURE_REASONS,
65− tryPromiseService,
66− userStoreReasonFromCause,
67− withServiceLogging,
68−} from "./errors";
69−
70−/** A WorkOS user, as the mirror stores it. `updatedAt` is WorkOS's own. */
71−export interface WorkOsMirrorUser {
72− readonly id: string;
73− readonly email: string;
74− readonly firstName: string | null;
75− readonly lastName: string | null;
76− readonly avatarUrl: string | null;
77− readonly lastSignInAt: Date | null;
78− readonly updatedAt: Date;
79−}
80−
81−/**
82− * A WorkOS organization membership, as the mirror stores it. `id` is the
83− * WorkOS `om_…`; `accountId` the WorkOS user id; `updatedAt` is WorkOS's own.
84− * The organization row must already be mirrored (`upsertOrganization`) — a
85− * membership of an unknown org is a `query` failure, not a silent skip.
86− */
87−export interface WorkOsMirrorMembership {
88− readonly id: string;
89− readonly accountId: string;
90− readonly organizationId: string;
91− readonly role: string;
92− readonly status: MemberStatus;
93− readonly updatedAt: Date;
94−}
95−
96−/**
97− * What identifies a membership to a delete: the WorkOS `om_…` id AND the
98− * (account, organization) pair it belongs to. The pair is the row's key,
99− * and a delete must be able to mint the row as a tombstone when the mirror
100− * has not seen the membership yet — an id alone cannot.
101− */
102−export type WorkOsMirrorMembershipRef = Pick<
103− WorkOsMirrorMembership,
104− "id" | "accountId" | "organizationId"
105−>;
106−
107−/**
108− * The public failure of every mirror write: which call, and how it failed
109− * (classified from the driver cause the same way `UserStoreError` is).
110− */
111−export class WorkOsMirrorError extends Schema.TaggedErrorClass<WorkOsMirrorError>()(
112− "WorkOsMirrorError",
113− {
114− operation: Schema.String,
115− reason: Schema.Literals(USER_STORE_FAILURE_REASONS),
116− },
117− { httpApiStatus: 500 },
118−) {
119− override get message(): string {
120− return `workos mirror ${this.operation} failed: ${this.reason}`;
121− }
122−}
123−
124−export interface WorkOsMirrorShape {
125− /**
126− * Insert or refresh a user row. `false` when the payload was refused and
127− * the row left untouched: the stored row is newer than `updatedAt`, or is
128− * a deletion tombstone stamped at `updatedAt` or later.
129− */
130− readonly upsertUser: (user: WorkOsMirrorUser) => Effect.Effect<boolean, WorkOsMirrorError>;
131− /**
132− * Insert or refresh a membership row, minting the bare account row first so
133− * the foreign key holds when the membership arrives before its user. `false`
134− * when the payload was refused: the stored row is newer than `updatedAt`,
135− * or is a tombstone stamped at `updatedAt` or later — or the membership id
136− * is recorded in the deletion ledger (`membership_tombstones`, written by
137− * `deleteMembership`), whatever the payload is stamped: WorkOS never
138− * reuses a deleted `om_…` id, so no payload naming a recorded id is
139− * current — or the account is a deletion tombstone (`deleteUser`),
140− * whatever the payload is stamped: WorkOS never reuses a user id either,
141− * so a deleted user has no memberships to mirror.
142− */
143− readonly upsertMembership: (
144− membership: WorkOsMirrorMembership,
145− ) => Effect.Effect<boolean, WorkOsMirrorError>;
146− /**
147− * Tombstone the membership as deleted: the row stays (or is minted, when
148− * the mirror has not seen the membership yet), `inactive`, carrying the
149− * deleted WorkOS id and stamped so that only a payload newer than the
150− * stamp can reactivate the row. The stamp is on WorkOS's clock, so it
151− * orders against every payload the mirror is fed: `deletedAt` is an
152− * instant at or after the membership's last reported state — a deletion
153− * event's `createdAt`, or the `updatedAt` WorkOS last reported for the
154− * membership when Executor deletes it (WorkOS answers a delete with no
155− * time) — or `null` when the caller holds no WorkOS instant at all: the
156− * row then keeps the stamp it holds, the last state WorkOS reported for
157− * this membership. Never a local clock: read after WorkOS answered, it
158− * can post-date a replacement membership WorkOS created for the same
159− * member meanwhile, and a tombstone stamped with it would refuse that
160− * replacement for good. Every payload of the deleted membership a feeder
161− * could have fetched is at or before the stamp (a tombstone wins a tie),
162− * and every replacement is after it. A row the mirror does not hold and
163− * no instant to stamp it with takes the current time: there is no WorkOS
164− * stamp to keep. Matches the row by the membership's IDENTITY, never by
165− * timestamp: `false` when the row already carries this tombstone or a
166− * later one (a replayed delete), or holds ANOTHER membership id — the
167− * member re-added in WorkOS under a new id, which stands whether the
168− * replacement was mirrored before or after this delete was stamped. In
169− * EVERY case the deleted id is recorded in the deletion ledger
170− * (`membership_tombstones`) — a row holding another id is left alone, but
171− * the id this delete names is still refused to every later write, so a
172− * membership the mirror never held under its own id (replaced and deleted
173− * in WorkOS before the mirror saw it) cannot be inserted live by a scan
174− * that listed it before the deletion. `true` when the delete changed
175− * anything: the row was tombstoned, or the id was newly recorded. The
176− * organization row must already be mirrored, as for `upsertMembership`.
177− */
178− readonly deleteMembership: (
179− membership: WorkOsMirrorMembershipRef,
180− deletedAt: Date | null,
181− ) => Effect.Effect<boolean, WorkOsMirrorError>;
182− /**
183− * Tombstone a deleted WorkOS user: every membership of the account is
184− * tombstoned as by `deleteMembership`, and the account row is kept (it
185− * anchors foreign keys) — or minted, when the mirror has not seen the user
186− * yet — with its profile cleared and stamped `deletedAt`, so a stale user
187− * payload cannot restore it. `false` when the account already carries
188− * this tombstone or a later one (a replayed delete).
189− */
190− readonly deleteUser: (
191− accountId: string,
192− deletedAt: Date,
193− ) => Effect.Effect<boolean, WorkOsMirrorError>;
194− /** The id of the last WorkOS event applied, or `null` before the first run. */
195− readonly getCursor: () => Effect.Effect<string | null, WorkOsMirrorError>;
196− /**
197− * Compare-and-set the cursor: advance to `next` only if it still reads
198− * `prev` (`null` = no cursor yet). `false` means another run moved it first
199− * — the caller must stop, it no longer owns the stream.
200− */
201− readonly setCursor: (
202− prev: string | null,
203− next: string,
204− ) => Effect.Effect<boolean, WorkOsMirrorError>;
205−}
206−
207−// The one events stream the reconciler follows. A row id rather than a
208−// singleton table so a second stream (another WorkOS environment, a replay)
209−// can be added without a schema change.
210−const EVENTS_CURSOR_ID = "events";
211−
212−// A tombstone keeps the row, marks it `inactive`, and moves its timestamp to
213−// the deletion time — never backwards, so a row that somehow carries a newer
214−// WorkOS timestamp keeps it and the upsert guard stays at least as strict.
215−// With no instant at all (`deletedAt` null, see `deleteMembership`) the row
216−// keeps the stamp it holds, and a row that has none takes the current time
217−// — the only instant known. (A raw `sql`
218−// fragment binds the Date without the column's driver mapping, so it is
219−// passed as ISO text and cast.)
220−const noEarlierThan = (column: AnyPgColumn, at: Date | null) =>
221− at === null
222− ? sql`coalesce(${column}, now())`
223− : sql`greatest(${column}, ${at.toISOString()}::timestamptz)`;
224−
225−const tombstone = (deletedAt: Date | null) => ({
226− status: "inactive" as const,
227− workosUpdatedAt: noEarlierThan(memberships.workosUpdatedAt, deletedAt),
228−});
229−
230−// Which stored membership rows a payload stamped `updatedAt` may overwrite:
231−// a row with no stamp (predating the mirror), any row stamped earlier, and a
232−// LIVE row stamped the same instant — feeders replay the same payload and
233−// must converge, not stall. An `inactive` row stamped the same instant is
234−// not overwritten: a deletion (or deactivation) at T beats an active payload
235−// at T, so an equal-timestamp payload can never undo a tombstone.
236−const membershipAcceptsPayload = (updatedAt: Date) =>
237− or(
238− isNull(memberships.workosUpdatedAt),
239− lt(memberships.workosUpdatedAt, updatedAt),
240− and(eq(memberships.workosUpdatedAt, updatedAt), ne(memberships.status, "inactive")),
241− );
242−
243−// A deleted user's account row, as `deleteUser` leaves it: the profile is
244−// cleared (every WorkOS user payload carries an email, so a stamped row with
245−// none was written by `deleteUser`) and the stamp is the deletion time. A row
246−// minted bare by `ensureAccount` has no stamp either, and is not a tombstone.
247−const accountIsTombstone = and(isNull(accounts.email), isNotNull(accounts.workosUpdatedAt));
248−
249−// Whether membership `id` is in the deletion ledger: a WorkOS id a delete
250−// has named, which never returns. Judged by identity alone — the ledger
251−// records WHEN for the record, not for ordering.
252−const membershipIdDeleted = async (db: DrizzleDb, id: string): Promise<boolean> => {
253− const rows = await db
254− .select({ membershipId: membershipTombstones.membershipId })
255− .from(membershipTombstones)
256− .where(eq(membershipTombstones.membershipId, id));
257− return rows.length > 0;
258−};
259−
260−// The same rule as for a membership row, for an account row. A tombstone
261−// takes no payload stamped at or before the deletion; a bare row takes any.
262−const accountAcceptsPayload = (updatedAt: Date) =>
263− or(
264− isNull(accounts.workosUpdatedAt),
265− lt(accounts.workosUpdatedAt, updatedAt),
266− and(eq(accounts.workosUpdatedAt, updatedAt), isNotNull(accounts.email)),
267− );
268−
269−// A delete is applied unless the row already carries this tombstone or a
270−// later one: a replayed deletion changes nothing and reports so. With no
271−// instant given, any tombstone the row carries is this one or later.
272−const notTombstonedSince = (deletedAt: Date | null) =>
273− or(
274− ne(memberships.status, "inactive"),
275− isNull(memberships.workosUpdatedAt),
276− deletedAt === null ? undefined : lt(memberships.workosUpdatedAt, deletedAt),
277− );
278−
279−// Which (account, organization) row a delete of membership `id` at
280−// `deletedAt` may tombstone: the row carrying THIS id — whatever its stamp,
281−// a deleted id is never reused — or a row with no id at all (written before
282−// the mirror recorded WorkOS ids; the delete fills the id in). Never a row
283−// under ANOTHER id: that is a different membership of the same account and
284−// organization, the member re-added in WorkOS after this one was removed,
285−// and it stands however the two are stamped. Its stamp cannot be trusted to
286−// order them: a removal Executor makes is stamped by a local clock, and one
287−// that reads the clock after WorkOS answered can post-date a replacement
288−// WorkOS created while the request was in flight. Identity orders them,
289−// timestamps do not. A row already carrying this tombstone or a later one is
290−// left alone (`notTombstonedSince`) so a replayed delete reports `false`.
291−const membershipDeletableBy = (id: string, deletedAt: Date | null) =>
292− and(
293− or(isNull(memberships.membershipId), eq(memberships.membershipId, id)),
294− notTombstonedSince(deletedAt),
295− );
296−
297−const makeService = (db: DrizzleDb): WorkOsMirrorShape => {
298− const run = <A>(op: string, fn: () => Promise<A>) =>
299− withServiceLogging(
300− `workos_mirror.${op}`,
301− (failure) =>
302− new WorkOsMirrorError({
303− operation: op,
304− reason: userStoreReasonFromCause(failure),
305− }),
306− tryPromiseService(fn),
307− );
308−
309− // Over `db` or a transaction handle (drizzle's is a `PgDatabase` too).
310− const ensureAccount = (on: DrizzleDb, id: string) =>
311− on.insert(accounts).values({ id }).onConflictDoNothing({ target: accounts.id });
312−
313− return {
314− upsertUser: (user) =>
315− run("upsertUser", async () => {
316− const written = await db
317− .insert(accounts)
318− .values({
319− id: user.id,
320− email: user.email,
321− firstName: user.firstName,
322− lastName: user.lastName,
323− avatarUrl: user.avatarUrl,
324− lastSignInAt: user.lastSignInAt,
325− workosUpdatedAt: user.updatedAt,
326− })
327− .onConflictDoUpdate({
328− target: accounts.id,
329− set: {
330− email: user.email,
331− firstName: user.firstName,
332− lastName: user.lastName,
333− avatarUrl: user.avatarUrl,
334− lastSignInAt: user.lastSignInAt,
335− workosUpdatedAt: user.updatedAt,
336− },
337− setWhere: accountAcceptsPayload(user.updatedAt),
338− })
339− .returning({ id: accounts.id });
340− return written.length > 0;
341− }),
342−
343− upsertMembership: (membership) =>
344− run("upsertMembership", async () => {
345− await ensureAccount(db, membership.accountId);
346− // Never for a DELETED user. The row guard below orders a payload
347− // against the membership row it would overwrite; a membership the
348− // mirror has not seen yet has no row, so the guard cannot refuse
349− // the INSERT — and a feeder that fetched the membership before the
350− // user was deleted and writes it after (a stalled login list, the
351− // backfill's older listing) would insert it live. The account
352− // tombstone is the one row a deleted user always leaves behind, so
353− // it is consulted first, by identity: WorkOS never reuses a user
354− // id, so no payload naming a tombstoned account is ever current.
355− const deleted = await db
356− .select({ id: accounts.id })
357− .from(accounts)
358− .where(and(eq(accounts.id, membership.accountId), accountIsTombstone));
359− if (deleted.length > 0) return false;
360− // Never under a DELETED membership id. The row guard below can only
361− // refuse against the id the row holds; a delete of THIS id that
362− // found the row under another id (see the header) left only the
363− // ledger entry behind, and that is what refuses the payload here.
364− if (await membershipIdDeleted(db, membership.id)) return false;
365− const written = await db
366− .insert(memberships)
367− .values({
368− accountId: membership.accountId,
369− organizationId: membership.organizationId,
370− membershipId: membership.id,
371− role: membership.role,
372− status: membership.status,
373− workosUpdatedAt: membership.updatedAt,
374− })
375− .onConflictDoUpdate({
376− target: [memberships.accountId, memberships.organizationId],
377− set: {
378− membershipId: membership.id,
379− role: membership.role,
380− status: membership.status,
381− workosUpdatedAt: membership.updatedAt,
382− },
383− setWhere: membershipAcceptsPayload(membership.updatedAt),
384− })
385− .returning({ accountId: memberships.accountId });
386− return written.length > 0;
387− }),
388−
389− // An upsert, like the membership write it guards against: a delete the
390− // mirror sees before the membership itself (the reconciler ahead of the
391− // backfill) must leave the tombstone behind, or the later, older payload
392− // would insert the row live.
393− deleteMembership: (membership, deletedAt) =>
394− run("deleteMembership", () =>
395− db.transaction(async (tx) => {
396− await ensureAccount(tx, membership.accountId);
397− // The ledger entry FIRST, whatever the row holds: this is the one
398− // record of the deletion that does not depend on the row carrying
399− // the deleted id. A replayed delete finds it there (`DO NOTHING`)
400− // and records nothing new.
401− const recorded = await tx
402− .insert(membershipTombstones)
403− .values({
404− membershipId: membership.id,
405− accountId: membership.accountId,
406− organizationId: membership.organizationId,
407− ...(deletedAt === null ? {} : { deletedAt }),
408− })
409− .onConflictDoNothing({ target: membershipTombstones.membershipId })
410− .returning({ membershipId: membershipTombstones.membershipId });
411− const tombstoned = await tx
412− .insert(memberships)
413− .values({
414− accountId: membership.accountId,
415− organizationId: membership.organizationId,
416− membershipId: membership.id,
417− status: "inactive",
418− workosUpdatedAt: deletedAt ?? new Date(),
419− })
420− .onConflictDoUpdate({
421− target: [memberships.accountId, memberships.organizationId],
422− set: { membershipId: membership.id, ...tombstone(deletedAt) },
423− setWhere: membershipDeletableBy(membership.id, deletedAt),
424− })
425− .returning({ accountId: memberships.accountId });
426− return recorded.length > 0 || tombstoned.length > 0;
427− }),
428− ),
429−
430− deleteUser: (accountId, deletedAt) =>
431− run("deleteUser", async () => {
432− // The account tombstone FIRST, then the memberships. A membership
433− // write checks the account tombstone before it inserts
434− // (`upsertMembership`), so a write racing this delete either sees
435− // the tombstone and refuses, or has landed before it and is caught
436− // by the membership tombstoning below. In the other order a write
437− // between the two statements would slip through live.
438− //
439− // The account tombstone: profile cleared, stamped with the deletion.
440− // Minted when absent, for the same reason as the membership one.
441− const cleared = await db
442− .insert(accounts)
443− .values({ id: accountId, workosUpdatedAt: deletedAt })
444− .onConflictDoUpdate({
445− target: accounts.id,
446− set: {
447− email: null,
448− firstName: null,
449− lastName: null,
450− avatarUrl: null,
451− workosUpdatedAt: noEarlierThan(accounts.workosUpdatedAt, deletedAt),
452− },
453− // Applied unless the row already carries this tombstone or a
454− // later one (a replayed delete).
455− setWhere: or(
456− isNotNull(accounts.email),
457− isNull(accounts.workosUpdatedAt),
458− lt(accounts.workosUpdatedAt, deletedAt),
459− ),
460− })
461− .returning({ id: accounts.id });
462− await db
463− .update(memberships)
464− .set(tombstone(deletedAt))
465− .where(and(eq(memberships.accountId, accountId), notTombstonedSince(deletedAt)));
466− return cleared.length > 0;
467− }),
468−
469− getCursor: () =>
470− run("getCursor", async () => {
471− const rows = await db
472− .select({ cursor: workosSync.cursor })
473− .from(workosSync)
474− .where(eq(workosSync.id, EVENTS_CURSOR_ID));
475− // No row yet is the same state as a row with no cursor: nothing applied.
476− return rows[0]?.cursor ?? null;
477− }),
478−
479− setCursor: (prev, next) =>
480− run("setCursor", async () => {
481− const now = new Date();
482− if (prev === null) {
483− // First advance: mint the row, or claim an existing row that still
484− // has no cursor. A row that already carries one belongs to another
485− // run and is left alone.
486− const written = await db
487− .insert(workosSync)
488− .values({ id: EVENTS_CURSOR_ID, cursor: next, updatedAt: now })
489− .onConflictDoUpdate({
490− target: workosSync.id,
491− set: { cursor: next, updatedAt: now },
492− setWhere: isNull(workosSync.cursor),
493− })
494− .returning({ id: workosSync.id });
495− return written.length > 0;
496− }
497− const written = await db
498− .update(workosSync)
499− .set({ cursor: next, updatedAt: now })
500− .where(and(eq(workosSync.id, EVENTS_CURSOR_ID), eq(workosSync.cursor, prev)))
501− .returning({ id: workosSync.id });
502− return written.length > 0;
503− }),
504− };
505−};
19+import { Context, Effect, Layer } from "effect";
20+
21+import { DbService } from "../db/db";
22+import { makeWorkOsMirrorStore, type WorkOsMirrorShape } from "./workos-mirror-store";
23+
24+export { WorkOsMirrorError } from "./errors";
25+export {
26+ mirrorMembershipFromWorkOs,
27+ mirrorUserFromWorkOs,
28+ type WorkOsMembershipPayload,
29+ type WorkOsMirrorMembership,
30+ type WorkOsMirrorMembershipRef,
31+ type WorkOsMirrorShape,
32+ type WorkOsMirrorUser,
33+ type WorkOsOrganizationScan,
34+ type WorkOsOrganizationScanWrites,
35+ type WorkOsScannedMember,
36+ type WorkOsUserPayload,
37+} from "./workos-mirror-store";
50638
50739 export class WorkOsMirror extends Context.Service<WorkOsMirror, WorkOsMirrorShape>()(
50840 "@executor-js/cloud/WorkOsMirror",
50941 ) {
510− static Live = Layer.effect(this)(Effect.map(DbService.asEffect(), ({ db }) => makeService(db)));
42+ static Live = Layer.effect(this)(
43+ Effect.map(DbService.asEffect(), ({ db }) => makeWorkOsMirrorStore(db)),
44+ );
51145 }
51246
51347 /**
@@ -516,4 +50,6 @@ export class WorkOsMirror extends Context.Service<WorkOsMirror, WorkOsMirrorShap · 2 unmodified lines
51650 * `makeUserStoreLayer` exists. See [[makeDbLayer]].
51751 */
51852 export const makeWorkOsMirrorLayer = (): Layer.Layer<WorkOsMirror, never, DbService> =>
519− Layer.effect(WorkOsMirror)(Effect.map(DbService.asEffect(), ({ db }) => makeService(db)));
53+ Layer.effect(WorkOsMirror)(
54+ Effect.map(DbService.asEffect(), ({ db }) => makeWorkOsMirrorStore(db)),
55+ );
apps/cloud/src/db/db.test.ts+42−4
@@ -94,7 +94,11 @@ describe("DbService", () => { · 93 unmodified lines
9494 Effect.gen(function* () {
9595 const { db } = yield* DbService;
9696 yield* Effect.promise(() =>
97− makeUserStore(db).upsertOrganization({ id: organizationId, name: "Acme" }),
97+ makeUserStore(db).upsertOrganization({
98+ id: organizationId,
99+ name: "Acme",
100+ updatedAt: new Date(),
101+ }),
98102 );
99103 }),
100104 ),
@@ -121,16 +125,24 @@ describe("DbService", () => { · 20 unmodified lines
121125 });
122126
123127 describe("upsertOrganization · slug is minted at insert", () => {
124− const upsert = (org: { id: string; name: string }) =>
128+ const upsert = (org: { id: string; name: string; updatedAt?: Date }) =>
125129 program(
126130 Effect.gen(function* () {
127131 const { db } = yield* DbService;
128− return yield* Effect.promise(() => makeUserStore(db).upsertOrganization(org));
132+ return yield* Effect.promise(() =>
133+ makeUserStore(db).upsertOrganization({
134+ updatedAt: new Date(),
135+ ...org,
136+ }),
137+ );
129138 }),
130139 );
131140
132141 it("mints a valid slug on insert", async () => {
133− const org = await upsert({ id: `org_${crypto.randomUUID()}`, name: "Slug Mint Co" });
142+ const org = await upsert({
143+ id: `org_${crypto.randomUUID()}`,
144+ name: "Slug Mint Co",
145+ });
134146 expect(org.slug, "a new org row is born with a slug").toBeTruthy();
135147 expect(isValidOrgSlug(org.slug), "the minted slug fits the URL grammar").toBe(true);
136148 });
@@ -143,6 +155,32 @@ describe("upsertOrganization · slug is minted at insert", () => { · 6 unmodified lines
143155 expect(renamed.name, "the name is refreshed on conflict").toBe("Renamed Org");
144156 });
145157
158+ it("refuses a name stamped earlier than the one it holds, and never renames a deleted org", async () => {
159+ const id = `org_${crypto.randomUUID()}`;
160+ const t1 = new Date("2026-01-01T00:00:00.000Z");
161+ const t2 = new Date("2026-01-02T00:00:00.000Z");
162+ const t3 = new Date("2026-01-03T00:00:00.000Z");
163+ await upsert({ id, name: "Original Name", updatedAt: t2 });
164+ // A payload fetched before the rename landed (a login that stalled).
165+ const stale = await upsert({ id, name: "Stale Name", updatedAt: t1 });
166+ expect(stale.name, "an older name never reverts a newer one").toBe("Original Name");
167+ const replay = await upsert({ id, name: "Replayed Name", updatedAt: t2 });
168+ expect(replay.name, "the same instant is accepted, so replays converge").toBe("Replayed Name");
169+ await program(
170+ Effect.gen(function* () {
171+ const { db } = yield* DbService;
172+ yield* Effect.promise(() => makeUserStore(db).deleteOrganizationCascade(id, t3));
173+ }),
174+ );
175+ const afterDelete = await upsert({
176+ id,
177+ name: "Resurrected Name",
178+ updatedAt: t3,
179+ });
180+ expect(afterDelete.deletedAt, "a deleted org stays deleted").toEqual(t3);
181+ expect(afterDelete.name, "and keeps its last name").toBe("Replayed Name");
182+ });
183+
146184 it("discriminates same-name collisions into distinct slugs", async () => {
147185 // Same name → same slug base; the second insert collides on the unique
148186 // index and gets a discriminated slug.
apps/cloud/src/db/org-deletion.test.ts+28−13
@@ -7,7 +7,8 @@ · 6 unmodified lines
77 // two orgs across every tenant table + blob namespace, purges one, and asserts:
88 // - every executor tenant table row for the target org is gone
99 // - org- and user-scoped secret blobs for the target org are gone
10−// - the identity row is gone and its memberships cascade with it
10+// - the org's memberships are gone; the identity row stays as a tombstone
11+// marked deleted (so a delayed feeder cannot re-mint the org live)
1112 // - a second org's data is completely untouched
1213 // - the blob prefix match escapes LIKE wildcards (a `_` in the org id must
1314 // not widen the match to a look-alike namespace)
@@ -196,11 +197,12 @@ const NOT_ORG_OWNED: Record<string, string> = { · 182 unmodified lines
196197 blob: "org-scoped by namespace prefix, purged via the LIKE match",
197198 // Instance-wide, not owned by any org.
198199 private_executor_cloud_settings: "singleton instance settings, not org-scoped",
199− // Identity mirror. `organizations` is deleted directly and `memberships`
200− // cascades from its FK; `accounts` deliberately outlives the org.
201− organizations: "the identity row itself, deleted directly",
202− memberships: "cascades from the organizations FK",
203− membership_tombstones: "cascades from the organizations FK",
200+ // Identity mirror. `organizations` is kept as a tombstone marked deleted,
201+ // `memberships` are deleted by organization id; `accounts` deliberately
202+ // outlives the org.
203+ organizations: "the identity row itself, kept as a tombstone marked deleted",
204+ memberships: "deleted by organization id",
205+ membership_tombstones: "deleted by organization id",
204206 accounts: "shared across orgs — deliberately survives",
205207 workos_sync: "the WorkOS Events API cursor, instance-wide and not org-scoped",
206208 };
@@ -231,14 +233,24 @@ describe("purgeOrganizationData", () => { · 24 unmodified lines
231233 // A look-alike blob that only an UNescaped `_` wildcard would match:
232234 // `o:<orgA>/…` with the underscore replaced by another char.
233235 const trapNs = `o:${orgA.replace("_", "X")}/plugin`;
236+ const now = new Date();
237+ const deletedAt = new Date("2026-01-02T00:00:00.000Z");
234238
235239 await program(
236240 Effect.gen(function* () {
237241 const { db } = yield* DbService;
238242 yield* Effect.promise(async () => {
239243 const store = makeUserStore(db);
240− await store.upsertOrganization({ id: orgA, name: "Delete Me" });
241− await store.upsertOrganization({ id: orgB, name: "Keep Me" });
244+ await store.upsertOrganization({
245+ id: orgA,
246+ name: "Delete Me",
247+ updatedAt: now,
248+ });
249+ await store.upsertOrganization({
250+ id: orgB,
251+ name: "Keep Me",
252+ updatedAt: now,
253+ });
242254 await store.ensureAccount(accountId);
243255 await db.insert(memberships).values({ accountId, organizationId: orgA });
244256 await db.insert(memberships).values({ accountId, organizationId: orgB });
@@ -257,7 +269,7 @@ describe("purgeOrganizationData", () => { · 12 unmodified lines
257269 await program(
258270 Effect.gen(function* () {
259271 const { db } = yield* DbService;
260− yield* Effect.promise(() => makeUserStore(db).deleteOrganizationCascade(orgA));
272+ yield* Effect.promise(() => makeUserStore(db).deleteOrganizationCascade(orgA, deletedAt));
261273 }),
262274 );
263275
@@ -267,10 +279,13 @@ describe("purgeOrganizationData", () => { · 3 unmodified lines
267279 yield* Effect.promise(async () => {
268280 const store = makeUserStore(db);
269281
270− // Target org: every tenant row + blob gone, identity gone, membership
271− // cascaded, but the shared account survives (it may join other orgs).
282+ // Target org: every tenant row + blob gone, memberships gone, the
283+ // identity row kept as a tombstone, and the shared account survives
284+ // (it may join other orgs).
272285 expect(await countTenantRows(db, orgA)).toBe(0);
273− expect(await store.getOrganization(orgA)).toBeNull();
286+ const tombstone = await store.getOrganization(orgA);
287+ expect(tombstone?.deletedAt, "the org row stays, marked deleted").toEqual(deletedAt);
288+ expect(tombstone?.name, "as it was").toBe("Delete Me");
274289 const orgAMemberships = await db
275290 .select()
276291 .from(memberships)
@@ -285,7 +300,7 @@ describe("purgeOrganizationData", () => { · 8 unmodified lines
285300
286301 // Second org: fully intact.
287302 expect(await countTenantRows(db, orgB)).toBeGreaterThan(0);
288− expect(await store.getOrganization(orgB)).not.toBeNull();
303+ expect((await store.getOrganization(orgB))?.deletedAt).toBeNull();
289304 const orgBMemberships = await db
290305 .select()
291306 .from(memberships)
apps/cloud/src/db/org-deletion.ts+30−7
@@ -10,11 +10,21 @@ · 9 unmodified lines
1010 //
1111 // External side effects (the WorkOS org, the Autumn customer) are NOT touched
1212 // here — the caller (auth handler) sequences those around this purge.
13+//
14+// The `organizations` row itself is NOT deleted: it stays as a TOMBSTONE,
15+// marked `deleted_at`, with its memberships removed. The membership mirror's
16+// feeders write whatever WorkOS payload they hold — a login that fetched its
17+// membership list before the deletion can write it after this purge — and
18+// the tombstone is what makes those writes refuse: `upsertOrganization`
19+// never re-mints or renames a marked organization, and the mirror never
20+// inserts a membership of one. Without it the login would insert a fresh,
21+// live organization row plus an active membership, and the deleted
22+// organization would authorize again.
1323
1424 import { eq, or, sql } from "drizzle-orm";
1525
1626 import type { DrizzleDb } from "./db";
17−import { organizations } from "./schema";
27+import { memberships, organizations } from "./schema";
1828 import {
1929 artifact,
2030 blob,
@@ -36,10 +46,16 @@ const escapeLike = (value: string): string => value.replace(/[\\%_]/g, "\\$&"); · 15 unmodified lines
3646
3747 /**
3848 * Delete all rows owned by `organizationId`: every executor tenant table, the
39− * org's secret blobs (org- and user-scoped), and the identity mirror row (which
40− * cascades to local `memberships`). Idempotent — a second run deletes nothing.
49+ * org's secret blobs (org- and user-scoped), and its local `memberships` —
50+ * and mark the identity row deleted as of `deletedAt` (an earlier mark
51+ * stands), keeping it as a tombstone. Idempotent — a second run deletes
52+ * nothing and keeps the first mark.
4153 */
42−export const purgeOrganizationData = (db: DrizzleDb, organizationId: string): Promise<void> =>
54+export const purgeOrganizationData = (
55+ db: DrizzleDb,
56+ organizationId: string,
57+ deletedAt: Date,
58+): Promise<void> =>
4359 db.transaction(async (tx) => {
4460 // Executor tenant tables — every row is scoped by `tenant = organizationId`.
4561 await tx.delete(tool).where(eq(tool.tenant, organizationId));
@@ -66,7 +82,14 @@ export const purgeOrganizationData = (db: DrizzleDb, organizationId: string): Pr · 20 unmodified lines
6682 ),
6783 );
6884
69− // Identity mirror — FK `ON DELETE CASCADE` removes local memberships too.
70− // `accounts` are intentionally left: a user may belong to other orgs.
71− await tx.delete(organizations).where(eq(organizations.id, organizationId));
85+ // Identity mirror: the memberships go, the organization row stays as a
86+ // tombstone (see the header). `accounts` are intentionally left: a user
87+ // may belong to other orgs.
88+ await tx.delete(memberships).where(eq(memberships.organizationId, organizationId));
89+ await tx
90+ .update(organizations)
91+ .set({
92+ deletedAt: sql`coalesce(${organizations.deletedAt}, ${deletedAt.toISOString()}::timestamptz)`,
93+ })
94+ .where(eq(organizations.id, organizationId));
7295 });
apps/cloud/src/db/schema.ts+71−5
@@ -9,7 +9,9 @@ · 8 unmodified lines
99 // - `organizations` — billing entity, scoping root for all domain data
1010 // - `memberships` — which accounts belong to which organizations, with
1111 // the WorkOS role and status
12−// - `workos_sync` — the WorkOS Events API cursor the reconciler resumes from
12+// - `workos_sync` — the WorkOS Events API cursor the reconciler resumes
13+// from, and the replay boundary the one-off backfill
14+// records
1315 //
1416 // The mirror is fed by login (the callback has the user + memberships in
1517 // hand), write-through on every Executor-initiated change, and the WorkOS
@@ -65,6 +67,41 @@ export const organizations = pgTable( · 49 unmodified lines
6567 id: text("id").primaryKey(),
6668 name: text("name").notNull(),
6769 slug: text("slug").notNull(),
70+ /**
71+ * When this organization's membership list was last FULLY scanned from
72+ * WorkOS (the one-off backfill, or the on-demand scan a seat count
73+ * triggers), or null if it never was. Until then the mirror may hold only
74+ * the members login and write-through happened to record, so a count read
75+ * from it is partial; every seat gate checks this mark first. Per
76+ * organization, never database-wide: an org mirrored lazily after a
77+ * backfill ran starts unmarked and is scanned on its first count. The
78+ * mark also orders membership writes: a payload stamped before it is
79+ * refused, since the scan was the full listing at that instant and a
80+ * membership it did not contain was revoked before it — before the
81+ * events replay boundary, so nothing would tombstone it again.
82+ */
83+ backfilledAt: timestamp("backfilled_at", { withTimezone: true }),
84+ /**
85+ * When this organization was deleted, or null while it is live. Set by
86+ * cloud's own deletion flow and by the `organization.deleted` event, and
87+ * KEPT by the local purge (`db/org-deletion.ts`), which removes the
88+ * organization's memberships and tenant data but leaves this row as a
89+ * tombstone: a feeder that fetched a membership before the deletion and
90+ * writes it after (a login that stalled across the purge) finds the
91+ * tombstone and does not re-mint the organization live. A marked
92+ * organization is never renamed and authorizes nobody.
93+ */
94+ deletedAt: timestamp("deleted_at", { withTimezone: true }),
95+ /**
96+ * The instant the stored `name` is known to have been the organization's
97+ * name in WorkOS: the WorkOS `updatedAt` of the organization payload that
98+ * wrote it, or — for a name learned from a membership list at sign-in,
99+ * which carries no organization timestamp — the instant that list was
100+ * fetched. A name write stamped earlier than this is refused
101+ * (`upsertOrganization`), so a sign-in whose list predates a rename cannot
102+ * revert it. Null only on rows written before the stamp existed.
103+ */
104+ workosUpdatedAt: timestamp("workos_updated_at", { withTimezone: true }),
68105 createdAt: timestamp("created_at", { withTimezone: true }).notNull().defaultNow(),
69106 },
70107 (t) => ({
@@ -157,13 +194,42 @@ export const membershipTombstones = pgTable( · 86 unmodified lines
157194 );
158195
159196 /**
160− * The WorkOS Events API cursor. One row per stream (`id` names the stream;
161− * the reconciler uses `"events"`), holding the id of the last event applied.
162− * Advanced only by compare-and-set, so two concurrent reconciler runs cannot
163− * both believe they own the stream: the loser's CAS fails and it stops.
197+ * The WorkOS Events API sync state. One row per stream (`id` names the
198+ * stream; the reconciler uses `"events"`), holding the id of the last event
199+ * applied. Advanced only by compare-and-set, so two concurrent reconciler
200+ * runs cannot both believe they own the stream: the loser's CAS fails and it
201+ * stops.
202+ *
203+ * `range_start` on the `"events"` row is the REPLAY BOUNDARY: the instant the
204+ * FIRST completed one-off backfill (`scripts/backfill-workos-mirror.ts`) began
205+ * reading WorkOS. Everything before it is covered by that backfill; the
206+ * reconciler's first run (no cursor yet) reads the events stream from here,
207+ * so a revocation between the backfill and the first run is never skipped.
208+ * Written once: a backfill that fails part-way records nothing, and a later
209+ * completed one keeps it, because the backfill does not refresh everything
210+ * the events stream carries (organization renames, deleted users'
211+ * profiles) — those between two runs are replayed from the first boundary.
212+ * Without a cursor or a boundary the reconciler does not guess; it waits for
213+ * the backfill.
214+ *
215+ * `backfill_completed_at` is when a backfill run first wrote EVERY live
216+ * organization (`scripts/backfill-workos-mirror.ts` completing, or refusing
217+ * an organization only because a later listing was already applied). Until
218+ * it is set, the mirror may lack members who have not signed in since it
219+ * shipped, so a membership check read from it would deny them: it is the
220+ * first half of the mirror-readiness mark the authorization path consults
221+ * before trusting the mirror over WorkOS. Write-once — a later completed run
222+ * keeps the first instant, so readiness never flips back. Per-organization
223+ * completeness for the seat gates is tracked separately
224+ * (`organizations.backfilled_at`).
225+ *
226+ * Migration 0019 seeds the boundary and the completion mark on a database
227+ * with no organizations, where there is nothing to backfill.
164228 */
165229 export const workosSync = pgTable("workos_sync", {
166230 id: text("id").primaryKey(),
167231 cursor: text("cursor"),
232+ rangeStart: timestamp("range_start", { withTimezone: true }),
233+ backfillCompletedAt: timestamp("backfill_completed_at", { withTimezone: true }),
168234 updatedAt: timestamp("updated_at", { withTimezone: true }).notNull().defaultNow(),
169235 });
apps/cloud/src/extensions/billing/route.node.test.ts+9−0
@@ -55,18 +55,27 @@ const stubUsers = Layer.succeed(UserStoreService)({ · 54 unmodified lines
5555 upsertOrganization: async (org: { id: string; name: string }) => ({
5656 ...org,
5757 slug: org.id,
58+ backfilledAt: null,
59+ deletedAt: null,
60+ workosUpdatedAt: null,
5861 createdAt,
5962 }),
6063 getOrganization: async (id: string) => ({
6164 id,
6265 name: `Org ${id}`,
6366 slug: id,
67+ backfilledAt: null,
68+ deletedAt: null,
69+ workosUpdatedAt: null,
6470 createdAt,
6571 }),
6672 getOrganizationBySlug: async (slug: string) => ({
6773 id: slug === URL_SLUG ? URL_ORG : "org_outsider",
6874 name: `Org ${slug}`,
6975 slug,
76+ backfilledAt: null,
77+ deletedAt: null,
78+ workosUpdatedAt: null,
7079 createdAt,
7180 }),
7281 deleteOrganizationCascade: async () => {},
apps/cloud/src/extensions/routes.ts+4−1
@@ -28,6 +28,7 @@ import { AccountApi, AdminUsersApi } from "@executor-js/api"; · 27 unmodified lines
2828 import { requestScopedMiddleware } from "@executor-js/api/server";
2929
3030 import { UserStoreService } from "../auth/context";
31+import { WorkOsMirror } from "../auth/workos-mirror";
3132 import {
3233 CloudAuthPublicHandlers,
3334 CloudSessionAuthHandlers,
@@ -72,7 +73,9 @@ const spec = OpenApi.fromApi(CloudOpenApi); · 38 unmodified lines
7273 * read it — the few app-only billing touchpoints. It is NOT on the neutral boot
7374 * core.
7475 */
75−export const makeCloudExtensionRoutes = (rsLive: Layer.Layer<DbService | UserStoreService>) => {
76+export const makeCloudExtensionRoutes = (
77+ rsLive: Layer.Layer<DbService | UserStoreService | WorkOsMirror>,
78+) => {
7679 // Session routes (login / callback / me / switch-org / …). Handlers yield
7780 // `UserStoreService` directly; the per-request DB combine keeps the postgres
7881 // socket request-scoped.