Authorize from the mirror unconditionally; alert on a stale reconciler (#2031)

Rhys Sullivan committed 1 month ago

Commit a72e51d1 parent a6cdcf131 files · +652 −811Browse files

Unified · Split

31 files

.changeset/mirror-readiness-removal.md +5 −0
.claude/skills/prod-telemetry/SKILL.md +22 −11
apps/cloud/src/account/account-api.ts +5 −8
apps/cloud/src/account/org-api-key-revoke.node.test.ts +0 −8
apps/cloud/src/account/workos-account-service.ts +1 −8
apps/cloud/src/admin/admin-users-api.node.test.ts +6 −16
apps/cloud/src/admin/admin-users-api.ts +5 −9
apps/cloud/src/api/layers.ts +7 −19
apps/cloud/src/api/protected-api-key-auth.node.test.ts +3 −11
apps/cloud/src/api/protected-jwt-auth.node.test.ts +3 −11
apps/cloud/src/api/protected.ts +1 −4
apps/cloud/src/api/router.ts +1 −4
apps/cloud/src/auth/doc-gate.ts +3 −3
apps/cloud/src/auth/mirror-feeders.node.test.ts +50 −43
apps/cloud/src/auth/mirror-readiness-store.ts +17 −20
apps/cloud/src/auth/mirror-readiness.ts +0 −69
apps/cloud/src/auth/org-api-key-auth.node.test.ts +5 −32
apps/cloud/src/auth/org-selector-auth.node.test.ts +31 −166
apps/cloud/src/auth/organization.ts +41 −99
apps/cloud/src/auth/workos-auth-provider.ts +15 −21
apps/cloud/src/auth/workos-events-runner.node.test.ts +81 −0
apps/cloud/src/auth/workos-events-runner.ts +59 −3
apps/cloud/src/auth/workos-events-sync.node.test.ts +35 −21
apps/cloud/src/auth/workos-mirror.node.test.ts +37 −14
apps/cloud/src/extensions/billing/route.node.test.ts +1 −10
apps/cloud/src/extensions/routes.ts +4 −5
apps/cloud/src/mcp/auth-provider.ts +16 −11
apps/cloud/src/mcp/auth.ts +6 −5
apps/cloud/src/org/auth-middleware.ts +8 −11
apps/cloud/src/org/handlers.test.ts +48 −64
e2e/cloud/mcp-workos-blip-session-survival.test.ts +136 −105

.changeset/mirror-readiness-removal.md+5−0

@@ -0,0 +1,5 @@
	1	+---
	2	+"@executor-js/cloud": patch
	3	+---
	4	+
	5	+`authorizeOrganization` now reads the local membership mirror unconditionally: the per-request readiness check (`MirrorReadiness`) and its live WorkOS `listUserMemberships` fallback are gone from the request path entirely. The backfill is complete and permanent, and an organization that predates the mirror is still covered by the existing on-demand scan (`ensureOrganizationBackfilled`). A stalled reconciler is now an operational alert instead of a per-request fallback: after each run, the cron checks the mirror's `drained_at` heartbeat and, if it has fallen behind the lag budget, logs a structured error and reports it to Sentry. The deploy gate (`scripts/ensure-workos-mirror-ready.ts`) is unchanged — it still refuses to ship while the mirror is unready — and `drained_at` keeps being written by every reconciler run.

.claude/skills/prod-telemetry/SKILL.md+22−11

@@ -57,22 +57,33 @@ join the same traces via traceparent). · 56 unmodified lines
57	57	`execute`/`execute-action` calls `mcp.execute.code` (the script itself,
58	58	capped at 10k chars — cloud-only content capture; local/self-host
59	59	telemetry never records content).
60		−- `auth.authorize_organization` — every membership authorization.
61		− `mirror.ready` (bool: the local membership mirror answered; `false` =
62		− the request fell back to a live WorkOS read) and `mirror.readiness`
63		− (why: `ready`, `backfill pending: …`, `reconciler stale: …`). The
64		− mirror's write spans are `workos_mirror.<op>`; the reconciler run is
	60	+- `auth.authorize_organization` — every membership authorization. Reads the
	61	+ local membership mirror unconditionally; there is no per-request readiness
	62	+ check and no WorkOS fallback, so this span carries no readiness attribute.
	63	+ The mirror's write spans are `workos_mirror.<op>`; the reconciler run is
65	64	`workos_events.sync`. `workos_sync.drained_at` in the prod DB is the
66		− reconciler heartbeat.
	65	+ reconciler heartbeat, and a stalled reconciler now raises its own error
	66	+ from the cron (see below) rather than showing up as a fallback here.
67	67
68		−Recipe — membership-mirror fallback rate (should be ~0 after cutover):
	68	+**Recipe — reconciler heartbeat (ticks should land roughly every minute; a
	69	+gap wider than the 10-minute lag budget means the cron alert should already
	70	+have fired — see `workos_events: reconciler stale` below):**
69	71
70	72	```apl
71	73	['executor-cloud']
72		−\| where _time > ago(1h) and name == "auth.authorize_organization"
73		−\| extend ready = tobool(['attributes.custom']['mirror.ready'])
74		−\| extend why = tostring(['attributes.custom']['mirror.readiness'])
75		−\| summarize n = count() by ready, why
	74	+\| where _time > ago(1h) and name == "workos_events.sync"
	75	+\| summarize n = count() by bin(_time, 1m)
	76	+\| sort by _time desc
	77	+```
	78	+
	79	+**Recipe — stale-reconciler alerts (should be empty; each row is one paging
	80	+event):**
	81	+
	82	+```apl
	83	+['executor-cloud']
	84	+\| where _time > ago(1d) and ['status.message'] contains "workos_events: reconciler stale"
	85	+\| project _time, trace_id, msg = tostring(['status.message'])
	86	+\| sort by _time desc
76	87	```
77	88
78	89	Recipe — error signatures by class (the daily-digest query):

apps/cloud/src/account/account-api.ts+5−8

@@ -10,7 +10,6 @@ import { · 9 unmodified lines
10	10
11	11	import { ApiKeyService } from "../auth/api-keys";
12	12	import { UserStoreService } from "../auth/context";
13		−import { MirrorReadiness } from "../auth/mirror-readiness";
14	13	import { WorkOsMirror } from "../auth/workos-mirror";
15	14	import { sessionFromSealed, type Session } from "../auth/middleware";
16	15	import { WorkOSClient } from "../auth/workos";
@@ -51,7 +50,9 @@ import { AccountCaller, workosAccountProvider } from "./workos-account-service"; · 34 unmodified lines
51	50	// `UserStoreService` / `WorkOsMirror` / `MemberDirectory` are supplied by the
52	51	// combined `rsLive` layer.
53	52	// `ApiKeyService.WorkOS` is built here on top of the boot `WorkOSClient`.
54		−const AccountProviderMiddleware = HttpRouter.middleware<{ provides: AccountProvider }>()(
	53	+const AccountProviderMiddleware = HttpRouter.middleware<{
	54	+ provides: AccountProvider;
	55	+}>()(
55	56	Effect.gen(function* () {
56	57	// Long-lived services only (built once at boot). `UserStoreService` and
57	58	// `DbService` are NOT grabbed here — they come per request from the combined
@@ -100,15 +101,11 @@ const AccountProviderMiddleware = HttpRouter.middleware<{ provides: AccountProvi · 42 unmodified lines
100	101	* (the seat-gate) stays a residual requirement, satisfied by the app `boot`.
101	102	*/
102	103	export const workosAccountMiddleware = (
103		− rsLive: Layer.Layer<
104		− DbService \| UserStoreService \| WorkOsMirror \| MemberDirectory \| MirrorReadiness
105		− >,
	104	+ rsLive: Layer.Layer<DbService \| UserStoreService \| WorkOsMirror \| MemberDirectory>,
106	105	) => AccountProviderMiddleware.combine(requestScopedMiddleware(rsLive)).layer;
107	106
108	107	export const makeAccountApiLive = (
109		− rsLive: Layer.Layer<
110		− DbService \| UserStoreService \| WorkOsMirror \| MemberDirectory \| MirrorReadiness
111		− >,
	108	+ rsLive: Layer.Layer<DbService \| UserStoreService \| WorkOsMirror \| MemberDirectory>,
112	109	) => {
113	110	// Cloud builds the WorkOS `AccountProvider` INSIDE the request body (so it
114	111	// closes over the per-request postgres socket), so it can't be a self-

apps/cloud/src/account/org-api-key-revoke.node.test.ts+0−8

@@ -6,7 +6,6 @@ import { AccountError, AccountForbidden } from "@executor-js/api"; · 5 unmodified lines
6	6
7	7	import { ApiKeyService, OrgApiKeyNotFound } from "../auth/api-keys";
8	8	import { UserStoreService } from "../auth/context";
9		−import { MirrorReadiness, MirrorReadinessState } from "../auth/mirror-readiness";
10	9	import { ORG_SELECTOR_HEADER } from "../auth/organization";
11	10	import { WorkOSClient, type WorkOSClientService } from "../auth/workos";
12	11	import { WorkOsMirror } from "../auth/workos-mirror";
@@ -132,12 +131,6 @@ const stubMirror = Layer.succeed(WorkOsMirror)({ · 119 unmodified lines
132	131	// The mirror as the directory reads it: both are active members of ORG, and
133	132	// only ADMIN carries the `admin` role. Revoke reads the caller's membership
134	133	// (the org check and the admin gate) and nothing else.
135		−// The mirror is READY in these tests (backfill complete, reconciler caught
136		−// up), so membership is read from the stubbed directory, never from WorkOS.
137		−const stubReadiness = Layer.succeed(MirrorReadiness)({
138		− state: () => Effect.succeed(MirrorReadinessState.Ready()),
139		−});
140		−
141	134	const stubDirectory = Layer.succeed(MemberDirectory)({
142	135	membership: (accountId, organizationId) =>
143	136	Effect.succeed(
@@ -201,7 +194,6 @@ const providerWith = (accountId: string) => { · 57 unmodified lines
201	194	stubUsers,
202	195	stubMirror,
203	196	stubDirectory,
204		− stubReadiness,
205	197	stubApiKeys,
206	198	stubAutumn,
207	199	Layer.succeed(AccountCaller)({ session: session(accountId) }),

apps/cloud/src/account/workos-account-service.ts+1−8

@@ -11,7 +11,6 @@ import { · 10 unmodified lines
11	11	import { ApiKeyService } from "../auth/api-keys";
12	12	import { UserStoreService } from "../auth/context";
13	13	import type { Session } from "../auth/middleware";
14		−import { MirrorReadiness } from "../auth/mirror-readiness";
15	14	import { WorkOSClient } from "../auth/workos";
16	15	import { ensureOrganizationBackfilled, mirrorInvitedMember } from "../auth/mirror-feeders";
17	16	import { WorkOsMirror, mirrorMembershipFromWorkOs } from "../auth/workos-mirror";
@@ -72,7 +71,6 @@ export const workosAccountProvider: Layer.Layer< · 54 unmodified lines
72	71	\| UserStoreService
73	72	\| WorkOsMirror
74	73	\| MemberDirectory
75		− \| MirrorReadiness
76	74	\| ApiKeyService
77	75	\| AutumnService
78	76	\| AccountCaller
@@ -103,12 +101,7 @@ export const workosAccountProvider: Layer.Layer< · 24 unmodified lines
103	101	// erased to `R = never`, as the neutral AccountProvider shape requires.
104	102	// Provided per method below.
105	103	const ctx = yield* Effect.context<
106		− \| WorkOSClient
107		− \| UserStoreService
108		− \| AutumnService
109		− \| MemberDirectory
110		− \| MirrorReadiness
111		− \| WorkOsMirror
	104	+ WorkOSClient \| UserStoreService \| AutumnService \| MemberDirectory \| WorkOsMirror
112	105	>();
113	106
114	107	// Unauthenticated (missing/invalid session) => AccountUnauthorized, exactly

apps/cloud/src/admin/admin-users-api.node.test.ts+6−16

@@ -6,7 +6,6 @@ import { MemberDirectory, type DirectoryMember } from "@executor-js/api/server"; · 5 unmodified lines
6	6
7	7	import { ApiKeyService } from "../auth/api-keys";
8	8	import { UserStoreService } from "../auth/context";
9		−import { MirrorReadiness, MirrorReadinessState } from "../auth/mirror-readiness";
10	9	import { ORG_SELECTOR_HEADER } from "../auth/organization";
11	10	import { WorkOSClient, type WorkOSClientService } from "../auth/workos";
12	11	import { WorkOsMirror, type WorkOsMirrorShape } from "../auth/workos-mirror";
@@ -49,12 +48,6 @@ const memberships = new Map<string, DirectoryMember>([ · 36 unmodified lines
49	48	["user_invited_admin", mirrored("user_invited_admin", { role: "admin", status: "pending" })],
50	49	]);
51	50
52		−// The mirror is READY in these tests (backfill complete, reconciler caught
53		−// up), so membership is read from the stubbed directory, never from WorkOS.
54		−const stubReadiness = Layer.succeed(MirrorReadiness)({
55		− state: () => Effect.succeed(MirrorReadinessState.Ready()),
56		−});
57		−
58	51	const stubDirectory = Layer.succeed(MemberDirectory)({
59	52	membership: (accountId, organizationId) =>
60	53	Effect.succeed(organizationId === ORG ? (memberships.get(accountId) ?? null) : null),
@@ -149,7 +142,11 @@ const stubWorkOS = (userId: string) => · 88 unmodified lines
149	142	get: (_target, prop) => {
150	143	if (prop === "authenticateRequest") {
151	144	return () =>
152		− Effect.succeed({ userId, email: `${userId}@placeholder.test`, organizationId: null });
	145	+ Effect.succeed({
	146	+ userId,
	147	+ email: `${userId}@placeholder.test`,
	148	+ organizationId: null,
	149	+ });
153	150	}
154	151	return () => Effect.die(`unexpected WorkOSClient.${String(prop)} call`);
155	152	},
@@ -163,14 +160,7 @@ const authorizeAs = (userId: string) => · 7 unmodified lines
163	160	}),
164	161	).pipe(
165	162	Effect.provide(
166		− Layer.mergeAll(
167		− stubDirectory,
168		− stubApiKeys,
169		− stubUsers,
170		− stubWorkOS(userId),
171		− stubMirror,
172		− stubReadiness,
173		− ),
	163	+ Layer.mergeAll(stubDirectory, stubApiKeys, stubUsers, stubWorkOS(userId), stubMirror),
174	164	),
175	165	);
176	166

apps/cloud/src/admin/admin-users-api.ts+5−9

@@ -56,7 +56,6 @@ import type { Executor } from "@executor-js/sdk"; · 55 unmodified lines
56	56
57	57	import { ApiKeyService } from "../auth/api-keys";
58	58	import { UserStoreService } from "../auth/context";
59		−import { MirrorReadiness } from "../auth/mirror-readiness";
60	59	import { WorkOsMirror } from "../auth/workos-mirror";
61	60	import { isPlatformAuth, resolveBearerAuth } from "../auth/workos-auth-provider";
62	61	import { orgSelectorFromRequest, authorizeOrganizationSelector } from "../auth/organization";
@@ -77,7 +76,7 @@ export const authorizeTenant = ( · 14 unmodified lines
77	76	): Effect.Effect<
78	77	string,
79	78	AdminUsersUnauthorized \| AdminUsersForbidden,
80		− WorkOSClient \| ApiKeyService \| UserStoreService \| MemberDirectory \| MirrorReadiness \| WorkOsMirror
	79	+ WorkOSClient \| ApiKeyService \| UserStoreService \| MemberDirectory \| WorkOsMirror
81	80	> =>
82	81	Effect.gen(function* () {
83	82	// (1) The bearer path. `resolveBearerAuth` (not `resolveApiKeyPrincipal`,
@@ -138,7 +137,6 @@ const withPlatformView = <A, E extends AdminUsersError \| AdminUserNotFound = Adm · 54 unmodified lines
138	137	\| ApiKeyService
139	138	\| UserStoreService
140	139	\| MemberDirectory
141		− \| MirrorReadiness
142	140	\| WorkOsMirror
143	141	\| DbProvider
144	142	\| PluginsProvider
@@ -178,7 +176,6 @@ export const workosAdminUsersProvider: Layer.Layer< · 33 unmodified lines
178	176	\| ApiKeyService
179	177	\| UserStoreService
180	178	\| MemberDirectory
181		− \| MirrorReadiness
182	179	\| WorkOsMirror
183	180	\| DbProvider
184	181	\| PluginsProvider
@@ -190,7 +187,6 @@ export const workosAdminUsersProvider: Layer.Layer< · 5 unmodified lines
190	187	\| ApiKeyService
191	188	\| UserStoreService
192	189	\| MemberDirectory
193		− \| MirrorReadiness
194	190	\| WorkOsMirror
195	191	\| DbProvider
196	192	\| PluginsProvider
@@ -241,7 +237,9 @@ export const workosAdminUsersProvider: Layer.Layer< · 44 unmodified lines
241	237	// per-request `DbService`/`UserStoreService`/`MemberDirectory` (and the
242	238	// execution seams built over them) are supplied by the combined
243	239	// `requestScopedMiddleware`.
244		−const AdminUsersProviderMiddleware = HttpRouter.middleware<{ provides: AdminUsersProvider }>()(
	240	+const AdminUsersProviderMiddleware = HttpRouter.middleware<{
	241	+ provides: AdminUsersProvider;
	242	+}>()(
245	243	Effect.gen(function* () {
246	244	const longLived = yield* Effect.context<WorkOSClient \| ApiKeyService>();
247	245	return (httpEffect) =>
@@ -265,9 +263,7 @@ const AdminUsersProviderMiddleware = HttpRouter.middleware<{ provides: AdminUser · 17 unmodified lines
265	263	* `/api` prefix as the rest of the cloud router.
266	264	*/
267	265	export const makeCloudAdminUsersRoutes = (
268		− rsLive: Layer.Layer<
269		− DbService \| UserStoreService \| MemberDirectory \| MirrorReadiness \| WorkOsMirror
270		− >,
	266	+ rsLive: Layer.Layer<DbService \| UserStoreService \| MemberDirectory \| WorkOsMirror>,
271	267	options: Parameters<typeof makeAdminUsersApiLayer>[1] = {},
272	268	) =>
273	269	makeAdminUsersApiLayer(

apps/cloud/src/api/layers.ts+7−19

@@ -11,7 +11,6 @@ import { · 10 unmodified lines
11	11	import { SessionAuthLive } from "../auth/middleware-live";
12	12	import { UserStoreService } from "../auth/context";
13	13	import { cloudMemberDirectoryLayer } from "../auth/member-directory";
14		−import { MirrorReadiness } from "../auth/mirror-readiness";
15	14	import { WorkOsMirror } from "../auth/workos-mirror";
16	15	import {
17	16	CloudAuthPublicHandlers,
@@ -36,23 +35,14 @@ const WorkOsMirrorLive = WorkOsMirror.Live.pipe(Layer.provide(DbLive)); · 18 unmodified lines
36	35	// The shared `MemberDirectory` read seam over the membership mirror — the
37	36	// same per-request socket the mirror writes through.
38	37	const MemberDirectoryLive = cloudMemberDirectoryLayer.pipe(Layer.provide(DbLive));
39		−// Whether the mirror may authorize this request at all (backfill complete,
40		−// reconciler caught up) — read on the same socket before the membership row.
41		−const MirrorReadinessLive = MirrorReadiness.Live.pipe(Layer.provide(DbLive));
42	38
43	39	// Per-request layer. Anything that opens an I/O object (postgres.js socket,
44	40	// fetch stream readers, anything backed by a `Writable`) MUST live here —
45	41	// `provideRequestScoped` rebuilds it per request so Cloudflare Workers'
46	42	// I/O isolation is satisfied. See `api.request-scope.test.ts`.
47	43	export const RequestScopedServicesLive: Layer.Layer<
48		− DbService \| UserStoreService \| WorkOsMirror \| MemberDirectory \| MirrorReadiness
49		−> = Layer.mergeAll(
50		− DbLive,
51		− UserStoreLive,
52		− WorkOsMirrorLive,
53		− MemberDirectoryLive,
54		− MirrorReadinessLive,
55		−);
	44	+ DbService \| UserStoreService \| WorkOsMirror \| MemberDirectory
	45	+> = Layer.mergeAll(DbLive, UserStoreLive, WorkOsMirrorLive, MemberDirectoryLive);
56	46
57	47	// Boot-scoped layer. Built once at worker boot, reused across requests.
58	48	// Safe for config, in-memory caches, the global tracer provider, and
@@ -77,9 +67,7 @@ export const BootSharedServices = Layer.mergeAll( · 18 unmodified lines
77	67	// handler reads it for the free-organizations-per-user limit gate — one of the
78	68	// few app-only billing touchpoints. (It is NOT on the neutral boot core.)
79	69	export const makeNonProtectedApiLive = (
80		− rsLive: Layer.Layer<
81		− DbService \| UserStoreService \| WorkOsMirror \| MemberDirectory \| MirrorReadiness
82		− >,
	70	+ rsLive: Layer.Layer<DbService \| UserStoreService \| WorkOsMirror \| MemberDirectory>,
83	71	) =>
84	72	HttpApiBuilder.layer(NonProtectedApi).pipe(
85	73	Layer.provide(Layer.mergeAll(CloudAuthPublicHandlers, CloudSessionAuthHandlers)),
@@ -95,9 +83,7 @@ export const makeNonProtectedApiLive = ( · 9 unmodified lines
95	83	// gates on billing, so `AutumnService.Default` is provided here, not on the
96	84	// neutral boot core.
97	85	export const makeOrgApiLive = (
98		− rsLive: Layer.Layer<
99		− DbService \| UserStoreService \| MemberDirectory \| MirrorReadiness \| WorkOsMirror
100		− >,
	86	+ rsLive: Layer.Layer<DbService \| UserStoreService \| MemberDirectory \| WorkOsMirror>,
101	87	) =>
102	88	HttpApiBuilder.layer(OrgHttpApi).pipe(
103	89	Layer.provide(OrgHandlers),
@@ -143,7 +129,9 @@ export const OrgApiLive = makeOrgApiLive(RequestScopedServicesLive); · 39 unmodified lines
143	129	// folded into `.layer` here; the rest of the router (`makeApiLive` in
144	130	// `./router.ts`, `./protected.ts`, the test harness) re-provides the same
145	131	// shared `RouterConfigLive` directly.
146		−const protectedApi = makeProtectedApiLayer(cloudPlugins, { errorCapture: ErrorCaptureLive });
	132	+const protectedApi = makeProtectedApiLayer(cloudPlugins, {
	133	+ errorCapture: ErrorCaptureLive,
	134	+});
147	135
148	136	export const ProtectedCloudApi = protectedApi.api;
149	137	export const ProtectedCloudApiHandlers = protectedApi.handlers;

apps/cloud/src/api/protected-api-key-auth.node.test.ts+3−11

@@ -5,7 +5,6 @@ import { MemberDirectory } from "@executor-js/api/server"; · 4 unmodified lines
5	5
6	6	import { ApiKeyService } from "../auth/api-keys";
7	7	import { UserStoreService } from "../auth/context";
8		−import { MirrorReadiness, MirrorReadinessState } from "../auth/mirror-readiness";
9	8	import { WorkOSClient, type WorkOSClientService } from "../auth/workos";
10	9	import { WorkOsMirror, type WorkOsMirrorShape } from "../auth/workos-mirror";
11	10	import { resolveProtectedPrincipal } from "./protected";
@@ -55,13 +54,8 @@ const stubWorkOS = Layer.succeed( · 43 unmodified lines
55	54	);
56	55
57	56	// The mirror as the directory reads it: user_123 holds an active membership in
58		−// org_123 and nothing else. Membership is never read from WorkOS.
59		−// The mirror is READY in these tests (backfill complete, reconciler caught
60		−// up), so membership is read from the stubbed directory, never from WorkOS.
61		−const stubReadiness = Layer.succeed(MirrorReadiness)({
62		− state: () => Effect.succeed(MirrorReadinessState.Ready()),
63		−});
64		−
	57	+// org_123 and nothing else. Membership is always read from the mirror, never
	58	+// from WorkOS.
65	59	const stubDirectory = Layer.succeed(MemberDirectory)({
66	60	membership: (accountId, organizationId) =>
67	61	Effect.succeed(
@@ -136,9 +130,7 @@ const stubMirror = Layer.succeed( · 68 unmodified lines
136	130
137	131	const run = (request: Request) =>
138	132	resolveProtectedPrincipal(request).pipe(
139		− Effect.provide(
140		− Layer.mergeAll(stubApiKeys, stubWorkOS, stubUsers, stubDirectory, stubMirror, stubReadiness),
141		− ),
	133	+ Effect.provide(Layer.mergeAll(stubApiKeys, stubWorkOS, stubUsers, stubDirectory, stubMirror)),
142	134	);
143	135
144	136	describe("protected API key auth", () => {

apps/cloud/src/api/protected-jwt-auth.node.test.ts+3−11

@@ -6,7 +6,6 @@ import { MemberDirectory } from "@executor-js/api/server"; · 5 unmodified lines
6	6
7	7	import { ApiKeyService } from "../auth/api-keys";
8	8	import { UserStoreService } from "../auth/context";
9		−import { MirrorReadiness, MirrorReadinessState } from "../auth/mirror-readiness";
10	9	import type { JwtBearerConfig } from "../auth/workos-auth-provider";
11	10	import { WorkOSClient, type WorkOSClientService } from "../auth/workos";
12	11	import { WorkOsMirror, type WorkOsMirrorShape } from "../auth/workos-mirror";
@@ -72,13 +71,8 @@ const stubWorkOS = Layer.succeed( · 59 unmodified lines
72	71	);
73	72
74	73	// The mirror as the directory reads it: user_123 holds an active membership in
75		−// org_123 and nothing else. Membership is never read from WorkOS.
76		−// The mirror is READY in these tests (backfill complete, reconciler caught
77		−// up), so membership is read from the stubbed directory, never from WorkOS.
78		−const stubReadiness = Layer.succeed(MirrorReadiness)({
79		− state: () => Effect.succeed(MirrorReadinessState.Ready()),
80		−});
81		−
	74	+// org_123 and nothing else. Membership is always read from the mirror, never
	75	+// from WorkOS.
82	76	const stubDirectory = Layer.succeed(MemberDirectory)({
83	77	membership: (accountId, organizationId) =>
84	78	Effect.succeed(
@@ -153,9 +147,7 @@ const stubMirror = Layer.succeed( · 68 unmodified lines
153	147
154	148	const run = (request: Request, jwt: JwtBearerConfig) =>
155	149	resolveProtectedPrincipal(request, jwt).pipe(
156		− Effect.provide(
157		− Layer.mergeAll(stubApiKeys, stubWorkOS, stubUsers, stubDirectory, stubMirror, stubReadiness),
158		− ),
	150	+ Effect.provide(Layer.mergeAll(stubApiKeys, stubWorkOS, stubUsers, stubDirectory, stubMirror)),
159	151	);
160	152
161	153	const request = (token: string) =>

apps/cloud/src/api/protected.ts+1−4

@@ -16,7 +16,6 @@ import { · 15 unmodified lines
16	16	import { cloudPlugins, type CloudPlugins } from "../plugins";
17	17	import { ApiKeyService } from "../auth/api-keys";
18	18	import { UserStoreService } from "../auth/context";
19		−import { MirrorReadiness } from "../auth/mirror-readiness";
20	19	import { WorkOsMirror } from "../auth/workos-mirror";
21	20	import { cloudIdentityFailureStrategy, workosIdentityLayer } from "../auth/workos-auth-provider";
22	21	import { AutumnService } from "../extensions/billing/service";
@@ -97,9 +96,7 @@ const ExecutionStackMiddleware = makeExecutionStackMiddleware< · 74 unmodified lines
97	96	// account seat-gate, and the createOrganization free-limit gate each provide it
98	97	// where they run.)
99	98	export const makeProtectedApiLive = (
100		− rsLive: Layer.Layer<
101		− DbService \| UserStoreService \| MemberDirectory \| MirrorReadiness \| WorkOsMirror
102		− >,
	99	+ rsLive: Layer.Layer<DbService \| UserStoreService \| MemberDirectory \| WorkOsMirror>,
103	100	) => {
104	101	// The neutral `IdentityProvider`, built per request: it reads `UserStoreService`
105	102	// + `MemberDirectory` from `rsLive` and the WorkOS control plane (`WorkOSClient` + `ApiKeyService`,

apps/cloud/src/api/router.ts+1−4

@@ -8,7 +8,6 @@ import { · 7 unmodified lines
8	8	} from "@executor-js/api/server";
9	9
10	10	import { UserStoreService } from "../auth/context";
11		−import { MirrorReadiness } from "../auth/mirror-readiness";
12	11	import { WorkOsMirror } from "../auth/workos-mirror";
13	12	import { DbService } from "../db/db";
14	13	import { makeAccountApiLive } from "../account/account-api";
@@ -36,9 +35,7 @@ import { makeProtectedApiLive } from "./protected"; · 21 unmodified lines
36	35	// assert per-request semantics — see
37	36	// `apps/cloud/src/api.request-scope.node.test.ts`.
38	37	export const makeApiLive = (
39		− requestScopedLive: Layer.Layer<
40		− DbService \| UserStoreService \| WorkOsMirror \| MemberDirectory \| MirrorReadiness
41		− >,
	38	+ requestScopedLive: Layer.Layer<DbService \| UserStoreService \| WorkOsMirror \| MemberDirectory>,
42	39	) => {
43	40	const BillingRoutesLive = AutumnRoutesLive.pipe(
44	41	Layer.provide(requestScopedMiddleware(requestScopedLive).layer),

apps/cloud/src/auth/doc-gate.ts+3−3

@@ -42,7 +42,6 @@ import { makeUserStoreLayer, UserStoreService } from "./context"; · 41 unmodified lines
42	42	import { parseCookie } from "./cookies";
43	43	import { LAST_ORG_COOKIE } from "./last-org-cookie";
44	44	import { makeMemberDirectoryLayer } from "./member-directory";
45		−import { makeMirrorReadinessLayer } from "./mirror-readiness";
46	45	import { makeWorkOsMirrorLayer } from "./workos-mirror";
47	46	import { sealedSessionDisplayName } from "./middleware";
48	47	import { authorizeOrganizationSelector } from "./organization";
@@ -184,7 +183,6 @@ const authorizeLastOrgSlug = async ( · 135 unmodified lines
184	183	Layer.mergeAll(
185	184	makeUserStoreLayer(),
186	185	makeMemberDirectoryLayer(),
187		− makeMirrorReadinessLayer(),
188	186	makeWorkOsMirrorLayer(),
189	187	).pipe(Layer.provide(dbLive)),
190	188	),
@@ -258,7 +256,9 @@ export const authGateMiddleware = createMiddleware({ type: "request" }).server( · 67 unmodified lines
258	256	// the client AuthGate makes mid-session, made here before the document
259	257	// exists so the app shell is never painted for an org-less session.
260	258	if (!session.organizationId && !ONBOARDING_PATHS.has(pathname)) {
261		− return redirect("/create-org", { refreshedSession: session.refreshedSession });
	259	+ return redirect("/create-org", {
	260	+ refreshedSession: session.refreshedSession,
	261	+ });
262	262	}
263	263
264	264	// A BARE console path (no org slug in the URL) canonicalizes onto the org

apps/cloud/src/auth/mirror-feeders.node.test.ts+50−43

@@ -87,7 +87,6 @@ import { encodeLoginState } from "./login-state"; · 86 unmodified lines
87	87	import { cloudMemberDirectoryLayer } from "./member-directory";
88	88	import { SessionAuthLive } from "./middleware-live";
89	89	import { mirrorSignIn } from "./mirror-feeders";
90		−import { MirrorReadiness, MirrorReadinessState } from "./mirror-readiness";
91	90	import {
92	91	ORG_SELECTOR_HEADER,
93	92	authorizeOrganization,
@@ -172,14 +171,6 @@ const seedOrganization = (id: string) => · 78 unmodified lines
172	171	),
173	172	);
174	173
175		−// The mirror is READY throughout (backfill complete, reconciler caught up):
176		−// every membership read below is against the mirror, never WorkOS. The
177		−// readiness rule itself is pinned in workos-mirror.node.test.ts and the
178		−// fallback in org-selector-auth.node.test.ts.
179		−const readyMirror = Layer.succeed(MirrorReadiness)({
180		− state: () => Effect.succeed(MirrorReadinessState.Ready()),
181		−});
182		−
183	174	const stubAutumn = Layer.succeed(AutumnService)({
184	175	use: () => Effect.die("feeders do not read billing"),
185	176	ensureCustomer: () => Effect.void,
@@ -500,7 +491,12 @@ describe("a delayed sign-in feeder", () => { · 314 unmodified lines
500	491	const rejoinedAt = "2026-01-03T00:00:00.000Z";
501	492	yield* mirrorSignIn(
502	493	workosUser(userId),
503		− [workosMembership(userId, org, { id: `om_${userId}_${org}_2`, updatedAt: rejoinedAt })],
	494	+ [
	495	+ workosMembership(userId, org, {
	496	+ id: `om_${userId}_${org}_2`,
	497	+ updatedAt: rejoinedAt,
	498	+ }),
	499	+ ],
504	500	new Date(rejoinedAt),
505	501	);
506	502	return { membership, rejoined: yield* readMembership(userId, org) };
@@ -581,20 +577,14 @@ describe("session handlers read membership from the mirror", () => { · 74 unmodified lines
581	577	DbService \| UserStoreService \| WorkOsMirror \| MemberDirectory
582	578	>;
583	579	readonly autumn?: Layer.Layer<AutumnService>;
584		− /** The mirror's readiness for this request; ready unless a test says otherwise. */
585		− readonly readiness?: Layer.Layer<MirrorReadiness>;
586	580	} = {},
587	581	) =>
588	582	HttpRouter.toWebHandler(
589	583	HttpApiBuilder.layer(NonProtectedApi).pipe(
590	584	Layer.provide(Layer.mergeAll(CloudAuthPublicHandlers, CloudSessionAuthHandlers)),
591	585	Layer.provide(
592		− requestScopedMiddleware(
593		− Layer.mergeAll(
594		− options.services ?? RequestScopedServicesLive,
595		− options.readiness ?? readyMirror,
596		− ),
597		− ).layer,
	586	+ requestScopedMiddleware(Layer.mergeAll(options.services ?? RequestScopedServicesLive))
	587	+ .layer,
598	588	),
599	589	Layer.provideMerge(SessionAuthLive),
600	590	Layer.provideMerge(options.autumn ?? stubAutumn),
@@ -627,9 +617,10 @@ describe("session handlers read membership from the mirror", () => { · 26 unmodified lines
627	617	);
628	618
629	619	/**
630		− * `authorizeOrganization` over the live stores and a READY mirror, as every
631		− * protected request runs it; `workos` serves whatever the check may read
632		− * from WorkOS (nothing, by default: any read dies).
	620	+ * `authorizeOrganization` over the live stores, as every protected request
	621	+ * runs it: membership is read from the mirror unconditionally; `workos`
	622	+ * serves whatever the check may read from WorkOS (nothing, by default: any
	623	+ * read dies).
633	624	*/
634	625	const authorize = (
635	626	userId: string,
@@ -639,12 +630,9 @@ describe("session handlers read membership from the mirror", () => { · 3 unmodified lines
639	630	Effect.runPromise(
640	631	authorizeOrganization(userId, org).pipe(
641	632	Effect.provide(
642		− Layer.mergeAll(
643		− UserStoreService.Live,
644		− WorkOsMirror.Live,
645		− cloudMemberDirectoryLayer,
646		− readyMirror,
647		− ).pipe(Layer.provideMerge(DbService.Live)),
	633	+ Layer.mergeAll(UserStoreService.Live, WorkOsMirror.Live, cloudMemberDirectoryLayer).pipe(
	634	+ Layer.provideMerge(DbService.Live),
	635	+ ),
648	636	),
649	637	Effect.provide(workos),
650	638	Effect.scoped,
@@ -665,7 +653,12 @@ describe("session handlers read membership from the mirror", () => { · 14 unmodified lines
665	653	purges.push(op);
666	654	}).pipe(
667	655	Effect.flatMap(() =>
668		− Effect.fail(new UserStoreError({ operation: op, reason: "connection_closed" })),
	656	+ Effect.fail(
	657	+ new UserStoreError({
	658	+ operation: op,
	659	+ reason: "connection_closed",
	660	+ }),
	661	+ ),
669	662	),
670	663	)
671	664	: live.use(op, fn),
@@ -914,7 +907,7 @@ describe("session handlers read membership from the mirror", () => { · 242 unmodified lines
914	907	expect(await authorized(admin, org)).toBe(false);
915	908	});
916	909
917		− it("finishes on a retry while the mirror is not ready, after WorkOS already deleted the org", async () => {
	910	+ it("finishes on a retry after WorkOS already deleted the org", async () => {
918	911	const admin = freshId("user");
919	912	const member = freshId("user");
920	913	const org = freshId("org");
@@ -931,19 +924,17 @@ describe("session handlers read membership from the mirror", () => { · 10 unmodified lines
931	924	expect(first.status).toBe(500);
932	925	expect(purges).toEqual(["deleteOrganizationCascade"]);
933	926
934		− // The reconciler stalls before the retry. WorkOS no longer lists the org
935		− // or the admin's membership in it — and the fallback must not ask it:
936		− // the stub dies on `listUserMemberships`. The admin's own mirror row,
937		− // which the failed purge left behind, is what admits the retry.
	927	+ // WorkOS no longer has the org to delete a second time. The admin's own
	928	+ // mirror row, which the failed purge left behind, is what admits the
	929	+ // retry — membership is read from the mirror unconditionally.
938	930	const retry = sessionHandler(admin, {
939	931	autumn: deletingAutumn,
940		− readiness: Layer.succeed(MirrorReadiness)({
941		− state: () => Effect.succeed(MirrorReadinessState.ReconcilerStale({ drainedAt: null })),
942		− }),
943		− workos: { deleteOrganization: () => Effect.fail(new WorkOSError({ status: 404 })) },
	932	+ workos: {
	933	+ deleteOrganization: () => Effect.fail(new WorkOSError({ status: 404 })),
	934	+ },
944	935	});
945	936	const second = await retry(deleteOrganizationRequest(org));
946		− expect(second.status, "the retry is admitted from the mirror, not WorkOS").toBe(200);
	937	+ expect(second.status, "the retry is admitted from the mirror").toBe(200);
947	938	expect(await second.json()).toEqual({ success: true });
948	939	expect(await readMembers(org), "and the purge ran").toEqual([]);
949	940	expect((await readOrganization(org))?.deletedAt).not.toBeNull();
@@ -961,7 +952,9 @@ describe("session handlers read membership from the mirror", () => { · 11 unmodified lines
961	952	calls.push(`getUserOrgMembership:${userId}`);
962	953	return Effect.succeed(
963	954	userId === memberId
964		− ? (workosMembership(userId, organizationId, { role: { slug: "admin" } }) as never)
	955	+ ? (workosMembership(userId, organizationId, {
	956	+ role: { slug: "admin" },
	957	+ }) as never)
965	958	: null,
966	959	);
967	960	},
@@ -1089,7 +1082,9 @@ describe("session handlers read membership from the mirror", () => { · 121 unmodified lines
1089	1082	);
1090	1083
1091	1084	expect(response.status).toBe(200);
1092		− const body = (await response.json()) as { organizations: { id: string; slug: string }[] };
	1085	+ const body = (await response.json()) as {
	1086	+ organizations: { id: string; slug: string }[];
	1087	+ };
1093	1088	expect(body.organizations.map((o) => [o.id, o.slug])).toEqual([[live, liveSlug]]);
1094	1089	});
1095	1090	});
@@ -1156,7 +1151,6 @@ describe("account service writes through to the mirror", () => { · 60 unmodified lines
1156	1151	UserStoreService.Live,
1157	1152	WorkOsMirror.Live,
1158	1153	cloudMemberDirectoryLayer,
1159		− readyMirror,
1160	1154	);
1161	1155	return workosAccountProvider.pipe(
1162	1156	Layer.provide(
@@ -1778,7 +1772,17 @@ describe("backfill", () => { · 615 unmodified lines
1778	1772	);
1779	1773
1780	1774	const counts = await runBackfill(
1781		− new Map([[org, [workosMembership(paused, org, { status: "inactive", updatedAt: T2 })]]]),
	1775	+ new Map([
	1776	+ [
	1777	+ org,
	1778	+ [
	1779	+ workosMembership(paused, org, {
	1780	+ status: "inactive",
	1781	+ updatedAt: T2,
	1782	+ }),
	1783	+ ],
	1784	+ ],
	1785	+ ]),
1782	1786	false,
1783	1787	);
1784	1788	expect(counts, "the deactivated membership is written, not tombstoned").toMatchObject({
@@ -1904,7 +1908,10 @@ describe("backfill", () => { · 119 unmodified lines
1904	1908	[orgB, [workosMembership(staying, orgB)]],
1905	1909	]);
1906	1910	const retried = await runBackfill(attemptB, false);
1907		− expect(retried).toMatchObject({ organizations: 2, membershipsTombstoned: 1 });
	1911	+ expect(retried).toMatchObject({
	1912	+ organizations: 2,
	1913	+ membershipsTombstoned: 1,
	1914	+ });
1908	1915	expect(
1909	1916	await syncState(),
1910	1917	"the retry keeps the first attempt's boundary instead of taking a later one",

apps/cloud/src/auth/mirror-readiness-store.ts+17−20

@@ -1,33 +1,30 @@
1	1	// ---------------------------------------------------------------------------
2		−// Mirror READINESS: whether the local membership mirror may be trusted as
3		−// the membership authority for a request, or WorkOS must still be asked.
	2	+// Mirror READINESS: whether the local membership mirror has ever been fit to
	3	+// authorize from, per the ORIGINAL cutover rule. The request path
	4	+// (`organization.ts`) no longer consults this — it authorizes from the
	5	+// mirror unconditionally, because the one-off backfill is complete and
	6	+// permanent and a pre-mirror organization is covered by the on-demand scan
	7	+// (`ensureOrganizationBackfilled`). What remains is the deploy gate
	8	+// (`scripts/ensure-workos-mirror-ready.ts`), which still refuses to ship a
	9	+// build that trusts the mirror while it is unready, and the reconciler's own
	10	+// staleness alert (`workos-events-runner.ts`), which reads `drainedAt` after
	11	+// each run and raises a Sentry error when the drain has fallen behind the lag
	12	+// budget below — a stalled reconciler is now an operational page, not a
	13	+// per-request fallback.
4	14	//
5		−// The mirror is fed by login, write-through, and the Events API reconciler
6		−// (`workos-mirror.ts`), and is complete only once the one-off backfill has
7		−// written every organization and the reconciler has caught up to the
8		−// present. Before that, two things go wrong if it is trusted anyway:
9		−// - a member who has not signed in since the mirror shipped has no row
10		−// yet, and every protected request of theirs is refused — the backfill
11		−// is what writes them;
12		−// - a member revoked in the WorkOS dashboard while the reconciler was not
13		−// running still holds an active row, and keeps their access until the
14		−// stream is replayed — the reconciler is what tombstones them.
15		−// So readiness is BOTH: the backfill's completion mark
	15	+// Readiness is BOTH: the backfill's completion mark
16	16	// (`workos_sync.backfill_completed_at`, written once by a run that covered
17	17	// every live organization) AND a recent drain of the events stream
18	18	// (`workos_sync.drained_at`, moved forward by every reconciler run that read
19	19	// the stream to its end). The lag budget bounds how far behind the reconciler
20	20	// may be: it runs every minute, so a mark older than the budget means it has
21	21	// stalled (WorkOS unreachable, the cron not deployed, a backlog draining over
22		−// many runs) and the mirror may be missing revocations. While either half is
23		−// missing the authorization path reads membership from WorkOS instead
24		−// (`organization.ts`), exactly as it did before the cutover; nothing is
25		−// denied or granted on the mirror's word.
	22	+// many runs) and the mirror may be missing revocations.
26	23	//
27	24	// The rule and the row read live here, free of `cloudflare:workers`, so the
28		−// deploy gate (`scripts/ensure-workos-mirror-ready.ts`) applies the SAME rule
29		−// over a plain postgres.js connection under bun before the build that trusts
30		−// the mirror goes live. The request-scoped service is `mirror-readiness.ts`.
	25	+// deploy gate applies the SAME rule over a plain postgres.js connection under
	26	+// bun before a build goes live, and the reconciler's alert applies the SAME
	27	+// `drainedAt` age check the gate does.
31	28	// ---------------------------------------------------------------------------
32	29
33	30	import { eq } from "drizzle-orm";

apps/cloud/src/auth/mirror-readiness.ts+0−69

@@ -1,69 +0,0 @@
1		−// ---------------------------------------------------------------------------
2		−// MirrorReadiness — the request-scoped service that answers whether the
3		−// membership mirror may authorize this request (see
4		−// `mirror-readiness-store.ts` for the rule and why it exists).
5		−//
6		−// Per-request layer shape, like `UserStoreService` and `WorkOsMirror`: it
7		−// reads the request's postgres socket, so it is rebuilt per request
8		−// (`RequestScopedServicesLive`) and never shared across Workers requests. One
9		−// indexed point read per authorization, on the same socket the membership
10		−// read uses next.
11		−// ---------------------------------------------------------------------------
12		−
13		−import { Clock, Context, Effect, Layer } from "effect";
14		−
15		−import { DbService, type DrizzleDb } from "../db/db";
16		−import {
17		− WorkOsMirrorError,
18		− tryPromiseService,
19		− userStoreReasonFromCause,
20		− withServiceLogging,
21		−} from "./errors";
22		−import { readMirrorReadiness, type MirrorReadinessState } from "./mirror-readiness-store";
23		−
24		−export {
25		− MIRROR_RECONCILER_LAG_BUDGET,
26		− MirrorReadinessState,
27		− describeMirrorReadiness,
28		− mirrorReadinessFrom,
29		− type MirrorReadinessRow,
30		−} from "./mirror-readiness-store";
31		−
32		−export interface MirrorReadinessShape {
33		− /**
34		− * The mirror's readiness as of now. Fails with `WorkOsMirrorError` when the
35		− * row cannot be read — the caller must not treat that as either ready or
36		− * not; it is the same infra failure as any other mirror read.
37		− */
38		− readonly state: () => Effect.Effect<MirrorReadinessState, WorkOsMirrorError>;
39		−}
40		−
41		−const makeService = (db: DrizzleDb): MirrorReadinessShape => ({
42		− state: () =>
43		− Effect.flatMap(Clock.currentTimeMillis, (millis) =>
44		− withServiceLogging(
45		− "workos_mirror.readiness",
46		− (failure) =>
47		− new WorkOsMirrorError({
48		− operation: "readiness",
49		− reason: userStoreReasonFromCause(failure),
50		− }),
51		− tryPromiseService(() => readMirrorReadiness(db, new Date(millis))),
52		− ),
53		− ),
54		−});
55		−
56		−export class MirrorReadiness extends Context.Service<MirrorReadiness, MirrorReadinessShape>()(
57		− "@executor-js/cloud/MirrorReadiness",
58		−) {
59		− static Live = Layer.effect(this)(Effect.map(DbService.asEffect(), ({ db }) => makeService(db)));
60		−}
61		−
62		−/**
63		− * A FRESH `MirrorReadiness` layer (new layer value per call), for a service
64		− * built once but invoked across many Workers requests — the MCP
65		− * org-authorization seam and the document gate — for the same reason
66		− * `makeUserStoreLayer` exists. See [[makeDbLayer]].
67		− */
68		−export const makeMirrorReadinessLayer = (): Layer.Layer<MirrorReadiness, never, DbService> =>
69		− Layer.effect(MirrorReadiness)(Effect.map(DbService.asEffect(), ({ db }) => makeService(db)));

apps/cloud/src/auth/org-api-key-auth.node.test.ts+5−32

@@ -5,7 +5,6 @@ import { MemberDirectory, NoOrganization } from "@executor-js/api/server"; · 4 unmodified lines
5	5
6	6	import { ApiKeyService } from "./api-keys";
7	7	import { UserStoreService } from "./context";
8		−import { MirrorReadiness, MirrorReadinessState } from "./mirror-readiness";
9	8	import { WorkOSClient, type WorkOSClientService } from "./workos";
10	9	import { WorkOsMirror, type WorkOsMirrorShape } from "./workos-mirror";
11	10	import { isPlatformAuth, resolveApiKeyPrincipal, resolveBearerAuth } from "./workos-auth-provider";
@@ -70,13 +69,8 @@ const stubWorkOS = Layer.succeed( · 58 unmodified lines
70	69	);
71	70
72	71	// The mirror as the directory reads it: user_123 holds an active membership in
73		−// org_123 and nothing else. Membership is never read from WorkOS.
74		−// The mirror is READY in these tests (backfill complete, reconciler caught
75		−// up), so membership is read from the stubbed directory, never from WorkOS.
76		−const stubReadiness = Layer.succeed(MirrorReadiness)({
77		− state: () => Effect.succeed(MirrorReadinessState.Ready()),
78		−});
79		−
	72	+// org_123 and nothing else. Membership is always read from the mirror, never
	73	+// from WorkOS.
80	74	const stubDirectory = Layer.succeed(MemberDirectory)({
81	75	membership: (accountId, organizationId) =>
82	76	Effect.succeed(
@@ -149,14 +143,7 @@ const stubMirror = Layer.succeed( · 66 unmodified lines
149	143	}),
150	144	);
151	145
152		−const layers = Layer.mergeAll(
153		− stubApiKeys,
154		− stubWorkOS,
155		− stubUsers,
156		− stubDirectory,
157		− stubMirror,
158		− stubReadiness,
159		−);
	146	+const layers = Layer.mergeAll(stubApiKeys, stubWorkOS, stubUsers, stubDirectory, stubMirror);
160	147
161	148	const bearer = (token: string) =>
162	149	new Request("https://executor.test/api/tools", {
@@ -224,14 +211,7 @@ describe("org-level API keys", () => { · 61 unmodified lines
224	211	const exit = yield* Effect.exit(
225	212	resolveBearerAuth(bearer("valid_org_key")).pipe(
226	213	Effect.provide(
227		− Layer.mergeAll(
228		− stubApiKeys,
229		− stubWorkOS,
230		− deletedOrgUsers,
231		− stubDirectory,
232		− stubMirror,
233		− stubReadiness,
234		− ),
	214	+ Layer.mergeAll(stubApiKeys, stubWorkOS, deletedOrgUsers, stubDirectory, stubMirror),
235	215	),
236	216	),
237	217	);
@@ -292,14 +272,7 @@ describe("org-level API keys", () => { · 54 unmodified lines
292	272	});
293	273	const auth = yield* resolveBearerAuth(bearer("valid_org_key")).pipe(
294	274	Effect.provide(
295		− Layer.mergeAll(
296		− stubApiKeys,
297		− stubWorkOS,
298		− stubUsers,
299		− noMembershipReads,
300		− stubMirror,
301		− stubReadiness,
302		− ),
	275	+ Layer.mergeAll(stubApiKeys, stubWorkOS, stubUsers, noMembershipReads, stubMirror),
303	276	),
304	277	);
305	278

apps/cloud/src/auth/org-selector-auth.node.test.ts+31−166

@@ -6,7 +6,6 @@ import { MemberDirectory, type DirectoryMember } from "@executor-js/api/server"; · 5 unmodified lines
6	6
7	7	import { ApiKeyService } from "./api-keys";
8	8	import { UserStoreService } from "./context";
9		−import { MirrorReadiness, MirrorReadinessState } from "./mirror-readiness";
10	9	import { AUTHORIZE_ORGANIZATION_SPAN } from "./organization";
11	10	import { resolveSessionPrincipal } from "./workos-auth-provider";
12	11	import { WorkOSClient, type WorkOSClientService } from "./workos";
@@ -68,12 +67,6 @@ const memberships = new Map<string, DirectoryMember>([ · 55 unmodified lines
68	67	[PENDING_ORG, mirrored(PENDING_ORG, { status: "pending" })],
69	68	]);
70	69
71		−// The mirror is READY in these tests (backfill complete, reconciler caught
72		−// up), so membership is read from the stubbed directory, never from WorkOS.
73		−const stubReadiness = Layer.succeed(MirrorReadiness)({
74		− state: () => Effect.succeed(MirrorReadinessState.Ready()),
75		−});
76		−
77	70	const stubDirectory = Layer.succeed(MemberDirectory)({
78	71	membership: (accountId, organizationId) =>
79	72	Effect.succeed(accountId === MEMBER ? (memberships.get(organizationId) ?? null) : null),
@@ -167,85 +160,28 @@ const stubMirror = Layer.succeed( · 87 unmodified lines
167	160	}),
168	161	);
169	162
170		−const run = (
171		− headers: Record<string, string>,
172		− readiness: Layer.Layer<MirrorReadiness> = stubReadiness,
173		−) =>
	163	+const run = (headers: Record<string, string>) =>
174	164	resolveSessionPrincipal(new Request("https://executor.test/api/tools", { headers })).pipe(
175		− Effect.provide(
176		− Layer.mergeAll(stubApiKeys, stubWorkOS, stubUsers, stubDirectory, stubMirror, readiness),
177		− ),
	165	+ Effect.provide(Layer.mergeAll(stubApiKeys, stubWorkOS, stubUsers, stubDirectory, stubMirror)),
178	166	);
179	167
180		−// The mirror before the cutover has landed: the backfill has not covered
181		−// every organization, or the reconciler has not drained recently.
182		−const unreadyMirror = (state: MirrorReadinessState) =>
183		− Layer.succeed(MirrorReadiness)({ state: () => Effect.succeed(state) });
184		−
185	168	/**
186		− * A WorkOS that answers the pre-cutover membership list for MEMBER — active
187		− * in SESSION_ORG only, as a member — and records the calls, so the fallback
188		− * is assertable: with the mirror unready the list is read from WorkOS and
189		− * the directory (which says MEMBER is active in URL_ORG too) is never asked.
190		− */
191		−const workosMemberships = (calls: string[]) =>
192		− Layer.succeed(
193		− WorkOSClient,
194		− new Proxy({} as WorkOSClientService, {
195		− get: (_t, prop) => {
196		− if (prop === "authenticateRequest") {
197		− return () =>
198		− Effect.succeed({
199		− userId: MEMBER,
200		− email: "u@e2e.test",
201		− organizationId: SESSION_ORG,
202		− });
203		− }
204		− if (prop === "listUserMemberships") {
205		− return (userId: string) =>
206		− Effect.sync(() => {
207		− calls.push(`listUserMemberships:${userId}`);
208		− return {
209		− object: "list" as const,
210		− data: [
211		− {
212		− id: `om_${MEMBER}_${SESSION_ORG}`,
213		− userId: MEMBER,
214		− organizationId: SESSION_ORG,
215		− status: "active",
216		− role: { slug: "member" },
217		− },
218		− ] as never[],
219		− listMetadata: { before: null, after: null },
220		− };
221		− });
222		− }
223		− return () => Effect.die(`unexpected WorkOSClient.${String(prop)} call`);
224		− },
225		− }),
226		− );
227		−
228		−const unreadDirectory = Layer.succeed(MemberDirectory)({
229		− membership: () => Effect.die("an unready mirror must not be asked for membership"),
230		− membershipById: () => Effect.die("an unready mirror must not be asked for membership"),
231		− membershipsOf: () => Effect.die("an unready mirror must not be asked for membership"),
232		− members: () => Effect.die("an unready mirror must not be asked for membership"),
233		− membersById: () => Effect.die("an unready mirror must not be asked for membership"),
234		− findByEmail: () => Effect.die("an unready mirror must not be asked for membership"),
235		−});
236		−
237		−/**
238		− * A tracer that keeps every span's attributes, so the readiness decision the
239		− * authorization stamps on its span (`mirror.ready`, `mirror.readiness`) is
240		− * assertable: that attribute is what production counts the fallback by.
	169	+ * A tracer that keeps every span's attributes, so the authorization span
	170	+ * itself — {@link AUTHORIZE_ORGANIZATION_SPAN} — is assertable.
241	171	*/
242	172	const makeRecordingTracer = () => {
243		− const spans: { readonly name: string; readonly attributes: Map<string, unknown> }[] = [];
	173	+ const spans: {
	174	+ readonly name: string;
	175	+ readonly attributes: Map<string, unknown>;
	176	+ }[] = [];
244	177	const tracer: Tracer.Tracer = {
245	178	span: (options) => {
246	179	const attributes = new Map<string, unknown>();
247	180	spans.push({ name: options.name, attributes });
248		− let status: Tracer.SpanStatus = { _tag: "Started", startTime: options.startTime };
	181	+ let status: Tracer.SpanStatus = {
	182	+ _tag: "Started",
	183	+ startTime: options.startTime,
	184	+ };
249	185	return {
250	186	_tag: "Span",
251	187	name: options.name,
@@ -261,7 +197,12 @@ const makeRecordingTracer = () => { · 9 unmodified lines
261	197	sampled: options.sampled,
262	198	kind: options.kind,
263	199	end: (endTime, exit) => {
264		− status = { _tag: "Ended", startTime: options.startTime, endTime, exit };
	200	+ status = {
	201	+ _tag: "Ended",
	202	+ startTime: options.startTime,
	203	+ endTime,
	204	+ exit,
	205	+ };
265	206	},
266	207	attribute: (key, value) => {
267	208	attributes.set(key, value);
@@ -275,24 +216,6 @@ const makeRecordingTracer = () => { · 7 unmodified lines
275	216	return { tracer, attributesOf };
276	217	};
277	218
278		−const runAgainstWorkOs = (headers: Record<string, string>, state: MirrorReadinessState) => {
279		− const calls: string[] = [];
280		− return resolveSessionPrincipal(new Request("https://executor.test/api/tools", { headers }))
281		− .pipe(
282		− Effect.provide(
283		− Layer.mergeAll(
284		− stubApiKeys,
285		− workosMemberships(calls),
286		− stubUsers,
287		− unreadDirectory,
288		− stubMirror,
289		− unreadyMirror(state),
290		− ),
291		− ),
292		− )
293		− .pipe(Effect.map((principal) => ({ principal, calls })));
294		−};
295		−
296	219	describe("resolveSessionPrincipal · URL org selector", () => {
297	220	it.effect("fails closed when no selector header is sent", () =>
298	221	Effect.gen(function* () {
@@ -358,79 +281,21 @@ describe("resolveSessionPrincipal · URL org selector", () => { · 59 unmodified lines
358	281	);
359	282	});
360	283
361		−// The mirror authorizes only while it is READY (`mirror-readiness.ts`):
362		−// the backfill has written every organization and the reconciler has drained
363		−// the stream within its lag budget. Until then the membership check reads
364		−// WorkOS, as it did before the cutover — so a member the backfill has not
365		−// written yet is not locked out, and a stale mirror row cannot grant access
366		−// WorkOS has revoked.
367		−describe("resolveSessionPrincipal · mirror readiness", () => {
368		− const unready: readonly [string, MirrorReadinessState][] = [
369		− ["the backfill has not completed", MirrorReadinessState.BackfillPending()],
370		− ["the reconciler has never drained", MirrorReadinessState.ReconcilerStale({ drainedAt: null })],
371		− [
372		− "the reconciler's last drain is older than the budget",
373		− MirrorReadinessState.ReconcilerStale({ drainedAt: createdAt }),
374		− ],
375		− ];
376		−
377		− for (const [why, state] of unready) {
378		− it.effect(`reads membership from WorkOS, not the mirror, while ${why}`, () =>
379		− Effect.gen(function* () {
380		− // WorkOS says MEMBER is active in SESSION_ORG: authorized there...
381		− const granted = yield* runAgainstWorkOs(
382		− { cookie: "wos-session=x", "x-executor-organization": SESSION_ORG },
383		− state,
384		− );
385		− expect(granted.principal.organizationId).toBe(SESSION_ORG);
386		− expect(granted.principal.orgRole, "the role comes from WorkOS's list too").toBe("member");
387		− expect(granted.calls).toEqual([`listUserMemberships:${MEMBER}`]);
388		−
389		− // ...and NOT in URL_ORG, even though the (unready) mirror holds an
390		− // active admin membership there: the mirror's word is not taken.
391		− const refused = yield* Effect.flip(
392		− runAgainstWorkOs({ cookie: "wos-session=x", "x-executor-organization": URL_SLUG }, state),
393		− );
394		− expect(refused).toMatchObject({ _tag: "NoOrganization" });
395		− }),
396		− );
397		− }
398		−
399		− it.effect("reads the mirror once it is ready, without any WorkOS membership call", () =>
	284	+// Membership is read from the local mirror unconditionally — there is no
	285	+// readiness gate and no per-request WorkOS fallback (`auth/organization.ts`).
	286	+// The span every authorization runs under is still pinned here; a stalled
	287	+// reconciler is now an operational alert (`workos-events-runner.ts`), not a
	288	+// request-path branch, so it has no span attribute left to assert on.
	289	+describe("resolveSessionPrincipal · authorization span", () => {
	290	+ it.effect("runs the authorization under its span", () =>
400	291	Effect.gen(function* () {
401		− // `stubWorkOS` dies on any call past session authentication, so a
402		− // resolved principal here proves the list was never requested.
403		− const principal = yield* run(
404		− { cookie: "wos-session=x", "x-executor-organization": URL_SLUG },
405		− stubReadiness,
406		− );
	292	+ const recorder = makeRecordingTracer();
	293	+ const principal = yield* run({
	294	+ cookie: "wos-session=x",
	295	+ "x-executor-organization": URL_SLUG,
	296	+ }).pipe(Effect.withTracer(recorder.tracer));
407	297	expect(principal.organizationId).toBe(URL_ORG);
408		− expect(principal.orgRole).toBe("admin");
409		− }),
410		− );
411		−
412		− it.effect("stamps which source answered on the authorization span", () =>
413		− Effect.gen(function* () {
414		− // The fallback is counted in production by this attribute, not by
415		− // grepping the warning it logs beside it — so both branches must set it.
416		− const ready = makeRecordingTracer();
417		− yield* run({ cookie: "wos-session=x", "x-executor-organization": URL_SLUG }).pipe(
418		− Effect.withTracer(ready.tracer),
419		− );
420		− const readyAttrs = ready.attributesOf(AUTHORIZE_ORGANIZATION_SPAN);
421		− expect(readyAttrs?.get("mirror.ready"), "the mirror answered").toBe(true);
422		− expect(readyAttrs?.get("mirror.readiness")).toBe("ready");
423		−
424		− const stale = makeRecordingTracer();
425		− yield* runAgainstWorkOs(
426		− { cookie: "wos-session=x", "x-executor-organization": SESSION_ORG },
427		− MirrorReadinessState.ReconcilerStale({ drainedAt: null }),
428		− ).pipe(Effect.withTracer(stale.tracer));
429		− const staleAttrs = stale.attributesOf(AUTHORIZE_ORGANIZATION_SPAN);
430		− expect(staleAttrs?.get("mirror.ready"), "WorkOS answered").toBe(false);
431		− expect(String(staleAttrs?.get("mirror.readiness")), "and the span says why").toContain(
432		− "reconciler",
433		− );
	298	+ expect(recorder.attributesOf(AUTHORIZE_ORGANIZATION_SPAN)).toBeDefined();
434	299	}),
435	300	);
436	301	});

apps/cloud/src/auth/organization.ts+41−99

@@ -18,7 +18,6 @@ import { EXECUTOR_ORG_SELECTOR_HEADER } from "@executor-js/sdk/shared"; · 17 unmodified lines
18	18
19	19	import { UserStoreService } from "./context";
20	20	import { ensureOrganizationBackfilled } from "./mirror-feeders";
21		−import { MirrorReadiness, MirrorReadinessState, describeMirrorReadiness } from "./mirror-readiness";
22	21	import type { Organization } from "./user-store";
23	22	import { WorkOSClient } from "./workos";
24	23
@@ -94,12 +93,11 @@ export const markOrganizationDeleted = (organizationId: string) => · 69 unmodified lines
94	93	// until their access token naturally expired (~10 min) if the session were
95	94	// trusted on its own.
96	95	//
97		−// To close that gap, membership is verified on every protected request — but
	96	+// To close that gap, membership is verified on every protected request
98	97	// against the LOCAL mirror of WorkOS memberships (`memberships` join
99		−// `accounts`, read through the shared `MemberDirectory`), never against WorkOS
100		−// itself. This used to be one `listUserMemberships` call per request (2026-07:
101		−// deliberately NOT cached, because a positive TTL cache is exactly what would
102		−// re-open the revocation gap). The mirror is not a cache with a TTL; it is a
	98	+// `accounts`, read through the shared `MemberDirectory`) — never against
	99	+// WorkOS itself, and unconditionally: there is no readiness gate and no
	100	+// per-request WorkOS fallback. The mirror is not a cache with a TTL; it is a
103	101	// replica whose freshness is defined by its feeders:
104	102	// - login (`auth/handlers.ts` callback): the user and every membership WorkOS
105	103	// lists for them, from the list the callback already fetches;
@@ -117,45 +115,39 @@ export const markOrganizationDeleted = (organizationId: string) => · 11 unmodified lines
117	115	// not finish refuses every session at once — its membership rows are still
118	116	// there, live, until the purge removes them, and must not authorize anyone.
119	117	//
120		−// The mirror is trusted only while it is READY (`mirror-readiness.ts`): the
121		−// one-off backfill has written every organization, and the events reconciler
122		−// has drained the stream within its lag budget. Until both hold, membership is
123		−// read from WorkOS (`listUserMemberships`, one call per request) exactly as
124		−// before the cutover — a member the backfill has not written yet must not be
125		−// locked out, and a member revoked in the dashboard while the reconciler was
126		−// down must not be let in on a stale row. The readiness row is one indexed
127		−// point read on the same socket; the deploy gate
128		−// (`scripts/ensure-workos-mirror-ready.ts`) applies the same rule before this
129		−// build goes live, so in steady state the fallback is never taken. A
130		−// readiness or mirror read failure fails the request (500), never a silent
131		−// fallback in either direction. The one org the fallback never asks WorkOS
132		−// about is one the mirror holds as DELETED: WorkOS no longer has it (or is
133		−// about to not), so its answer is "no member" for everyone — including the
134		−// admin whose deletion failed part-way and must retry it (below). That
135		−// membership is read from the mirror, whose rows are exactly what the purge
136		−// has not removed yet, ready or not; a refused caller gets null either way.
	118	+// The one-off backfill is complete and permanent, and an organization that
	119	+// predates it is covered on demand (below), so there is nothing left for a
	120	+// per-request readiness check to gate. What can still go wrong is the events
	121	+// reconciler falling behind — a member revoked in the WorkOS dashboard would
	122	+// keep a stale active row until it catches up. That is now an OPERATIONAL
	123	+// concern, not a request-path fallback: the reconciler itself
	124	+// (`workos-events-runner.ts`) checks its own drain lag after every run and
	125	+// raises a Sentry error when it has stalled, so it is fixed by paging someone,
	126	+// not by asking WorkOS on every request. The deploy gate
	127	+// (`scripts/ensure-workos-mirror-ready.ts`) separately refuses to ship a build
	128	+// that trusts the mirror while it is unready, using the same rule
	129	+// (`mirror-readiness-store.ts`).
137	130	//
138		−// Readiness is database-wide; completeness is PER ORGANIZATION. An
139		−// organization whose row was minted after the backfill ran — lazily by a
140		−// request (`resolveOrganization`), or by a first login — carries no
141		−// `backfilled_at`, and the mirror holds only the memberships login and
142		−// write-through happened to record for it: a member who has not signed in
143		−// since would be refused on a row that was never written. So the org row is
144		−// read FIRST, and an unmarked live organization is scanned from WorkOS
145		−// (`ensureOrganizationBackfilled`: one membership listing plus one `getUser`
146		−// per member, then the mark) BEFORE its mirror is read — the same on-demand
147		−// scan the seat gates run. One-time per organization: the scan marks the
148		−// row, and this branch is never taken for it again. An organization the
149		−// mirror does not hold at all — one that predates the mirror and that nobody
150		−// has signed in to since (a CLI or MCP token names it, and the JWT path has
151		−// no login feeder), or one created in the WorkOS dashboard — is reachable by
152		−// neither the backfill (which lists the mirror's organizations) nor the
153		−// reconciler (which starts at the replay boundary), so it is resolved on
154		−// demand HERE: WorkOS is asked for the caller's own membership in it first
155		−// (`getUserOrgMembership`, a read scoped to this caller — never a listing
156		−// of the org), and only a member's answer mints the row
157		−// (`resolveOrganization`) and scans it as above. A non-member mints nothing:
158		−// a signed-in caller cannot create the row of an arbitrary WorkOS
	131	+// Completeness is PER ORGANIZATION. An organization whose row was minted
	132	+// after the backfill ran — lazily by a request (`resolveOrganization`), or by
	133	+// a first login — carries no `backfilled_at`, and the mirror holds only the
	134	+// memberships login and write-through happened to record for it: a member
	135	+// who has not signed in since would be refused on a row that was never
	136	+// written. So the org row is read FIRST, and an unmarked live organization is
	137	+// scanned from WorkOS (`ensureOrganizationBackfilled`: one membership listing
	138	+// plus one `getUser` per member, then the mark) BEFORE its mirror is read —
	139	+// the same on-demand scan the seat gates run. One-time per organization: the
	140	+// scan marks the row, and this branch is never taken for it again. An
	141	+// organization the mirror does not hold at all — one that predates the
	142	+// mirror and that nobody has signed in to since (a CLI or MCP token names it,
	143	+// and the JWT path has no login feeder), or one created in the WorkOS
	144	+// dashboard — is reachable by neither the backfill (which lists the mirror's
	145	+// organizations) nor the reconciler (which starts at the replay boundary), so
	146	+// it is resolved on demand HERE: WorkOS is asked for the caller's own
	147	+// membership in it first (`getUserOrgMembership`, a read scoped to this
	148	+// caller — never a listing of the org), and only a member's answer mints the
	149	+// row (`resolveOrganization`) and scans it as above. A non-member mints
	150	+// nothing: a signed-in caller cannot create the row of an arbitrary WorkOS
159	151	// organization by naming its id. An organization marked deleted is never
160	152	// scanned: WorkOS no longer has it, and its rows are the purge's to remove,
161	153	// not a listing's to refresh.
@@ -189,20 +181,6 @@ const activeMembershipFromMirror = (userId: string, organizationId: string) => · 27 unmodified lines
189	181	return active;
190	182	});
191	183
192		−// The pre-cutover read, kept for the window in which the mirror is not yet
193		−// ready: WorkOS's own membership list for the user, one call per request.
194		−const activeMembershipFromWorkOs = (userId: string, organizationId: string) =>
195		− Effect.gen(function* () {
196		− const workos = yield* WorkOSClient;
197		− const memberships = yield* workos.listUserMemberships(userId);
198		− const membership = memberships.data.find(
199		− (m) => m.organizationId === organizationId && m.status === "active",
200		− );
201		− if (!membership) return null;
202		− const active: ActiveMembership = { role: membership.role.slug };
203		− return active;
204		− });
205		−
206	184	// The authorized organization, or null for one marked deleted (unless the
207	185	// caller is the deletion flow). The membership already names the caller's
208	186	// role — surfaced normalized so identity resolution can bind the executor's
@@ -236,14 +214,7 @@ const heldOrResolvedForMember = (userId: string, organizationId: string) => · 27 unmodified lines
236	214	return yield* resolveOrganization(organizationId);
237	215	});
238	216
239		−/**
240		− * The span every membership authorization runs under, with the readiness
241		− * decision stamped on it so the fallback can be counted rather than grepped:
242		− * `mirror.ready` (boolean, which source answered) and `mirror.readiness` (the
243		− * state's description, why). Query Axiom for `name == "auth.authorize_organization"`
244		− * and `mirror.ready == false` to see how many requests are on the WorkOS
245		− * fallback and for which reason.
246		− */
	217	+/** The span every membership authorization runs under. */
247	218	export const AUTHORIZE_ORGANIZATION_SPAN = "auth.authorize_organization";
248	219
249	220	export const authorizeOrganization = (
@@ -252,43 +223,14 @@ export const authorizeOrganization = ( · 2 unmodified lines
252	223	options: AuthorizeOrganizationOptions = {},
253	224	) =>
254	225	Effect.gen(function* () {
255		− const readiness = yield* MirrorReadiness;
256		− const state = yield* readiness.state();
257		− const ready = MirrorReadinessState.$is("Ready")(state);
258		− yield* Effect.annotateCurrentSpan({
259		− "mirror.ready": ready,
260		− "mirror.readiness": describeMirrorReadiness(state),
261		− });
262		− if (!ready) {
263		− yield* Effect.logWarning(
264		− "authorizeOrganization: membership mirror not ready; membership read from WorkOS",
265		− { readiness: describeMirrorReadiness(state) },
266		− );
267		− // A marked organization is the mirror's to answer for (see above):
268		− // WorkOS lists no member of it, and the deletion retry must still
269		− // get in.
270		− const users = yield* UserStoreService;
271		− const held = yield* users.use("getOrganization", (s) => s.getOrganization(organizationId));
272		− if (held?.deletedAt != null) {
273		− if (options.deleted !== "allow") return null;
274		− const membership = yield* activeMembershipFromMirror(userId, organizationId);
275		− if (!membership) return null;
276		− return authorized(held, membership, options);
277		− }
278		− const membership = yield* activeMembershipFromWorkOs(userId, organizationId);
279		− if (!membership) return null;
280		− // The row read above is reused: `resolveOrganization` would read it a
281		− // second time, and a request reads the organization row ONCE (the MCP
282		− // session DO relies on that — see e2e `mcp-session-cold-init`).
283		− const org = held ?? (yield* resolveOrganization(organizationId));
284		− return authorized(org, membership, options);
285		− }
286		−
287	226	const org = yield* heldOrResolvedForMember(userId, organizationId);
288	227	if (!org) return null;
289	228	// An unmarked live organization is scanned before its mirror is read
290	229	// (see above). The row returned below still shows the mark as it was
291		− // read; nothing past this point reads it.
	230	+ // read; nothing past this point reads it. A marked-deleted organization
	231	+ // is never scanned, so the deletion retry (`deleted: "allow"`) reaches
	232	+ // `authorized()` below with the membership row the purge has not removed
	233	+ // yet.
292	234	if (org.deletedAt === null && org.backfilledAt === null) {
293	235	yield* ensureOrganizationBackfilled(organizationId);
294	236	}

apps/cloud/src/auth/workos-auth-provider.ts+15−21

@@ -20,16 +20,16 @@ · 19 unmodified lines
20	20	// - session org not authorized -> NoOrganization 403 no_organization
21	21	// - no auth header -> falls through to the sealed-session path
22	22	// The org-resolution infra errors (`UserStoreError` / `WorkOSError` /
23		−// `MemberDirectoryError` / `WorkOsMirrorError`, the last from the mirror
24		−// readiness read) are `Effect.die`d so they surface as 500 defects — the
25		−// same status the old inline resolver produced when those bubbled up.
	23	+// `MemberDirectoryError` / `WorkOsMirrorError`) are `Effect.die`d so they
	24	+// surface as 500 defects — the same status the old inline resolver produced
	25	+// when those bubbled up.
26	26	//
27		−// The per-request `UserStoreService` + `MemberDirectory` + `MirrorReadiness`
28		−// + `WorkOsMirror` (read by the org-resolution path: the org row, whether
29		−// the mirror may be trusted, the caller's mirrored membership, and the
30		−// on-demand scan of an organization the backfill never covered) stay
31		−// REQUIREMENTS OF THE LAYER, satisfied by the facade's per-request DB combine —
32		−// NOT function-level requirements (that is what forced a forked tag before).
	27	+// The per-request `UserStoreService` + `MemberDirectory` + `WorkOsMirror`
	28	+// (read by the org-resolution path: the org row, the caller's mirrored
	29	+// membership, and the on-demand scan of an organization the backfill never
	30	+// covered) stay REQUIREMENTS OF THE LAYER, satisfied by the facade's
	31	+// per-request DB combine — NOT function-level requirements (that is what
	32	+// forced a forked tag before).
33	33	// ---------------------------------------------------------------------------
34	34
35	35	import { Effect, Layer } from "effect";
@@ -54,7 +54,6 @@ import type { · 18 unmodified lines
54	54
55	55	import { ApiKeyService } from "./api-keys";
56	56	import { workosApiJwtBearerConfig } from "./api-jwt-bearer";
57		−import { MirrorReadiness } from "./mirror-readiness";
58	57	import { WorkOsMirror } from "./workos-mirror";
59	58	import { BEARER_PREFIX } from "./bearer";
60	59	import {
@@ -215,7 +214,7 @@ export const resolveBearerAuth = ( · 154 unmodified lines
215	214	\| WorkOSError
216	215	\| WorkOsMirrorError
217	216	\| MemberDirectoryError,
218		− WorkOSClient \| ApiKeyService \| UserStoreService \| MemberDirectory \| MirrorReadiness \| WorkOsMirror
	217	+ WorkOSClient \| ApiKeyService \| UserStoreService \| MemberDirectory \| WorkOsMirror
219	218	> =>
220	219	Effect.gen(function* () {
221	220	const authHeader = request.headers.get("authorization");
@@ -301,7 +300,7 @@ export const resolveApiKeyPrincipal = ( · 79 unmodified lines
301	300	\| WorkOSError
302	301	\| WorkOsMirrorError
303	302	\| MemberDirectoryError,
304		− WorkOSClient \| ApiKeyService \| UserStoreService \| MemberDirectory \| MirrorReadiness \| WorkOsMirror
	303	+ WorkOSClient \| ApiKeyService \| UserStoreService \| MemberDirectory \| WorkOsMirror
305	304	> =>
306	305	Effect.gen(function* () {
307	306	const auth = yield* resolveBearerAuth(request, jwt);
@@ -383,7 +382,7 @@ export const resolveProtectedPrincipal = ( · 75 unmodified lines
383	382	\| WorkOSError
384	383	\| WorkOsMirrorError
385	384	\| MemberDirectoryError,
386		− WorkOSClient \| ApiKeyService \| UserStoreService \| MemberDirectory \| MirrorReadiness \| WorkOsMirror
	385	+ WorkOSClient \| ApiKeyService \| UserStoreService \| MemberDirectory \| WorkOsMirror
387	386	> =>
388	387	Effect.gen(function* () {
389	388	const bearerPrincipal = yield* resolveApiKeyPrincipal(request, jwt);
@@ -404,17 +403,12 @@ export const resolveProtectedPrincipal = ( · 14 unmodified lines
404	403	export const workosIdentityLayer: Layer.Layer<
405	404	IdentityProvider,
406	405	never,
407		− WorkOSClient \| ApiKeyService \| UserStoreService \| MemberDirectory \| MirrorReadiness \| WorkOsMirror
	406	+ WorkOSClient \| ApiKeyService \| UserStoreService \| MemberDirectory \| WorkOsMirror
408	407	> = Layer.effect(
409	408	IdentityProvider,
410	409	Effect.gen(function* () {
411	410	const context = yield* Effect.context<
412		− \| WorkOSClient
413		− \| ApiKeyService
414		− \| UserStoreService
415		− \| MemberDirectory
416		− \| MirrorReadiness
417		− \| WorkOsMirror
	411	+ WorkOSClient \| ApiKeyService \| UserStoreService \| MemberDirectory \| WorkOsMirror
418	412	>();
419	413	return IdentityProvider.of({
420	414	authenticate: (request) =>
@@ -430,7 +424,7 @@ export const workosIdentityLayer: Layer.Layer< · 9 unmodified lines
430	424	WorkOSError: (error) => Effect.die(error),
431	425	// oxlint-disable-next-line executor/no-effect-escape-hatch -- boundary: membership-mirror read failure -> 500 defect, same class as the store failure above
432	426	MemberDirectoryError: (error) => Effect.die(error),
433		− // oxlint-disable-next-line executor/no-effect-escape-hatch -- boundary: mirror-readiness read failure -> 500 defect, same class as the store failure above
	427	+ // oxlint-disable-next-line executor/no-effect-escape-hatch -- boundary: membership-mirror read failure -> 500 defect, same class as the store failure above
434	428	WorkOsMirrorError: (error) => Effect.die(error),
435	429	}),
436	430	Effect.provide(context),

apps/cloud/src/auth/workos-events-runner.node.test.ts+81−0

@@ -0,0 +1,81 @@
	1	+// ---------------------------------------------------------------------------
	2	+// A reconciler run that does not end `"drained"` can still leave the mirror
	3	+// fresh (another run drained it moments ago) or leave it stale (nothing has
	4	+// drained inside the lag budget). `alertOnStaleReconciler` is the ONLY place
	5	+// that distinction is now reported — the request path no longer reads
	6	+// readiness at all — so this pins both branches directly against it: a
	7	+// fresh `drainedAt` logs nothing, and a stale or absent `drainedAt` logs a
	8	+// structured error. `captureCauseEffect` is not swapped out: it calls
	9	+// `Sentry.captureException` directly, is a no-op in this uninitialized test
	10	+// environment, and its call path is exercised for real rather than mocked.
	11	+// ---------------------------------------------------------------------------
	12	+
	13	+import { describe, expect, it } from "@effect/vitest";
	14	+import { Cause, Duration, Effect, Layer, Logger } from "effect";
	15	+
	16	+import { WorkOsMirror, type WorkOsMirrorShape } from "./workos-mirror";
	17	+import { alertOnStaleReconciler } from "./workos-events-runner";
	18	+import type { WorkOsEventsSyncReport } from "./workos-events-sync";
	19	+
	20	+const capturingLogger = (sink: Array<string>) =>
	21	+ Logger.make<unknown, void>((options) => {
	22	+ sink.push(String(options.message));
	23	+ sink.push(Cause.pretty(options.cause));
	24	+ });
	25	+
	26	+const reportEndingWith = (stopped: WorkOsEventsSyncReport["stopped"]): WorkOsEventsSyncReport => ({
	27	+ pages: 1,
	28	+ events: 0,
	29	+ applied: 0,
	30	+ stale: 0,
	31	+ absent: 0,
	32	+ stopped,
	33	+ cursor: null,
	34	+});
	35	+
	36	+const stubMirror = (drainedAt: Date \| null) =>
	37	+ Layer.succeed(
	38	+ WorkOsMirror,
	39	+ new Proxy({} as WorkOsMirrorShape, {
	40	+ get: (_target, prop) => {
	41	+ if (prop === "drainedAt") return () => Effect.succeed(drainedAt);
	42	+ return () => Effect.die(`unexpected WorkOsMirror.${String(prop)} call`);
	43	+ },
	44	+ }),
	45	+ );
	46	+
	47	+const run = (report: WorkOsEventsSyncReport, drainedAt: Date \| null) => {
	48	+ const logged: string[] = [];
	49	+ return Effect.runPromise(
	50	+ alertOnStaleReconciler(report).pipe(
	51	+ Effect.provide(stubMirror(drainedAt)),
	52	+ Effect.provide(Logger.layer([capturingLogger(logged)])),
	53	+ ),
	54	+ ).then(() => logged);
	55	+};
	56	+
	57	+describe("alertOnStaleReconciler", () => {
	58	+ it("does not alert when the run itself drained the stream", async () => {
	59	+ // A `"drained"` run just called `markDrained`, so it is healthy by
	60	+ // definition and `drainedAt` is never read.
	61	+ const logged = await run(reportEndingWith("drained"), null);
	62	+ expect(logged).toEqual([]);
	63	+ });
	64	+
	65	+ it("does not alert when the mirror drained inside the lag budget", async () => {
	66	+ const fresh = new Date(Date.now() - Duration.toMillis(Duration.minutes(1)));
	67	+ const logged = await run(reportEndingWith("page_budget"), fresh);
	68	+ expect(logged).toEqual([]);
	69	+ });
	70	+
	71	+ it("alerts when the mirror has not drained inside the lag budget", async () => {
	72	+ const stale = new Date(Date.now() - Duration.toMillis(Duration.minutes(11)));
	73	+ const logged = await run(reportEndingWith("page_budget"), stale);
	74	+ expect(logged.some((line) => line.includes("workos_events: reconciler stale"))).toBe(true);
	75	+ });
	76	+
	77	+ it("alerts when the mirror has never drained", () =>
	78	+ run(reportEndingWith("awaiting_backfill"), null).then((logged) => {
	79	+ expect(logged.some((line) => line.includes("workos_events: reconciler stale"))).toBe(true);
	80	+ }));
	81	+});

apps/cloud/src/auth/workos-events-runner.ts+59−3

@@ -13,17 +13,28 @@ · 12 unmodified lines
13	13	// A failing run is captured (Sentry + structured log) and swallowed here:
14	14	// neither entry has a caller to report to, and the run is retried by the
15	15	// next cron tick from the last committed cursor.
	16	+//
	17	+// A run that does not fail outright can still leave the mirror stale: one
	18	+// that stops short of `"drained"` (page budget, cursor contention) means the
	19	+// reconciler did not catch up this tick, and one stuck on
	20	+// `"awaiting_backfill"` past the budget means the boundary row the backfill
	21	+// was supposed to hand off was lost — both are read from `drainedAt` after
	22	+// the run and, past `MIRROR_RECONCILER_LAG_BUDGET`, reported the same way a
	23	+// thrown failure is: a structured error log plus a Sentry capture. This is
	24	+// the ONLY place a stale reconciler surfaces now; the request path
	25	+// (`auth/organization.ts`) no longer checks readiness or falls back.
16	26	// ---------------------------------------------------------------------------
17	27
18		−import { Effect, Layer } from "effect";
	28	+import { Clock, Data, Duration, Effect, Layer } from "effect";
19	29
20	30	import { captureCauseEffect } from "../observability";
21	31	import { WorkerTelemetryLive } from "../observability/telemetry";
22	32	import { makeDbLayer } from "../db/db";
23	33	import { makeUserStoreLayer } from "./context";
	34	+import { MIRROR_RECONCILER_LAG_BUDGET } from "./mirror-readiness-store";
24	35	import { CoreSharedServices } from "./workos";
25		−import { syncWorkOsEvents } from "./workos-events-sync";
26		−import { makeWorkOsMirrorLayer } from "./workos-mirror";
	36	+import { syncWorkOsEvents, type WorkOsEventsSyncReport } from "./workos-events-sync";
	37	+import { makeWorkOsMirrorLayer, WorkOsMirror } from "./workos-mirror";
27	38
28	39	const makeSyncServices = () => {
29	40	const dbLive = makeDbLayer();
@@ -34,14 +45,59 @@ const makeSyncServices = () => { · 4 unmodified lines
34	45	);
35	46	};
36	47
	48	+const LAG_BUDGET_MS = Duration.toMillis(MIRROR_RECONCILER_LAG_BUDGET);
	49	+
	50	+/**
	51	+ * The mirror has not drained the WorkOS events stream inside its lag budget.
	52	+ * Its own tagged error so Sentry groups every occurrence under one issue,
	53	+ * with the run's outcome and the heartbeat's age as the fields to read.
	54	+ */
	55	+export class WorkOsReconcilerStale extends Data.TaggedError("WorkOsReconcilerStale")<{
	56	+ readonly stopped: WorkOsEventsSyncReport["stopped"];
	57	+ readonly drainedAt: string \| null;
	58	+ readonly ageMs: number \| null;
	59	+}> {}
	60	+
	61	+/**
	62	+ * After a run that did not end `"drained"`, check whether the mirror has
	63	+ * fallen behind its lag budget and, if so, report it the way a failed run is
	64	+ * reported: a structured error log plus a Sentry capture. A run ending
	65	+ * `"drained"` just wrote `markDrained` and is healthy by definition, so it is
	66	+ * never checked. Cheap by design: one `drainedAt` read per run (about once a
	67	+ * minute).
	68	+ */
	69	+export const alertOnStaleReconciler = Effect.fn("workos_events.alert_on_stale_reconciler")(
	70	+ function* (report: WorkOsEventsSyncReport) {
	71	+ if (report.stopped === "drained") return;
	72	+
	73	+ const mirror = yield* WorkOsMirror;
	74	+ const drainedAt = yield* mirror.drainedAt();
	75	+ const now = yield* Clock.currentTimeMillis;
	76	+ const ageMs = drainedAt === null ? null : now - drainedAt.getTime();
	77	+ if (ageMs !== null && ageMs <= LAG_BUDGET_MS) return;
	78	+
	79	+ const stale = new WorkOsReconcilerStale({
	80	+ stopped: report.stopped,
	81	+ drainedAt: drainedAt === null ? null : drainedAt.toISOString(),
	82	+ ageMs,
	83	+ });
	84	+ yield* Effect.logError("workos_events: reconciler stale", stale);
	85	+ yield* captureCauseEffect(stale);
	86	+ },
	87	+);
	88	+
37	89	/**
38	90	* One reconciler pass over fresh request-scoped services. Resolves when the
39	91	* pass ends, whether it drained the stream, stopped at the page budget,
40	92	* yielded to another run, or failed (a failure is reported, never thrown).
	93	+ * A run that ends short of `"drained"` and leaves `drainedAt` past the lag
	94	+ * budget is ALSO reported (see {@link alertOnStaleReconciler}) — a stalled
	95	+ * reconciler is now an operational alert, not a per-request fallback.
41	96	*/
42	97	export const runWorkOsEventsSync = (): Promise<void> =>
43	98	Effect.runPromise(
44	99	syncWorkOsEvents().pipe(
	100	+ Effect.tap((report) => alertOnStaleReconciler(report)),
45	101	Effect.asVoid,
46	102	Effect.provide(makeSyncServices()),
47	103	Effect.scoped,

apps/cloud/src/auth/workos-events-sync.node.test.ts+35−21

@@ -62,7 +62,6 @@ import { UserStoreService } from "./context"; · 61 unmodified lines
62	62	import { WorkOSError } from "./errors";
63	63	import { cloudMemberDirectoryLayer } from "./member-directory";
64	64	import { mirrorSignIn } from "./mirror-feeders";
65		−import { MirrorReadiness, MirrorReadinessState } from "./mirror-readiness";
66	65	import { authorizeOrganization } from "./organization";
67	66	import { WorkOSClient, type WorkOSClientService, type WorkOSListEventsOptions } from "./workos";
68	67	import {
@@ -202,26 +201,14 @@ const profiles = (reads: string[] = []): Partial<WorkOSClientService> => ({ · 133 unmodified lines
202	201	});
203	202
204	203	const DbLive = DbService.Live;
205		−// The mirror is READY here (the authorization checks below read the mirror,
206		−// not WorkOS); the readiness rule is pinned in workos-mirror.node.test.ts.
207		−const readyMirror = Layer.succeed(MirrorReadiness)({
208		− state: () => Effect.succeed(MirrorReadinessState.Ready()),
209		−});
210		−
	204	+// The authorization checks below always read the mirror, never WorkOS.
211	205	const MirrorServices = Layer.mergeAll(
212	206	WorkOsMirror.Live,
213	207	UserStoreService.Live,
214	208	cloudMemberDirectoryLayer,
215		− readyMirror,
216	209	).pipe(Layer.provideMerge(DbLive));
217	210
218		−type Services =
219		− \| WorkOsMirror
220		− \| UserStoreService
221		− \| MemberDirectory
222		− \| MirrorReadiness
223		− \| DbService
224		− \| WorkOSClient;
	211	+type Services = WorkOsMirror \| UserStoreService \| MemberDirectory \| DbService \| WorkOSClient;
225	212
226	213	const run = <A, E>(
227	214	body: Effect.Effect<A, E, Services>,
@@ -549,7 +536,10 @@ describe("applyEvent", () => { · 321 unmodified lines
549	536	const promoted = yield* applyEvent(
550	537	membershipEvent(
551	538	"organization_membership.updated",
552		− workosMembership(joiner, org, { role: { slug: "admin" }, updatedAt: T2 }),
	539	+ workosMembership(joiner, org, {
	540	+ role: { slug: "admin" },
	541	+ updatedAt: T2,
	542	+ }),
553	543	),
554	544	);
555	545	// A member WorkOS no longer has (their `user.deleted` is further down
@@ -558,7 +548,15 @@ describe("applyEvent", () => { · 2 unmodified lines
558	548	membershipEvent("organization_membership.created", workosMembership(gone, org)),
559	549	);
560	550	const goneRow = yield* readMembership(gone, org);
561		− return { knownJoins, knownRow, joins, joinerRow, promoted, goneJoins, goneRow };
	551	+ return {
	552	+ knownJoins,
	553	+ knownRow,
	554	+ joins,
	555	+ joinerRow,
	556	+ promoted,
	557	+ goneJoins,
	558	+ goneRow,
	559	+ };
562	560	}),
563	561	stubWorkOS({
564	562	getUser: (userId) =>
@@ -581,7 +579,10 @@ describe("applyEvent", () => { · 16 unmodified lines
581	579	expect(result.joinerRow?.email).toBe(`${joiner}@placeholder.test`);
582	580	expect(result.promoted).toBe("applied");
583	581	expect(result.goneJoins, "a member WorkOS no longer has is still mirrored").toBe("applied");
584		− expect(result.goneRow).toMatchObject({ membershipId: `om_${gone}_${org}`, name: null });
	582	+ expect(result.goneRow).toMatchObject({
	583	+ membershipId: `om_${gone}_${org}`,
	584	+ name: null,
	585	+ });
585	586	expect(reads, "one read per unprofiled member, none for a profiled one").toEqual([
586	587	joiner,
587	588	gone,
@@ -601,7 +602,9 @@ describe("applyEvent", () => { · 13 unmodified lines
601	602	Effect.provide(
602	603	Layer.mergeAll(
603	604	MirrorServices,
604		− stubWorkOS({ getUser: () => Effect.fail(new WorkOSError({ status: 503 })) }),
	605	+ stubWorkOS({
	606	+ getUser: () => Effect.fail(new WorkOSError({ status: 503 })),
	607	+ }),
605	608	),
606	609	),
607	610	Effect.scoped,
@@ -731,7 +734,11 @@ describe("applyEvent", () => { · 123 unmodified lines
731	734	// deletion, and stalled past it now writes what it holds.
732	735	yield* mirrorSignIn(
733	736	workosUser(userId),
734		− [workosMembership(userId, org, { organizationName: "Never Mirrored" })],
	737	+ [
	738	+ workosMembership(userId, org, {
	739	+ organizationName: "Never Mirrored",
	740	+ }),
	741	+ ],
735	742	new Date(T1),
736	743	);
737	744	const afterLogin = yield* readOrganization(org);
@@ -1108,7 +1115,14 @@ describe("syncWorkOsEvents", () => { · 370 unmodified lines
1108	1115	const cursor = yield* mirror.getCursor();
1109	1116	const membership = yield* readMembership(userId, org);
1110	1117	const drainedAfter = yield* mirror.drainedAt();
1111		− return { report, cursor, intruder, membership, drainedBefore, drainedAfter };
	1118	+ return {
	1119	+ report,
	1120	+ cursor,
	1121	+ intruder,
	1122	+ membership,
	1123	+ drainedBefore,
	1124	+ drainedAfter,
	1125	+ };
1112	1126	}),
1113	1127	);
1114	1128	expect(requests, "the second page is never read").toHaveLength(1);

apps/cloud/src/auth/workos-mirror.node.test.ts+37−14

@@ -63,11 +63,10 @@ import { · 62 unmodified lines
63	63	import { makeWorkOsMirrorStore } from "./workos-mirror-store";
64	64	import {
65	65	MIRROR_RECONCILER_LAG_BUDGET,
66		− MirrorReadiness,
67	66	MirrorReadinessState,
68		− makeMirrorReadinessLayer,
69	67	mirrorReadinessFrom,
70		−} from "./mirror-readiness";
	68	+ readMirrorReadiness,
	69	+} from "./mirror-readiness-store";
71	70
72	71	const DbLive = DbService.Live;
73	72	const Services = Layer.mergeAll(
@@ -243,7 +242,11 @@ describe("WorkOsMirror upserts", () => { · 169 unmodified lines
243	242	// the row holds — the payload a timestamp guard would let through.
244	243	// Same id: the membership is deleted, it never returns.
245	244	const newerSameId = yield* mirror.upsertMembership(
246		− membership(org, id, { id: membershipId, role: "admin", updatedAt: T3 }),
	245	+ membership(org, id, {
	246	+ id: membershipId,
	247	+ role: "admin",
	248	+ updatedAt: T3,
	249	+ }),
247	250	);
248	251	const afterNewerSameId = yield* directory.membership(id, org, ["inactive"]);
249	252	// The member re-added in WorkOS: a payload newer than the deletion,
@@ -299,7 +302,10 @@ describe("WorkOsMirror upserts", () => { · 49 unmodified lines
299	302	result.newerSameId,
300	303	"a payload of the deleted id stamped AFTER the removal is refused: identity, not time",
301	304	).toBe(false);
302		− expect(result.afterNewerSameId).toMatchObject({ status: "inactive", role: "member" });
	305	+ expect(result.afterNewerSameId).toMatchObject({
	306	+ status: "inactive",
	307	+ role: "member",
	308	+ });
303	309	expect(result.readded, "a replacement under a new id reactivates").toBe(true);
304	310	expect(result.afterReadd?.status).toBe("active");
305	311	expect(result.lateDelete, "a replayed deletion of the OLD id is refused").toBe(false);
@@ -516,7 +522,11 @@ describe("WorkOsMirror upserts", () => { · 210 unmodified lines
516	522	// with status inactive): the row is inactive but still WorkOS's, with
517	523	// no `deleted_at` to protect it.
518	524	const deactivated = yield* mirror.upsertMembership(
519		− membership(org, id, { id: membershipId, status: "inactive", updatedAt: T2 }),
	525	+ membership(org, id, {
	526	+ id: membershipId,
	527	+ status: "inactive",
	528	+ updatedAt: T2,
	529	+ }),
520	530	);
521	531	const whileInactive = yield* directory.membership(id, org);
522	532	// A payload older than the deactivation cannot undo it, nor one
@@ -529,7 +539,11 @@ describe("WorkOsMirror upserts", () => { · 6 unmodified lines
529	539	);
530	540	// WorkOS reactivates it, same id, newer stamp: live again.
531	541	const reactivated = yield* mirror.upsertMembership(
532		− membership(org, id, { id: membershipId, role: "admin", updatedAt: T3 }),
	542	+ membership(org, id, {
	543	+ id: membershipId,
	544	+ role: "admin",
	545	+ updatedAt: T3,
	546	+ }),
533	547	);
534	548	const afterReactivate = yield* directory.membership(id, org);
535	549	return {
@@ -634,7 +648,10 @@ describe("WorkOsMirror upserts", () => { · 98 unmodified lines
634	648	result.rejoined,
635	649	"a membership of a deleted user is refused however it is stamped: identity, not time",
636	650	).toBe(false);
637		− expect(result.afterRejoin).toMatchObject({ status: "inactive", name: null });
	651	+ expect(result.afterRejoin).toMatchObject({
	652	+ status: "inactive",
	653	+ name: null,
	654	+ });
638	655	expect(result.unknown, "deleting an unseen user leaves a tombstone").toBe(true);
639	656	expect(result.unseenProfile, "which the older profile cannot fill").toBe(false);
640	657	expect(
@@ -883,19 +900,20 @@ describe("mirror readiness", () => { · 242 unmodified lines
883	900	const result = await run(
884	901	Effect.gen(function* () {
885	902	const mirror = yield* WorkOsMirror;
886		− const readiness = yield* MirrorReadiness;
	903	+ const { db } = yield* DbService;
	904	+ const readiness = () => Effect.promise(() => readMirrorReadiness(db, new Date()));
887	905	yield* clearEventsRow;
888		− const noRow = yield* readiness.state();
	906	+ const noRow = yield* readiness();
889	907	yield* mirror.setReplayBoundary(T1);
890	908	yield* mirror.markBackfillCompleted(T1);
891		− const backfilledOnly = yield* readiness.state();
	909	+ const backfilledOnly = yield* readiness();
892	910	// A drain as of now: what a reconciler run that just read the stream
893	911	// to its end records.
894	912	const drainedAt = new Date();
895	913	yield* mirror.markDrained(drainedAt);
896		− const ready = yield* readiness.state();
	914	+ const ready = yield* readiness();
897	915	return { noRow, backfilledOnly, ready };
898		− }).pipe(Effect.provide(makeMirrorReadinessLayer().pipe(Layer.provide(DbLive)))),
	916	+ }),
899	917	);
900	918	expect(result.noRow).toEqual(MirrorReadinessState.BackfillPending());
901	919	expect(result.backfilledOnly).toEqual(
@@ -1006,7 +1024,12 @@ describe("WorkOsMirror backfill sync state", () => { · 104 unmodified lines
1006	1024	yield* mirror.applyOrganizationScan({
1007	1025	organizationId: org,
1008	1026	listedAt: T2,
1009		− members: [{ user: user(kept), membership: membership(org, kept, { updatedAt: T1 }) }],
	1027	+ members: [
	1028	+ {
	1029	+ user: user(kept),
	1030	+ membership: membership(org, kept, { updatedAt: T1 }),
	1031	+ },
	1032	+ ],
1010	1033	});
1011	1034	// The stalled login resumes and writes what it holds: refused, the
1012	1035	// revocation predates the scan and nothing would ever undo the row.

apps/cloud/src/extensions/billing/route.node.test.ts+1−10

@@ -4,7 +4,6 @@ import { Effect, Layer } from "effect"; · 3 unmodified lines
4	4	import { MemberDirectory } from "@executor-js/api/server";
5	5
6	6	import { UserStoreService } from "../../auth/context";
7		−import { MirrorReadiness, MirrorReadinessState } from "../../auth/mirror-readiness";
8	7	import { WorkOSClient, type WorkOSClientService } from "../../auth/workos";
9	8	import { WorkOsMirror, type WorkOsMirrorShape } from "../../auth/workos-mirror";
10	9	import { resolveBillingOrganization } from "./route";
@@ -40,12 +39,6 @@ const stubWorkOS = Layer.succeed( · 29 unmodified lines
40	39	);
41	40
42	41	// MEMBER is active in both orgs, as the mirror reports it.
43		−// The mirror is READY in these tests (backfill complete, reconciler caught
44		−// up), so membership is read from the stubbed directory, never from WorkOS.
45		−const stubReadiness = Layer.succeed(MirrorReadiness)({
46		− state: () => Effect.succeed(MirrorReadinessState.Ready()),
47		−});
48		−
49	42	const stubDirectory = Layer.succeed(MemberDirectory)({
50	43	membership: (accountId, organizationId) =>
51	44	Effect.succeed(
@@ -122,9 +115,7 @@ const run = (headers: Record<string, string>) => · 70 unmodified lines
122	115	resolveBillingOrganization(
123	116	new Request("https://executor.test/api/billing/customer", { headers }),
124	117	{ userId: MEMBER },
125		− ).pipe(
126		− Effect.provide(Layer.mergeAll(stubWorkOS, stubUsers, stubDirectory, stubMirror, stubReadiness)),
127		− );
	118	+ ).pipe(Effect.provide(Layer.mergeAll(stubWorkOS, stubUsers, stubDirectory, stubMirror)));
128	119
129	120	describe("billing route org selector", () => {
130	121	it.effect("fails closed when no selector header is sent", () =>

apps/cloud/src/extensions/routes.ts+4−5

@@ -31,7 +31,6 @@ import { AccountApi, AdminUsersApi } from "@executor-js/api"; · 30 unmodified lines
31	31	import { requestScopedMiddleware, type MemberDirectory } from "@executor-js/api/server";
32	32
33	33	import { UserStoreService } from "../auth/context";
34		−import { MirrorReadiness } from "../auth/mirror-readiness";
35	34	import { WorkOsMirror } from "../auth/workos-mirror";
36	35	import {
37	36	CloudAuthPublicHandlers,
@@ -80,9 +79,7 @@ const spec = OpenApi.fromApi(CloudOpenApi); · 42 unmodified lines
80	79	* core.
81	80	*/
82	81	export const makeCloudExtensionRoutes = (
83		− rsLive: Layer.Layer<
84		− DbService \| UserStoreService \| WorkOsMirror \| MemberDirectory \| MirrorReadiness
85		− >,
	82	+ rsLive: Layer.Layer<DbService \| UserStoreService \| WorkOsMirror \| MemberDirectory>,
86	83	) => {
87	84	// Session routes (login / callback / me / switch-org / …). Handlers yield
88	85	// `UserStoreService` directly; the per-request DB combine keeps the postgres
@@ -119,7 +116,9 @@ export const makeCloudExtensionRoutes = ( · 30 unmodified lines
119	116	// rather than on the protected API because the protected plane's middleware
120	117	// binds a product-view executor to one acting member — this one authorizes an
121	118	// org key (or an admin session) and builds a subject-less platform view.
122		− const AdminUsersRoutes = makeCloudAdminUsersRoutes(rsLive, { router: apiPrefixedRouter });
	119	+ const AdminUsersRoutes = makeCloudAdminUsersRoutes(rsLive, {
	120	+ router: apiPrefixedRouter,
	121	+ });
123	122
124	123	// The WorkOS webhook needs no per-request DB layer: it verifies the
125	124	// signature with the boot `WorkOSClient` and detaches a reconciler pass

apps/cloud/src/mcp/auth-provider.ts+16−11

@@ -2,7 +2,7 @@ · 1 unmodified line
2	2	// Cloud McpAuthProvider adapter — the cloud analog of selfHostMcpAuthProviderLayer.
3	3	//
4	4	// Folds the entire cloud edge auth/authz surface (WorkOS JWT verify + API-key
5		−// bearer + per-request org-liveness check + the two OAuth discovery docs) into
	5	+// bearer + per-request membership check + the two OAuth discovery docs) into
6	6	// ONE `McpAuthProvider` Layer behind the shared host-mcp envelope.
7	7	//
8	8	// `authenticate(request)` runs on EVERY /mcp request and resolves a typed
@@ -10,13 +10,16 @@ · 1 unmodified line
10	10	// - missing bearer -> Unauthorized (challenge: Bearer resource_metadata=…)
11	11	// - invalid token/api key -> Unauthorized (challenge: Bearer error="invalid_token" …)
12	12	// - transient JWKS OR membership-lookup infra -> Unavailable (caught here;
13		−// envelope renders a retryable 503 -32001). A WorkOS blip during the live
14		−// org check is a TRANSIENT failure, not evidence the org is gone, so it
15		−// must NOT reach the Forbidden/destroy path below.
	13	+// envelope renders a retryable 503 -32001). The membership check reads
	14	+// the local mirror (`auth/organization.ts`), so the infra that can fail
	15	+// here is the database — or WorkOS, on the one path that still asks it
	16	+// (an organization the mirror has never seen). Either is TRANSIENT, not
	17	+// evidence the org is gone, and must NOT reach the Forbidden/destroy
	18	+// path below.
16	19	// - no org / revoked org -> Forbidden ("No organization in session …", -32001).
17	20	// This requires a POSITIVE determination (the lookup SUCCEEDED and the org
18	21	// is absent), never a failed lookup. Because authenticate reads the
19		−// mcp-session-id header to do the live org check, the envelope's
	22	+// mcp-session-id header to do the membership check, the envelope's
20	23	// dispose-on-Forbidden-with-sessionId path reproduces the old inline
21	24	// clearExistingSession.
22	25	// - verified + org allowed -> Authenticated(principal)
@@ -70,15 +73,17 @@ const TOOLKIT_PROTECTED_RESOURCE_METADATA_PATH = `${PROTECTED_RESOURCE_METADATA_ · 47 unmodified lines
70	73
71	74	const NO_ORGANIZATION_MESSAGE = "No organization in session — log in via the web app first";
72	75
73		−// A transient WorkOS failure (429 / 5xx / timeout / network) during the live
74		−// membership lookup must NOT masquerade as "org revoked" — but the failure
75		−// channel alone is not enough to tell them apart: WorkOS also answers with
76		−// DEFINITIVE 4xx denials (401 revoked/invalid API key, 403, 404 deleted org)
77		−// that the SDK throws as typed exceptions. So the classification is:
	76	+// A transient failure during the membership lookup must NOT masquerade as
	77	+// "org revoked". The lookup reads the local mirror, so in steady state its
	78	+// only failure is the database; the one path that still asks WorkOS (an
	79	+// organization the mirror has never seen, resolved for a caller WorkOS
	80	+// confirms as its member) can also fail with a DEFINITIVE 4xx denial (401
	81	+// revoked/invalid API key, 403, 404 deleted org) that the SDK throws as a
	82	+// typed exception. So the classification is:
78	83	// - lookup SUCCEEDS with `null` -> genuine absence -> Forbidden
79	84	// - lookup FAILS with WorkOS 401/403/404 -> definitive denial -> Forbidden
80	85	// (fail CLOSED: WorkOS answered and said no; retrying cannot help)
81		−// - lookup FAILS any other way (429/5xx/timeout/network/no status)
	86	+// - lookup FAILS any other way (database, 429/5xx/timeout/network/no status)
82	87	// -> transient -> retryable 503, session preserved
83	88	// The status rides on `WorkOSError.status` (threaded from the SDK exception at
84	89	// the service boundary in auth/workos.ts); `isDefinitiveWorkOSDenial` is the

apps/cloud/src/mcp/auth.ts+6−5

@@ -18,7 +18,6 @@ import { BEARER_PREFIX } from "../auth/bearer"; · 17 unmodified lines
18	18	import { authorizeOrganization } from "../auth/organization";
19	19	import { UserStoreService, makeUserStoreLayer } from "../auth/context";
20	20	import { makeMemberDirectoryLayer } from "../auth/member-directory";
21		−import { makeMirrorReadinessLayer } from "../auth/mirror-readiness";
22	21	import { makeWorkOsMirrorLayer } from "../auth/workos-mirror";
23	22	import { CoreSharedServices } from "../auth/workos";
24	23	import { makeDbLayer } from "../db/db";
@@ -211,13 +210,11 @@ const makeMcpOrganizationAuthServices = () => { · 186 unmodified lines
211	210	const dbLive = makeDbLayer();
212	211	const userStoreLive = makeUserStoreLayer().pipe(Layer.provide(dbLive));
213	212	const memberDirectoryLive = makeMemberDirectoryLayer().pipe(Layer.provide(dbLive));
214		− const mirrorReadinessLive = makeMirrorReadinessLayer().pipe(Layer.provide(dbLive));
215	213	const workOsMirrorLive = makeWorkOsMirrorLayer().pipe(Layer.provide(dbLive));
216	214	return Layer.mergeAll(
217	215	dbLive,
218	216	userStoreLive,
219	217	memberDirectoryLive,
220		− mirrorReadinessLive,
221	218	workOsMirrorLive,
222	219	CoreSharedServices,
223	220	);
@@ -329,7 +326,9 @@ export const McpAuthLive = Layer.effect( · 105 unmodified lines
329	326	if (!verified) return mcpUnauthorized("invalid_token", "The access token is invalid");
330	327	if (Predicate.isTagged(verified, "Unauthorized")) return verified;
331	328	if (!verified.accountId) {
332		− yield* Effect.annotateCurrentSpan({ "mcp.auth.outcome": "missing_subject" });
	329	+ yield* Effect.annotateCurrentSpan({
	330	+ "mcp.auth.outcome": "missing_subject",
	331	+ });
333	332	return mcpUnauthorized("invalid_token", "The access token is invalid");
334	333	}
335	334	yield* Effect.annotateCurrentSpan({
@@ -344,7 +343,9 @@ export const McpAuthLive = Layer.effect( · 8 unmodified lines
344	343	verifyBearer: Effect.fn("mcp.auth.verify_bearer")(function* (request) {
345	344	const authHeader = request.headers.get("authorization");
346	345	if (!authHeader?.startsWith(BEARER_PREFIX)) {
347		− yield* Effect.annotateCurrentSpan({ "mcp.auth.outcome": "missing_bearer" });
	346	+ yield* Effect.annotateCurrentSpan({
	347	+ "mcp.auth.outcome": "missing_bearer",
	348	+ });
348	349	return mcpUnauthorized("missing_bearer");
349	350	}
350	351	const token = authHeader.slice(BEARER_PREFIX.length).trim();

apps/cloud/src/org/auth-middleware.ts+8−11

@@ -9,7 +9,6 @@ import { · 8 unmodified lines
9	9
10	10	import { UserStoreService } from "../auth/context";
11	11	import { sessionFromSealed } from "../auth/middleware";
12		−import { MirrorReadiness } from "../auth/mirror-readiness";
13	12	import { WorkOsMirror } from "../auth/workos-mirror";
14	13	import { ORG_SELECTOR_HEADER, authorizeOrganizationSelector } from "../auth/organization";
15	14	import { WorkOSClient } from "../auth/workos";
@@ -35,19 +34,19 @@ const noOrganization = () => · 19 unmodified lines
35	34
36	35	/**
37	36	* The caller's role in the session org, as `authorizeOrganizationSelector`
38		− * read it for THIS request: from the mirror while the mirror is ready, from
39		− * WorkOS otherwise (`auth/organization.ts`). Provided beside `AuthContext` —
40		− * the shared seam, which carries no role — so the domain handlers' admin gate
41		− * is this one value, never a second read of the mirror that would skip the
42		− * readiness rule and admit a demoted admin on a stale row while the
43		− * reconciler is behind.
	37	+ * read it for THIS request from the membership mirror (`auth/organization.ts`).
	38	+ * Provided beside `AuthContext` — the shared seam, which carries no role — so
	39	+ * the domain handlers' admin gate is this one value, never a second read of
	40	+ * the mirror.
44	41	*/
45	42	export class OrgMemberRole extends Context.Service<
46	43	OrgMemberRole,
47	44	{ readonly memberRole: "admin" \| "member" }
48	45	>()("@executor-js/cloud/OrgMemberRole") {}
49	46
50		−const OrgAuthMiddleware = HttpRouter.middleware<{ provides: AuthContext \| OrgMemberRole }>()(
	47	+const OrgAuthMiddleware = HttpRouter.middleware<{
	48	+ provides: AuthContext \| OrgMemberRole;
	49	+}>()(
51	50	Effect.gen(function* () {
52	51	const captured = yield* Effect.context<WorkOSClient>();
53	52	const workos = yield* WorkOSClient;
@@ -93,7 +92,5 @@ const OrgAuthMiddleware = HttpRouter.middleware<{ provides: AuthContext \| OrgMem · 39 unmodified lines
93	92	);
94	93
95	94	export const orgAuthMiddleware = (
96		− rsLive: Layer.Layer<
97		− DbService \| UserStoreService \| MemberDirectory \| MirrorReadiness \| WorkOsMirror
98		− >,
	95	+ rsLive: Layer.Layer<DbService \| UserStoreService \| MemberDirectory \| WorkOsMirror>,
99	96	) => OrgAuthMiddleware.combine(requestScopedMiddleware(rsLive)).layer;

apps/cloud/src/org/handlers.test.ts+48−64

@@ -5,7 +5,6 @@ import { HttpApiBuilder } from "effect/unstable/httpapi"; · 4 unmodified lines
5	5
6	6	import { AuthContext, MemberDirectory, type DirectoryMember } from "@executor-js/api/server";
7	7	import { UserStoreService } from "../auth/context";
8		−import { MirrorReadiness, MirrorReadinessState } from "../auth/mirror-readiness";
9	8	import { ORG_SELECTOR_HEADER } from "../auth/organization";
10	9	import { WorkOSClient, type WorkOSClientService } from "../auth/workos";
11	10	import { WorkOsMirror, type WorkOsMirrorShape } from "../auth/workos-mirror";
@@ -23,8 +22,8 @@ import { OrgHandlers, assertDomainInSessionOrg, requireAdmin } from "./handlers" · 11 unmodified lines
23	22	// share — the REAL `requireAdmin` and `assertDomainInSessionOrg` exported from
24	23	// `org/handlers.ts`, so a change to the gate cannot pass on a stale copy — and
25	24	// the admin gate's SOURCE: the role `orgAuthMiddleware` resolved for the
26		−// request, so a stale mirror row cannot admit a demoted admin while the mirror
27		−// is not trusted (the readiness rule in `auth/organization.ts`).
	25	+// request, read from the local membership mirror unconditionally
	26	+// (`auth/organization.ts`).
28	27	// ---------------------------------------------------------------------------
29	28
30	29	// eslint-disable-next-line @typescript-eslint/no-explicit-any -- test stub needs wide function types
@@ -99,7 +98,11 @@ describe("Org domain handlers", () => { · 68 unmodified lines
99	98	Effect.provide(
100	99	provide("admin", {
101	100	getOrganizationDomain: () =>
102		− Effect.succeed({ id: "dom_1", organizationId: "org_1", domain: "acme.test" }),
	101	+ Effect.succeed({
	102	+ id: "dom_1",
	103	+ organizationId: "org_1",
	104	+ domain: "acme.test",
	105	+ }),
103	106	}),
104	107	),
105	108	),
@@ -113,7 +116,11 @@ describe("Org domain handlers", () => { · 7 unmodified lines
113	116	Effect.provide(
114	117	provide("admin", {
115	118	getOrganizationDomain: () =>
116		− Effect.succeed({ id: "dom_other", organizationId: "org_2", domain: "evil.test" }),
	119	+ Effect.succeed({
	120	+ id: "dom_other",
	121	+ organizationId: "org_2",
	122	+ domain: "evil.test",
	123	+ }),
117	124	}),
118	125	),
119	126	),
@@ -136,11 +143,12 @@ describe("Org domain handlers", () => { · 16 unmodified lines
136	143
137	144	// ---------------------------------------------------------------------------
138	145	// The admin gate over HTTP, through `orgAuthMiddleware`: the role the gate
139		−// sees is the one the middleware resolved through `authorizeOrganizationSelector`
140		−// — the mirror while it is ready, WorkOS otherwise. The mirror row below is
141		−// STALE: it still says `admin` for a caller WorkOS has demoted to `member`.
142		−// While the mirror is not trusted (the reconciler is behind), WorkOS's answer
143		−// must decide, and the delete must be refused.
	146	+// sees is the one the middleware resolved through `authorizeOrganizationSelector`,
	147	+// which reads the local membership mirror unconditionally — there is no
	148	+// readiness gate and no WorkOS fallback (`auth/organization.ts`). The mirror
	149	+// row below is the sole source of the role, so `stubWorkOS` here serves only
	150	+// session authentication and dies on anything else, proving membership is
	151	+// never re-checked against WorkOS.
144	152	// ---------------------------------------------------------------------------
145	153
146	154	const ORG = "org_1";
@@ -148,33 +156,30 @@ const CALLER = "user_caller"; · 1 unmodified line
148	156	const DOMAIN = "dom_1";
149	157	const createdAt = new Date("2026-01-01T00:00:00.000Z");
150	158
151		−// The mirror's row for the caller: an active admin — stale once WorkOS has
152		−// demoted them and the reconciler has not landed the change yet.
153		−const staleAdminRow: DirectoryMember = {
	159	+// The mirror's row for the caller, at whatever role a test sets.
	160	+const callerRow = (role: "admin" \| "member"): DirectoryMember => ({
154	161	accountId: CALLER,
155	162	membershipId: `om_${CALLER}_${ORG}`,
156	163	organizationId: ORG,
157	164	email: null,
158	165	name: null,
159	166	avatarUrl: null,
160		− role: "admin",
	167	+ role,
161	168	status: "active",
162	169	lastActiveAt: null,
163		−};
164		−
165		−const unread = (why: string) => () => Effect.die(why);
166		−const stubDirectory = Layer.succeed(MemberDirectory)({
167		− membership: (accountId, organizationId) =>
168		− Effect.succeed(accountId === CALLER && organizationId === ORG ? staleAdminRow : null),
169		− membershipById: unread("the org plane does not look up by membership id"),
170		− membershipsOf: unread("the org plane reads one membership, not the list"),
171		− members: unread("the org plane does not list members"),
172		− membersById: unread("the org plane does not batch members"),
173		− findByEmail: unread("the org plane does not resolve emails"),
174	170	});
175	171
176		−const readiness = (state: MirrorReadinessState) =>
177		− Layer.succeed(MirrorReadiness)({ state: () => Effect.succeed(state) });
	172	+const unread = (why: string) => () => Effect.die(why);
	173	+const stubDirectory = (role: "admin" \| "member") =>
	174	+ Layer.succeed(MemberDirectory)({
	175	+ membership: (accountId, organizationId) =>
	176	+ Effect.succeed(accountId === CALLER && organizationId === ORG ? callerRow(role) : null),
	177	+ membershipById: unread("the org plane does not look up by membership id"),
	178	+ membershipsOf: unread("the org plane reads one membership, not the list"),
	179	+ members: unread("the org plane does not list members"),
	180	+ membersById: unread("the org plane does not batch members"),
	181	+ findByEmail: unread("the org plane does not resolve emails"),
	182	+ });
178	183
179	184	const organizationRow = (id: string) => ({
180	185	id,
@@ -226,22 +231,16 @@ const stubAutumn = Layer.succeed(AutumnService)({ · 45 unmodified lines
226	231	setMemberSeats: unread("the delete does not count seats"),
227	232	});
228	233
229		−// WorkOS as the org plane sees it: the caller's session, their CURRENT
230		−// membership list (demoted to member), and the domain to delete.
231		−const workosWithCallerAs = (role: "admin" \| "member", deleted: string[]) =>
	234	+// WorkOS as the org plane sees it: session authentication and the domain to
	235	+// delete. `stubWorkOS` dies on anything else, so a `listUserMemberships` call
	236	+// would fail the test — membership is never re-checked against WorkOS.
	237	+const workosForCaller = (deleted: string[]) =>
232	238	stubWorkOS({
233	239	authenticateSealedSession: () =>
234		− Effect.succeed({ userId: CALLER, email: "caller@placeholder.test", organizationId: ORG }),
235		− listUserMemberships: () =>
236	240	Effect.succeed({
237		− data: [
238		− {
239		− id: staleAdminRow.membershipId,
240		− organizationId: ORG,
241		− status: "active",
242		− role: { slug: role },
243		− },
244		− ],
	241	+ userId: CALLER,
	242	+ email: "caller@placeholder.test",
	243	+ organizationId: ORG,
245	244	}),
246	245	getOrganizationDomain: () =>
247	246	Effect.succeed({ id: DOMAIN, organizationId: ORG, domain: "acme.test" }),
@@ -251,8 +250,8 @@ const workosWithCallerAs = (role: "admin" \| "member", deleted: string[]) => · 3 unmodified lines
251	250	}),
252	251	});
253	252
254		−const orgApp = (state: MirrorReadinessState, workos: Layer.Layer<WorkOSClient>) => {
255		− const rsLive = Layer.mergeAll(stubDb, stubUsers, stubDirectory, stubMirror, readiness(state));
	253	+const orgApp = (role: "admin" \| "member", workos: Layer.Layer<WorkOSClient>) => {
	254	+ const rsLive = Layer.mergeAll(stubDb, stubUsers, stubDirectory(role), stubMirror);
256	255	const App = HttpApiBuilder.layer(OrgHttpApi).pipe(
257	256	Layer.provide(OrgHandlers),
258	257	Layer.provide(orgAuthMiddleware(rsLive)),
@@ -268,9 +267,9 @@ afterAll(async () => { · 9 unmodified lines
268	267	await Promise.all(apps.map((app) => app.dispose()));
269	268	});
270	269
271		−const deleteDomain = async (state: MirrorReadinessState, role: "admin" \| "member") => {
	270	+const deleteDomain = async (role: "admin" \| "member") => {
272	271	const deleted: string[] = [];
273		− const app = orgApp(state, workosWithCallerAs(role, deleted));
	272	+ const app = orgApp(role, workosForCaller(deleted));
274	273	apps.push(app);
275	274	const response = await app.handler(
276	275	new Request(`https://executor.test/org/domains/${DOMAIN}`, {
@@ -285,30 +284,15 @@ const deleteDomain = async (state: MirrorReadinessState, role: "admin" \| "member · 8 unmodified lines
285	284	};
286	285
287	286	describe("Org domain handlers over HTTP: the admin gate is the authorized role", () => {
288		− it("lets a mirrored admin delete a domain while the mirror is ready", async () => {
289		− const { status, deleted } = await deleteDomain(MirrorReadinessState.Ready(), "member");
290		− // WorkOS is not consulted for membership while the mirror is ready: the
291		− // mirror row (admin) decides, and the demotion lands through the
292		− // reconciler within its lag budget.
	287	+ it("lets a mirrored admin delete a domain", async () => {
	288	+ const { status, deleted } = await deleteDomain("admin");
293	289	expect(status).toBe(200);
294	290	expect(deleted).toEqual([DOMAIN]);
295	291	});
296	292
297		− it("refuses a demoted admin while the mirror is not ready, however stale the mirror row is", async () => {
298		− const { status, deleted } = await deleteDomain(
299		− MirrorReadinessState.ReconcilerStale({ drainedAt: null }),
300		− "member",
301		− );
302		− expect(status, "WorkOS says member; the stale admin row does not grant the delete").toBe(403);
	293	+ it("refuses a mirrored plain member, without ever consulting WorkOS", async () => {
	294	+ const { status, deleted } = await deleteDomain("member");
	295	+ expect(status).toBe(403);
303	296	expect(deleted).toEqual([]);
304	297	});
305		−
306		− it("lets an admin WorkOS confirms delete a domain while the mirror is not ready", async () => {
307		− const { status, deleted } = await deleteDomain(
308		− MirrorReadinessState.ReconcilerStale({ drainedAt: null }),
309		− "admin",
310		− );
311		− expect(status).toBe(200);
312		− expect(deleted).toEqual([DOMAIN]);
313		− });
314	298	});

e2e/cloud/mcp-workos-blip-session-survival.test.ts+136−105

@@ -1,25 +1,25 @@
1		−// Cloud: how the per-request live-membership check classifies WorkOS failures,
2		−// pinned in BOTH directions at the real upstream (faults armed on the WorkOS
3		−// emulator's membership endpoint — the same emulator the product's real WorkOS
4		−// SDK talks to; no product code or stubs touched):
	1	+// Cloud: an MCP session's relationship to WorkOS after the membership mirror.
5	2	//
6		−// 1. A TRANSIENT WorkOS outage (5xx/timeout) must NOT destroy a live MCP
7		−// session. This is the churn-risk defect: a WorkOS blip used to collapse to
8		−// Forbidden, and a Forbidden carrying a session id schedules the session
9		−// Durable Object for destruction (in-flight executions, paused approvals,
10		−// undelivered results — all gone). For a shared-API-key org a single blip
11		−// could mass-condemn every session at once. Contract: the blip request fails
12		−// RETRYABLY (503 + Retry-After), and once WorkOS recovers the SAME session
13		−// id keeps serving requests.
	3	+// Membership is authorized from the local mirror on every /mcp request
	4	+// (`auth/organization.ts`); WorkOS is a write target and an event source, not
	5	+// a per-request read. Two contracts follow, pinned here at the real upstream
	6	+// (faults armed on the WorkOS emulator's membership endpoint — the same
	7	+// emulator the product's real WorkOS SDK talks to; no product code or stubs
	8	+// touched):
14	9	//
15		−// 2. A DEFINITIVE WorkOS denial (401 — the revoked/invalid API key answer) must
16		−// fail CLOSED: Forbidden, session condemned. Retrying cannot help; treating
17		−// it as transient would preserve sessions indefinitely for a revoked
18		−// customer (the fail-open inversion the adversarial review caught).
	10	+// 1. A WorkOS OUTAGE is INVISIBLE to a live session. Before the mirror, a
	11	+// 5xx from the membership lookup had to be classified as transient (a
	12	+// retryable 503 that left the session alive) so a blip could not
	13	+// mass-condemn every session of a shared-API-key org. Now the request never
	14	+// asks WorkOS at all: a request issued during the outage is a plain 200,
	15	+// and the SAME session id keeps serving afterwards. The fault is armed on
	16	+// the exact endpoint the old check hit, so an unnoticed regression back to
	17	+// a per-request WorkOS read would fail this as a 503 (or worse, a 403).
19	18	//
20		−// Red/green for (1): pre-fix, the outage request returns a session-destroying
21		−// Forbidden and the post-outage request gets 404 "reconnect". With the fix the
22		−// outage request is a 503 and the post-outage request is a clean 200.
	19	+// 2. A REVOKED membership still fails CLOSED. The mirror is not a cache with a
	20	+// TTL: a removal made through the product writes the mirror in the same
	21	+// request, so the removed member's next /mcp request is a Forbidden, the
	22	+// session is condemned, and the id is dead. Retrying cannot help.
23	23	import { expect } from "@effect/vitest";
24	24	import { Effect } from "effect";
25	25
@@ -29,6 +29,7 @@ import { scenario } from "../src/scenario"; · 3 unmodified lines
29	29	import { Mcp, Target } from "../src/services";
30	30	import type { Identity } from "../src/target";
31	31	import { WORKOS_EMULATOR_PORT } from "../targets/cloud";
	32	+import { cookieOf, joinOrg, orgSelectorOf } from "./support/session";
32	33
33	34	const JSON_AND_SSE = "application/json, text/event-stream";
34	35	const PROTOCOL_VERSION = "2025-03-26";
@@ -105,11 +106,11 @@ const openSession = async (mcpUrl: string, bearer: string): Promise<string> => { · 70 unmodified lines
105	106	return sessionId;
106	107	};
107	108
108		−// The live membership check is `GET /user_management/organization_memberships`
109		−// (WorkOS `listOrganizationMemberships`). A bounded count covers the outage
110		−// request without leaking into later (post-clear) requests; we also clear
111		−// explicitly. `times` is generous so any internal retry inside the one faulted
112		−// request still sees the outage, but the finalizer removes whatever remains.
	109	+// The endpoint the pre-mirror per-request check hit
	110	+// (`GET /user_management/organization_memberships`). Armed to prove it is no
	111	+// longer on the request path: a request that reached it would fail. `times`
	112	+// is generous so any retry inside a faulted request still sees the outage; the
	113	+// finalizer removes whatever remains.
113	114	const MEMBERSHIP_FAULT = {
114	115	match: {
115	116	method: "GET",
@@ -119,22 +120,8 @@ const MEMBERSHIP_FAULT = { · 3 unmodified lines
119	120	times: 8,
120	121	} as const;
121	122
122		−// The definitive-denial counterpart: WorkOS ANSWERS the membership lookup with
123		−// 401 — the shape of a revoked/invalid API key. Not a blip; must fail closed.
124		−const MEMBERSHIP_DENIAL_FAULT = {
125		− match: {
126		− method: "GET",
127		− pathPattern: "/user_management/organization_memberships*",
128		− },
129		− response: {
130		− status: 401,
131		− body: { message: "Could not authorize the request. Maybe your API key is invalid?" },
132		− },
133		− times: 8,
134		−} as const;
135		−
136	123	scenario(
137		− "MCP sessions · a transient WorkOS outage 503s retryably and leaves the session alive",
	124	+ "MCP sessions · a WorkOS outage is invisible to a live session, which is authorized from the mirror",
138	125	{},
139	126	Effect.gen(function* () {
140	127	const target = yield* Target;
@@ -151,106 +138,150 @@ scenario( · 10 unmodified lines
151	138	const healthy = yield* Effect.promise(() =>
152	139	mcpPost(target.mcpUrl, { bearer, sessionId, body: toolsList(2) }),
153	140	);
154		− expect(healthy.status, "the session serves requests before the blip").toBe(200);
	141	+ expect(healthy.status, "the session serves requests before the outage").toBe(200);
155	142	yield* Effect.promise(() => healthy.text());
156	143
157	144	yield* Effect.gen(function* () {
158		− // The blip: WorkOS membership lookups start failing with 503.
	145	+ // The outage: WorkOS membership lookups would fail with 503 — if
	146	+ // anything asked.
159	147	yield* Effect.promise(() => workos.faults.arm(MEMBERSHIP_FAULT));
160	148
161		− // A request issued DURING the outage. The membership lookup fails
162		− // transiently — this must be a retryable 503, NOT a Forbidden (which
163		− // would condemn the session).
	149	+ // A request issued DURING the outage. Membership is read from the
	150	+ // mirror, so WorkOS is never consulted and the request is a plain
	151	+ // success — not a retryable 503 (the pre-mirror contract) and never a
	152	+ // Forbidden (which would condemn the session).
164	153	const duringOutage = yield* Effect.promise(() =>
165	154	mcpPost(target.mcpUrl, { bearer, sessionId, body: toolsList(3) }),
166	155	);
167		− const outageBody = (yield* Effect.promise(() => duringOutage.json())) as JsonRpcError;
168	156	expect(
169	157	duringOutage.status,
170		− "a WorkOS blip is a retryable 503, not a session-destroying error",
171		− ).toBe(503);
172		− expect(
173		− duringOutage.status,
174		− "the blip is NOT surfaced as a 404 reconnect (which would mean the session was destroyed)",
175		− ).not.toBe(404);
176		− expect(
177		− outageBody.error.code,
178		− "the 503 is a JSON-RPC error envelope the transport retries",
179		− ).toBe(-32001);
180		− expect(
181		− duringOutage.headers.get("retry-after"),
182		− "the 503 advertises a Retry-After so clients back off",
183		− ).toEqual(expect.any(String));
	158	+ "a WorkOS outage does not touch a request: membership comes from the mirror",
	159	+ ).toBe(200);
	160	+ yield* Effect.promise(() => duringOutage.text());
184	161	}).pipe(
185		− // Always lift the outage, even if an assertion above fails, so the
186		− // recovery request runs against a healthy WorkOS.
	162	+ // Always lift the outage, even if an assertion above fails.
187	163	Effect.ensuring(Effect.promise(() => workos.faults.clear())),
188	164	);
189	165
190		− // WorkOS has recovered. The SAME session id must still serve requests: the
191		− // blip left it untouched. On the pre-fix code this is a 404 (the outage
192		− // request destroyed the DO); with the fix it is a clean 200.
	166	+ // The SAME session id keeps serving after the outage: nothing condemned it.
193	167	const afterOutage = yield* Effect.promise(() =>
194	168	mcpPost(target.mcpUrl, { bearer, sessionId, body: toolsList(4) }),
195	169	);
196		− expect(
197		− afterOutage.status,
198		− "the session survived the blip and resumes work once WorkOS recovers",
199		− ).toBe(200);
	170	+ expect(afterOutage.status, "the session is untouched by the outage").toBe(200);
200	171	yield* Effect.promise(() => afterOutage.text());
201	172	}),
202	173	);
203	174
204	175	scenario(
205		− "MCP sessions · a definitive WorkOS denial fails closed and condemns the session",
	176	+ "MCP sessions · a revoked membership fails closed on the next request and condemns the session",
206	177	{},
207	178	Effect.gen(function* () {
208	179	const target = yield* Target;
209	180	const mcp = yield* Mcp;
210		− const identity = yield* target.newIdentity();
211		− const bearer = yield* mcp.mintBearer(emailOf(identity));
212	181
213		− const workos = yield* Effect.promise(() =>
214		− connectEmulator({ baseUrl: `http://127.0.0.1:${WORKOS_EMULATOR_PORT}` }),
215		− );
	182	+ // An admin's org with one plain member, joined through the real invite →
	183	+ // accept flow. The member is the one whose access is revoked.
	184	+ const admin = yield* target.newIdentity();
	185	+ const invitee = yield* target.newIdentity({ org: false });
	186	+ const member = yield* joinOrg(target, admin, invitee);
	187	+ const bearer = yield* mcp.mintBearer(emailOf(member));
	188	+ const orgSelector = orgSelectorOf(member);
216	189
217		− // A healthy session doing real work before the denial.
218		− const sessionId = yield* Effect.promise(() => openSession(target.mcpUrl, bearer));
219		− const healthy = yield* Effect.promise(() =>
220		− mcpPost(target.mcpUrl, { bearer, sessionId, body: toolsList(2) }),
221		− );
222		− expect(healthy.status, "the session serves requests before the denial").toBe(200);
	190	+ // The member's healthy session doing real work before the revocation.
	191	+ const mcpUrl = `${target.mcpUrl}`;
	192	+ const withOrg = (body: unknown, sessionId?: string) =>
	193	+ fetch(mcpUrl, {
	194	+ method: "POST",
	195	+ headers: {
	196	+ accept: JSON_AND_SSE,
	197	+ "content-type": "application/json",
	198	+ authorization: `Bearer ${bearer}`,
	199	+ "x-executor-mcp-organization": orgSelector,
	200	+ ...(sessionId ? { "mcp-session-id": sessionId } : {}),
	201	+ },
	202	+ body: JSON.stringify(body),
	203	+ });
	204	+ const initialize = yield* Effect.promise(() => withOrg(INITIALIZE_REQUEST));
	205	+ const sessionId = initialize.headers.get("mcp-session-id");
	206	+ yield* Effect.promise(() => initialize.text());
	207	+ expect(initialize.status, "the member opens a session in the org").toBe(200);
	208	+ if (!sessionId) throw new Error("initialize returned no session id");
	209	+ const initialized = yield* Effect.promise(() => withOrg(INITIALIZED_NOTIFICATION, sessionId));
	210	+ yield* Effect.promise(() => initialized.text());
	211	+ const healthy = yield* Effect.promise(() => withOrg(toolsList(2), sessionId));
	212	+ expect(healthy.status, "the session serves requests before the revocation").toBe(200);
223	213	yield* Effect.promise(() => healthy.text());
224	214
225		− yield* Effect.gen(function* () {
226		− // WorkOS starts ANSWERING the membership lookup with 401 — the
227		− // revoked/invalid API key shape. Deterministic denial, not a blip.
228		− yield* Effect.promise(() => workos.faults.arm(MEMBERSHIP_DENIAL_FAULT));
	215	+ // The admin removes the member through the product. The removal writes
	216	+ // the mirror in the same request (a deletion tombstone keyed to the
	217	+ // WorkOS membership id), so no reconciler tick is needed for it to land.
	218	+ const members = yield* Effect.promise(async () => {
	219	+ const response = await fetch(new URL("/api/account/members", target.baseUrl), {
	220	+ headers: { ...(admin.headers ?? {}) },
	221	+ });
	222	+ if (!response.ok) throw new Error(`/api/account/members failed (${response.status})`);
	223	+ return (await response.json()) as {
	224	+ readonly members: ReadonlyArray<{ readonly id: string; readonly isCurrentUser: boolean }>;
	225	+ };
	226	+ });
	227	+ const removed = members.members.find((row) => !row.isCurrentUser);
	228	+ if (!removed) throw new Error("the joined member is not listed in the org");
	229	+ const removal = yield* Effect.promise(() =>
	230	+ fetch(new URL(`/api/account/members/${removed.id}`, target.baseUrl), {
	231	+ method: "DELETE",
	232	+ headers: { ...(admin.headers ?? {}), origin: new URL(target.baseUrl).origin },
	233	+ }),
	234	+ );
	235	+ expect(removal.status, "the admin removes the member").toBe(200);
	236	+ yield* Effect.promise(() => removal.text());
229	237
230		− const denied = yield* Effect.promise(() =>
231		− mcpPost(target.mcpUrl, { bearer, sessionId, body: toolsList(3) }),
232		− );
233		− const deniedBody = (yield* Effect.promise(() => denied.json())) as JsonRpcError;
234		− expect(
235		− denied.status,
236		− "a definitive WorkOS denial fails closed as Forbidden, never a retryable 503",
237		− ).toBe(403);
238		− expect(deniedBody.error.code, "the denial is a JSON-RPC error envelope").toBe(-32001);
239		− }).pipe(Effect.ensuring(Effect.promise(() => workos.faults.clear())));
	238	+ // The removed member's NEXT request on the live session: a positive
	239	+ // determination from the mirror that they hold no active membership — a
	240	+ // real Forbidden, which condemns the session.
	241	+ const denied = yield* Effect.promise(() => withOrg(toolsList(3), sessionId));
	242	+ const deniedBody = (yield* Effect.promise(() => denied.json())) as JsonRpcError;
	243	+ expect(denied.status, "a revoked member fails closed as Forbidden on the next request").toBe(
	244	+ 403,
	245	+ );
	246	+ expect(deniedBody.error.code, "the denial is a JSON-RPC error envelope").toBe(-32001);
240	247
241		− // The Forbidden carried the session id, so the session was condemned: the
242		− // id must NOT serve requests once WorkOS recovers. If this returned 200 the
243		− // fail-closed contract is broken (a revoked customer kept a live session).
244		− const afterDenial = yield* Effect.promise(() =>
245		− mcpPost(target.mcpUrl, { bearer, sessionId, body: toolsList(4) }),
	248	+ // While revoked, every further request is refused at the gate — still a
	249	+ // Forbidden, never a 200 (a removed member kept a live session) and never
	250	+ // a retryable 503 (nothing about this is transient).
	251	+ const stillDenied = yield* Effect.promise(() => withOrg(toolsList(4), sessionId));
	252	+ expect(stillDenied.status, "a revoked member stays refused, deterministically").toBe(403);
	253	+ yield* Effect.promise(() => stillDenied.text());
	254	+
	255	+ // The Forbidden carried the session id, so the session was condemned. A
	256	+ // caller the gate admits proves it: the admin, still a member, presents
	257	+ // the condemned id with their own bearer. Had the id survived, the answer
	258	+ // would be the ownership Forbidden (-32003: the session belongs to someone
	259	+ // else); condemned, it is dead and the client is told to reconnect.
	260	+ const adminBearer = yield* mcp.mintBearer(emailOf(admin));
	261	+ const condemned = yield* Effect.promise(() =>
	262	+ fetch(mcpUrl, {
	263	+ method: "POST",
	264	+ headers: {
	265	+ accept: JSON_AND_SSE,
	266	+ "content-type": "application/json",
	267	+ authorization: `Bearer ${adminBearer}`,
	268	+ "x-executor-mcp-organization": orgSelectorOf(admin),
	269	+ "mcp-session-id": sessionId,
	270	+ },
	271	+ body: JSON.stringify(toolsList(5)),
	272	+ }),
246	273	);
247		− expect(
248		− afterDenial.status,
249		− "the condemned session id is dead after a definitive denial (reconnect required)",
250		− ).toBe(404);
251		− const afterBody = (yield* Effect.promise(() => afterDenial.json())) as JsonRpcError;
252		− expect(afterBody.error.message, "the client is told to reconnect").toMatch(
	274	+ expect(condemned.status, "the condemned session id is dead (reconnect required)").toBe(404);
	275	+ const condemnedBody = (yield* Effect.promise(() => condemned.json())) as JsonRpcError;
	276	+ expect(condemnedBody.error.message, "the client is told to reconnect").toMatch(
253	277	/timed out\|reconnect\|not found/i,
254	278	);
	279	+
	280	+ // The admin's own access is unaffected by removing someone else.
	281	+ const adminStillIn = yield* Effect.promise(() =>
	282	+ fetch(new URL("/api/account/me", target.baseUrl), { headers: { cookie: cookieOf(admin) } }),
	283	+ );
	284	+ expect(adminStillIn.status, "the admin keeps their access").toBe(200);
	285	+ yield* Effect.promise(() => adminStillIn.text());
255	286	}),
256	287	);