History

Give each request its own execution stack build

Concurrent requests shared one database provider build, so a request could use a connection owned by another request and lose it when that request finished.

Rhys Sullivan committed 2 months ago

Commit a2ed2ab1 parent bc8dd395 files · +341 −7Browse files

Unified · Split

5 files
  1. .changeset/request-scoped-execution-stack-build.md +11 −0
  2. apps/cloud/src/account/account-api.ts +4 −1
  3. apps/cloud/src/admin/admin-users-api.ts +4 −1
  4. apps/cloud/src/api.request-scope.node.test.ts +309 −3
  5. packages/core/api/src/server/execution-stack-middleware.ts +13 −2
.changeset/request-scoped-execution-stack-build.md+11−0
@@ -0,0 +1,11 @@
1+---
2+"executor": patch
3+---
4+
5+**Concurrent API requests no longer share one database provider build**
6+
7+Every HTTP request gets its own database connection, opened when the request fiber's scope opens and closed when it closes. The middleware that builds a request's execution stack, however, captures the boot fiber's context once at layer-construction time and re-applies it to every request. A captured context carries Effect's current memoization map, and re-applying it replaced the fresh per-request map with the boot one — which every in-flight request in the isolate shares.
8+
9+The per-request stack build then memoized itself there. Sequential requests still rebuilt, because the memo entry is released once the request that built it finishes, so the problem was confined to requests that overlap: the second request reused the first one's stack build, and with it the first one's database connection. A request could therefore issue queries on a connection it did not own, and lose that connection mid-flight when the owner finished and closed it — typically surfacing as a failed read after a slow outbound call, on a request that had already read successfully.
10+
11+The stack is now built with a request-local memoization scope, so overlapping requests each build their own stack over their own connection. The captured context still carries the long-lived services it exists to carry. The two other per-request provider builds that ran under a captured context — the account provider and the admin-users provider — are built the same way, for the same reason.
apps/cloud/src/account/account-api.ts+4−1
@@ -72,13 +72,16 @@ const AccountProviderMiddleware = HttpRouter.middleware<{ provides: AccountProvi · 71 unmodified lines
7272
7373 // Built inside the request body so the WorkOS account service closes
7474 // over the per-request `UserStoreService` (postgres socket) supplied by
75− // the combined request-scoped layer.
75+ // the combined request-scoped layer. `local` keeps that promise: the
76+ // `longLived` context re-applied below carries the boot `CurrentMemoMap`,
77+ // so a shared build would hand overlapping requests one another's socket.
7678 const accountProvider = yield* Effect.provide(
7779 AccountProvider.asEffect(),
7880 workosAccountProvider.pipe(
7981 Layer.provide(ApiKeyService.WorkOS),
8082 Layer.provide(Layer.succeed(AccountCaller)({ session })),
8183 ),
84+ { local: true },
8285 );
8386 return yield* Effect.provideService(httpEffect, AccountProvider, accountProvider);
8487 }).pipe(Effect.provideContext(longLived));
apps/cloud/src/admin/admin-users-api.ts+4−1
@@ -330,10 +330,13 @@ const AdminUsersProviderMiddleware = HttpRouter.middleware<{ provides: AdminUser · 329 unmodified lines
330330 return (httpEffect) =>
331331 Effect.gen(function* () {
332332 // Built inside the request body so the execution seams close over the
333− // per-request postgres socket.
333+ // per-request postgres socket. `local` keeps that promise: the
334+ // `longLived` context re-applied below carries the boot `CurrentMemoMap`,
335+ // so a shared build would hand overlapping requests one another's socket.
334336 const provider = yield* Effect.provide(
335337 AdminUsersProvider.asEffect(),
336338 workosAdminUsersProvider.pipe(Layer.provide(CloudExecutionSeamsLayer)),
339+ { local: true },
337340 );
338341 return yield* Effect.provideService(httpEffect, AdminUsersProvider, provider);
339342 }).pipe(Effect.provideContext(longLived));
apps/cloud/src/api.request-scope.node.test.ts+309−3
@@ -21,11 +21,32 @@ · 20 unmodified lines
2121 // for `acquireRelease`.
2222 // ---------------------------------------------------------------------------
2323
24−import { describe, it, expect } from "@effect/vitest";
24+import { describe, it, expect, beforeEach } from "@effect/vitest";
2525 import { Context, Effect, Layer } from "effect";
26−import { HttpRouter, HttpServer, HttpServerResponse } from "effect/unstable/http";
26+import {
27+ HttpRouter,
28+ HttpServer,
29+ HttpServerRequest,
30+ HttpServerResponse,
31+} from "effect/unstable/http";
2732
28−import { requestScopedMiddleware } from "@executor-js/api/server";
33+import {
34+ CodeExecutorProvider,
35+ DbProvider,
36+ EngineDecoratorNoop,
37+ ExecutorService,
38+ HostConfig,
39+ PluginsProvider,
40+ makeExecutionStackMiddleware,
41+ requestScopedMiddleware,
42+ textFailureStrategy,
43+ type CodeExecutor,
44+ type IdentityFailure,
45+ type Principal,
46+} from "@executor-js/api/server";
47+import { collectTables } from "@executor-js/sdk";
48+import { resetSubjectTouchCache } from "@executor-js/sdk/host-internal";
49+import { createSqliteTestFumaDb, type SqliteTestFumaDb } from "@executor-js/sdk/testing";
2950
3051 import { RequestScopedServicesLive } from "./api/layers";
3152 import { makeApiLive } from "./api/router";
@@ -195,3 +216,288 @@ describe("makeApiLive (prod handler factory) request scoping", () => { · 163 unmodified lines
195216 expect(counts.releases).toBe(2);
196217 });
197218 });
219+
220+// ---------------------------------------------------------------------------
221+// `ExecutionStackMiddleware` request scoping.
222+//
223+// `requestScopedMiddleware` gives every request a fresh `MemoMap`, so the
224+// per-request DB handle is genuinely per request (the suites above). But
225+// `makeExecutionStackMiddleware` captures the BOOT fiber's context once, at
226+// layer-construction time, and re-applies it to every request with
227+// `Effect.provideContext`. That captured context carries Effect's
228+// `CurrentMemoMap`, and `provideContext`'s merge lets the BOOT map overwrite
229+// the fresh per-request one. The per-request `Effect.provide(stackLayer)` then
230+// memoizes its build in the boot map, which every in-flight request fiber in
231+// the isolate shares.
232+//
233+// Sequential requests still rebuild (the memo entry is refcounted by observer
234+// and drops to zero when the request scope closes), so only OVERLAPPING
235+// requests are affected: the second request reuses the first request's stack
236+// build, and therefore the first request's database handle. On Cloudflare
237+// Workers that is a cross-request I/O violation; and when the owning request
238+// finishes, its scope finalizer closes the connection out from under the
239+// borrower, whose next query fails.
240+//
241+// These tests stand the real middleware up over per-request in-memory
242+// databases and pin all three symptoms.
243+// ---------------------------------------------------------------------------
244+
245+interface TestDbHandle {
246+ readonly id: number;
247+ readonly db: SqliteTestFumaDb;
248+ closed: boolean;
249+}
250+
251+class TestDb extends Context.Service<TestDb, TestDbHandle>()("test/TestDb") {}
252+
253+interface StackProbeState {
254+ /**
255+ * Databases created up front, one per request the case will make. Handing a
256+ * ready database to the request-scoped layer keeps its acquire synchronous,
257+ * so the first request to arrive is reliably the one that owns handle 1 —
258+ * which is what makes the borrow direction deterministic below.
259+ */
260+ readonly pool: readonly SqliteTestFumaDb[];
261+ /** Handles acquired by `requestScopedMiddleware` — one per request. */
262+ readonly handles: TestDbHandle[];
263+ /** How many times the stack layer's `DbProvider` was actually built. */
264+ builds: number;
265+ /** The handle id each stack build read. One entry per build. */
266+ readonly builtWithHandleId: number[];
267+ /**
268+ * What each request could see about connection ownership at the moment it
269+ * performed its read-after-await, keyed by that request's await duration.
270+ */
271+ readonly lateReadOwnership: Map<
272+ number,
273+ {
274+ readonly ownHandleId: number;
275+ readonly ownClosed: boolean;
276+ readonly otherClosed: readonly boolean[];
277+ }
278+ >;
279+}
280+
281+const noopCodeExecutor: CodeExecutor = {
282+ execute: () => Effect.succeed({ result: undefined, logs: [] }),
283+};
284+
285+const stackProbePrincipal: Principal = {
286+ kind: "member",
287+ accountId: "user_request_scope",
288+ organizationId: "org_request_scope",
289+ organizationName: "Request Scope Test Org",
290+ email: "request-scope@test.local",
291+ name: "Request Scope",
292+ avatarUrl: null,
293+ roles: ["admin"],
294+};
295+
296+/**
297+ * The per-request DB handle, in the slot cloud's postgres.js socket occupies:
298+ * acquired when the request fiber's scope opens, closed when it closes.
299+ */
300+const makeTestDbLive = (state: StackProbeState) =>
301+ Layer.effect(TestDb)(
302+ Effect.acquireRelease(
303+ Effect.sync(() => {
304+ const index = state.handles.length;
305+ const handle: TestDbHandle = {
306+ id: index + 1,
307+ db: state.pool[index] as SqliteTestFumaDb,
308+ closed: false,
309+ };
310+ state.handles.push(handle);
311+ return handle;
312+ }),
313+ (handle) =>
314+ // The request's own scope ends its connection, exactly as the host
315+ // closes the postgres socket when the request finishes.
316+ Effect.promise(async () => {
317+ handle.closed = true;
318+ await handle.db.close();
319+ }),
320+ ),
321+ );
322+
323+/**
324+ * The host's `stackLayer` seam, reduced to the parts that matter: a
325+ * `DbProvider` derived from the REQUEST-scoped handle (exactly cloud's
326+ * arrangement) plus inert stand-ins for the rest. The 5ms sleep widens the
327+ * window in which two request fibers overlap on the layer build, which is what
328+ * an isolate serving concurrent requests does on its own.
329+ */
330+const makeTestStackLayer = (state: StackProbeState) =>
331+ Layer.mergeAll(
332+ Layer.effect(DbProvider)(
333+ Effect.gen(function* () {
334+ const handle = yield* TestDb;
335+ yield* Effect.sleep("5 millis");
336+ state.builds += 1;
337+ state.builtWithHandleId.push(handle.id);
338+ return handle.db;
339+ }),
340+ ),
341+ Layer.succeed(PluginsProvider)({ plugins: () => [] }),
342+ Layer.succeed(HostConfig)({ allowLocalNetwork: false, oauthCallbackPath: "/oauth/callback" }),
343+ Layer.succeed(CodeExecutorProvider)(noopCodeExecutor),
344+ EngineDecoratorNoop,
345+ );
346+
347+/**
348+ * `GET /probe?delay=N`: one cheap read, a pause standing in for a slow
349+ * outbound call, then a second read. That is the production shape — an early
350+ * query succeeds, the request awaits something slow, and by the time it queries
351+ * again the connection it borrowed has been closed by its real owner.
352+ */
353+const makeStackProbeRoutes = (state: StackProbeState) =>
354+ HttpRouter.add(
355+ "GET",
356+ "/probe",
357+ Effect.gen(function* () {
358+ const request = yield* HttpServerRequest.HttpServerRequest;
359+ const delayMs = Number(
360+ new URL(request.url, "http://test.local").searchParams.get("delay") ?? "0",
361+ );
362+ const executor = yield* ExecutorService;
363+ const read = () =>
364+ executor.connections.list().pipe(
365+ Effect.as("ok" as const),
366+ Effect.catchCause(() => Effect.succeed("failed" as const)),
367+ );
368+
369+ // The handle this request acquired for itself, whatever order the
370+ // request fibers happened to acquire in.
371+ const own = yield* TestDb;
372+
373+ const early = yield* read();
374+ yield* Effect.sleep(`${delayMs} millis`);
375+ state.lateReadOwnership.set(delayMs, {
376+ ownHandleId: own.id,
377+ ownClosed: own.closed,
378+ otherClosed: state.handles.filter((handle) => handle.id !== own.id).map((h) => h.closed),
379+ });
380+ const late = yield* read();
381+ return HttpServerResponse.jsonUnsafe({ early, late });
382+ }),
383+ );
384+
385+const makeStackProbeHandler = (state: StackProbeState) => {
386+ const ExecutionStackMiddleware = makeExecutionStackMiddleware<
387+ readonly [],
388+ IdentityFailure,
389+ never,
390+ TestDb,
391+ never,
392+ never
393+ >({
394+ plugins: [],
395+ authenticate: () => Effect.succeed(stackProbePrincipal),
396+ strategy: textFailureStrategy,
397+ stackLayer: makeTestStackLayer(state),
398+ });
399+
400+ const App = makeStackProbeRoutes(state).pipe(
401+ // Exactly cloud's composition: the stack middleware with the per-request
402+ // DB layer combined in, so `DbProvider` is built over a handle owned by the
403+ // request fiber's scope.
404+ Layer.provide(
405+ ExecutionStackMiddleware.combine(requestScopedMiddleware(makeTestDbLive(state))).layer,
406+ ),
407+ Layer.provideMerge(HttpServer.layerServices),
408+ );
409+ return HttpRouter.toWebHandler(App, { disableLogger: true }).handler;
410+};
411+
412+const makeStackProbeState = async (requestCount: number): Promise<StackProbeState> => {
413+ const pool: SqliteTestFumaDb[] = [];
414+ for (let index = 0; index < requestCount; index += 1) {
415+ pool.push(await createSqliteTestFumaDb({ tables: collectTables() }));
416+ }
417+ return {
418+ pool,
419+ handles: [],
420+ builds: 0,
421+ builtWithHandleId: [],
422+ lateReadOwnership: new Map(),
423+ };
424+};
425+
426+describe("ExecutionStackMiddleware request scoping", () => {
427+ beforeEach(() => {
428+ // `makeExecutionStack` touches the subject once per build, behind a
429+ // process-local throttle. Reset it so every case does the same work.
430+ resetSubjectTouchCache();
431+ });
432+
433+ it("builds the execution stack once per concurrent request", async () => {
434+ const state = await makeStackProbeState(3);
435+ const handler = makeStackProbeHandler(state);
436+
437+ const responses = await Promise.all([
438+ handler(new Request("http://test.local/probe"), Context.empty()),
439+ handler(new Request("http://test.local/probe"), Context.empty()),
440+ handler(new Request("http://test.local/probe"), Context.empty()),
441+ ]);
442+
443+ expect(responses.map((response) => response.status)).toEqual([200, 200, 200]);
444+ // Three requests, three request-scoped handles, three stack builds. A lower
445+ // build count means some request served its work on a stack built for a
446+ // different request.
447+ expect(state.handles.length).toBe(3);
448+ expect(state.builds).toBe(3);
449+ });
450+
451+ it("never builds a request's stack over another request's database handle", async () => {
452+ const state = await makeStackProbeState(3);
453+ const handler = makeStackProbeHandler(state);
454+
455+ await Promise.all([
456+ handler(new Request("http://test.local/probe"), Context.empty()),
457+ handler(new Request("http://test.local/probe"), Context.empty()),
458+ handler(new Request("http://test.local/probe"), Context.empty()),
459+ ]);
460+
461+ // Every acquired handle must be the one its own request's stack was built
462+ // over: three builds reading three distinct handles, covering all three.
463+ const acquired = state.handles.map((handle) => handle.id);
464+ expect(acquired).toEqual([1, 2, 3]);
465+ expect([...state.builtWithHandleId].sort()).toEqual(acquired);
466+ expect(new Set(state.builtWithHandleId).size).toBe(state.handles.length);
467+ });
468+
469+ it("keeps a slow request's connection usable after a faster overlapping request ends", async () => {
470+ const state = await makeStackProbeState(2);
471+ const handler = makeStackProbeHandler(state);
472+
473+ // The fast request is served first, so it acquires handle 1. It finishes
474+ // while the slow request is still awaiting, which is the window in which a
475+ // borrowed connection gets pulled out from under its borrower.
476+ const [fast, slow] = await Promise.all([
477+ handler(new Request("http://test.local/probe?delay=0"), Context.empty()),
478+ handler(new Request("http://test.local/probe?delay=80"), Context.empty()),
479+ ]);
480+
481+ // Read, slow await, read again — both reads succeed for both requests.
482+ expect(await fast.json()).toEqual({ early: "ok", late: "ok" });
483+ expect(await slow.json()).toEqual({ early: "ok", late: "ok" });
484+
485+ // Ownership at the read-after-await is the real assertion. The fast
486+ // request has ended and released its connection by then, so the slow
487+ // request is reading in exactly the window that breaks in production —
488+ // and its OWN connection is still open, because that is the one its stack
489+ // was built over.
490+ //
491+ // Sharing one build inverts this: the slow request's own connection is
492+ // released while it is still running, and it keeps reading on the fast
493+ // request's instead.
494+ const slowView = state.lateReadOwnership.get(80);
495+ expect(slowView?.ownClosed).toBe(false);
496+ expect(slowView?.otherClosed).toEqual([true]);
497+
498+ // Each request built its stack over its own connection.
499+ expect(state.builds).toBe(2);
500+ expect([...state.builtWithHandleId].sort()).toEqual([1, 2]);
501+ expect(state.lateReadOwnership.get(0)?.ownHandleId).not.toBe(slowView?.ownHandleId);
502+ });
503+});
packages/core/api/src/server/execution-stack-middleware.ts+13−2
@@ -178,6 +178,13 @@ export const makeExecutionStackMiddleware = < · 177 unmodified lines
178178 | PluginExtensionServices<TPlugins>;
179179 }>()(
180180 Effect.gen(function* () {
181+ // Captured ONCE, at layer-build time, so the per-request body can carry
182+ // the boot-scoped `RCapture` services. Note what else rides along: a
183+ // captured context also holds Effect's `CurrentMemoMap`, and the
184+ // `Effect.provideContext(captured)` below re-applies it to every request
185+ // fiber — overwriting the fresh per-request map the host installed. Every
186+ // per-request `Effect.provide` in this body must therefore build with
187+ // `{ local: true }`; see the two `options.stackLayer` sites.
181188 const captured = yield* Effect.context<RCapture>();
182189 return (httpEffect) =>
183190 Effect.gen(function* () {
@@ -216,7 +223,7 @@ export const makeExecutionStackMiddleware = < · 32 unmodified lines
216223 const { executor } = yield* makePlatformExecutionStack<TPlugins>(
217224 resolved.organizationId,
218225 ).pipe(
219− Effect.provide(options.stackLayer),
226+ Effect.provide(options.stackLayer, { local: true }),
220227 Effect.withSpan("executor.stack.http.resolve_platform"),
221228 );
222229 return yield* httpEffect.pipe(
@@ -240,7 +247,7 @@ export const makeExecutionStackMiddleware = < · 17 unmodified lines
240247 resolved.organizationId,
241248 resolved.organizationName,
242249 ).pipe(
243− Effect.provide(options.stackLayer),
250+ Effect.provide(options.stackLayer, { local: true }),
244251 Effect.provideService(RequestWebOrigin, {
245252 origin: requestWebOriginFromRequest(webRequest),
246253 }),
@@ -281,6 +288,10 @@ export const makeExecutionStackMiddleware = < · 34 unmodified lines
281288 );
282289 // Provide the boot-captured context; uncaptured deps (cloud's
283290 // request-scoped `DbService`) remain residual and flow through here.
291+ // This also reinstates the BOOT `CurrentMemoMap` on the request
292+ // fiber, which is why the stack builds above are `{ local: true }`:
293+ // without that, concurrent requests memoize into one shared map and
294+ // reuse a single stack build — and so a single database connection.
284295 }).pipe(Effect.provideContext(captured as Context.Context<RCapture>));
285296 }),
286297 );

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t