History

Give each request its own execution stack build

Concurrent requests shared one database provider build, so a request could use a connection owned by another request and lose it when that request finished.

Rhys Sullivan committed 2 months ago

Commit a2ed2ab1 parent bc8dd395 files · +341 −7Browse files

Unified · Split

5 files
  1. .changeset/request-scoped-execution-stack-build.md +11 −0
  2. apps/cloud/src/account/account-api.ts +4 −1
  3. apps/cloud/src/admin/admin-users-api.ts +4 −1
  4. apps/cloud/src/api.request-scope.node.test.ts +309 −3
  5. packages/core/api/src/server/execution-stack-middleware.ts +13 −2
.changeset/request-scoped-execution-stack-build.md+11−0
@@ -0,0 +1,11 @@
1---
2"executor": patch
3---
4
5**Concurrent API requests no longer share one database provider build**
6
7Every HTTP request gets its own database connection, opened when the request fiber's scope opens and closed when it closes. The middleware that builds a request's execution stack, however, captures the boot fiber's context once at layer-construction time and re-applies it to every request. A captured context carries Effect's current memoization map, and re-applying it replaced the fresh per-request map with the boot one — which every in-flight request in the isolate shares.
8
9The per-request stack build then memoized itself there. Sequential requests still rebuilt, because the memo entry is released once the request that built it finishes, so the problem was confined to requests that overlap: the second request reused the first one's stack build, and with it the first one's database connection. A request could therefore issue queries on a connection it did not own, and lose that connection mid-flight when the owner finished and closed it — typically surfacing as a failed read after a slow outbound call, on a request that had already read successfully.
10
11The stack is now built with a request-local memoization scope, so overlapping requests each build their own stack over their own connection. The captured context still carries the long-lived services it exists to carry. The two other per-request provider builds that ran under a captured context — the account provider and the admin-users provider — are built the same way, for the same reason.
apps/cloud/src/account/account-api.ts+4−1
@@ -72,13 +72,16 @@ const AccountProviderMiddleware = HttpRouter.middleware<{ provides: AccountProvi · 71 unmodified lines
72 72
73 // Built inside the request body so the WorkOS account service closes73 // Built inside the request body so the WorkOS account service closes
74 // over the per-request `UserStoreService` (postgres socket) supplied by74 // over the per-request `UserStoreService` (postgres socket) supplied by
75 // the combined request-scoped layer.75 // the combined request-scoped layer. `local` keeps that promise: the
76 // `longLived` context re-applied below carries the boot `CurrentMemoMap`,
77 // so a shared build would hand overlapping requests one another's socket.
76 const accountProvider = yield* Effect.provide(78 const accountProvider = yield* Effect.provide(
77 AccountProvider.asEffect(),79 AccountProvider.asEffect(),
78 workosAccountProvider.pipe(80 workosAccountProvider.pipe(
79 Layer.provide(ApiKeyService.WorkOS),81 Layer.provide(ApiKeyService.WorkOS),
80 Layer.provide(Layer.succeed(AccountCaller)({ session })),82 Layer.provide(Layer.succeed(AccountCaller)({ session })),
81 ),83 ),
84 { local: true },
82 );85 );
83 return yield* Effect.provideService(httpEffect, AccountProvider, accountProvider);86 return yield* Effect.provideService(httpEffect, AccountProvider, accountProvider);
84 }).pipe(Effect.provideContext(longLived));87 }).pipe(Effect.provideContext(longLived));
apps/cloud/src/admin/admin-users-api.ts+4−1
@@ -330,10 +330,13 @@ const AdminUsersProviderMiddleware = HttpRouter.middleware<{ provides: AdminUser · 329 unmodified lines
330 return (httpEffect) =>330 return (httpEffect) =>
331 Effect.gen(function* () {331 Effect.gen(function* () {
332 // Built inside the request body so the execution seams close over the332 // Built inside the request body so the execution seams close over the
333 // per-request postgres socket.333 // per-request postgres socket. `local` keeps that promise: the
334 // `longLived` context re-applied below carries the boot `CurrentMemoMap`,
335 // so a shared build would hand overlapping requests one another's socket.
334 const provider = yield* Effect.provide(336 const provider = yield* Effect.provide(
335 AdminUsersProvider.asEffect(),337 AdminUsersProvider.asEffect(),
336 workosAdminUsersProvider.pipe(Layer.provide(CloudExecutionSeamsLayer)),338 workosAdminUsersProvider.pipe(Layer.provide(CloudExecutionSeamsLayer)),
339 { local: true },
337 );340 );
338 return yield* Effect.provideService(httpEffect, AdminUsersProvider, provider);341 return yield* Effect.provideService(httpEffect, AdminUsersProvider, provider);
339 }).pipe(Effect.provideContext(longLived));342 }).pipe(Effect.provideContext(longLived));
apps/cloud/src/api.request-scope.node.test.ts+309−3
@@ -21,11 +21,32 @@ · 20 unmodified lines
21// for `acquireRelease`.21// for `acquireRelease`.
22// ---------------------------------------------------------------------------22// ---------------------------------------------------------------------------
23 23
24import { describe, it, expect } from "@effect/vitest";24import { describe, it, expect, beforeEach } from "@effect/vitest";
25import { Context, Effect, Layer } from "effect";25import { Context, Effect, Layer } from "effect";
26import { HttpRouter, HttpServer, HttpServerResponse } from "effect/unstable/http";26import {
27 HttpRouter,
28 HttpServer,
29 HttpServerRequest,
30 HttpServerResponse,
31} from "effect/unstable/http";
27 32
28import { requestScopedMiddleware } from "@executor-js/api/server";33import {
34 CodeExecutorProvider,
35 DbProvider,
36 EngineDecoratorNoop,
37 ExecutorService,
38 HostConfig,
39 PluginsProvider,
40 makeExecutionStackMiddleware,
41 requestScopedMiddleware,
42 textFailureStrategy,
43 type CodeExecutor,
44 type IdentityFailure,
45 type Principal,
46} from "@executor-js/api/server";
47import { collectTables } from "@executor-js/sdk";
48import { resetSubjectTouchCache } from "@executor-js/sdk/host-internal";
49import { createSqliteTestFumaDb, type SqliteTestFumaDb } from "@executor-js/sdk/testing";
29 50
30import { RequestScopedServicesLive } from "./api/layers";51import { RequestScopedServicesLive } from "./api/layers";
31import { makeApiLive } from "./api/router";52import { makeApiLive } from "./api/router";
@@ -195,3 +216,288 @@ describe("makeApiLive (prod handler factory) request scoping", () => { · 163 unmodified lines
195 expect(counts.releases).toBe(2);216 expect(counts.releases).toBe(2);
196 });217 });
197});218});
219
220// ---------------------------------------------------------------------------
221// `ExecutionStackMiddleware` request scoping.
222//
223// `requestScopedMiddleware` gives every request a fresh `MemoMap`, so the
224// per-request DB handle is genuinely per request (the suites above). But
225// `makeExecutionStackMiddleware` captures the BOOT fiber's context once, at
226// layer-construction time, and re-applies it to every request with
227// `Effect.provideContext`. That captured context carries Effect's
228// `CurrentMemoMap`, and `provideContext`'s merge lets the BOOT map overwrite
229// the fresh per-request one. The per-request `Effect.provide(stackLayer)` then
230// memoizes its build in the boot map, which every in-flight request fiber in
231// the isolate shares.
232//
233// Sequential requests still rebuild (the memo entry is refcounted by observer
234// and drops to zero when the request scope closes), so only OVERLAPPING
235// requests are affected: the second request reuses the first request's stack
236// build, and therefore the first request's database handle. On Cloudflare
237// Workers that is a cross-request I/O violation; and when the owning request
238// finishes, its scope finalizer closes the connection out from under the
239// borrower, whose next query fails.
240//
241// These tests stand the real middleware up over per-request in-memory
242// databases and pin all three symptoms.
243// ---------------------------------------------------------------------------
244
245interface TestDbHandle {
246 readonly id: number;
247 readonly db: SqliteTestFumaDb;
248 closed: boolean;
249}
250
251class TestDb extends Context.Service<TestDb, TestDbHandle>()("test/TestDb") {}
252
253interface StackProbeState {
254 /**
255 * Databases created up front, one per request the case will make. Handing a
256 * ready database to the request-scoped layer keeps its acquire synchronous,
257 * so the first request to arrive is reliably the one that owns handle 1 —
258 * which is what makes the borrow direction deterministic below.
259 */
260 readonly pool: readonly SqliteTestFumaDb[];
261 /** Handles acquired by `requestScopedMiddleware` — one per request. */
262 readonly handles: TestDbHandle[];
263 /** How many times the stack layer's `DbProvider` was actually built. */
264 builds: number;
265 /** The handle id each stack build read. One entry per build. */
266 readonly builtWithHandleId: number[];
267 /**
268 * What each request could see about connection ownership at the moment it
269 * performed its read-after-await, keyed by that request's await duration.
270 */
271 readonly lateReadOwnership: Map<
272 number,
273 {
274 readonly ownHandleId: number;
275 readonly ownClosed: boolean;
276 readonly otherClosed: readonly boolean[];
277 }
278 >;
279}
280
281const noopCodeExecutor: CodeExecutor = {
282 execute: () => Effect.succeed({ result: undefined, logs: [] }),
283};
284
285const stackProbePrincipal: Principal = {
286 kind: "member",
287 accountId: "user_request_scope",
288 organizationId: "org_request_scope",
289 organizationName: "Request Scope Test Org",
290 email: "request-scope@test.local",
291 name: "Request Scope",
292 avatarUrl: null,
293 roles: ["admin"],
294};
295
296/**
297 * The per-request DB handle, in the slot cloud's postgres.js socket occupies:
298 * acquired when the request fiber's scope opens, closed when it closes.
299 */
300const makeTestDbLive = (state: StackProbeState) =>
301 Layer.effect(TestDb)(
302 Effect.acquireRelease(
303 Effect.sync(() => {
304 const index = state.handles.length;
305 const handle: TestDbHandle = {
306 id: index + 1,
307 db: state.pool[index] as SqliteTestFumaDb,
308 closed: false,
309 };
310 state.handles.push(handle);
311 return handle;
312 }),
313 (handle) =>
314 // The request's own scope ends its connection, exactly as the host
315 // closes the postgres socket when the request finishes.
316 Effect.promise(async () => {
317 handle.closed = true;
318 await handle.db.close();
319 }),
320 ),
321 );
322
323/**
324 * The host's `stackLayer` seam, reduced to the parts that matter: a
325 * `DbProvider` derived from the REQUEST-scoped handle (exactly cloud's
326 * arrangement) plus inert stand-ins for the rest. The 5ms sleep widens the
327 * window in which two request fibers overlap on the layer build, which is what
328 * an isolate serving concurrent requests does on its own.
329 */
330const makeTestStackLayer = (state: StackProbeState) =>
331 Layer.mergeAll(
332 Layer.effect(DbProvider)(
333 Effect.gen(function* () {
334 const handle = yield* TestDb;
335 yield* Effect.sleep("5 millis");
336 state.builds += 1;
337 state.builtWithHandleId.push(handle.id);
338 return handle.db;
339 }),
340 ),
341 Layer.succeed(PluginsProvider)({ plugins: () => [] }),
342 Layer.succeed(HostConfig)({ allowLocalNetwork: false, oauthCallbackPath: "/oauth/callback" }),
343 Layer.succeed(CodeExecutorProvider)(noopCodeExecutor),
344 EngineDecoratorNoop,
345 );
346
347/**
348 * `GET /probe?delay=N`: one cheap read, a pause standing in for a slow
349 * outbound call, then a second read. That is the production shape — an early
350 * query succeeds, the request awaits something slow, and by the time it queries
351 * again the connection it borrowed has been closed by its real owner.
352 */
353const makeStackProbeRoutes = (state: StackProbeState) =>
354 HttpRouter.add(
355 "GET",
356 "/probe",
357 Effect.gen(function* () {
358 const request = yield* HttpServerRequest.HttpServerRequest;
359 const delayMs = Number(
360 new URL(request.url, "http://test.local").searchParams.get("delay") ?? "0",
361 );
362 const executor = yield* ExecutorService;
363 const read = () =>
364 executor.connections.list().pipe(
365 Effect.as("ok" as const),
366 Effect.catchCause(() => Effect.succeed("failed" as const)),
367 );
368
369 // The handle this request acquired for itself, whatever order the
370 // request fibers happened to acquire in.
371 const own = yield* TestDb;
372
373 const early = yield* read();
374 yield* Effect.sleep(`${delayMs} millis`);
375 state.lateReadOwnership.set(delayMs, {
376 ownHandleId: own.id,
377 ownClosed: own.closed,
378 otherClosed: state.handles.filter((handle) => handle.id !== own.id).map((h) => h.closed),
379 });
380 const late = yield* read();
381 return HttpServerResponse.jsonUnsafe({ early, late });
382 }),
383 );
384
385const makeStackProbeHandler = (state: StackProbeState) => {
386 const ExecutionStackMiddleware = makeExecutionStackMiddleware<
387 readonly [],
388 IdentityFailure,
389 never,
390 TestDb,
391 never,
392 never
393 >({
394 plugins: [],
395 authenticate: () => Effect.succeed(stackProbePrincipal),
396 strategy: textFailureStrategy,
397 stackLayer: makeTestStackLayer(state),
398 });
399
400 const App = makeStackProbeRoutes(state).pipe(
401 // Exactly cloud's composition: the stack middleware with the per-request
402 // DB layer combined in, so `DbProvider` is built over a handle owned by the
403 // request fiber's scope.
404 Layer.provide(
405 ExecutionStackMiddleware.combine(requestScopedMiddleware(makeTestDbLive(state))).layer,
406 ),
407 Layer.provideMerge(HttpServer.layerServices),
408 );
409 return HttpRouter.toWebHandler(App, { disableLogger: true }).handler;
410};
411
412const makeStackProbeState = async (requestCount: number): Promise<StackProbeState> => {
413 const pool: SqliteTestFumaDb[] = [];
414 for (let index = 0; index < requestCount; index += 1) {
415 pool.push(await createSqliteTestFumaDb({ tables: collectTables() }));
416 }
417 return {
418 pool,
419 handles: [],
420 builds: 0,
421 builtWithHandleId: [],
422 lateReadOwnership: new Map(),
423 };
424};
425
426describe("ExecutionStackMiddleware request scoping", () => {
427 beforeEach(() => {
428 // `makeExecutionStack` touches the subject once per build, behind a
429 // process-local throttle. Reset it so every case does the same work.
430 resetSubjectTouchCache();
431 });
432
433 it("builds the execution stack once per concurrent request", async () => {
434 const state = await makeStackProbeState(3);
435 const handler = makeStackProbeHandler(state);
436
437 const responses = await Promise.all([
438 handler(new Request("http://test.local/probe"), Context.empty()),
439 handler(new Request("http://test.local/probe"), Context.empty()),
440 handler(new Request("http://test.local/probe"), Context.empty()),
441 ]);
442
443 expect(responses.map((response) => response.status)).toEqual([200, 200, 200]);
444 // Three requests, three request-scoped handles, three stack builds. A lower
445 // build count means some request served its work on a stack built for a
446 // different request.
447 expect(state.handles.length).toBe(3);
448 expect(state.builds).toBe(3);
449 });
450
451 it("never builds a request's stack over another request's database handle", async () => {
452 const state = await makeStackProbeState(3);
453 const handler = makeStackProbeHandler(state);
454
455 await Promise.all([
456 handler(new Request("http://test.local/probe"), Context.empty()),
457 handler(new Request("http://test.local/probe"), Context.empty()),
458 handler(new Request("http://test.local/probe"), Context.empty()),
459 ]);
460
461 // Every acquired handle must be the one its own request's stack was built
462 // over: three builds reading three distinct handles, covering all three.
463 const acquired = state.handles.map((handle) => handle.id);
464 expect(acquired).toEqual([1, 2, 3]);
465 expect([...state.builtWithHandleId].sort()).toEqual(acquired);
466 expect(new Set(state.builtWithHandleId).size).toBe(state.handles.length);
467 });
468
469 it("keeps a slow request's connection usable after a faster overlapping request ends", async () => {
470 const state = await makeStackProbeState(2);
471 const handler = makeStackProbeHandler(state);
472
473 // The fast request is served first, so it acquires handle 1. It finishes
474 // while the slow request is still awaiting, which is the window in which a
475 // borrowed connection gets pulled out from under its borrower.
476 const [fast, slow] = await Promise.all([
477 handler(new Request("http://test.local/probe?delay=0"), Context.empty()),
478 handler(new Request("http://test.local/probe?delay=80"), Context.empty()),
479 ]);
480
481 // Read, slow await, read again — both reads succeed for both requests.
482 expect(await fast.json()).toEqual({ early: "ok", late: "ok" });
483 expect(await slow.json()).toEqual({ early: "ok", late: "ok" });
484
485 // Ownership at the read-after-await is the real assertion. The fast
486 // request has ended and released its connection by then, so the slow
487 // request is reading in exactly the window that breaks in production —
488 // and its OWN connection is still open, because that is the one its stack
489 // was built over.
490 //
491 // Sharing one build inverts this: the slow request's own connection is
492 // released while it is still running, and it keeps reading on the fast
493 // request's instead.
494 const slowView = state.lateReadOwnership.get(80);
495 expect(slowView?.ownClosed).toBe(false);
496 expect(slowView?.otherClosed).toEqual([true]);
497
498 // Each request built its stack over its own connection.
499 expect(state.builds).toBe(2);
500 expect([...state.builtWithHandleId].sort()).toEqual([1, 2]);
501 expect(state.lateReadOwnership.get(0)?.ownHandleId).not.toBe(slowView?.ownHandleId);
502 });
503});
packages/core/api/src/server/execution-stack-middleware.ts+13−2
@@ -178,6 +178,13 @@ export const makeExecutionStackMiddleware = < · 177 unmodified lines
178 | PluginExtensionServices<TPlugins>;178 | PluginExtensionServices<TPlugins>;
179 }>()(179 }>()(
180 Effect.gen(function* () {180 Effect.gen(function* () {
181 // Captured ONCE, at layer-build time, so the per-request body can carry
182 // the boot-scoped `RCapture` services. Note what else rides along: a
183 // captured context also holds Effect's `CurrentMemoMap`, and the
184 // `Effect.provideContext(captured)` below re-applies it to every request
185 // fiber — overwriting the fresh per-request map the host installed. Every
186 // per-request `Effect.provide` in this body must therefore build with
187 // `{ local: true }`; see the two `options.stackLayer` sites.
181 const captured = yield* Effect.context<RCapture>();188 const captured = yield* Effect.context<RCapture>();
182 return (httpEffect) =>189 return (httpEffect) =>
183 Effect.gen(function* () {190 Effect.gen(function* () {
@@ -216,7 +223,7 @@ export const makeExecutionStackMiddleware = < · 32 unmodified lines
216 const { executor } = yield* makePlatformExecutionStack<TPlugins>(223 const { executor } = yield* makePlatformExecutionStack<TPlugins>(
217 resolved.organizationId,224 resolved.organizationId,
218 ).pipe(225 ).pipe(
219 Effect.provide(options.stackLayer),226 Effect.provide(options.stackLayer, { local: true }),
220 Effect.withSpan("executor.stack.http.resolve_platform"),227 Effect.withSpan("executor.stack.http.resolve_platform"),
221 );228 );
222 return yield* httpEffect.pipe(229 return yield* httpEffect.pipe(
@@ -240,7 +247,7 @@ export const makeExecutionStackMiddleware = < · 17 unmodified lines
240 resolved.organizationId,247 resolved.organizationId,
241 resolved.organizationName,248 resolved.organizationName,
242 ).pipe(249 ).pipe(
243 Effect.provide(options.stackLayer),250 Effect.provide(options.stackLayer, { local: true }),
244 Effect.provideService(RequestWebOrigin, {251 Effect.provideService(RequestWebOrigin, {
245 origin: requestWebOriginFromRequest(webRequest),252 origin: requestWebOriginFromRequest(webRequest),
246 }),253 }),
@@ -281,6 +288,10 @@ export const makeExecutionStackMiddleware = < · 34 unmodified lines
281 );288 );
282 // Provide the boot-captured context; uncaptured deps (cloud's289 // Provide the boot-captured context; uncaptured deps (cloud's
283 // request-scoped `DbService`) remain residual and flow through here.290 // request-scoped `DbService`) remain residual and flow through here.
291 // This also reinstates the BOOT `CurrentMemoMap` on the request
292 // fiber, which is why the stack builds above are `{ local: true }`:
293 // without that, concurrent requests memoize into one shared map and
294 // reuse a single stack build — and so a single database connection.
284 }).pipe(Effect.provideContext(captured as Context.Context<RCapture>));295 }).pipe(Effect.provideContext(captured as Context.Context<RCapture>));
285 }),296 }),
286 );297 );

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t