| @@ -21,11 +21,32 @@ · 20 unmodified lines |
| 21 | 21 | // for `acquireRelease`. |
| 22 | 22 | // --------------------------------------------------------------------------- |
| 23 | 23 | |
| 24 | | −import { describe, it, expect } from "@effect/vitest"; |
| 24 | +import { describe, it, expect, beforeEach } from "@effect/vitest"; |
| 25 | 25 | import { Context, Effect, Layer } from "effect"; |
| 26 | | −import { HttpRouter, HttpServer, HttpServerResponse } from "effect/unstable/http"; |
| 26 | +import { |
| 27 | + HttpRouter, |
| 28 | + HttpServer, |
| 29 | + HttpServerRequest, |
| 30 | + HttpServerResponse, |
| 31 | +} from "effect/unstable/http"; |
| 27 | 32 | |
| 28 | | −import { requestScopedMiddleware } from "@executor-js/api/server"; |
| 33 | +import { |
| 34 | + CodeExecutorProvider, |
| 35 | + DbProvider, |
| 36 | + EngineDecoratorNoop, |
| 37 | + ExecutorService, |
| 38 | + HostConfig, |
| 39 | + PluginsProvider, |
| 40 | + makeExecutionStackMiddleware, |
| 41 | + requestScopedMiddleware, |
| 42 | + textFailureStrategy, |
| 43 | + type CodeExecutor, |
| 44 | + type IdentityFailure, |
| 45 | + type Principal, |
| 46 | +} from "@executor-js/api/server"; |
| 47 | +import { collectTables } from "@executor-js/sdk"; |
| 48 | +import { resetSubjectTouchCache } from "@executor-js/sdk/host-internal"; |
| 49 | +import { createSqliteTestFumaDb, type SqliteTestFumaDb } from "@executor-js/sdk/testing"; |
| 29 | 50 | |
| 30 | 51 | import { RequestScopedServicesLive } from "./api/layers"; |
| 31 | 52 | import { makeApiLive } from "./api/router"; |
| @@ -195,3 +216,288 @@ describe("makeApiLive (prod handler factory) request scoping", () => { · 163 unmodified lines |
| 195 | 216 | expect(counts.releases).toBe(2); |
| 196 | 217 | }); |
| 197 | 218 | }); |
| 219 | + |
| 220 | +// --------------------------------------------------------------------------- |
| 221 | +// `ExecutionStackMiddleware` request scoping. |
| 222 | +// |
| 223 | +// `requestScopedMiddleware` gives every request a fresh `MemoMap`, so the |
| 224 | +// per-request DB handle is genuinely per request (the suites above). But |
| 225 | +// `makeExecutionStackMiddleware` captures the BOOT fiber's context once, at |
| 226 | +// layer-construction time, and re-applies it to every request with |
| 227 | +// `Effect.provideContext`. That captured context carries Effect's |
| 228 | +// `CurrentMemoMap`, and `provideContext`'s merge lets the BOOT map overwrite |
| 229 | +// the fresh per-request one. The per-request `Effect.provide(stackLayer)` then |
| 230 | +// memoizes its build in the boot map, which every in-flight request fiber in |
| 231 | +// the isolate shares. |
| 232 | +// |
| 233 | +// Sequential requests still rebuild (the memo entry is refcounted by observer |
| 234 | +// and drops to zero when the request scope closes), so only OVERLAPPING |
| 235 | +// requests are affected: the second request reuses the first request's stack |
| 236 | +// build, and therefore the first request's database handle. On Cloudflare |
| 237 | +// Workers that is a cross-request I/O violation; and when the owning request |
| 238 | +// finishes, its scope finalizer closes the connection out from under the |
| 239 | +// borrower, whose next query fails. |
| 240 | +// |
| 241 | +// These tests stand the real middleware up over per-request in-memory |
| 242 | +// databases and pin all three symptoms. |
| 243 | +// --------------------------------------------------------------------------- |
| 244 | + |
| 245 | +interface TestDbHandle { |
| 246 | + readonly id: number; |
| 247 | + readonly db: SqliteTestFumaDb; |
| 248 | + closed: boolean; |
| 249 | +} |
| 250 | + |
| 251 | +class TestDb extends Context.Service<TestDb, TestDbHandle>()("test/TestDb") {} |
| 252 | + |
| 253 | +interface StackProbeState { |
| 254 | + /** |
| 255 | + * Databases created up front, one per request the case will make. Handing a |
| 256 | + * ready database to the request-scoped layer keeps its acquire synchronous, |
| 257 | + * so the first request to arrive is reliably the one that owns handle 1 — |
| 258 | + * which is what makes the borrow direction deterministic below. |
| 259 | + */ |
| 260 | + readonly pool: readonly SqliteTestFumaDb[]; |
| 261 | + /** Handles acquired by `requestScopedMiddleware` — one per request. */ |
| 262 | + readonly handles: TestDbHandle[]; |
| 263 | + /** How many times the stack layer's `DbProvider` was actually built. */ |
| 264 | + builds: number; |
| 265 | + /** The handle id each stack build read. One entry per build. */ |
| 266 | + readonly builtWithHandleId: number[]; |
| 267 | + /** |
| 268 | + * What each request could see about connection ownership at the moment it |
| 269 | + * performed its read-after-await, keyed by that request's await duration. |
| 270 | + */ |
| 271 | + readonly lateReadOwnership: Map< |
| 272 | + number, |
| 273 | + { |
| 274 | + readonly ownHandleId: number; |
| 275 | + readonly ownClosed: boolean; |
| 276 | + readonly otherClosed: readonly boolean[]; |
| 277 | + } |
| 278 | + >; |
| 279 | +} |
| 280 | + |
| 281 | +const noopCodeExecutor: CodeExecutor = { |
| 282 | + execute: () => Effect.succeed({ result: undefined, logs: [] }), |
| 283 | +}; |
| 284 | + |
| 285 | +const stackProbePrincipal: Principal = { |
| 286 | + kind: "member", |
| 287 | + accountId: "user_request_scope", |
| 288 | + organizationId: "org_request_scope", |
| 289 | + organizationName: "Request Scope Test Org", |
| 290 | + email: "request-scope@test.local", |
| 291 | + name: "Request Scope", |
| 292 | + avatarUrl: null, |
| 293 | + roles: ["admin"], |
| 294 | +}; |
| 295 | + |
| 296 | +/** |
| 297 | + * The per-request DB handle, in the slot cloud's postgres.js socket occupies: |
| 298 | + * acquired when the request fiber's scope opens, closed when it closes. |
| 299 | + */ |
| 300 | +const makeTestDbLive = (state: StackProbeState) => |
| 301 | + Layer.effect(TestDb)( |
| 302 | + Effect.acquireRelease( |
| 303 | + Effect.sync(() => { |
| 304 | + const index = state.handles.length; |
| 305 | + const handle: TestDbHandle = { |
| 306 | + id: index + 1, |
| 307 | + db: state.pool[index] as SqliteTestFumaDb, |
| 308 | + closed: false, |
| 309 | + }; |
| 310 | + state.handles.push(handle); |
| 311 | + return handle; |
| 312 | + }), |
| 313 | + (handle) => |
| 314 | + // The request's own scope ends its connection, exactly as the host |
| 315 | + // closes the postgres socket when the request finishes. |
| 316 | + Effect.promise(async () => { |
| 317 | + handle.closed = true; |
| 318 | + await handle.db.close(); |
| 319 | + }), |
| 320 | + ), |
| 321 | + ); |
| 322 | + |
| 323 | +/** |
| 324 | + * The host's `stackLayer` seam, reduced to the parts that matter: a |
| 325 | + * `DbProvider` derived from the REQUEST-scoped handle (exactly cloud's |
| 326 | + * arrangement) plus inert stand-ins for the rest. The 5ms sleep widens the |
| 327 | + * window in which two request fibers overlap on the layer build, which is what |
| 328 | + * an isolate serving concurrent requests does on its own. |
| 329 | + */ |
| 330 | +const makeTestStackLayer = (state: StackProbeState) => |
| 331 | + Layer.mergeAll( |
| 332 | + Layer.effect(DbProvider)( |
| 333 | + Effect.gen(function* () { |
| 334 | + const handle = yield* TestDb; |
| 335 | + yield* Effect.sleep("5 millis"); |
| 336 | + state.builds += 1; |
| 337 | + state.builtWithHandleId.push(handle.id); |
| 338 | + return handle.db; |
| 339 | + }), |
| 340 | + ), |
| 341 | + Layer.succeed(PluginsProvider)({ plugins: () => [] }), |
| 342 | + Layer.succeed(HostConfig)({ allowLocalNetwork: false, oauthCallbackPath: "/oauth/callback" }), |
| 343 | + Layer.succeed(CodeExecutorProvider)(noopCodeExecutor), |
| 344 | + EngineDecoratorNoop, |
| 345 | + ); |
| 346 | + |
| 347 | +/** |
| 348 | + * `GET /probe?delay=N`: one cheap read, a pause standing in for a slow |
| 349 | + * outbound call, then a second read. That is the production shape — an early |
| 350 | + * query succeeds, the request awaits something slow, and by the time it queries |
| 351 | + * again the connection it borrowed has been closed by its real owner. |
| 352 | + */ |
| 353 | +const makeStackProbeRoutes = (state: StackProbeState) => |
| 354 | + HttpRouter.add( |
| 355 | + "GET", |
| 356 | + "/probe", |
| 357 | + Effect.gen(function* () { |
| 358 | + const request = yield* HttpServerRequest.HttpServerRequest; |
| 359 | + const delayMs = Number( |
| 360 | + new URL(request.url, "http://test.local").searchParams.get("delay") ?? "0", |
| 361 | + ); |
| 362 | + const executor = yield* ExecutorService; |
| 363 | + const read = () => |
| 364 | + executor.connections.list().pipe( |
| 365 | + Effect.as("ok" as const), |
| 366 | + Effect.catchCause(() => Effect.succeed("failed" as const)), |
| 367 | + ); |
| 368 | + |
| 369 | + // The handle this request acquired for itself, whatever order the |
| 370 | + // request fibers happened to acquire in. |
| 371 | + const own = yield* TestDb; |
| 372 | + |
| 373 | + const early = yield* read(); |
| 374 | + yield* Effect.sleep(`${delayMs} millis`); |
| 375 | + state.lateReadOwnership.set(delayMs, { |
| 376 | + ownHandleId: own.id, |
| 377 | + ownClosed: own.closed, |
| 378 | + otherClosed: state.handles.filter((handle) => handle.id !== own.id).map((h) => h.closed), |
| 379 | + }); |
| 380 | + const late = yield* read(); |
| 381 | + return HttpServerResponse.jsonUnsafe({ early, late }); |
| 382 | + }), |
| 383 | + ); |
| 384 | + |
| 385 | +const makeStackProbeHandler = (state: StackProbeState) => { |
| 386 | + const ExecutionStackMiddleware = makeExecutionStackMiddleware< |
| 387 | + readonly [], |
| 388 | + IdentityFailure, |
| 389 | + never, |
| 390 | + TestDb, |
| 391 | + never, |
| 392 | + never |
| 393 | + >({ |
| 394 | + plugins: [], |
| 395 | + authenticate: () => Effect.succeed(stackProbePrincipal), |
| 396 | + strategy: textFailureStrategy, |
| 397 | + stackLayer: makeTestStackLayer(state), |
| 398 | + }); |
| 399 | + |
| 400 | + const App = makeStackProbeRoutes(state).pipe( |
| 401 | + // Exactly cloud's composition: the stack middleware with the per-request |
| 402 | + // DB layer combined in, so `DbProvider` is built over a handle owned by the |
| 403 | + // request fiber's scope. |
| 404 | + Layer.provide( |
| 405 | + ExecutionStackMiddleware.combine(requestScopedMiddleware(makeTestDbLive(state))).layer, |
| 406 | + ), |
| 407 | + Layer.provideMerge(HttpServer.layerServices), |
| 408 | + ); |
| 409 | + return HttpRouter.toWebHandler(App, { disableLogger: true }).handler; |
| 410 | +}; |
| 411 | + |
| 412 | +const makeStackProbeState = async (requestCount: number): Promise<StackProbeState> => { |
| 413 | + const pool: SqliteTestFumaDb[] = []; |
| 414 | + for (let index = 0; index < requestCount; index += 1) { |
| 415 | + pool.push(await createSqliteTestFumaDb({ tables: collectTables() })); |
| 416 | + } |
| 417 | + return { |
| 418 | + pool, |
| 419 | + handles: [], |
| 420 | + builds: 0, |
| 421 | + builtWithHandleId: [], |
| 422 | + lateReadOwnership: new Map(), |
| 423 | + }; |
| 424 | +}; |
| 425 | + |
| 426 | +describe("ExecutionStackMiddleware request scoping", () => { |
| 427 | + beforeEach(() => { |
| 428 | + // `makeExecutionStack` touches the subject once per build, behind a |
| 429 | + // process-local throttle. Reset it so every case does the same work. |
| 430 | + resetSubjectTouchCache(); |
| 431 | + }); |
| 432 | + |
| 433 | + it("builds the execution stack once per concurrent request", async () => { |
| 434 | + const state = await makeStackProbeState(3); |
| 435 | + const handler = makeStackProbeHandler(state); |
| 436 | + |
| 437 | + const responses = await Promise.all([ |
| 438 | + handler(new Request("http://test.local/probe"), Context.empty()), |
| 439 | + handler(new Request("http://test.local/probe"), Context.empty()), |
| 440 | + handler(new Request("http://test.local/probe"), Context.empty()), |
| 441 | + ]); |
| 442 | + |
| 443 | + expect(responses.map((response) => response.status)).toEqual([200, 200, 200]); |
| 444 | + // Three requests, three request-scoped handles, three stack builds. A lower |
| 445 | + // build count means some request served its work on a stack built for a |
| 446 | + // different request. |
| 447 | + expect(state.handles.length).toBe(3); |
| 448 | + expect(state.builds).toBe(3); |
| 449 | + }); |
| 450 | + |
| 451 | + it("never builds a request's stack over another request's database handle", async () => { |
| 452 | + const state = await makeStackProbeState(3); |
| 453 | + const handler = makeStackProbeHandler(state); |
| 454 | + |
| 455 | + await Promise.all([ |
| 456 | + handler(new Request("http://test.local/probe"), Context.empty()), |
| 457 | + handler(new Request("http://test.local/probe"), Context.empty()), |
| 458 | + handler(new Request("http://test.local/probe"), Context.empty()), |
| 459 | + ]); |
| 460 | + |
| 461 | + // Every acquired handle must be the one its own request's stack was built |
| 462 | + // over: three builds reading three distinct handles, covering all three. |
| 463 | + const acquired = state.handles.map((handle) => handle.id); |
| 464 | + expect(acquired).toEqual([1, 2, 3]); |
| 465 | + expect([...state.builtWithHandleId].sort()).toEqual(acquired); |
| 466 | + expect(new Set(state.builtWithHandleId).size).toBe(state.handles.length); |
| 467 | + }); |
| 468 | + |
| 469 | + it("keeps a slow request's connection usable after a faster overlapping request ends", async () => { |
| 470 | + const state = await makeStackProbeState(2); |
| 471 | + const handler = makeStackProbeHandler(state); |
| 472 | + |
| 473 | + // The fast request is served first, so it acquires handle 1. It finishes |
| 474 | + // while the slow request is still awaiting, which is the window in which a |
| 475 | + // borrowed connection gets pulled out from under its borrower. |
| 476 | + const [fast, slow] = await Promise.all([ |
| 477 | + handler(new Request("http://test.local/probe?delay=0"), Context.empty()), |
| 478 | + handler(new Request("http://test.local/probe?delay=80"), Context.empty()), |
| 479 | + ]); |
| 480 | + |
| 481 | + // Read, slow await, read again — both reads succeed for both requests. |
| 482 | + expect(await fast.json()).toEqual({ early: "ok", late: "ok" }); |
| 483 | + expect(await slow.json()).toEqual({ early: "ok", late: "ok" }); |
| 484 | + |
| 485 | + // Ownership at the read-after-await is the real assertion. The fast |
| 486 | + // request has ended and released its connection by then, so the slow |
| 487 | + // request is reading in exactly the window that breaks in production — |
| 488 | + // and its OWN connection is still open, because that is the one its stack |
| 489 | + // was built over. |
| 490 | + // |
| 491 | + // Sharing one build inverts this: the slow request's own connection is |
| 492 | + // released while it is still running, and it keeps reading on the fast |
| 493 | + // request's instead. |
| 494 | + const slowView = state.lateReadOwnership.get(80); |
| 495 | + expect(slowView?.ownClosed).toBe(false); |
| 496 | + expect(slowView?.otherClosed).toEqual([true]); |
| 497 | + |
| 498 | + // Each request built its stack over its own connection. |
| 499 | + expect(state.builds).toBe(2); |
| 500 | + expect([...state.builtWithHandleId].sort()).toEqual([1, 2]); |
| 501 | + expect(state.lateReadOwnership.get(0)?.ownHandleId).not.toBe(slowView?.ownHandleId); |
| 502 | + }); |
| 503 | +}); |