tools.search loads the full tool list on every call to score against it. The MCP session DO survives the whole CLI session, so repeated searches share one DB fetch. Wrap executor.tools.list in a plain Map keyed by JSON.stringify(filter); DO idle timeout (5m) caps staleness. Subsequent searches drop from ~1.5s p50 to ~0ms.
Unified · Split
| @@ -9,22 +9,53 @@ import { Effect } from "effect"; · 8 unmodified lines | |||
| 9 | 9 | ||
| 10 | import { createExecutionEngine } from "@executor/execution"; | 10 | import { createExecutionEngine } from "@executor/execution"; |
| 11 | import { makeDynamicWorkerExecutor } from "@executor/runtime-dynamic-worker"; | 11 | import { makeDynamicWorkerExecutor } from "@executor/runtime-dynamic-worker"; |
| 12 | import type { Tool, ToolListFilter } from "@executor/sdk"; | ||
| 12 | 13 | ||
| 13 | import { withExecutionUsageTracking } from "../api/execution-usage"; | 14 | import { withExecutionUsageTracking } from "../api/execution-usage"; |
| 14 | import { AutumnService } from "./autumn"; | 15 | import { AutumnService } from "./autumn"; |
| 15 | import { createScopedExecutor } from "./executor"; | 16 | import { createScopedExecutor } from "./executor"; |
| 16 | 17 | ||
| 18 | // In-memory tools.list cache on the DO. `tools.search` scans the full | ||
| 19 | // list on every call, so caching it collapses the N calls in a session | ||
| 20 | // into one DB fetch. DO lifetime caps cache staleness at | ||
| 21 | // SESSION_TIMEOUT_MS (5m idle) — no TTL needed. | ||
| 22 | const makeToolsListCache = <E, R>( | ||
| 23 | inner: (filter?: ToolListFilter) => Effect.Effect<readonly Tool[], E, R>, | ||
| 24 | ) => { | ||
| 25 | const cache = new Map<string, readonly Tool[]>(); | ||
| 26 | return (filter?: ToolListFilter) => | ||
| 27 | Effect.gen(function* () { | ||
| 28 | const key = JSON.stringify(filter ?? null); | ||
| 29 | const hit = cache.get(key); | ||
| 30 | if (hit) { | ||
| 31 | yield* Effect.annotateCurrentSpan({ "cache.state": "hit" }); | ||
| 32 | return hit; | ||
| 33 | } | ||
| 34 | const value = yield* inner(filter); | ||
| 35 | cache.set(key, value); | ||
| 36 | yield* Effect.annotateCurrentSpan({ "cache.state": "miss" }); | ||
| 37 | return value; | ||
| 38 | }).pipe(Effect.withSpan("executor.tools.list.cached")); | ||
| 39 | }; | ||
| 40 | |||
| 17 | export const makeExecutionStack = ( | 41 | export const makeExecutionStack = ( |
| 18 | userId: string, | 42 | userId: string, |
| 19 | organizationId: string, | 43 | organizationId: string, |
| 20 | organizationName: string, | 44 | organizationName: string, |
| 21 | ) => | 45 | ) => |
| 22 | Effect.gen(function* () { | 46 | Effect.gen(function* () { |
| 23 | const executor = yield* createScopedExecutor( | 47 | const rawExecutor = yield* createScopedExecutor( |
| 24 | userId, | 48 | userId, |
| 25 | organizationId, | 49 | organizationId, |
| 26 | organizationName, | 50 | organizationName, |
| 27 | ).pipe(Effect.withSpan("McpSessionDO.createScopedExecutor")); | 51 | ).pipe(Effect.withSpan("McpSessionDO.createScopedExecutor")); |
| 52 | const executor = { | ||
| 53 | ...rawExecutor, | ||
| 54 | tools: { | ||
| 55 | ...rawExecutor.tools, | ||
| 56 | list: makeToolsListCache(rawExecutor.tools.list), | ||
| 57 | }, | ||
| 58 | }; | ||
| 28 | const codeExecutor = makeDynamicWorkerExecutor({ loader: env.LOADER }); | 59 | const codeExecutor = makeDynamicWorkerExecutor({ loader: env.LOADER }); |
| 29 | const autumn = yield* AutumnService; | 60 | const autumn = yield* AutumnService; |
| 30 | const engine = withExecutionUsageTracking( | 61 | const engine = withExecutionUsageTracking( |