History

Speed up and tune CI wall-clock (#1296)

* Tune CI wall-clock and hygiene * Fix CI review findings * Scope cloud e2e idle overrides

Rhys Sullivan committed 4 months ago

Commit 87ed7c91 parent 892382710 files · +245 −69Browse files

Unified · Split

10 files
  1. .github/workflows/ci.yml +93 −47
  2. apps/cloud/src/env-augment.d.ts +2 −0
  3. apps/cloud/src/mcp/session-durable-object.ts +17 −0
  4. e2e/cloud/mcp-client-sessions.test.ts +21 −13
  5. e2e/setup/cloud.boot.ts +2 −0
  6. e2e/setup/cloud.globalsetup.ts +2 −0
  7. e2e/setup/mcp-session-timeouts.ts +34 −0
  8. packages/hosts/cloudflare/src/mcp/agent-session-durable-object.ts +47 −6
  9. packages/hosts/cloudflare/src/mcp/session-alarm-policy.test.ts +14 −0
  10. packages/hosts/cloudflare/src/mcp/session-alarm-policy.ts +13 −3
.github/workflows/ci.yml+93−47
@@ -7,13 +7,58 @@ on: · 6 unmodified lines
7 - main7 - main
8 8
9concurrency:9concurrency:
10 group: ci-${{ github.event_name == 'push' && format('{0}-{1}', github.ref, github.sha) || github.ref }}10 group: ci-${{ github.ref }}
11 cancel-in-progress: ${{ github.event_name == 'pull_request' }}11 cancel-in-progress: true
12 12
13jobs:13jobs:
14 changes:
15 name: Changed paths
16 runs-on: blacksmith-4vcpu-ubuntu-2404
17 timeout-minutes: 10
18 outputs:
19 desktop_smoke: ${{ github.event_name != 'pull_request' || steps.filter.outputs.desktop_smoke == 'true' }}
20 selfhost_docker_smoke: ${{ github.event_name != 'pull_request' || steps.filter.outputs.selfhost_docker_smoke == 'true' }}
21 steps:
22 - uses: actions/checkout@v4
23
24 - name: Filter changed paths
25 id: filter
26 if: github.event_name == 'pull_request'
27 uses: dorny/paths-filter@v3
28 with:
29 filters: |
30 desktop_smoke:
31 - ".github/workflows/**"
32 - "bun.lock"
33 - "package.json"
34 - "turbo.json"
35 - "apps/desktop/**"
36 - "apps/local/**"
37 - "apps/cli/**"
38 - "packages/app/**"
39 - "packages/core/**"
40 - "packages/hosts/mcp/**"
41 - "packages/kernel/runtime-quickjs/**"
42 - "packages/plugins/**"
43 - "packages/react/**"
44 selfhost_docker_smoke:
45 - ".github/workflows/**"
46 - ".dockerignore"
47 - "bun.lock"
48 - "package.json"
49 - "turbo.json"
50 - "apps/host-selfhost/**"
51 - "packages/app/**"
52 - "packages/core/**"
53 - "packages/hosts/mcp/**"
54 - "packages/kernel/runtime-quickjs/**"
55 - "packages/plugins/**"
56 - "packages/react/**"
57
14 format:58 format:
15 name: Format59 name: Format
16 runs-on: blacksmith-4vcpu-ubuntu-240460 runs-on: blacksmith-4vcpu-ubuntu-2404
61 timeout-minutes: 10
17 steps:62 steps:
18 - uses: actions/checkout@v463 - uses: actions/checkout@v4
19 64
@@ -29,20 +74,14 @@ jobs: · 9 unmodified lines
29 restore-keys: |74 restore-keys: |
30 ${{ runner.os }}-bun-1.3.11-75 ${{ runner.os }}-bun-1.3.11-
31 76
32 # No prebuilt better-sqlite3 binary matches this runner, so `bun install`77 - run: bun install --frozen-lockfile --ignore-scripts
33 # builds it from source via node-gyp, whose undici needs Node 22.10+
34 # (webidl.markAsUncloneable). Pin the same runtime the test/e2e jobs use.
35 - uses: actions/setup-node@v4
36 with:
37 node-version: 22
38
39 - run: bun install --frozen-lockfile
40 78
41 - run: bun run format:check79 - run: bun run format:check
42 80
43 lint:81 lint:
44 name: Lint82 name: Lint
45 runs-on: blacksmith-4vcpu-ubuntu-240483 runs-on: blacksmith-4vcpu-ubuntu-2404
84 timeout-minutes: 10
46 steps:85 steps:
47 - uses: actions/checkout@v486 - uses: actions/checkout@v4
48 87
@@ -58,20 +97,14 @@ jobs: · 9 unmodified lines
58 restore-keys: |97 restore-keys: |
59 ${{ runner.os }}-bun-1.3.11-98 ${{ runner.os }}-bun-1.3.11-
60 99
61 # No prebuilt better-sqlite3 binary matches this runner, so `bun install`100 - run: bun install --frozen-lockfile --ignore-scripts
62 # builds it from source via node-gyp, whose undici needs Node 22.10+
63 # (webidl.markAsUncloneable). Pin the same runtime the test/e2e jobs use.
64 - uses: actions/setup-node@v4
65 with:
66 node-version: 22
67
68 - run: bun install --frozen-lockfile
69 101
70 - run: bun run lint102 - run: bun run lint
71 103
72 typecheck:104 typecheck:
73 name: Typecheck105 name: Typecheck
74 runs-on: blacksmith-4vcpu-ubuntu-2404106 runs-on: blacksmith-4vcpu-ubuntu-2404
107 timeout-minutes: 15
75 steps:108 steps:
76 - uses: actions/checkout@v4109 - uses: actions/checkout@v4
77 110
@@ -89,10 +122,10 @@ jobs: · 11 unmodified lines
89 122
90 # No prebuilt better-sqlite3 binary matches this runner, so `bun install`123 # No prebuilt better-sqlite3 binary matches this runner, so `bun install`
91 # builds it from source via node-gyp, whose undici needs Node 22.10+124 # builds it from source via node-gyp, whose undici needs Node 22.10+
92 # (webidl.markAsUncloneable). Pin the same runtime the test/e2e jobs use.125 # (webidl.markAsUncloneable). Pin the same Node 24 runtime used by release.
93 - uses: actions/setup-node@v4126 - uses: actions/setup-node@v4
94 with:127 with:
95 node-version: 22128 node-version: 24
96 129
97 - run: bun install --frozen-lockfile130 - run: bun install --frozen-lockfile
98 131
@@ -101,8 +134,10 @@ jobs: · 2 unmodified lines
101 test:134 test:
102 name: Test135 name: Test
103 runs-on: blacksmith-4vcpu-ubuntu-2404136 runs-on: blacksmith-4vcpu-ubuntu-2404
137 timeout-minutes: 15
138 # Tuned for Blacksmith's 4 vCPU runners.
104 env:139 env:
105 TURBO_TEST_CONCURRENCY: 3140 TURBO_TEST_CONCURRENCY: 4
106 steps:141 steps:
107 - uses: actions/checkout@v4142 - uses: actions/checkout@v4
108 143
@@ -120,10 +155,10 @@ jobs: · 11 unmodified lines
120 155
121 # apps/cloud's test script invokes `node` directly; undici 8.x (pulled156 # apps/cloud's test script invokes `node` directly; undici 8.x (pulled
122 # in by @cloudflare/vitest-pool-workers) calls webidl.markAsUncloneable157 # in by @cloudflare/vitest-pool-workers) calls webidl.markAsUncloneable
123 # which only exists in Node 22.10+. Pin a known-good runtime.158 # which only exists in Node 22.10+. Pin the Node 24 CI runtime.
124 - uses: actions/setup-node@v4159 - uses: actions/setup-node@v4
125 with:160 with:
126 node-version: 22161 node-version: 24
127 162
128 - run: bun install --frozen-lockfile163 - run: bun install --frozen-lockfile
129 164
@@ -135,21 +170,13 @@ jobs: · 5 unmodified lines
135 fail-fast: false170 fail-fast: false
136 matrix:171 matrix:
137 include:172 include:
138 # Cloud is SHARDED: each shard boots its own fresh dev stack. The173 # Each cloud shard boots its own fresh dev stack. On 4 vCPU runners,
139 # cloud dev server degrades after a few minutes of sustained suite174 # four fatter shards keep the longest shard below selfhost while saving
140 # load on 2-core runners (the SSE/OTel memory growth being175 # four runner boots and four warm cache restores.
141 # instrumented on main) — requests start failing partway through and176 - { target: cloud, shard: 1/4, shard-name: 1of4 }
142 # everything after dies with connection errors. Short shards on177 - { target: cloud, shard: 2/4, shard-name: 2of4 }
143 # fresh boots stay under that threshold; re-merge into fewer jobs178 - { target: cloud, shard: 3/4, shard-name: 3of4 }
144 # once the degradation is fixed.179 - { target: cloud, shard: 4/4, shard-name: 4of4 }
145 - { target: cloud, shard: 1/8, shard-name: 1of8 }
146 - { target: cloud, shard: 2/8, shard-name: 2of8 }
147 - { target: cloud, shard: 3/8, shard-name: 3of8 }
148 - { target: cloud, shard: 4/8, shard-name: 4of8 }
149 - { target: cloud, shard: 5/8, shard-name: 5of8 }
150 - { target: cloud, shard: 6/8, shard-name: 6of8 }
151 - { target: cloud, shard: 7/8, shard-name: 7of8 }
152 - { target: cloud, shard: 8/8, shard-name: 8of8 }
153 - target: selfhost180 - target: selfhost
154 runs-on: blacksmith-4vcpu-ubuntu-2404181 runs-on: blacksmith-4vcpu-ubuntu-2404
155 timeout-minutes: 30182 timeout-minutes: 30
@@ -169,10 +196,10 @@ jobs: · 13 unmodified lines
169 ${{ runner.os }}-bun-1.3.11-196 ${{ runner.os }}-bun-1.3.11-
170 197
171 # The dev stacks spawn Node sidecars (vite/workerd tooling); pin the198 # The dev stacks spawn Node sidecars (vite/workerd tooling); pin the
172 # same known-good runtime the unit-test job uses.199 # same Node 24 runtime the release and publish workflows use.
173 - uses: actions/setup-node@v4200 - uses: actions/setup-node@v4
174 with:201 with:
175 node-version: 22202 node-version: 24
176 203
177 - run: bun install --frozen-lockfile204 - run: bun install --frozen-lockfile
178 205
@@ -181,6 +208,8 @@ jobs: · 2 unmodified lines
181 with:208 with:
182 path: ~/.cache/ms-playwright209 path: ~/.cache/ms-playwright
183 key: ${{ runner.os }}-playwright-1.60.0210 key: ${{ runner.os }}-playwright-1.60.0
211 restore-keys: |
212 ${{ runner.os }}-playwright-
184 213
185 # Install from e2e so bunx resolves ITS pinned playwright (the version214 # Install from e2e so bunx resolves ITS pinned playwright (the version
186 # the tests run against) rather than floating to the latest.215 # the tests run against) rather than floating to the latest.
@@ -190,11 +219,20 @@ jobs: · 3 unmodified lines
190 219
191 # The globalsetup boots the target's own dev server (ports are claimed220 # The globalsetup boots the target's own dev server (ports are claimed
192 # per checkout, so this is hermetic) and tears it down after the run.221 # per checkout, so this is hermetic) and tears it down after the run.
193 # --retry=2: browser scenarios time out sporadically on 2-core runners222 # --retry=2: browser scenarios can still hit isolated waitFor timeouts
194 # (single-test waitFor timeouts, not systemic failures); a retry on the223 # (single-test waitFor timeouts, not systemic failures); a retry on the
195 # same booted stack clears them.224 # same booted stack clears them.
196 - name: Run ${{ matrix.target }} scenarios225 - name: Run cloud scenarios
197 run: bunx vitest run --project ${{ matrix.target }} --retry=2 ${{ matrix.shard && format('--shard={0}', matrix.shard) || '' }}226 if: matrix.target == 'cloud'
227 env:
228 MCP_SESSION_TIMEOUT_MS: "3000"
229 MCP_PAUSED_SESSION_IDLE_TIMEOUT_MS: "6000"
230 run: bunx vitest run --project cloud --retry=2 ${{ matrix.shard && format('--shard={0}', matrix.shard) || '' }}
231 working-directory: e2e
232
233 - name: Run selfhost scenarios
234 if: matrix.target == 'selfhost'
235 run: bunx vitest run --project selfhost --retry=2
198 working-directory: e2e236 working-directory: e2e
199 237
200 # Failed runs keep their trace.zip / session.mp4 / step screenshots in238 # Failed runs keep their trace.zip / session.mp4 / step screenshots in
@@ -230,10 +268,10 @@ jobs: · 29 unmodified lines
230 ${{ runner.os }}-bun-1.3.11-268 ${{ runner.os }}-bun-1.3.11-
231 269
232 # The local scenarios boot a real `executor web` (which spawns a Node270 # The local scenarios boot a real `executor web` (which spawns a Node
233 # sidecar) and some drive a browser, so pin Node 22 and install Chromium.271 # sidecar) and some drive a browser, so pin Node 24 and install Chromium.
234 - uses: actions/setup-node@v4272 - uses: actions/setup-node@v4
235 with:273 with:
236 node-version: 22274 node-version: 24
237 275
238 - run: bun install --frozen-lockfile276 - run: bun install --frozen-lockfile
239 277
@@ -242,6 +280,8 @@ jobs: · 2 unmodified lines
242 with:280 with:
243 path: ~/.cache/ms-playwright281 path: ~/.cache/ms-playwright
244 key: ${{ runner.os }}-playwright-1.60.0282 key: ${{ runner.os }}-playwright-1.60.0
283 restore-keys: |
284 ${{ runner.os }}-playwright-
245 285
246 # `chromium` and the new `chromium-headless-shell` ship as separate286 # `chromium` and the new `chromium-headless-shell` ship as separate
247 # downloads; the browser-driven scenarios launch the headless shell.287 # downloads; the browser-driven scenarios launch the headless shell.
@@ -265,7 +305,10 @@ jobs: · 17 unmodified lines
265 305
266 desktop-smoke:306 desktop-smoke:
267 name: Desktop smoke build307 name: Desktop smoke build
308 needs: changes
309 if: needs.changes.outputs.desktop_smoke == 'true'
268 runs-on: blacksmith-4vcpu-ubuntu-2404310 runs-on: blacksmith-4vcpu-ubuntu-2404
311 timeout-minutes: 20
269 steps:312 steps:
270 - uses: actions/checkout@v4313 - uses: actions/checkout@v4
271 314
@@ -283,10 +326,10 @@ jobs: · 11 unmodified lines
283 326
284 # No prebuilt better-sqlite3 binary matches this runner, so `bun install`327 # No prebuilt better-sqlite3 binary matches this runner, so `bun install`
285 # builds it from source via node-gyp, whose undici needs Node 22.10+328 # builds it from source via node-gyp, whose undici needs Node 22.10+
286 # (webidl.markAsUncloneable). Pin the same runtime the test/e2e jobs use.329 # (webidl.markAsUncloneable). Pin the same Node 24 CI runtime.
287 - uses: actions/setup-node@v4330 - uses: actions/setup-node@v4
288 with:331 with:
289 node-version: 22332 node-version: 24
290 333
291 - run: bun install --frozen-lockfile334 - run: bun install --frozen-lockfile
292 335
@@ -305,7 +348,10 @@ jobs: · 12 unmodified lines
305 348
306 selfhost-docker-smoke:349 selfhost-docker-smoke:
307 name: Self-host Docker image350 name: Self-host Docker image
351 needs: changes
352 if: needs.changes.outputs.selfhost_docker_smoke == 'true'
308 runs-on: blacksmith-4vcpu-ubuntu-2404353 runs-on: blacksmith-4vcpu-ubuntu-2404
354 timeout-minutes: 20
309 steps:355 steps:
310 - uses: actions/checkout@v4356 - uses: actions/checkout@v4
311 357
apps/cloud/src/env-augment.d.ts+2−0
@@ -58,6 +58,8 @@ declare global { · 57 unmodified lines
58 EXECUTOR_MCP_DEBUG?: string;58 EXECUTOR_MCP_DEBUG?: string;
59 MCP_AUTHKIT_DOMAIN?: string;59 MCP_AUTHKIT_DOMAIN?: string;
60 MCP_RESOURCE_ORIGIN?: string;60 MCP_RESOURCE_ORIGIN?: string;
61 MCP_SESSION_TIMEOUT_MS?: string;
62 MCP_PAUSED_SESSION_IDLE_TIMEOUT_MS?: string;
61 NODE_ENV?: string;63 NODE_ENV?: string;
62 64
63 // Shared with frontend65 // Shared with frontend
apps/cloud/src/mcp/session-durable-object.ts+17−0
@@ -77,6 +77,13 @@ const LONG_LIVED_DB_IDLE_TIMEOUT_SECONDS = 5; · 76 unmodified lines
77const LONG_LIVED_DB_MAX_LIFETIME_SECONDS = 120;77const LONG_LIVED_DB_MAX_LIFETIME_SECONDS = 120;
78const TELEMETRY_FLUSH_TIMEOUT_MS = 1_000;78const TELEMETRY_FLUSH_TIMEOUT_MS = 1_000;
79 79
80const positiveMilliseconds = (raw: string | undefined): number | undefined => {
81 if (!raw) return undefined;
82 const parsed = Number(raw);
83 if (!Number.isFinite(parsed) || parsed <= 0) return undefined;
84 return Math.floor(parsed);
85};
86
80type CloudSessionDbHandle = DbServiceShape & {87type CloudSessionDbHandle = DbServiceShape & {
81 readonly sql: Sql;88 readonly sql: Sql;
82 readonly end: () => Promise<void>;89 readonly end: () => Promise<void>;
@@ -158,6 +165,16 @@ const makeSessionServices = (dbHandle: CloudSessionDbHandle) => { · 75 unmodified lines
158// ---------------------------------------------------------------------------165// ---------------------------------------------------------------------------
159 166
160export class McpSessionDOSqlite extends McpAgentSessionDOBase<Env, CloudSessionDbHandle> {167export class McpSessionDOSqlite extends McpAgentSessionDOBase<Env, CloudSessionDbHandle> {
168 protected override sessionTimeoutMs(): number {
169 return positiveMilliseconds(env.MCP_SESSION_TIMEOUT_MS) ?? super.sessionTimeoutMs();
170 }
171
172 protected override maxPausedSessionIdleMs(): number {
173 return (
174 positiveMilliseconds(env.MCP_PAUSED_SESSION_IDLE_TIMEOUT_MS) ?? super.maxPausedSessionIdleMs()
175 );
176 }
177
161 protected override openSessionDb(): CloudSessionDbHandle {178 protected override openSessionDb(): CloudSessionDbHandle {
162 return makeDbHandle({179 return makeDbHandle({
163 idleTimeout: LONG_LIVED_DB_IDLE_TIMEOUT_SECONDS,180 idleTimeout: LONG_LIVED_DB_IDLE_TIMEOUT_SECONDS,
e2e/cloud/mcp-client-sessions.test.ts+21−13
@@ -18,6 +18,7 @@ import { StreamableHTTPClientTransport } from "@modelcontextprotocol/sdk/client/ · 17 unmodified lines
18import { scenario } from "../src/scenario";18import { scenario } from "../src/scenario";
19import { Api, Mcp, Target } from "../src/services";19import { Api, Mcp, Target } from "../src/services";
20import type { Identity } from "../src/target";20import type { Identity } from "../src/target";
21import { configuredMcpPausedSessionIdleTimeoutMs } from "../setup/mcp-session-timeouts";
21 22
22const coreApi = composePluginApi([] as const);23const coreApi = composePluginApi([] as const);
23 24
@@ -216,17 +217,19 @@ return JSON.stringify(result); · 192 unmodified lines
216const executionIdOf = (text: string): string | undefined =>217const executionIdOf = (text: string): string | undefined =>
217 /\bexecutionId:\s*(\S+)/.exec(text)?.[1];218 /\bexecutionId:\s*(\S+)/.exec(text)?.[1];
218 219
219// The session DO tears its runtime down after 5 minutes without a request220const IDLE_TEARDOWN_BUFFER_MS = 2_000;
220// (SESSION_TIMEOUT_MS in McpSessionDOBase) and rebuilds it from storage on221const IDLE_TEARDOWN_GAP_MS = configuredMcpPausedSessionIdleTimeoutMs() + IDLE_TEARDOWN_BUFFER_MS;
221// the next one — the same engine-state wipe a workerd eviction or a deploy222const IDLE_TEARDOWN_SCENARIO_TIMEOUT_MS = IDLE_TEARDOWN_GAP_MS + 120_000;
222// causes. Paused approvals deliberately do NOT survive this (durable pause223
223// state is out of scope); the contract is that an expired pause fails with224// The session DO tears its runtime down after the configured idle ceiling and
224// recovery guidance and the session keeps working.225// rebuilds it from storage on the next request. The e2e harness shortens that
225const IDLE_TEARDOWN_GAP = "6 minutes";226// ceiling when it owns the cloud dev stack. Paused approvals deliberately do
227// not survive this (durable pause state is out of scope); the contract is that
228// an expired pause fails with recovery guidance and the session keeps working.
226 229
227scenario(230scenario(
228 "MCP sessions · an approval paused past the idle window expires with re-run guidance, not a dead end",231 "MCP sessions · an approval paused past the idle window expires with re-run guidance, not a dead end",
229 { timeout: 480_000 },232 { timeout: IDLE_TEARDOWN_SCENARIO_TIMEOUT_MS },
230 Effect.gen(function* () {233 Effect.gen(function* () {
231 const target = yield* Target;234 const target = yield* Target;
232 const { client } = yield* Api;235 const { client } = yield* Api;
@@ -242,6 +245,8 @@ scenario( · 9 unmodified lines
242 yield* Effect.gen(function* () {245 yield* Effect.gen(function* () {
243 const first = yield* Effect.promise(() => connectClient(target.mcpUrl, bearer));246 const first = yield* Effect.promise(() => connectClient(target.mcpUrl, bearer));
244 const sessionId = first.transport.sessionId;247 const sessionId = first.transport.sessionId;
248 expect(sessionId, "the first client got a session id").toEqual(expect.any(String));
249 if (sessionId === undefined) return yield* Effect.die("missing session id");
245 const paused = yield* Effect.promise(() =>250 const paused = yield* Effect.promise(() =>
246 first.client.callTool({ name: "execute", arguments: { code: GATED_CODE } }),251 first.client.callTool({ name: "execute", arguments: { code: GATED_CODE } }),
247 ).pipe(Effect.ensuring(closeQuietly(first)));252 ).pipe(Effect.ensuring(closeQuietly(first)));
@@ -251,24 +256,26 @@ scenario( · 3 unmodified lines
251 );256 );
252 const executionId = executionIdOf(pausedText);257 const executionId = executionIdOf(pausedText);
253 expect(executionId, "the paused result carries the executionId").toEqual(expect.any(String));258 expect(executionId, "the paused result carries the executionId").toEqual(expect.any(String));
259 if (executionId === undefined) return yield* Effect.die("missing paused execution id");
254 260
255 // The user thinks the approval over for longer than the session keeps261 // The user thinks the approval over for longer than the session keeps
256 // its runtime warm; the pause is gone when they come back.262 // its runtime warm; the pause is gone when they come back.
257 yield* Effect.sleep(IDLE_TEARDOWN_GAP);263 yield* Effect.sleep(IDLE_TEARDOWN_GAP_MS);
258 264
259 const second = yield* Effect.promise(() => connectClient(target.mcpUrl, bearer, sessionId));265 const second = yield* Effect.promise(() => connectClient(target.mcpUrl, bearer, sessionId));
260 yield* Effect.gen(function* () {266 yield* Effect.gen(function* () {
261 const resumed = yield* Effect.promise(() =>267 const resumed = yield* Effect.promise(() =>
262 second.client.callTool({268 second.client.callTool({
263 name: "resume",269 name: "resume",
264 arguments: { executionId: executionId ?? "", action: "accept", content: "{}" },270 arguments: { executionId, action: "accept", content: "{}" },
265 }),271 }),
266 );272 );
267 expect(resumed.isError, "the expired resume is an error, not a silent success").toBe(true);273 const resumedText = textOf(resumed);
268 expect(274 expect(
269 textOf(resumed),275 resumedText,
270 "the error tells the model how to recover, not just that the pause is gone",276 "the error tells the model how to recover, not just that the pause is gone",
271 ).toContain("run the execute tool again");277 ).toContain("run the execute tool again");
278 expect(resumed.isError, "the expired resume is an error, not a silent success").toBe(true);
272 279
273 // The advertised recovery actually works on the same session: a fresh280 // The advertised recovery actually works on the same session: a fresh
274 // execute pauses with a NEW id (stale ids are never reused), and281 // execute pauses with a NEW id (stale ids are never reused), and
@@ -282,11 +289,12 @@ scenario( · 7 unmodified lines
282 );289 );
283 const freshExecutionId = executionIdOf(reExecutedText);290 const freshExecutionId = executionIdOf(reExecutedText);
284 expect(freshExecutionId, "the re-run mints a different executionId").not.toBe(executionId);291 expect(freshExecutionId, "the re-run mints a different executionId").not.toBe(executionId);
292 if (freshExecutionId === undefined) return yield* Effect.die("missing fresh execution id");
285 293
286 const resumedFresh = yield* Effect.promise(() =>294 const resumedFresh = yield* Effect.promise(() =>
287 second.client.callTool({295 second.client.callTool({
288 name: "resume",296 name: "resume",
289 arguments: { executionId: freshExecutionId ?? "", action: "accept", content: "{}" },297 arguments: { executionId: freshExecutionId, action: "accept", content: "{}" },
290 }),298 }),
291 );299 );
292 expect(resumedFresh.isError, "the fresh approval resumes to completion").not.toBe(true);300 expect(resumedFresh.isError, "the fresh approval resumes to completion").not.toBe(true);
e2e/setup/cloud.boot.ts+2−0
@@ -90,6 +90,8 @@ export const bootCloud = async (options: CloudBootOptions): Promise<CloudBooted> · 89 unmodified lines
90 // The AuthKit domain (MCP OAuth metadata + JWKS) is the emulator too.90 // The AuthKit domain (MCP OAuth metadata + JWKS) is the emulator too.
91 MCP_AUTHKIT_DOMAIN: workosUrl,91 MCP_AUTHKIT_DOMAIN: workosUrl,
92 MCP_RESOURCE_ORIGIN: options.publicUrl,92 MCP_RESOURCE_ORIGIN: options.publicUrl,
93 MCP_SESSION_TIMEOUT_MS: process.env.MCP_SESSION_TIMEOUT_MS,
94 MCP_PAUSED_SESSION_IDLE_TIMEOUT_MS: process.env.MCP_PAUSED_SESSION_IDLE_TIMEOUT_MS,
93 ALLOW_LOCAL_NETWORK: "true",95 ALLOW_LOCAL_NETWORK: "true",
94 // Shrink the per-org hourly execution cap (prod default 1000) to a number96 // Shrink the per-org hourly execution cap (prod default 1000) to a number
95 // the rate-limit-backstop scenario can actually exhaust with real97 // the rate-limit-backstop scenario can actually exhaust with real
e2e/setup/cloud.globalsetup.ts+2−0
@@ -10,6 +10,7 @@ import { claimAndBoot } from "../src/ports"; · 9 unmodified lines
10import { E2E_COOKIE_PASSWORD, E2E_WORKOS_CLIENT_ID } from "../targets/cloud";10import { E2E_COOKIE_PASSWORD, E2E_WORKOS_CLIENT_ID } from "../targets/cloud";
11import { waitForHttp } from "./boot";11import { waitForHttp } from "./boot";
12import { bootCloud } from "./cloud.boot";12import { bootCloud } from "./cloud.boot";
13import { ensureE2eMcpSessionTimeoutEnv } from "./mcp-session-timeouts";
13import { bootMotel, motelExporterEnv } from "./motel";14import { bootMotel, motelExporterEnv } from "./motel";
14import { RUNS_DIR } from "../src/scenario";15import { RUNS_DIR } from "../src/scenario";
15 16
@@ -31,6 +32,7 @@ export default async function setup(): Promise<(() => Promise<void>) | void> { · 15 unmodified lines
31 }32 }
32 33
33 if (bootLogFile) mkdirSync(resolve(bootLogFile, ".."), { recursive: true });34 if (bootLogFile) mkdirSync(resolve(bootLogFile, ".."), { recursive: true });
35 ensureE2eMcpSessionTimeoutEnv();
34 36
35 // Suite-owned trace store — every run captures distributed traces. Booted37 // Suite-owned trace store — every run captures distributed traces. Booted
36 // once outside the retry: it binds its own OS-assigned port (not a claimed38 // once outside the retry: it binds its own OS-assigned port (not a claimed
e2e/setup/mcp-session-timeouts.ts+34−0
@@ -0,0 +1,34 @@
1const DEFAULT_E2E_MCP_SESSION_TIMEOUT_MS = 3_000;
2const DEFAULT_E2E_MCP_PAUSED_SESSION_IDLE_TIMEOUT_MS = 6_000;
3const PRODUCTION_MCP_PAUSED_SESSION_IDLE_TIMEOUT_MS = 9 * 60 * 1000;
4
5export const MCP_SESSION_TIMEOUT_ENV = "MCP_SESSION_TIMEOUT_MS";
6export const MCP_PAUSED_SESSION_IDLE_TIMEOUT_ENV = "MCP_PAUSED_SESSION_IDLE_TIMEOUT_MS";
7
8const positiveMilliseconds = (raw: string | undefined): number | undefined => {
9 if (!raw) return undefined;
10 const parsed = Number(raw);
11 if (!Number.isFinite(parsed) || parsed <= 0) return undefined;
12 return Math.floor(parsed);
13};
14
15export const ensureE2eMcpSessionTimeoutEnv = (): {
16 readonly sessionTimeoutMs: number;
17 readonly pausedSessionIdleTimeoutMs: number;
18} => {
19 const sessionTimeoutMs =
20 positiveMilliseconds(process.env[MCP_SESSION_TIMEOUT_ENV]) ??
21 DEFAULT_E2E_MCP_SESSION_TIMEOUT_MS;
22 const pausedSessionIdleTimeoutMs =
23 positiveMilliseconds(process.env[MCP_PAUSED_SESSION_IDLE_TIMEOUT_ENV]) ??
24 DEFAULT_E2E_MCP_PAUSED_SESSION_IDLE_TIMEOUT_MS;
25
26 process.env[MCP_SESSION_TIMEOUT_ENV] = String(sessionTimeoutMs);
27 process.env[MCP_PAUSED_SESSION_IDLE_TIMEOUT_ENV] = String(pausedSessionIdleTimeoutMs);
28
29 return { sessionTimeoutMs, pausedSessionIdleTimeoutMs };
30};
31
32export const configuredMcpPausedSessionIdleTimeoutMs = (): number =>
33 positiveMilliseconds(process.env[MCP_PAUSED_SESSION_IDLE_TIMEOUT_ENV]) ??
34 PRODUCTION_MCP_PAUSED_SESSION_IDLE_TIMEOUT_MS;
packages/hosts/cloudflare/src/mcp/agent-session-durable-object.ts+47−6
@@ -18,6 +18,7 @@ import { defaultMcpResource, type McpResource } from "@executor-js/host-mcp"; · 17 unmodified lines
18 18
19import type { IncomingPropagationHeaders, McpElicitationMode } from "./do-headers";19import type { IncomingPropagationHeaders, McpElicitationMode } from "./do-headers";
20import {20import {
21 MAX_PAUSED_SESSION_IDLE_MS,
21 SESSION_TIMEOUT_MS,22 SESSION_TIMEOUT_MS,
22 decideSessionAlarm,23 decideSessionAlarm,
23 pausedLeaseExtensionLog,24 pausedLeaseExtensionLog,
@@ -234,6 +235,14 @@ export abstract class McpAgentSessionDOBase< · 210 unmodified lines
234 return undefined;235 return undefined;
235 }236 }
236 237
238 protected sessionTimeoutMs(): number {
239 return SESSION_TIMEOUT_MS;
240 }
241
242 protected maxPausedSessionIdleMs(): number {
243 return MAX_PAUSED_SESSION_IDLE_MS;
244 }
245
237 protected readonly browserApprovalStore: BrowserApprovalStore = {246 protected readonly browserApprovalStore: BrowserApprovalStore = {
238 takeResponse: (executionId) => this.takeApprovalResponse(executionId),247 takeResponse: (executionId) => this.takeApprovalResponse(executionId),
239 waitForResponse: (executionId) => this.waitForApprovalResponse(executionId),248 waitForResponse: (executionId) => this.waitForApprovalResponse(executionId),
@@ -310,7 +319,7 @@ export abstract class McpAgentSessionDOBase< · 70 unmodified lines
310 this.lastActivityMs = now;319 this.lastActivityMs = now;
311 await Promise.all([320 await Promise.all([
312 this.ctx.storage.put(LAST_ACTIVITY_KEY, now),321 this.ctx.storage.put(LAST_ACTIVITY_KEY, now),
313 this.ctx.storage.setAlarm(now + SESSION_TIMEOUT_MS),322 this.ctx.storage.setAlarm(now + this.sessionTimeoutMs()),
314 ]);323 ]);
315 }324 }
316 325
@@ -337,6 +346,27 @@ export abstract class McpAgentSessionDOBase< · 20 unmodified lines
337 );346 );
338 }347 }
339 348
349 private async disposeIdleRuntime(input: {
350 readonly idleMs: number;
351 readonly pausedExecutionCount: number;
352 }): Promise<void> {
353 console.info(
354 JSON.stringify({
355 event: "mcp_session_idle_runtime_dispose",
356 sessionId: this.sessionId,
357 idleMs: input.idleMs,
358 pausedExecutionCount: input.pausedExecutionCount,
359 }),
360 );
361 await Effect.runPromise(this.closeRuntime());
362 await Effect.runPromise(
363 Effect.all([
364 Effect.ignore(Effect.tryPromise(() => this.ctx.storage.deleteAlarm())),
365 Effect.ignore(Effect.tryPromise(() => this.ctx.storage.delete(LAST_ACTIVITY_KEY))),
366 ]),
367 );
368 }
369
340 private resolveAndStoreSessionMeta(token: McpSessionInit) {370 private resolveAndStoreSessionMeta(token: McpSessionInit) {
341 const self = this;371 const self = this;
342 return Effect.gen(function* () {372 return Effect.gen(function* () {
@@ -476,9 +506,15 @@ export abstract class McpAgentSessionDOBase< · 133 unmodified lines
476 Effect.gen(function* () {506 Effect.gen(function* () {
477 const sessionMeta = yield* self.loadSessionMeta();507 const sessionMeta = yield* self.loadSessionMeta();
478 if (!sessionMeta) return "not_found" as const;508 if (!sessionMeta) return "not_found" as const;
479 yield* Effect.promise(() => self.markActivity()).pipe(509 if (self.initialized) {
480 Effect.withSpan("McpSessionDO.markActivity"),510 yield* Effect.promise(() => self.markActivity()).pipe(
481 );511 Effect.withSpan("McpSessionDO.markActivity"),
512 );
513 } else {
514 yield* Effect.promise(() => self.onStart()).pipe(
515 Effect.withSpan("McpSessionDO.restore_transport_runtime"),
516 );
517 }
482 return identity.accountId === sessionMeta.userId &&518 return identity.accountId === sessionMeta.userId &&
483 identity.organizationId === sessionMeta.organizationId519 identity.organizationId === sessionMeta.organizationId
484 ? ("ok" as const)520 ? ("ok" as const)
@@ -576,7 +612,12 @@ export abstract class McpAgentSessionDOBase< · 91 unmodified lines
576 const lastActivityMs = await this.loadLastActivity();612 const lastActivityMs = await this.loadLastActivity();
577 const idleMs = lastActivityMs > 0 ? Date.now() - lastActivityMs : 0;613 const idleMs = lastActivityMs > 0 ? Date.now() - lastActivityMs : 0;
578 const pausedExecutionCount = await this.pausedExecutionCount();614 const pausedExecutionCount = await this.pausedExecutionCount();
579 const decision = decideSessionAlarm({ idleMs, pausedExecutionCount });615 const decision = decideSessionAlarm({
616 idleMs,
617 pausedExecutionCount,
618 sessionTimeoutMs: this.sessionTimeoutMs(),
619 maxPausedSessionIdleMs: this.maxPausedSessionIdleMs(),
620 });
580 621
581 if (decision.kind === "idle_within_timeout") {622 if (decision.kind === "idle_within_timeout") {
582 await super.alarm();623 await super.alarm();
@@ -598,7 +639,7 @@ export abstract class McpAgentSessionDOBase< · 15 unmodified lines
598 return;639 return;
599 }640 }
600 641
601 await this.destroy();642 await this.disposeIdleRuntime({ idleMs, pausedExecutionCount });
602 }643 }
603 644
604 private validateApprovalIdentity(645 private validateApprovalIdentity(
packages/hosts/cloudflare/src/mcp/session-alarm-policy.test.ts+14−0
@@ -47,6 +47,20 @@ describe("decideSessionAlarm", () => { · 46 unmodified lines
47 });47 });
48 });48 });
49 49
50 it("caps the extension at the configured paused idle ceiling", () => {
51 expect(
52 decideSessionAlarm({
53 idleMs: 3_000,
54 pausedExecutionCount: 1,
55 sessionTimeoutMs: 3_000,
56 maxPausedSessionIdleMs: 6_000,
57 }),
58 ).toEqual({
59 kind: "extend_paused_lease",
60 leaseMs: 3_000,
61 });
62 });
63
50 it("destroys a paused session once the lease cap elapses", () => {64 it("destroys a paused session once the lease cap elapses", () => {
51 expect(65 expect(
52 decideSessionAlarm({66 decideSessionAlarm({
packages/hosts/cloudflare/src/mcp/session-alarm-policy.ts+13−3
@@ -23,12 +23,22 @@ export type SessionAlarmDecision = · 22 unmodified lines
23export const decideSessionAlarm = (input: {23export const decideSessionAlarm = (input: {
24 readonly idleMs: number;24 readonly idleMs: number;
25 readonly pausedExecutionCount: number;25 readonly pausedExecutionCount: number;
26 readonly sessionTimeoutMs?: number;
27 readonly maxPausedSessionIdleMs?: number;
26}): SessionAlarmDecision => {28}): SessionAlarmDecision => {
27 if (input.idleMs < SESSION_TIMEOUT_MS) {29 const sessionTimeoutMs = input.sessionTimeoutMs ?? SESSION_TIMEOUT_MS;
30 const maxPausedSessionIdleMs = input.maxPausedSessionIdleMs ?? MAX_PAUSED_SESSION_IDLE_MS;
31 if (input.idleMs < sessionTimeoutMs) {
28 return { kind: "idle_within_timeout" };32 return { kind: "idle_within_timeout" };
29 }33 }
30 if (input.pausedExecutionCount > 0 && input.idleMs < MAX_PAUSED_SESSION_IDLE_MS) {34 if (input.pausedExecutionCount > 0 && input.idleMs < maxPausedSessionIdleMs) {
31 return { kind: "extend_paused_lease", leaseMs: PAUSED_EXECUTION_LEASE_MS };35 return {
36 kind: "extend_paused_lease",
37 leaseMs: Math.max(
38 1,
39 Math.min(PAUSED_EXECUTION_LEASE_MS, maxPausedSessionIdleMs - input.idleMs),
40 ),
41 };
32 }42 }
33 return { kind: "destroy_idle_session" };43 return { kind: "destroy_idle_session" };
34};44};