History

Suspend sandbox execution deadlines while a tool dispatch awaits the host (#1437)

Rhys Sullivan committed 3 months ago

Commit 99c808f1 parent af95edb10 files · +473 −73Browse files

Unified · Split

10 files
  1. .changeset/suspend-private-sandbox-deadlines.md +6 −0
  2. .changeset/suspend-sandbox-deadlines.md +5 −0
  3. e2e/scenarios/resume-after-sandbox-deadline.test.ts +150 −0
  4. packages/core/execution/src/tool-invoker.test.ts +19 −10
  5. packages/kernel/runtime-deno-subprocess/src/index.test.ts +43 −11
  6. packages/kernel/runtime-deno-subprocess/src/index.ts +47 −28
  7. packages/kernel/runtime-dynamic-worker/src/invocation.test.ts +58 −0
  8. packages/kernel/runtime-dynamic-worker/src/module-template.ts +24 −4
  9. packages/kernel/runtime-quickjs/src/index.test.ts +48 −0
  10. packages/kernel/runtime-quickjs/src/index.ts +73 −20
.changeset/suspend-private-sandbox-deadlines.md+6−0
@@ -0,0 +1,6 @@
1---
2"@executor-js/runtime-dynamic-worker": patch
3"@executor-js/runtime-deno-subprocess": patch
4---
5
6Suspend sandbox execution deadlines while tool calls await the host, and reset the autonomous-compute budget after each dispatch returns.
.changeset/suspend-sandbox-deadlines.md+5−0
@@ -0,0 +1,5 @@
1---
2"@executor-js/runtime-quickjs": patch
3---
4
5Suspend sandbox execution deadlines while tool calls await the host, and reset the autonomous-compute budget after each dispatch returns.
e2e/scenarios/resume-after-sandbox-deadline.test.ts+150−0
@@ -0,0 +1,150 @@
1// Cross-target: approval durability against the sandbox clock — the "a human
2// is allowed to take a few minutes" promise. A paused execution suspends the
3// sandbox mid-call, but the sandbox's execution deadline (5 minutes) used to
4// keep ticking while the human decided. Each pause advertises its own
5// approval window, so with chained approvals a later window legitimately
6// extends past the sandbox's absolute clock: the human answers every window
7// on time, yet the fiber is already dead and the final approval lands on an
8// unknown execution.
9//
10// The journey drives exactly that shape: ONE execution with TWO approval
11// gates. The first approval is granted late in its window (~3.5 min), so the
12// second pause's window reaches well past the sandbox's 5-minute mark. The
13// second approval arrives ~5.75 min after execution start — inside its OWN
14// advertised window, but past the old absolute deadline. Deliberately slow
15// (~6 min): the elapsed time IS the subject under test. A single-pause
16// variant cannot express this cross-target — hosts that advertise a
17// 4-minute window would expire it legitimately before the sandbox clock
18// even matters.
19//
20// The gate is `policies.create`'s own `requiresApproval` annotation
21// (hermetic, same device as policy-tool-approval.test.ts); both approvals
22// happen in the same MCP session, so the only variable is elapsed time.
23import { randomUUID } from "node:crypto";
24
25import { expect } from "@effect/vitest";
26import { Effect } from "effect";
27import { composePluginApi } from "@executor-js/api/server";
28
29import { scenario } from "../src/scenario";
30import { Api, Mcp, Target } from "../src/services";
31import { configuredMcpPausedSessionIdleTimeoutMs } from "../setup/mcp-session-timeouts";
32
33const coreApi = composePluginApi([] as const);
34
35// Grant the first approval at 3.5 min — late but inside its 4-minute window.
36// The second pause then opens a fresh window reaching ~7.5 min.
37const FIRST_APPROVAL_DELAY_MS = 3.5 * 60_000;
38// Grant the second approval 2.25 min later: ~5.75 min after execution start,
39// past the sandbox's 5-minute budget but inside the second window.
40const SECOND_APPROVAL_DELAY_MS = 2.25 * 60_000;
41
42/** Sandbox code that creates two policies through the approval-gated core
43 * tool. Patterns are unique-per-run and match no real tool, so the rules are
44 * inert even if leaked. */
45const createPoliciesCode = (firstPattern: string, secondPattern: string) => `
46const first = await tools.executor.coreTools.policies.create({
47 owner: "user",
48 pattern: ${JSON.stringify(firstPattern)},
49 action: "block",
50});
51const second = await tools.executor.coreTools.policies.create({
52 owner: "user",
53 pattern: ${JSON.stringify(secondPattern)},
54 action: "block",
55});
56return JSON.stringify({ first: first.ok, second: second.ok });
57`;
58
59// The journey spans ~6 real minutes of paused waiting, so the host must keep
60// the paused session alive that long. The suite's default e2e override shrinks
61// the paused-session idle teardown to seconds (to keep teardown tests fast),
62// which would evict the session mid-scenario for reasons unrelated to the
63// clock under test — require the production-like window instead.
64const PAUSED_IDLE_WINDOW_TOO_SHORT =
65 configuredMcpPausedSessionIdleTimeoutMs() < 8 * 60_000
66 ? `the target's paused-session idle teardown (${configuredMcpPausedSessionIdleTimeoutMs()}ms) evicts the session before this ~6-minute journey completes; boot the target with MCP_PAUSED_SESSION_IDLE_TIMEOUT_MS >= 480000 to run it`
67 : undefined;
68
69scenario(
70 "MCP · chained approvals granted within their windows survive the sandbox clock",
71 { timeout: 480_000, skip: PAUSED_IDLE_WINDOW_TOO_SHORT },
72 Effect.gen(function* () {
73 const target = yield* Target;
74 const apiSurface = yield* Api;
75 const mcp = yield* Mcp;
76 const identity = yield* target.newIdentity();
77 const client = yield* apiSurface.client(coreApi, identity);
78 const runId = randomUUID().slice(0, 8);
79 const firstPattern = `resume-deadline-a-${runId}.*`;
80 const secondPattern = `resume-deadline-b-${runId}.*`;
81
82 const cleanup = client.policies.list().pipe(
83 Effect.flatMap((list) =>
84 Effect.forEach(
85 list.filter((p) => p.pattern === firstPattern || p.pattern === secondPattern),
86 (p) =>
87 client.policies
88 .remove({ params: { policyId: p.id }, payload: { owner: "user" } })
89 .pipe(Effect.ignore),
90 ),
91 ),
92 Effect.ignore,
93 );
94
95 yield* Effect.gen(function* () {
96 const session = mcp.session(identity);
97 const tools = yield* session.listTools();
98 expect(tools).toContain("execute");
99
100 const paused = yield* session.call("execute", {
101 code: createPoliciesCode(firstPattern, secondPattern),
102 });
103 expect(paused.text, "the first gated call paused for approval").toContain("Execution paused");
104 const firstMatch = /\bexecutionId:\s*(\S+)/.exec(paused.text);
105 expect(firstMatch, "the first pause carries an executionId").not.toBeNull();
106
107 // The human takes most of the first approval window before deciding.
108 yield* Effect.sleep(`${FIRST_APPROVAL_DELAY_MS} millis`);
109
110 const secondPaused = yield* session.call("resume", {
111 executionId: firstMatch![1]!,
112 action: "accept",
113 content: JSON.stringify({}),
114 });
115 expect(
116 secondPaused.text,
117 "the on-time first approval reaches its pause and surfaces the second gate",
118 ).toContain("Execution paused");
119 const secondMatch = /\bexecutionId:\s*(\S+)/.exec(secondPaused.text);
120 expect(secondMatch, "the second pause carries an executionId").not.toBeNull();
121
122 // The second decision lands past the sandbox's 5-minute budget (counted
123 // from execution start) but well inside the second advertised window.
124 yield* Effect.sleep(`${SECOND_APPROVAL_DELAY_MS} millis`);
125
126 const resumed = yield* session.call("resume", {
127 executionId: secondMatch![1]!,
128 action: "accept",
129 content: JSON.stringify({}),
130 });
131
132 expect(
133 resumed.text,
134 "the second on-time approval reaches a live pause instead of a dead or unknown execution",
135 ).not.toMatch(/Paused execution is unknown|Paused execution expired|timed out after/);
136 expect(resumed.ok, "the fully approved execution completed without error").toBe(true);
137
138 // Both approvals took effect: the gated tool ran twice.
139 const afterApproval = yield* client.policies.list();
140 expect(
141 afterApproval.some((p) => p.pattern === firstPattern),
142 "the first gated tool ran after its approval",
143 ).toBe(true);
144 expect(
145 afterApproval.some((p) => p.pattern === secondPattern),
146 "the second gated tool ran after the late-but-in-window approval",
147 ).toBe(true);
148 }).pipe(Effect.ensuring(cleanup));
149 }),
150);
packages/core/execution/src/tool-invoker.test.ts+19−10
@@ -1743,28 +1743,37 @@ describe("pause/resume with multiple elicitations", () => { · 1742 unmodified lines
1743 { timeout: 10000 },1743 { timeout: 10000 },
1744 );1744 );
1745 1745
1746 // live clock: the sandbox timeout is a real timer, so the test must1746 // Live clock: the failing executor delays long enough for its detached tool
1747 // actually wait for it rather than suspend on the virtual TestClock.1747 // invocation to publish the pause before the executor settles on its own.
1748 it.live(1748 it.live(
1749 "a pause abandoned by a failing sandbox is dropped and its resume replays the failure outcome",1749 "a pause abandoned by a failing sandbox is dropped and its resume replays the failure outcome",
1750 () =>1750 () =>
1751 Effect.gen(function* () {1751 Effect.gen(function* () {
1752 const executor = yield* makeElicitingExecutor();1752 const executor = yield* makeElicitingExecutor();
1753 // Sandbox times out while suspended on the elicitation, so the fiber1753 const failingCodeExecutor = {
1754 // settles without a resume ever arriving.1754 execute: (_code, toolInvoker) =>
1755 Effect.gen(function* () {
1756 yield* Effect.forkChild(
1757 toolInvoker
1758 .invoke({ path: "api.org.main.singleApproval", args: {} })
1759 .pipe(Effect.ignore),
1760 );
1761 yield* Effect.sleep("100 millis");
1762 return { result: null, error: "sandbox failed after pausing" };
1763 }),
1764 } satisfies typeof codeExecutor;
1755 const engine = createExecutionEngine({1765 const engine = createExecutionEngine({
1756 executor,1766 executor,
1757 codeExecutor: makeQuickJsExecutor({ timeoutMs: 250 }),1767 codeExecutor: failingCodeExecutor,
1758 });1768 });
1759 const code = "return await tools.api.org.main.singleApproval({});";
1760 1769
1761 const outcome1 = yield* engine.executeWithPause(code);1770 const outcome1 = yield* engine.executeWithPause("ignored");
1762 expect(outcome1.status).toBe("paused");1771 expect(outcome1.status).toBe("paused");
1763 if (outcome1.status !== "paused") return;1772 if (outcome1.status !== "paused") return;
1764 const executionId = outcome1.execution.id;1773 const executionId = outcome1.execution.id;
1765 1774
1766 // Wait for the sandbox timeout to settle the detached fiber.1775 // Wait for the sandbox failure to settle the detached fiber.
1767 yield* Effect.sleep("600 millis");1776 yield* Effect.sleep("300 millis");
1768 1777
1769 // The dead pause must no longer be reported as live...1778 // The dead pause must no longer be reported as live...
1770 const stillPaused = yield* engine.getPausedExecution(executionId);1779 const stillPaused = yield* engine.getPausedExecution(executionId);
@@ -1774,7 +1783,7 @@ describe("pause/resume with multiple elicitations", () => { · 3 unmodified lines
1774 const late = yield* engine.resume(executionId, { action: "accept" });1783 const late = yield* engine.resume(executionId, { action: "accept" });
1775 expect(late?.status).toBe("completed");1784 expect(late?.status).toBe("completed");
1776 const completed = late as Extract<NonNullable<typeof late>, { status: "completed" }>;1785 const completed = late as Extract<NonNullable<typeof late>, { status: "completed" }>;
1777 expect(completed.result.error).toContain("timed out");1786 expect(completed.result.error).toBe("sandbox failed after pausing");
1778 }),1787 }),
1779 { timeout: 10000 },1788 { timeout: 10000 },
1780 );1789 );
packages/kernel/runtime-deno-subprocess/src/index.test.ts+43−11
@@ -234,19 +234,51 @@ describe.skipIf(!isDenoAvailable())("runtime-deno-subprocess", () => { · 233 unmodified lines
234 }),234 }),
235 );235 );
236 236
237 it.effect("respects timeout", () =>237 it("suspends the execution deadline while a tool dispatch is in flight", async () => {
238 Effect.gen(function* () {238 const timeoutMs = 300;
239 const executor = makeDenoSubprocessExecutor({239 const executor = makeDenoSubprocessExecutor({ timeoutMs });
240 timeoutMs: 500,240 const toolInvoker: SandboxToolInvoker = {
241 });241 invoke: () => Effect.sleep(timeoutMs * 3).pipe(Effect.as("slow result")),
242 const toolInvoker = makeTestInvoker({});242 };
243
244 const output = await Effect.runPromise(
245 executor.execute("return await tools.slow.wait({});", toolInvoker),
246 );
247
248 expect(output.error).toBeUndefined();
249 expect(output.result).toBe("slow result");
250 });
251
252 it("still times out continuous autonomous compute", async () => {
253 const timeoutMs = 300;
254 const executor = makeDenoSubprocessExecutor({ timeoutMs });
255 const toolInvoker = makeTestInvoker({});
243 256
244 const output = yield* executor.execute("await new Promise(() => {}); return 1;", toolInvoker);257 const output = await Effect.runPromise(
258 executor.execute("await new Promise(() => {}); return 1;", toolInvoker),
259 );
245 260
246 expect(output.result).toBeNull();261 expect(output.result).toBeNull();
247 expect(output.error).toContain("timed out");262 expect(output.error).toBe(`Deno subprocess execution timed out after ${timeoutMs}ms`);
248 }),263 });
249 );264
265 it("resets the execution deadline after a tool dispatch returns", async () => {
266 const timeoutMs = 300;
267 const executor = makeDenoSubprocessExecutor({ timeoutMs });
268 const toolInvoker: SandboxToolInvoker = {
269 invoke: () => Effect.sleep(timeoutMs * 3).pipe(Effect.as("done")),
270 };
271
272 const output = await Effect.runPromise(
273 executor.execute(
274 "await tools.slow.wait({}); await new Promise(() => {}); return 1;",
275 toolInvoker,
276 ),
277 );
278
279 expect(output.result).toBeNull();
280 expect(output.error).toBe(`Deno subprocess execution timed out after ${timeoutMs}ms`);
281 });
250 282
251 it.effect("network access is denied by default", () =>283 it.effect("network access is denied by default", () =>
252 Effect.gen(function* () {284 Effect.gen(function* () {
packages/kernel/runtime-deno-subprocess/src/index.ts+47−28
@@ -230,11 +230,21 @@ const executeInDeno = ( · 229 unmodified lines
230 catch: (cause) => new DenoSpawnError({ executable: denoExecutable, reason: cause }),230 catch: (cause) => new DenoSpawnError({ executable: denoExecutable, reason: cause }),
231 });231 });
232 232
233 const start = Date.now();
234 let inFlight = 0;
235 let lastReturnedAt = start;
236
233 // Send code to the subprocess237 // Send code to the subprocess
234 writeMessage(worker.stdin, { type: "start", code: recoveredBody, nonce });238 writeMessage(worker.stdin, { type: "start", code: recoveredBody, nonce });
235 239
236 // Set up timeout — kills process and completes the deferred240 // Measure only continuous autonomous subprocess compute. Tool dispatches
237 const timer = setTimeout(() => {241 // suspend the clock and each return grants a fresh timeout budget.
242 const timer = setInterval(() => {
243 const now = Date.now();
244 if (inFlight > 0 || now - Math.max(start, lastReturnedAt) < timeoutMs) {
245 return;
246 }
247
238 worker.dispose();248 worker.dispose();
239 runSync(249 runSync(
240 completeWith({250 completeWith({
@@ -242,7 +252,7 @@ const executeInDeno = ( · 1 unmodified line
242 error: `Deno subprocess execution timed out after ${timeoutMs}ms`,252 error: `Deno subprocess execution timed out after ${timeoutMs}ms`,
243 }),253 }),
244 );254 );
245 }, timeoutMs);255 }, 1_000);
246 256
247 // -----------------------------------------------------------------------257 // -----------------------------------------------------------------------
248 // Message processing fiber — tool calls happen here, inside Effect258 // Message processing fiber — tool calls happen here, inside Effect
@@ -255,30 +265,39 @@ const executeInDeno = ( · 6 unmodified lines
255 265
256 switch (msg.type) {266 switch (msg.type) {
257 case "tool_call": {267 case "tool_call": {
258 const toolResult = yield* toolInvoker268 // Symmetric bracket (mirrors ToolDispatcher.call): the decrement
259 .invoke({ path: msg.toolPath, args: msg.args })269 // must run even if the invoke or write dies, or the watchdog's
260 .pipe(270 // in-flight guard would suspend the timeout forever.
261 Effect.map(271 inFlight += 1;
262 (value): HostToWorkerMessage => ({272 yield* toolInvoker.invoke({ path: msg.toolPath, args: msg.args }).pipe(
263 type: "tool_result",273 Effect.map(
264 nonce,274 (value): HostToWorkerMessage => ({
265 requestId: msg.requestId,275 type: "tool_result",
266 ok: true,276 nonce,
267 value,277 requestId: msg.requestId,
268 }),278 ok: true,
269 ),279 value,
270 Effect.catchCause((cause) =>280 }),
271 Effect.succeed<HostToWorkerMessage>({281 ),
272 type: "tool_result",282 Effect.catchCause((cause) =>
273 nonce,283 Effect.succeed<HostToWorkerMessage>({
274 requestId: msg.requestId,284 type: "tool_result",
275 ok: false,285 nonce,
276 error: causeMessage(cause),286 requestId: msg.requestId,
277 }),287 ok: false,
278 ),288 error: causeMessage(cause),
279 );289 }),
280 290 ),
281 writeMessage(worker.stdin, toolResult);291 Effect.flatMap((toolResult) =>
292 Effect.sync(() => writeMessage(worker.stdin, toolResult)),
293 ),
294 Effect.ensuring(
295 Effect.sync(() => {
296 inFlight -= 1;
297 lastReturnedAt = Date.now();
298 }),
299 ),
300 );
282 break;301 break;
283 }302 }
284 303
@@ -318,7 +337,7 @@ const executeInDeno = ( · 33 unmodified lines
318 const output = yield* Deferred.await(result).pipe(337 const output = yield* Deferred.await(result).pipe(
319 Effect.ensuring(338 Effect.ensuring(
320 Effect.gen(function* () {339 Effect.gen(function* () {
321 clearTimeout(timer);340 clearInterval(timer);
322 yield* Fiber.interrupt(processFiber);341 yield* Fiber.interrupt(processFiber);
323 worker.dispose();342 worker.dispose();
324 }),343 }),
packages/kernel/runtime-dynamic-worker/src/invocation.test.ts+58−0
@@ -484,6 +484,64 @@ describe("makeDynamicWorkerExecutor", () => { · 483 unmodified lines
484 expect(result.result).toBe(30);484 expect(result.result).toBe(30);
485 });485 });
486 486
487 it("suspends the execution deadline while a tool dispatch is in flight", async () => {
488 const timeoutMs = 200;
489 const executor = makeDynamicWorkerExecutor({
490 loader,
491 timeoutMs,
492 hostTimeoutGraceMs: 5_000,
493 });
494 const invoker: SandboxToolInvoker = {
495 invoke: () => Effect.sleep(timeoutMs * 3).pipe(Effect.as("slow result")),
496 };
497
498 const result = await Effect.runPromise(
499 executor.execute("async () => await tools.slow.wait({})", invoker),
500 );
501
502 expect(result.error).toBeUndefined();
503 expect(result.result).toBe("slow result");
504 });
505
506 it("still times out continuous autonomous compute", async () => {
507 const timeoutMs = 200;
508 const executor = makeDynamicWorkerExecutor({
509 loader,
510 timeoutMs,
511 hostTimeoutGraceMs: 5_000,
512 });
513 const invoker = makeInvoker(() => null);
514
515 const result = await Effect.runPromise(
516 executor.execute("async () => await new Promise(() => {})", invoker),
517 );
518
519 expect(result.result).toBeNull();
520 expect(result.error).toBe(`Execution timed out after ${timeoutMs}ms`);
521 });
522
523 it("resets the execution deadline after a tool dispatch returns", async () => {
524 const timeoutMs = 200;
525 const executor = makeDynamicWorkerExecutor({
526 loader,
527 timeoutMs,
528 hostTimeoutGraceMs: 5_000,
529 });
530 const invoker: SandboxToolInvoker = {
531 invoke: () => Effect.sleep(timeoutMs * 3).pipe(Effect.as("done")),
532 };
533
534 const result = await Effect.runPromise(
535 executor.execute(
536 "async () => { await tools.slow.wait({}); await new Promise(() => {}); }",
537 invoker,
538 ),
539 );
540
541 expect(result.result).toBeNull();
542 expect(result.error).toBe(`Execution timed out after ${timeoutMs}ms`);
543 });
544
487 it("returns an execution error for circular tool args", async () => {545 it("returns an execution error for circular tool args", async () => {
488 const executor = makeDynamicWorkerExecutor({ loader });546 const executor = makeDynamicWorkerExecutor({ loader });
489 const invoker = makeInvoker(() => null);547 const invoker = makeInvoker(() => null);
packages/kernel/runtime-dynamic-worker/src/module-template.ts+24−4
@@ -22,6 +22,9 @@ export const buildExecutorModule = (body: string, timeoutMs: number): string => · 21 unmodified lines
22 "",22 "",
23 "export default class CodeExecutor extends WorkerEntrypoint {",23 "export default class CodeExecutor extends WorkerEntrypoint {",
24 " async evaluate(__dispatcher) {",24 " async evaluate(__dispatcher) {",
25 " const __start = Date.now();",
26 " let __inFlight = 0;",
27 " let __lastReturnedAt = __start;",
25 " const __logs = [];",28 " const __logs = [];",
26 " const __outputs = [];",29 " const __outputs = [];",
27 ' console.log = (...a) => { __logs.push(a.map(String).join(" ")); };',30 ' console.log = (...a) => { __logs.push(a.map(String).join(" ")); };',
@@ -196,7 +199,14 @@ export const buildExecutorModule = (body: string, timeoutMs: number): string => · 168 unmodified lines
196 " if (!toolPath) throw new Error('Tool path missing in invocation');",199 " if (!toolPath) throw new Error('Tool path missing in invocation');",
197 " return (async () => {",200 " return (async () => {",
198 " const encoded = await __encodeBinary(args[0]);",201 " const encoded = await __encodeBinary(args[0]);",
199 " const data = await __dispatcher.call(toolPath, encoded);",202 " __inFlight += 1;",
203 " let data;",
204 " try {",
205 " data = await __dispatcher.call(toolPath, encoded);",
206 " } finally {",
207 " __inFlight -= 1;",
208 " __lastReturnedAt = Date.now();",
209 " }",
200 " if (!data.ok) throw new Error(__publicToolErrorMessage(data.error) || 'Internal tool error');",210 " if (!data.ok) throw new Error(__publicToolErrorMessage(data.error) || 'Internal tool error');",
201 " return __decodeBinary(data.result);",211 " return __decodeBinary(data.result);",
202 " })();",212 " })();",
@@ -204,18 +214,28 @@ export const buildExecutorModule = (body: string, timeoutMs: number): string => · 1 unmodified line
204 " });",214 " });",
205 " const tools = __makeToolsProxy();",215 " const tools = __makeToolsProxy();",
206 "",216 "",
217 " let __watchdogInterval;",
218 " const __watchdog = new Promise((_, reject) => {",
219 " __watchdogInterval = setInterval(() => {",
220 " const now = Date.now();",
221 ` if (__inFlight === 0 && now - Math.max(__start, __lastReturnedAt) >= ${timeoutMs}) {`,
222 ` reject(new Error("Execution timed out after ${timeoutMs}ms"));`,
223 " }",
224 " }, 1000);",
225 " });",
226 "",
207 " try {",227 " try {",
208 " const result = await Promise.race([",228 " const result = await Promise.race([",
209 " (async () => {",229 " (async () => {",
210 body,230 body,
211 " })(),",231 " })(),",
212 " new Promise((_, reject) =>",232 " __watchdog,",
213 ` setTimeout(() => reject(new Error("Execution timed out after ${timeoutMs}ms")), ${timeoutMs})`,
214 " ),",
215 " ]);",233 " ]);",
216 " return { result, output: __outputs.length > 0 ? __outputs : undefined, logs: __logs };",234 " return { result, output: __outputs.length > 0 ? __outputs : undefined, logs: __logs };",
217 " } catch (err) {",235 " } catch (err) {",
218 " return { result: undefined, output: __outputs.length > 0 ? __outputs : undefined, error: __serializeThrownError(err), logs: __logs };",236 " return { result: undefined, output: __outputs.length > 0 ? __outputs : undefined, error: __serializeThrownError(err), logs: __logs };",
237 " } finally {",
238 " clearInterval(__watchdogInterval);",
219 " }",239 " }",
220 " }",240 " }",
221 "}",241 "}",
packages/kernel/runtime-quickjs/src/index.test.ts+48−0
@@ -150,6 +150,54 @@ describe("quickjs executor", () => { · 149 unmodified lines
150 }),150 }),
151 );151 );
152 152
153 it("suspends the execution deadline while a tool dispatch is in flight", async () => {
154 const timeoutMs = 100;
155 const slowInvoker: SandboxToolInvoker = {
156 invoke: () => Effect.sleep(timeoutMs * 3).pipe(Effect.as("slow result")),
157 };
158 const slowExecutor = makeQuickJsExecutor({ timeoutMs });
159
160 const result = await Effect.runPromise(
161 slowExecutor.execute("return await tools.slow.wait({});", slowInvoker),
162 );
163
164 expect(result.error).toBeUndefined();
165 expect(result.result).toBe("slow result");
166 });
167
168 it("still times out continuous autonomous compute", async () => {
169 const timeoutMs = 100;
170 const timedExecutor = makeQuickJsExecutor({ timeoutMs });
171
172 const result = await Effect.runPromise(
173 timedExecutor.execute("while (true) {}", makeTestInvoker({})),
174 );
175
176 expect(result.result).toBeNull();
177 expect(result.error).toBe(`QuickJS execution timed out after ${timeoutMs}ms`);
178 });
179
180 it("resets the execution deadline after a tool dispatch returns", async () => {
181 const timeoutMs = 100;
182 const slowInvoker: SandboxToolInvoker = {
183 invoke: () => Effect.sleep(timeoutMs * 3).pipe(Effect.as("done")),
184 };
185 const timedExecutor = makeQuickJsExecutor({ timeoutMs });
186
187 const result = await Effect.runPromise(
188 timedExecutor.execute(
189 `
190 await tools.slow.wait({});
191 while (true) {}
192 `,
193 slowInvoker,
194 ),
195 );
196
197 expect(result.result).toBeNull();
198 expect(result.error).toBe(`QuickJS execution timed out after ${timeoutMs}ms`);
199 });
200
153 it.effect("invokes multiple tools in sequence", () =>201 it.effect("invokes multiple tools in sequence", () =>
154 Effect.gen(function* () {202 Effect.gen(function* () {
155 const invoker = makeTestInvoker({203 const invoker = makeTestInvoker({
packages/kernel/runtime-quickjs/src/index.ts+73−20
@@ -10,7 +10,6 @@ import * as Data from "effect/Data"; · 9 unmodified lines
10import * as Effect from "effect/Effect";10import * as Effect from "effect/Effect";
11import {11import {
12 getQuickJS,12 getQuickJS,
13 shouldInterruptAfterDeadline,
14 type QuickJSContext,13 type QuickJSContext,
15 type QuickJSDeferredPromise,14 type QuickJSDeferredPromise,
16 type QuickJSHandle,15 type QuickJSHandle,
@@ -109,9 +108,45 @@ const looksLikeInterruptedError = (message: string): boolean => /\binterrupted\b · 92 unmodified lines
109const timeoutMessage = (timeoutMs: number): string =>108const timeoutMessage = (timeoutMs: number): string =>
110 `QuickJS execution timed out after ${timeoutMs}ms`;109 `QuickJS execution timed out after ${timeoutMs}ms`;
111 110
112const normalizeExecutionError = (cause: unknown, deadlineMs: number, timeoutMs: number): string => {111type DeadlineTracker = {
112 /** Current absolute deadline, or null while at least one dispatch is in flight. */
113 readonly deadlineMs: () => number | null;
114 readonly dispatchStarted: () => void;
115 readonly dispatchReturned: () => void;
116};
117
118const makeDeadlineTracker = (timeoutMs: number): DeadlineTracker => {
119 const start = Date.now();
120 let inFlight = 0;
121 let lastReturnedAt = start;
122
123 return {
124 deadlineMs: () => (inFlight > 0 ? null : Math.max(start, lastReturnedAt) + timeoutMs),
125 dispatchStarted: () => {
126 inFlight += 1;
127 },
128 dispatchReturned: () => {
129 inFlight -= 1;
130 lastReturnedAt = Date.now();
131 },
132 };
133};
134
135const shouldInterruptAfterDeadline =
136 (deadline: DeadlineTracker): (() => boolean) =>
137 () => {
138 const deadlineMs = deadline.deadlineMs();
139 return deadlineMs !== null && Date.now() >= deadlineMs;
140 };
141
142const normalizeExecutionError = (
143 cause: unknown,
144 deadline: DeadlineTracker,
145 timeoutMs: number,
146): string => {
113 const message = toErrorMessage(cause);147 const message = toErrorMessage(cause);
114 return Date.now() >= deadlineMs && looksLikeInterruptedError(message)148 const deadlineMs = deadline.deadlineMs();
149 return deadlineMs !== null && Date.now() >= deadlineMs && looksLikeInterruptedError(message)
115 ? timeoutMessage(timeoutMs)150 ? timeoutMessage(timeoutMs)
116 : message;151 : message;
117};152};
@@ -252,6 +287,7 @@ const createToolBridge = ( · 134 unmodified lines
252 toolInvoker: SandboxToolInvoker,287 toolInvoker: SandboxToolInvoker,
253 pendingDeferreds: Set<QuickJSDeferredPromise>,288 pendingDeferreds: Set<QuickJSDeferredPromise>,
254 runPromise: RunPromise,289 runPromise: RunPromise,
290 deadline: DeadlineTracker,
255): QuickJSHandle =>291): QuickJSHandle =>
256 context.newFunction("__executor_invokeTool", (pathHandle, argsHandle) => {292 context.newFunction("__executor_invokeTool", (pathHandle, argsHandle) => {
257 const path = context.getString(pathHandle);293 const path = context.getString(pathHandle);
@@ -265,8 +301,10 @@ const createToolBridge = ( · 7 unmodified lines
265 pendingDeferreds.delete(deferred);301 pendingDeferreds.delete(deferred);
266 });302 });
267 303
304 deadline.dispatchStarted();
268 void runPromise(toolInvoker.invoke({ path, args })).then(305 void runPromise(toolInvoker.invoke({ path, args })).then(
269 (value) => {306 (value) => {
307 deadline.dispatchReturned();
270 if (!deferred.alive) {308 if (!deferred.alive) {
271 return;309 return;
272 }310 }
@@ -282,6 +320,7 @@ const createToolBridge = ( · 9 unmodified lines
282 valueHandle.dispose();320 valueHandle.dispose();
283 },321 },
284 (cause) => {322 (cause) => {
323 deadline.dispatchReturned();
285 if (!deferred.alive) {324 if (!deferred.alive) {
286 return;325 return;
287 }326 }
@@ -308,11 +347,12 @@ const createToolBridge = ( · 20 unmodified lines
308const drainJobs = (347const drainJobs = (
309 context: QuickJSContext,348 context: QuickJSContext,
310 runtime: QuickJSRuntime,349 runtime: QuickJSRuntime,
311 deadlineMs: number,350 deadline: DeadlineTracker,
312 timeoutMs: number,351 timeoutMs: number,
313): void => {352): void => {
314 while (runtime.hasPendingJob()) {353 while (runtime.hasPendingJob()) {
315 if (Date.now() >= deadlineMs) {354 const deadlineMs = deadline.deadlineMs();
355 if (deadlineMs !== null && Date.now() >= deadlineMs) {
316 throw new Error(timeoutMessage(timeoutMs));356 throw new Error(timeoutMessage(timeoutMs));
317 }357 }
318 358
@@ -327,9 +367,16 @@ const drainJobs = ( · 8 unmodified lines
327 367
328const waitForDeferreds = async (368const waitForDeferreds = async (
329 pendingDeferreds: ReadonlySet<QuickJSDeferredPromise>,369 pendingDeferreds: ReadonlySet<QuickJSDeferredPromise>,
330 deadlineMs: number,370 deadline: DeadlineTracker,
331 timeoutMs: number,371 timeoutMs: number,
332): Promise<void> => {372): Promise<void> => {
373 const settled = Promise.race([...pendingDeferreds].map((deferred) => deferred.settled));
374 const deadlineMs = deadline.deadlineMs();
375 if (deadlineMs === null) {
376 await settled;
377 return;
378 }
379
333 const remainingMs = deadlineMs - Date.now();380 const remainingMs = deadlineMs - Date.now();
334 if (remainingMs <= 0) {381 if (remainingMs <= 0) {
335 throw new Error(timeoutMessage(timeoutMs));382 throw new Error(timeoutMessage(timeoutMs));
@@ -338,7 +385,7 @@ const waitForDeferreds = async ( · 2 unmodified lines
338 let timer: ReturnType<typeof setTimeout> | undefined;385 let timer: ReturnType<typeof setTimeout> | undefined;
339 try {386 try {
340 await Promise.race([387 await Promise.race([
341 Promise.race([...pendingDeferreds].map((deferred) => deferred.settled)),388 settled,
342 new Promise<never>((_, reject) => {389 new Promise<never>((_, reject) => {
343 timer = setTimeout(() => reject(new Error(timeoutMessage(timeoutMs))), remainingMs);390 timer = setTimeout(() => reject(new Error(timeoutMessage(timeoutMs))), remainingMs);
344 }),391 }),
@@ -354,17 +401,17 @@ const drainAsync = async ( · 9 unmodified lines
354 context: QuickJSContext,401 context: QuickJSContext,
355 runtime: QuickJSRuntime,402 runtime: QuickJSRuntime,
356 pendingDeferreds: ReadonlySet<QuickJSDeferredPromise>,403 pendingDeferreds: ReadonlySet<QuickJSDeferredPromise>,
357 deadlineMs: number,404 deadline: DeadlineTracker,
358 timeoutMs: number,405 timeoutMs: number,
359): Promise<void> => {406): Promise<void> => {
360 drainJobs(context, runtime, deadlineMs, timeoutMs);407 drainJobs(context, runtime, deadline, timeoutMs);
361 408
362 while (pendingDeferreds.size > 0) {409 while (pendingDeferreds.size > 0) {
363 await waitForDeferreds(pendingDeferreds, deadlineMs, timeoutMs);410 await waitForDeferreds(pendingDeferreds, deadline, timeoutMs);
364 drainJobs(context, runtime, deadlineMs, timeoutMs);411 drainJobs(context, runtime, deadline, timeoutMs);
365 }412 }
366 413
367 drainJobs(context, runtime, deadlineMs, timeoutMs);414 drainJobs(context, runtime, deadline, timeoutMs);
368};415};
369 416
370const evaluateInQuickJs = async (417const evaluateInQuickJs = async (
@@ -374,7 +421,7 @@ const evaluateInQuickJs = async ( · 3 unmodified lines
374 runPromise: RunPromise,421 runPromise: RunPromise,
375): Promise<ExecuteResult> => {422): Promise<ExecuteResult> => {
376 const timeoutMs = Math.max(100, options.timeoutMs ?? DEFAULT_TIMEOUT_MS);423 const timeoutMs = Math.max(100, options.timeoutMs ?? DEFAULT_TIMEOUT_MS);
377 const deadlineMs = Date.now() + timeoutMs;424 const deadline = makeDeadlineTracker(timeoutMs);
378 const logs: string[] = [];425 const logs: string[] = [];
379 const pendingDeferreds = new Set<QuickJSDeferredPromise>();426 const pendingDeferreds = new Set<QuickJSDeferredPromise>();
380 const QuickJS = await resolveQuickJS();427 const QuickJS = await resolveQuickJS();
@@ -384,7 +431,7 @@ const evaluateInQuickJs = async ( · 3 unmodified lines
384 runtime.setMemoryLimit(options.memoryLimitBytes ?? DEFAULT_MEMORY_LIMIT_BYTES);431 runtime.setMemoryLimit(options.memoryLimitBytes ?? DEFAULT_MEMORY_LIMIT_BYTES);
385 runtime.setMaxStackSize(options.maxStackSizeBytes ?? DEFAULT_MAX_STACK_SIZE_BYTES);432 runtime.setMaxStackSize(options.maxStackSizeBytes ?? DEFAULT_MAX_STACK_SIZE_BYTES);
386 433
387 runtime.setInterruptHandler(shouldInterruptAfterDeadline(deadlineMs));434 runtime.setInterruptHandler(shouldInterruptAfterDeadline(deadline));
388 435
389 const context = runtime.newContext();436 const context = runtime.newContext();
390 try {437 try {
@@ -392,7 +439,13 @@ const evaluateInQuickJs = async ( · 1 unmodified line
392 context.setProp(context.global, "__executor_log", logBridge);439 context.setProp(context.global, "__executor_log", logBridge);
393 logBridge.dispose();440 logBridge.dispose();
394 441
395 const toolBridge = createToolBridge(context, toolInvoker, pendingDeferreds, runPromise);442 const toolBridge = createToolBridge(
443 context,
444 toolInvoker,
445 pendingDeferreds,
446 runPromise,
447 deadline,
448 );
396 context.setProp(context.global, "__executor_invokeTool", toolBridge);449 context.setProp(context.global, "__executor_invokeTool", toolBridge);
397 toolBridge.dispose();450 toolBridge.dispose();
398 451
@@ -402,7 +455,7 @@ const evaluateInQuickJs = async ( · 3 unmodified lines
402 evaluated.error.dispose();455 evaluated.error.dispose();
403 return {456 return {
404 result: null,457 result: null,
405 error: normalizeExecutionError(error, deadlineMs, timeoutMs),458 error: normalizeExecutionError(error, deadline, timeoutMs),
406 logs,459 logs,
407 } satisfies ExecuteResult;460 } satisfies ExecuteResult;
408 }461 }
@@ -418,14 +471,14 @@ const evaluateInQuickJs = async ( · 9 unmodified lines
418 stateResult.error.dispose();471 stateResult.error.dispose();
419 return {472 return {
420 result: null,473 result: null,
421 error: normalizeExecutionError(error, deadlineMs, timeoutMs),474 error: normalizeExecutionError(error, deadline, timeoutMs),
422 logs,475 logs,
423 } satisfies ExecuteResult;476 } satisfies ExecuteResult;
424 }477 }
425 478
426 const stateHandle = stateResult.value;479 const stateHandle = stateResult.value;
427 try {480 try {
428 await drainAsync(context, runtime, pendingDeferreds, deadlineMs, timeoutMs);481 await drainAsync(context, runtime, pendingDeferreds, deadline, timeoutMs);
429 const state = readResultState(context, stateHandle);482 const state = readResultState(context, stateHandle);
430 if (!state.settled) {483 if (!state.settled) {
431 return {484 return {
@@ -439,7 +492,7 @@ const evaluateInQuickJs = async ( · 7 unmodified lines
439 if (typeof state.error !== "undefined") {492 if (typeof state.error !== "undefined") {
440 return {493 return {
441 result: null,494 result: null,
442 error: normalizeExecutionError(state.error, deadlineMs, timeoutMs),495 error: normalizeExecutionError(state.error, deadline, timeoutMs),
443 output: readOutputItems(context),496 output: readOutputItems(context),
444 logs,497 logs,
445 } satisfies ExecuteResult;498 } satisfies ExecuteResult;
@@ -466,7 +519,7 @@ const evaluateInQuickJs = async ( · 20 unmodified lines
466 } catch (cause) {519 } catch (cause) {
467 return {520 return {
468 result: null,521 result: null,
469 error: normalizeExecutionError(cause, deadlineMs, timeoutMs),522 error: normalizeExecutionError(cause, deadline, timeoutMs),
470 logs,523 logs,
471 } satisfies ExecuteResult;524 } satisfies ExecuteResult;
472 } finally {525 } finally {