Suspend sandbox execution deadlines while a tool dispatch awaits the host (#1437)

Rhys Sullivan committed 3 months ago

Commit 99c808f1 parent af95edb10 files · +473 −73Browse files

Unified · Split

10 files

.changeset/suspend-private-sandbox-deadlines.md +6 −0
.changeset/suspend-sandbox-deadlines.md +5 −0
e2e/scenarios/resume-after-sandbox-deadline.test.ts +150 −0
packages/core/execution/src/tool-invoker.test.ts +19 −10
packages/kernel/runtime-deno-subprocess/src/index.test.ts +43 −11
packages/kernel/runtime-deno-subprocess/src/index.ts +47 −28
packages/kernel/runtime-dynamic-worker/src/invocation.test.ts +58 −0
packages/kernel/runtime-dynamic-worker/src/module-template.ts +24 −4
packages/kernel/runtime-quickjs/src/index.test.ts +48 −0
packages/kernel/runtime-quickjs/src/index.ts +73 −20

.changeset/suspend-private-sandbox-deadlines.md+6−0

@@ -0,0 +1,6 @@
	1	+---
	2	+"@executor-js/runtime-dynamic-worker": patch
	3	+"@executor-js/runtime-deno-subprocess": patch
	4	+---
	5	+
	6	+Suspend sandbox execution deadlines while tool calls await the host, and reset the autonomous-compute budget after each dispatch returns.

.changeset/suspend-sandbox-deadlines.md+5−0

@@ -0,0 +1,5 @@
	1	+---
	2	+"@executor-js/runtime-quickjs": patch
	3	+---
	4	+
	5	+Suspend sandbox execution deadlines while tool calls await the host, and reset the autonomous-compute budget after each dispatch returns.

e2e/scenarios/resume-after-sandbox-deadline.test.ts+150−0

@@ -0,0 +1,150 @@
	1	+// Cross-target: approval durability against the sandbox clock — the "a human
	2	+// is allowed to take a few minutes" promise. A paused execution suspends the
	3	+// sandbox mid-call, but the sandbox's execution deadline (5 minutes) used to
	4	+// keep ticking while the human decided. Each pause advertises its own
	5	+// approval window, so with chained approvals a later window legitimately
	6	+// extends past the sandbox's absolute clock: the human answers every window
	7	+// on time, yet the fiber is already dead and the final approval lands on an
	8	+// unknown execution.
	9	+//
	10	+// The journey drives exactly that shape: ONE execution with TWO approval
	11	+// gates. The first approval is granted late in its window (~3.5 min), so the
	12	+// second pause's window reaches well past the sandbox's 5-minute mark. The
	13	+// second approval arrives ~5.75 min after execution start — inside its OWN
	14	+// advertised window, but past the old absolute deadline. Deliberately slow
	15	+// (~6 min): the elapsed time IS the subject under test. A single-pause
	16	+// variant cannot express this cross-target — hosts that advertise a
	17	+// 4-minute window would expire it legitimately before the sandbox clock
	18	+// even matters.
	19	+//
	20	+// The gate is `policies.create`'s own `requiresApproval` annotation
	21	+// (hermetic, same device as policy-tool-approval.test.ts); both approvals
	22	+// happen in the same MCP session, so the only variable is elapsed time.
	23	+import { randomUUID } from "node:crypto";
	24	+
	25	+import { expect } from "@effect/vitest";
	26	+import { Effect } from "effect";
	27	+import { composePluginApi } from "@executor-js/api/server";
	28	+
	29	+import { scenario } from "../src/scenario";
	30	+import { Api, Mcp, Target } from "../src/services";
	31	+import { configuredMcpPausedSessionIdleTimeoutMs } from "../setup/mcp-session-timeouts";
	32	+
	33	+const coreApi = composePluginApi([] as const);
	34	+
	35	+// Grant the first approval at 3.5 min — late but inside its 4-minute window.
	36	+// The second pause then opens a fresh window reaching ~7.5 min.
	37	+const FIRST_APPROVAL_DELAY_MS = 3.5 * 60_000;
	38	+// Grant the second approval 2.25 min later: ~5.75 min after execution start,
	39	+// past the sandbox's 5-minute budget but inside the second window.
	40	+const SECOND_APPROVAL_DELAY_MS = 2.25 * 60_000;
	41	+
	42	+/** Sandbox code that creates two policies through the approval-gated core
	43	+ * tool. Patterns are unique-per-run and match no real tool, so the rules are
	44	+ * inert even if leaked. */
	45	+const createPoliciesCode = (firstPattern: string, secondPattern: string) => `
	46	+const first = await tools.executor.coreTools.policies.create({
	47	+ owner: "user",
	48	+ pattern: ${JSON.stringify(firstPattern)},
	49	+ action: "block",
	50	+});
	51	+const second = await tools.executor.coreTools.policies.create({
	52	+ owner: "user",
	53	+ pattern: ${JSON.stringify(secondPattern)},
	54	+ action: "block",
	55	+});
	56	+return JSON.stringify({ first: first.ok, second: second.ok });
	57	+`;
	58	+
	59	+// The journey spans ~6 real minutes of paused waiting, so the host must keep
	60	+// the paused session alive that long. The suite's default e2e override shrinks
	61	+// the paused-session idle teardown to seconds (to keep teardown tests fast),
	62	+// which would evict the session mid-scenario for reasons unrelated to the
	63	+// clock under test — require the production-like window instead.
	64	+const PAUSED_IDLE_WINDOW_TOO_SHORT =
	65	+ configuredMcpPausedSessionIdleTimeoutMs() < 8 * 60_000
	66	+ ? `the target's paused-session idle teardown (${configuredMcpPausedSessionIdleTimeoutMs()}ms) evicts the session before this ~6-minute journey completes; boot the target with MCP_PAUSED_SESSION_IDLE_TIMEOUT_MS >= 480000 to run it`
	67	+ : undefined;
	68	+
	69	+scenario(
	70	+ "MCP · chained approvals granted within their windows survive the sandbox clock",
	71	+ { timeout: 480_000, skip: PAUSED_IDLE_WINDOW_TOO_SHORT },
	72	+ Effect.gen(function* () {
	73	+ const target = yield* Target;
	74	+ const apiSurface = yield* Api;
	75	+ const mcp = yield* Mcp;
	76	+ const identity = yield* target.newIdentity();
	77	+ const client = yield* apiSurface.client(coreApi, identity);
	78	+ const runId = randomUUID().slice(0, 8);
	79	+ const firstPattern = `resume-deadline-a-${runId}.*`;
	80	+ const secondPattern = `resume-deadline-b-${runId}.*`;
	81	+
	82	+ const cleanup = client.policies.list().pipe(
	83	+ Effect.flatMap((list) =>
	84	+ Effect.forEach(
	85	+ list.filter((p) => p.pattern === firstPattern \|\| p.pattern === secondPattern),
	86	+ (p) =>
	87	+ client.policies
	88	+ .remove({ params: { policyId: p.id }, payload: { owner: "user" } })
	89	+ .pipe(Effect.ignore),
	90	+ ),
	91	+ ),
	92	+ Effect.ignore,
	93	+ );
	94	+
	95	+ yield* Effect.gen(function* () {
	96	+ const session = mcp.session(identity);
	97	+ const tools = yield* session.listTools();
	98	+ expect(tools).toContain("execute");
	99	+
	100	+ const paused = yield* session.call("execute", {
	101	+ code: createPoliciesCode(firstPattern, secondPattern),
	102	+ });
	103	+ expect(paused.text, "the first gated call paused for approval").toContain("Execution paused");
	104	+ const firstMatch = /\bexecutionId:\s*(\S+)/.exec(paused.text);
	105	+ expect(firstMatch, "the first pause carries an executionId").not.toBeNull();
	106	+
	107	+ // The human takes most of the first approval window before deciding.
	108	+ yield* Effect.sleep(`${FIRST_APPROVAL_DELAY_MS} millis`);
	109	+
	110	+ const secondPaused = yield* session.call("resume", {
	111	+ executionId: firstMatch![1]!,
	112	+ action: "accept",
	113	+ content: JSON.stringify({}),
	114	+ });
	115	+ expect(
	116	+ secondPaused.text,
	117	+ "the on-time first approval reaches its pause and surfaces the second gate",
	118	+ ).toContain("Execution paused");
	119	+ const secondMatch = /\bexecutionId:\s*(\S+)/.exec(secondPaused.text);
	120	+ expect(secondMatch, "the second pause carries an executionId").not.toBeNull();
	121	+
	122	+ // The second decision lands past the sandbox's 5-minute budget (counted
	123	+ // from execution start) but well inside the second advertised window.
	124	+ yield* Effect.sleep(`${SECOND_APPROVAL_DELAY_MS} millis`);
	125	+
	126	+ const resumed = yield* session.call("resume", {
	127	+ executionId: secondMatch![1]!,
	128	+ action: "accept",
	129	+ content: JSON.stringify({}),
	130	+ });
	131	+
	132	+ expect(
	133	+ resumed.text,
	134	+ "the second on-time approval reaches a live pause instead of a dead or unknown execution",
	135	+ ).not.toMatch(/Paused execution is unknown\|Paused execution expired\|timed out after/);
	136	+ expect(resumed.ok, "the fully approved execution completed without error").toBe(true);
	137	+
	138	+ // Both approvals took effect: the gated tool ran twice.
	139	+ const afterApproval = yield* client.policies.list();
	140	+ expect(
	141	+ afterApproval.some((p) => p.pattern === firstPattern),
	142	+ "the first gated tool ran after its approval",
	143	+ ).toBe(true);
	144	+ expect(
	145	+ afterApproval.some((p) => p.pattern === secondPattern),
	146	+ "the second gated tool ran after the late-but-in-window approval",
	147	+ ).toBe(true);
	148	+ }).pipe(Effect.ensuring(cleanup));
	149	+ }),
	150	+);

packages/core/execution/src/tool-invoker.test.ts+19−10

@@ -1743,28 +1743,37 @@ describe("pause/resume with multiple elicitations", () => { · 1742 unmodified lines
1743	1743	{ timeout: 10000 },
1744	1744	);
1745	1745
1746		− // live clock: the sandbox timeout is a real timer, so the test must
1747		− // actually wait for it rather than suspend on the virtual TestClock.
	1746	+ // Live clock: the failing executor delays long enough for its detached tool
	1747	+ // invocation to publish the pause before the executor settles on its own.
1748	1748	it.live(
1749	1749	"a pause abandoned by a failing sandbox is dropped and its resume replays the failure outcome",
1750	1750	() =>
1751	1751	Effect.gen(function* () {
1752	1752	const executor = yield* makeElicitingExecutor();
1753		− // Sandbox times out while suspended on the elicitation, so the fiber
1754		− // settles without a resume ever arriving.
	1753	+ const failingCodeExecutor = {
	1754	+ execute: (_code, toolInvoker) =>
	1755	+ Effect.gen(function* () {
	1756	+ yield* Effect.forkChild(
	1757	+ toolInvoker
	1758	+ .invoke({ path: "api.org.main.singleApproval", args: {} })
	1759	+ .pipe(Effect.ignore),
	1760	+ );
	1761	+ yield* Effect.sleep("100 millis");
	1762	+ return { result: null, error: "sandbox failed after pausing" };
	1763	+ }),
	1764	+ } satisfies typeof codeExecutor;
1755	1765	const engine = createExecutionEngine({
1756	1766	executor,
1757		− codeExecutor: makeQuickJsExecutor({ timeoutMs: 250 }),
	1767	+ codeExecutor: failingCodeExecutor,
1758	1768	});
1759		− const code = "return await tools.api.org.main.singleApproval({});";
1760	1769
1761		− const outcome1 = yield* engine.executeWithPause(code);
	1770	+ const outcome1 = yield* engine.executeWithPause("ignored");
1762	1771	expect(outcome1.status).toBe("paused");
1763	1772	if (outcome1.status !== "paused") return;
1764	1773	const executionId = outcome1.execution.id;
1765	1774
1766		− // Wait for the sandbox timeout to settle the detached fiber.
1767		− yield* Effect.sleep("600 millis");
	1775	+ // Wait for the sandbox failure to settle the detached fiber.
	1776	+ yield* Effect.sleep("300 millis");
1768	1777
1769	1778	// The dead pause must no longer be reported as live...
1770	1779	const stillPaused = yield* engine.getPausedExecution(executionId);
@@ -1774,7 +1783,7 @@ describe("pause/resume with multiple elicitations", () => { · 3 unmodified lines
1774	1783	const late = yield* engine.resume(executionId, { action: "accept" });
1775	1784	expect(late?.status).toBe("completed");
1776	1785	const completed = late as Extract<NonNullable<typeof late>, { status: "completed" }>;
1777		− expect(completed.result.error).toContain("timed out");
	1786	+ expect(completed.result.error).toBe("sandbox failed after pausing");
1778	1787	}),
1779	1788	{ timeout: 10000 },
1780	1789	);

packages/kernel/runtime-deno-subprocess/src/index.test.ts+43−11

@@ -234,19 +234,51 @@ describe.skipIf(!isDenoAvailable())("runtime-deno-subprocess", () => { · 233 unmodified lines
234	234	}),
235	235	);
236	236
237		− it.effect("respects timeout", () =>
238		− Effect.gen(function* () {
239		− const executor = makeDenoSubprocessExecutor({
240		− timeoutMs: 500,
241		− });
242		− const toolInvoker = makeTestInvoker({});
	237	+ it("suspends the execution deadline while a tool dispatch is in flight", async () => {
	238	+ const timeoutMs = 300;
	239	+ const executor = makeDenoSubprocessExecutor({ timeoutMs });
	240	+ const toolInvoker: SandboxToolInvoker = {
	241	+ invoke: () => Effect.sleep(timeoutMs * 3).pipe(Effect.as("slow result")),
	242	+ };
	243	+
	244	+ const output = await Effect.runPromise(
	245	+ executor.execute("return await tools.slow.wait({});", toolInvoker),
	246	+ );
	247	+
	248	+ expect(output.error).toBeUndefined();
	249	+ expect(output.result).toBe("slow result");
	250	+ });
	251	+
	252	+ it("still times out continuous autonomous compute", async () => {
	253	+ const timeoutMs = 300;
	254	+ const executor = makeDenoSubprocessExecutor({ timeoutMs });
	255	+ const toolInvoker = makeTestInvoker({});
243	256
244		− const output = yield* executor.execute("await new Promise(() => {}); return 1;", toolInvoker);
	257	+ const output = await Effect.runPromise(
	258	+ executor.execute("await new Promise(() => {}); return 1;", toolInvoker),
	259	+ );
245	260
246		− expect(output.result).toBeNull();
247		− expect(output.error).toContain("timed out");
248		− }),
249		− );
	261	+ expect(output.result).toBeNull();
	262	+ expect(output.error).toBe(`Deno subprocess execution timed out after ${timeoutMs}ms`);
	263	+ });
	264	+
	265	+ it("resets the execution deadline after a tool dispatch returns", async () => {
	266	+ const timeoutMs = 300;
	267	+ const executor = makeDenoSubprocessExecutor({ timeoutMs });
	268	+ const toolInvoker: SandboxToolInvoker = {
	269	+ invoke: () => Effect.sleep(timeoutMs * 3).pipe(Effect.as("done")),
	270	+ };
	271	+
	272	+ const output = await Effect.runPromise(
	273	+ executor.execute(
	274	+ "await tools.slow.wait({}); await new Promise(() => {}); return 1;",
	275	+ toolInvoker,
	276	+ ),
	277	+ );
	278	+
	279	+ expect(output.result).toBeNull();
	280	+ expect(output.error).toBe(`Deno subprocess execution timed out after ${timeoutMs}ms`);
	281	+ });
250	282
251	283	it.effect("network access is denied by default", () =>
252	284	Effect.gen(function* () {

packages/kernel/runtime-deno-subprocess/src/index.ts+47−28

@@ -230,11 +230,21 @@ const executeInDeno = ( · 229 unmodified lines
230	230	catch: (cause) => new DenoSpawnError({ executable: denoExecutable, reason: cause }),
231	231	});
232	232
	233	+ const start = Date.now();
	234	+ let inFlight = 0;
	235	+ let lastReturnedAt = start;
	236	+
233	237	// Send code to the subprocess
234	238	writeMessage(worker.stdin, { type: "start", code: recoveredBody, nonce });
235	239
236		− // Set up timeout — kills process and completes the deferred
237		− const timer = setTimeout(() => {
	240	+ // Measure only continuous autonomous subprocess compute. Tool dispatches
	241	+ // suspend the clock and each return grants a fresh timeout budget.
	242	+ const timer = setInterval(() => {
	243	+ const now = Date.now();
	244	+ if (inFlight > 0 \|\| now - Math.max(start, lastReturnedAt) < timeoutMs) {
	245	+ return;
	246	+ }
	247	+
238	248	worker.dispose();
239	249	runSync(
240	250	completeWith({
@@ -242,7 +252,7 @@ const executeInDeno = ( · 1 unmodified line
242	252	error: `Deno subprocess execution timed out after ${timeoutMs}ms`,
243	253	}),
244	254	);
245		− }, timeoutMs);
	255	+ }, 1_000);
246	256
247	257	// -----------------------------------------------------------------------
248	258	// Message processing fiber — tool calls happen here, inside Effect
@@ -255,30 +265,39 @@ const executeInDeno = ( · 6 unmodified lines
255	265
256	266	switch (msg.type) {
257	267	case "tool_call": {
258		− const toolResult = yield* toolInvoker
259		− .invoke({ path: msg.toolPath, args: msg.args })
260		− .pipe(
261		− Effect.map(
262		− (value): HostToWorkerMessage => ({
263		− type: "tool_result",
264		− nonce,
265		− requestId: msg.requestId,
266		− ok: true,
267		− value,
268		− }),
269		− ),
270		− Effect.catchCause((cause) =>
271		− Effect.succeed<HostToWorkerMessage>({
272		− type: "tool_result",
273		− nonce,
274		− requestId: msg.requestId,
275		− ok: false,
276		− error: causeMessage(cause),
277		− }),
278		− ),
279		− );
280		−
281		− writeMessage(worker.stdin, toolResult);
	268	+ // Symmetric bracket (mirrors ToolDispatcher.call): the decrement
	269	+ // must run even if the invoke or write dies, or the watchdog's
	270	+ // in-flight guard would suspend the timeout forever.
	271	+ inFlight += 1;
	272	+ yield* toolInvoker.invoke({ path: msg.toolPath, args: msg.args }).pipe(
	273	+ Effect.map(
	274	+ (value): HostToWorkerMessage => ({
	275	+ type: "tool_result",
	276	+ nonce,
	277	+ requestId: msg.requestId,
	278	+ ok: true,
	279	+ value,
	280	+ }),
	281	+ ),
	282	+ Effect.catchCause((cause) =>
	283	+ Effect.succeed<HostToWorkerMessage>({
	284	+ type: "tool_result",
	285	+ nonce,
	286	+ requestId: msg.requestId,
	287	+ ok: false,
	288	+ error: causeMessage(cause),
	289	+ }),
	290	+ ),
	291	+ Effect.flatMap((toolResult) =>
	292	+ Effect.sync(() => writeMessage(worker.stdin, toolResult)),
	293	+ ),
	294	+ Effect.ensuring(
	295	+ Effect.sync(() => {
	296	+ inFlight -= 1;
	297	+ lastReturnedAt = Date.now();
	298	+ }),
	299	+ ),
	300	+ );
282	301	break;
283	302	}
284	303
@@ -318,7 +337,7 @@ const executeInDeno = ( · 33 unmodified lines
318	337	const output = yield* Deferred.await(result).pipe(
319	338	Effect.ensuring(
320	339	Effect.gen(function* () {
321		− clearTimeout(timer);
	340	+ clearInterval(timer);
322	341	yield* Fiber.interrupt(processFiber);
323	342	worker.dispose();
324	343	}),

packages/kernel/runtime-dynamic-worker/src/invocation.test.ts+58−0

@@ -484,6 +484,64 @@ describe("makeDynamicWorkerExecutor", () => { · 483 unmodified lines
484	484	expect(result.result).toBe(30);
485	485	});
486	486
	487	+ it("suspends the execution deadline while a tool dispatch is in flight", async () => {
	488	+ const timeoutMs = 200;
	489	+ const executor = makeDynamicWorkerExecutor({
	490	+ loader,
	491	+ timeoutMs,
	492	+ hostTimeoutGraceMs: 5_000,
	493	+ });
	494	+ const invoker: SandboxToolInvoker = {
	495	+ invoke: () => Effect.sleep(timeoutMs * 3).pipe(Effect.as("slow result")),
	496	+ };
	497	+
	498	+ const result = await Effect.runPromise(
	499	+ executor.execute("async () => await tools.slow.wait({})", invoker),
	500	+ );
	501	+
	502	+ expect(result.error).toBeUndefined();
	503	+ expect(result.result).toBe("slow result");
	504	+ });
	505	+
	506	+ it("still times out continuous autonomous compute", async () => {
	507	+ const timeoutMs = 200;
	508	+ const executor = makeDynamicWorkerExecutor({
	509	+ loader,
	510	+ timeoutMs,
	511	+ hostTimeoutGraceMs: 5_000,
	512	+ });
	513	+ const invoker = makeInvoker(() => null);
	514	+
	515	+ const result = await Effect.runPromise(
	516	+ executor.execute("async () => await new Promise(() => {})", invoker),
	517	+ );
	518	+
	519	+ expect(result.result).toBeNull();
	520	+ expect(result.error).toBe(`Execution timed out after ${timeoutMs}ms`);
	521	+ });
	522	+
	523	+ it("resets the execution deadline after a tool dispatch returns", async () => {
	524	+ const timeoutMs = 200;
	525	+ const executor = makeDynamicWorkerExecutor({
	526	+ loader,
	527	+ timeoutMs,
	528	+ hostTimeoutGraceMs: 5_000,
	529	+ });
	530	+ const invoker: SandboxToolInvoker = {
	531	+ invoke: () => Effect.sleep(timeoutMs * 3).pipe(Effect.as("done")),
	532	+ };
	533	+
	534	+ const result = await Effect.runPromise(
	535	+ executor.execute(
	536	+ "async () => { await tools.slow.wait({}); await new Promise(() => {}); }",
	537	+ invoker,
	538	+ ),
	539	+ );
	540	+
	541	+ expect(result.result).toBeNull();
	542	+ expect(result.error).toBe(`Execution timed out after ${timeoutMs}ms`);
	543	+ });
	544	+
487	545	it("returns an execution error for circular tool args", async () => {
488	546	const executor = makeDynamicWorkerExecutor({ loader });
489	547	const invoker = makeInvoker(() => null);

packages/kernel/runtime-dynamic-worker/src/module-template.ts+24−4

@@ -22,6 +22,9 @@ export const buildExecutorModule = (body: string, timeoutMs: number): string => · 21 unmodified lines
22	22	"",
23	23	"export default class CodeExecutor extends WorkerEntrypoint {",
24	24	" async evaluate(__dispatcher) {",
	25	+ " const __start = Date.now();",
	26	+ " let __inFlight = 0;",
	27	+ " let __lastReturnedAt = __start;",
25	28	" const __logs = [];",
26	29	" const __outputs = [];",
27	30	' console.log = (...a) => { __logs.push(a.map(String).join(" ")); };',
@@ -196,7 +199,14 @@ export const buildExecutorModule = (body: string, timeoutMs: number): string => · 168 unmodified lines
196	199	" if (!toolPath) throw new Error('Tool path missing in invocation');",
197	200	" return (async () => {",
198	201	" const encoded = await __encodeBinary(args[0]);",
199		− " const data = await __dispatcher.call(toolPath, encoded);",
	202	+ " __inFlight += 1;",
	203	+ " let data;",
	204	+ " try {",
	205	+ " data = await __dispatcher.call(toolPath, encoded);",
	206	+ " } finally {",
	207	+ " __inFlight -= 1;",
	208	+ " __lastReturnedAt = Date.now();",
	209	+ " }",
200	210	" if (!data.ok) throw new Error(__publicToolErrorMessage(data.error) \|\| 'Internal tool error');",
201	211	" return __decodeBinary(data.result);",
202	212	" })();",
@@ -204,18 +214,28 @@ export const buildExecutorModule = (body: string, timeoutMs: number): string => · 1 unmodified line
204	214	" });",
205	215	" const tools = __makeToolsProxy();",
206	216	"",
	217	+ " let __watchdogInterval;",
	218	+ " const __watchdog = new Promise((_, reject) => {",
	219	+ " __watchdogInterval = setInterval(() => {",
	220	+ " const now = Date.now();",
	221	+ ` if (__inFlight === 0 && now - Math.max(__start, __lastReturnedAt) >= ${timeoutMs}) {`,
	222	+ ` reject(new Error("Execution timed out after ${timeoutMs}ms"));`,
	223	+ " }",
	224	+ " }, 1000);",
	225	+ " });",
	226	+ "",
207	227	" try {",
208	228	" const result = await Promise.race([",
209	229	" (async () => {",
210	230	body,
211	231	" })(),",
212		− " new Promise((_, reject) =>",
213		− ` setTimeout(() => reject(new Error("Execution timed out after ${timeoutMs}ms")), ${timeoutMs})`,
214		− " ),",
	232	+ " __watchdog,",
215	233	" ]);",
216	234	" return { result, output: __outputs.length > 0 ? __outputs : undefined, logs: __logs };",
217	235	" } catch (err) {",
218	236	" return { result: undefined, output: __outputs.length > 0 ? __outputs : undefined, error: __serializeThrownError(err), logs: __logs };",
	237	+ " } finally {",
	238	+ " clearInterval(__watchdogInterval);",
219	239	" }",
220	240	" }",
221	241	"}",

packages/kernel/runtime-quickjs/src/index.test.ts+48−0

@@ -150,6 +150,54 @@ describe("quickjs executor", () => { · 149 unmodified lines
150	150	}),
151	151	);
152	152
	153	+ it("suspends the execution deadline while a tool dispatch is in flight", async () => {
	154	+ const timeoutMs = 100;
	155	+ const slowInvoker: SandboxToolInvoker = {
	156	+ invoke: () => Effect.sleep(timeoutMs * 3).pipe(Effect.as("slow result")),
	157	+ };
	158	+ const slowExecutor = makeQuickJsExecutor({ timeoutMs });
	159	+
	160	+ const result = await Effect.runPromise(
	161	+ slowExecutor.execute("return await tools.slow.wait({});", slowInvoker),
	162	+ );
	163	+
	164	+ expect(result.error).toBeUndefined();
	165	+ expect(result.result).toBe("slow result");
	166	+ });
	167	+
	168	+ it("still times out continuous autonomous compute", async () => {
	169	+ const timeoutMs = 100;
	170	+ const timedExecutor = makeQuickJsExecutor({ timeoutMs });
	171	+
	172	+ const result = await Effect.runPromise(
	173	+ timedExecutor.execute("while (true) {}", makeTestInvoker({})),
	174	+ );
	175	+
	176	+ expect(result.result).toBeNull();
	177	+ expect(result.error).toBe(`QuickJS execution timed out after ${timeoutMs}ms`);
	178	+ });
	179	+
	180	+ it("resets the execution deadline after a tool dispatch returns", async () => {
	181	+ const timeoutMs = 100;
	182	+ const slowInvoker: SandboxToolInvoker = {
	183	+ invoke: () => Effect.sleep(timeoutMs * 3).pipe(Effect.as("done")),
	184	+ };
	185	+ const timedExecutor = makeQuickJsExecutor({ timeoutMs });
	186	+
	187	+ const result = await Effect.runPromise(
	188	+ timedExecutor.execute(
	189	+ `
	190	+ await tools.slow.wait({});
	191	+ while (true) {}
	192	+ `,
	193	+ slowInvoker,
	194	+ ),
	195	+ );
	196	+
	197	+ expect(result.result).toBeNull();
	198	+ expect(result.error).toBe(`QuickJS execution timed out after ${timeoutMs}ms`);
	199	+ });
	200	+
153	201	it.effect("invokes multiple tools in sequence", () =>
154	202	Effect.gen(function* () {
155	203	const invoker = makeTestInvoker({

packages/kernel/runtime-quickjs/src/index.ts+73−20

@@ -10,7 +10,6 @@ import * as Data from "effect/Data"; · 9 unmodified lines
10	10	import * as Effect from "effect/Effect";
11	11	import {
12	12	getQuickJS,
13		− shouldInterruptAfterDeadline,
14	13	type QuickJSContext,
15	14	type QuickJSDeferredPromise,
16	15	type QuickJSHandle,
@@ -109,9 +108,45 @@ const looksLikeInterruptedError = (message: string): boolean => /\binterrupted\b · 92 unmodified lines
109	108	const timeoutMessage = (timeoutMs: number): string =>
110	109	`QuickJS execution timed out after ${timeoutMs}ms`;
111	110
112		−const normalizeExecutionError = (cause: unknown, deadlineMs: number, timeoutMs: number): string => {
	111	+type DeadlineTracker = {
	112	+ /** Current absolute deadline, or null while at least one dispatch is in flight. */
	113	+ readonly deadlineMs: () => number \| null;
	114	+ readonly dispatchStarted: () => void;
	115	+ readonly dispatchReturned: () => void;
	116	+};
	117	+
	118	+const makeDeadlineTracker = (timeoutMs: number): DeadlineTracker => {
	119	+ const start = Date.now();
	120	+ let inFlight = 0;
	121	+ let lastReturnedAt = start;
	122	+
	123	+ return {
	124	+ deadlineMs: () => (inFlight > 0 ? null : Math.max(start, lastReturnedAt) + timeoutMs),
	125	+ dispatchStarted: () => {
	126	+ inFlight += 1;
	127	+ },
	128	+ dispatchReturned: () => {
	129	+ inFlight -= 1;
	130	+ lastReturnedAt = Date.now();
	131	+ },
	132	+ };
	133	+};
	134	+
	135	+const shouldInterruptAfterDeadline =
	136	+ (deadline: DeadlineTracker): (() => boolean) =>
	137	+ () => {
	138	+ const deadlineMs = deadline.deadlineMs();
	139	+ return deadlineMs !== null && Date.now() >= deadlineMs;
	140	+ };
	141	+
	142	+const normalizeExecutionError = (
	143	+ cause: unknown,
	144	+ deadline: DeadlineTracker,
	145	+ timeoutMs: number,
	146	+): string => {
113	147	const message = toErrorMessage(cause);
114		− return Date.now() >= deadlineMs && looksLikeInterruptedError(message)
	148	+ const deadlineMs = deadline.deadlineMs();
	149	+ return deadlineMs !== null && Date.now() >= deadlineMs && looksLikeInterruptedError(message)
115	150	? timeoutMessage(timeoutMs)
116	151	: message;
117	152	};
@@ -252,6 +287,7 @@ const createToolBridge = ( · 134 unmodified lines
252	287	toolInvoker: SandboxToolInvoker,
253	288	pendingDeferreds: Set<QuickJSDeferredPromise>,
254	289	runPromise: RunPromise,
	290	+ deadline: DeadlineTracker,
255	291	): QuickJSHandle =>
256	292	context.newFunction("__executor_invokeTool", (pathHandle, argsHandle) => {
257	293	const path = context.getString(pathHandle);
@@ -265,8 +301,10 @@ const createToolBridge = ( · 7 unmodified lines
265	301	pendingDeferreds.delete(deferred);
266	302	});
267	303
	304	+ deadline.dispatchStarted();
268	305	void runPromise(toolInvoker.invoke({ path, args })).then(
269	306	(value) => {
	307	+ deadline.dispatchReturned();
270	308	if (!deferred.alive) {
271	309	return;
272	310	}
@@ -282,6 +320,7 @@ const createToolBridge = ( · 9 unmodified lines
282	320	valueHandle.dispose();
283	321	},
284	322	(cause) => {
	323	+ deadline.dispatchReturned();
285	324	if (!deferred.alive) {
286	325	return;
287	326	}
@@ -308,11 +347,12 @@ const createToolBridge = ( · 20 unmodified lines
308	347	const drainJobs = (
309	348	context: QuickJSContext,
310	349	runtime: QuickJSRuntime,
311		− deadlineMs: number,
	350	+ deadline: DeadlineTracker,
312	351	timeoutMs: number,
313	352	): void => {
314	353	while (runtime.hasPendingJob()) {
315		− if (Date.now() >= deadlineMs) {
	354	+ const deadlineMs = deadline.deadlineMs();
	355	+ if (deadlineMs !== null && Date.now() >= deadlineMs) {
316	356	throw new Error(timeoutMessage(timeoutMs));
317	357	}
318	358
@@ -327,9 +367,16 @@ const drainJobs = ( · 8 unmodified lines
327	367
328	368	const waitForDeferreds = async (
329	369	pendingDeferreds: ReadonlySet<QuickJSDeferredPromise>,
330		− deadlineMs: number,
	370	+ deadline: DeadlineTracker,
331	371	timeoutMs: number,
332	372	): Promise<void> => {
	373	+ const settled = Promise.race([...pendingDeferreds].map((deferred) => deferred.settled));
	374	+ const deadlineMs = deadline.deadlineMs();
	375	+ if (deadlineMs === null) {
	376	+ await settled;
	377	+ return;
	378	+ }
	379	+
333	380	const remainingMs = deadlineMs - Date.now();
334	381	if (remainingMs <= 0) {
335	382	throw new Error(timeoutMessage(timeoutMs));
@@ -338,7 +385,7 @@ const waitForDeferreds = async ( · 2 unmodified lines
338	385	let timer: ReturnType<typeof setTimeout> \| undefined;
339	386	try {
340	387	await Promise.race([
341		− Promise.race([...pendingDeferreds].map((deferred) => deferred.settled)),
	388	+ settled,
342	389	new Promise<never>((_, reject) => {
343	390	timer = setTimeout(() => reject(new Error(timeoutMessage(timeoutMs))), remainingMs);
344	391	}),
@@ -354,17 +401,17 @@ const drainAsync = async ( · 9 unmodified lines
354	401	context: QuickJSContext,
355	402	runtime: QuickJSRuntime,
356	403	pendingDeferreds: ReadonlySet<QuickJSDeferredPromise>,
357		− deadlineMs: number,
	404	+ deadline: DeadlineTracker,
358	405	timeoutMs: number,
359	406	): Promise<void> => {
360		− drainJobs(context, runtime, deadlineMs, timeoutMs);
	407	+ drainJobs(context, runtime, deadline, timeoutMs);
361	408
362	409	while (pendingDeferreds.size > 0) {
363		− await waitForDeferreds(pendingDeferreds, deadlineMs, timeoutMs);
364		− drainJobs(context, runtime, deadlineMs, timeoutMs);
	410	+ await waitForDeferreds(pendingDeferreds, deadline, timeoutMs);
	411	+ drainJobs(context, runtime, deadline, timeoutMs);
365	412	}
366	413
367		− drainJobs(context, runtime, deadlineMs, timeoutMs);
	414	+ drainJobs(context, runtime, deadline, timeoutMs);
368	415	};
369	416
370	417	const evaluateInQuickJs = async (
@@ -374,7 +421,7 @@ const evaluateInQuickJs = async ( · 3 unmodified lines
374	421	runPromise: RunPromise,
375	422	): Promise<ExecuteResult> => {
376	423	const timeoutMs = Math.max(100, options.timeoutMs ?? DEFAULT_TIMEOUT_MS);
377		− const deadlineMs = Date.now() + timeoutMs;
	424	+ const deadline = makeDeadlineTracker(timeoutMs);
378	425	const logs: string[] = [];
379	426	const pendingDeferreds = new Set<QuickJSDeferredPromise>();
380	427	const QuickJS = await resolveQuickJS();
@@ -384,7 +431,7 @@ const evaluateInQuickJs = async ( · 3 unmodified lines
384	431	runtime.setMemoryLimit(options.memoryLimitBytes ?? DEFAULT_MEMORY_LIMIT_BYTES);
385	432	runtime.setMaxStackSize(options.maxStackSizeBytes ?? DEFAULT_MAX_STACK_SIZE_BYTES);
386	433
387		− runtime.setInterruptHandler(shouldInterruptAfterDeadline(deadlineMs));
	434	+ runtime.setInterruptHandler(shouldInterruptAfterDeadline(deadline));
388	435
389	436	const context = runtime.newContext();
390	437	try {
@@ -392,7 +439,13 @@ const evaluateInQuickJs = async ( · 1 unmodified line
392	439	context.setProp(context.global, "__executor_log", logBridge);
393	440	logBridge.dispose();
394	441
395		− const toolBridge = createToolBridge(context, toolInvoker, pendingDeferreds, runPromise);
	442	+ const toolBridge = createToolBridge(
	443	+ context,
	444	+ toolInvoker,
	445	+ pendingDeferreds,
	446	+ runPromise,
	447	+ deadline,
	448	+ );
396	449	context.setProp(context.global, "__executor_invokeTool", toolBridge);
397	450	toolBridge.dispose();
398	451
@@ -402,7 +455,7 @@ const evaluateInQuickJs = async ( · 3 unmodified lines
402	455	evaluated.error.dispose();
403	456	return {
404	457	result: null,
405		− error: normalizeExecutionError(error, deadlineMs, timeoutMs),
	458	+ error: normalizeExecutionError(error, deadline, timeoutMs),
406	459	logs,
407	460	} satisfies ExecuteResult;
408	461	}
@@ -418,14 +471,14 @@ const evaluateInQuickJs = async ( · 9 unmodified lines
418	471	stateResult.error.dispose();
419	472	return {
420	473	result: null,
421		− error: normalizeExecutionError(error, deadlineMs, timeoutMs),
	474	+ error: normalizeExecutionError(error, deadline, timeoutMs),
422	475	logs,
423	476	} satisfies ExecuteResult;
424	477	}
425	478
426	479	const stateHandle = stateResult.value;
427	480	try {
428		− await drainAsync(context, runtime, pendingDeferreds, deadlineMs, timeoutMs);
	481	+ await drainAsync(context, runtime, pendingDeferreds, deadline, timeoutMs);
429	482	const state = readResultState(context, stateHandle);
430	483	if (!state.settled) {
431	484	return {
@@ -439,7 +492,7 @@ const evaluateInQuickJs = async ( · 7 unmodified lines
439	492	if (typeof state.error !== "undefined") {
440	493	return {
441	494	result: null,
442		− error: normalizeExecutionError(state.error, deadlineMs, timeoutMs),
	495	+ error: normalizeExecutionError(state.error, deadline, timeoutMs),
443	496	output: readOutputItems(context),
444	497	logs,
445	498	} satisfies ExecuteResult;
@@ -466,7 +519,7 @@ const evaluateInQuickJs = async ( · 20 unmodified lines
466	519	} catch (cause) {
467	520	return {
468	521	result: null,
469		− error: normalizeExecutionError(cause, deadlineMs, timeoutMs),
	522	+ error: normalizeExecutionError(cause, deadline, timeoutMs),
470	523	logs,
471	524	} satisfies ExecuteResult;
472	525	} finally {