Add convex-test dependency and implement access policy management

- Added `convex-test` as a development dependency in `package.json` and `bun.lock`. - Introduced access policy management in `service.test.ts`, including methods for listing and upserting access policies. - Enhanced runtime execution handling in `runtime-core.ts` and `vercel-sandbox-runtime.ts` to support new policy checks and improved error handling for script execution timeouts.

Rhys Sullivan committed 9 months ago

Commit 70e652d1 parent 9783c8d8 files · +671 −23Browse files

Unified · Split

8 files

executor/apps/server/src/adapters/in-process-execution-adapter.test.ts +78 −0
executor/apps/server/src/runtimes/runtime-core.test.ts +319 −0
executor/apps/server/src/runtimes/runtime-core.ts +11 −10
executor/apps/server/src/runtimes/vercel-sandbox-runtime.ts +13 −4
executor/apps/server/src/service.test.ts +108 −9
executor/bun.lock +3 −0
executor/convex/database.test.ts +138 −0
executor/package.json +1 −0

executor/apps/server/src/adapters/in-process-execution-adapter.test.ts+78−0

@@ -0,0 +1,78 @@
	1	+import { expect, test } from "bun:test";
	2	+import { APPROVAL_DENIED_PREFIX } from "../execution-constants";
	3	+import { InProcessExecutionAdapter } from "./in-process-execution-adapter";
	4	+
	5	+test("returns run mismatch without invoking tool", async () => {
	6	+ let called = 0;
	7	+ const adapter = new InProcessExecutionAdapter({
	8	+ runId: "run_expected",
	9	+ invokeTool: async () => {
	10	+ called += 1;
	11	+ return { ok: true };
	12	+ },
	13	+ emitOutput: () => {},
	14	+ });
	15	+
	16	+ const result = await adapter.invokeTool({
	17	+ runId: "run_other",
	18	+ callId: "call_1",
	19	+ toolPath: "utils.echo",
	20	+ input: {},
	21	+ });
	22	+
	23	+ expect(result.ok).toBe(false);
	24	+ if (!result.ok) {
	25	+ expect(result.error).toContain("Run mismatch");
	26	+ }
	27	+ expect(called).toBe(0);
	28	+});
	29	+
	30	+test("maps approval denied errors to denied result", async () => {
	31	+ const adapter = new InProcessExecutionAdapter({
	32	+ runId: "run_1",
	33	+ invokeTool: async () => {
	34	+ throw new Error(`${APPROVAL_DENIED_PREFIX}approval required`);
	35	+ },
	36	+ emitOutput: () => {},
	37	+ });
	38	+
	39	+ const result = await adapter.invokeTool({
	40	+ runId: "run_1",
	41	+ callId: "call_1",
	42	+ toolPath: "admin.delete_data",
	43	+ input: { id: "abc" },
	44	+ });
	45	+
	46	+ expect(result).toEqual({
	47	+ ok: false,
	48	+ denied: true,
	49	+ error: "approval required",
	50	+ });
	51	+});
	52	+
	53	+test("emits output only for matching run id", () => {
	54	+ const lines: string[] = [];
	55	+ const adapter = new InProcessExecutionAdapter({
	56	+ runId: "run_1",
	57	+ invokeTool: async () => null,
	58	+ emitOutput: (event) => {
	59	+ lines.push(`${event.stream}:${event.line}`);
	60	+ },
	61	+ });
	62	+
	63	+ adapter.emitOutput({
	64	+ runId: "run_other",
	65	+ stream: "stdout",
	66	+ line: "ignored",
	67	+ timestamp: Date.now(),
	68	+ });
	69	+
	70	+ adapter.emitOutput({
	71	+ runId: "run_1",
	72	+ stream: "stdout",
	73	+ line: "accepted",
	74	+ timestamp: Date.now(),
	75	+ });
	76	+
	77	+ expect(lines).toEqual(["stdout:accepted"]);
	78	+});

executor/apps/server/src/runtimes/runtime-core.test.ts+319−0

@@ -0,0 +1,319 @@
	1	+import { expect, test } from "bun:test";
	2	+import { runCodeWithAdapter } from "./runtime-core";
	3	+import { loadExternalTools } from "../tool-sources";
	4	+import type {
	5	+ ExecutionAdapter,
	6	+ RuntimeOutputEvent,
	7	+ SandboxExecutionRequest,
	8	+ ToolDefinition,
	9	+ ToolRunContext,
	10	+} from "../types";
	11	+
	12	+function request(code: string, timeoutMs = 1_000): SandboxExecutionRequest {
	13	+ return {
	14	+ taskId: `task_${crypto.randomUUID()}`,
	15	+ code,
	16	+ timeoutMs,
	17	+ };
	18	+}
	19	+
	20	+function createRuntimeAdapter(
	21	+ tools: Map<string, ToolDefinition>,
	22	+ outputEvents: RuntimeOutputEvent[],
	23	+): ExecutionAdapter {
	24	+ return {
	25	+ async invokeTool(call) {
	26	+ const tool = tools.get(call.toolPath);
	27	+ if (!tool) {
	28	+ return {
	29	+ ok: false,
	30	+ error: `Tool not found: ${call.toolPath}`,
	31	+ };
	32	+ }
	33	+
	34	+ try {
	35	+ const context: ToolRunContext = {
	36	+ taskId: call.runId,
	37	+ workspaceId: "ws_test",
	38	+ actorId: "actor_test",
	39	+ clientId: "web",
	40	+ isToolAllowed: () => true,
	41	+ };
	42	+ const value = await tool.run(call.input, context);
	43	+ return { ok: true, value };
	44	+ } catch (error) {
	45	+ return {
	46	+ ok: false,
	47	+ error: error instanceof Error ? error.message : String(error),
	48	+ };
	49	+ }
	50	+ },
	51	+ emitOutput(event) {
	52	+ outputEvents.push(event);
	53	+ },
	54	+ };
	55	+}
	56	+
	57	+test("executes tool calls and captures output", async () => {
	58	+ const outputEvents: RuntimeOutputEvent[] = [];
	59	+ const tools = new Map<string, ToolDefinition>([
	60	+ [
	61	+ "utils.echo",
	62	+ {
	63	+ path: "utils.echo",
	64	+ description: "Echo back input",
	65	+ approval: "auto",
	66	+ run: async (input) => ({ echoed: input }),
	67	+ },
	68	+ ],
	69	+ ]);
	70	+
	71	+ const result = await runCodeWithAdapter(
	72	+ request(`
	73	+ const out = await tools.utils.echo({ message: "hi" });
	74	+ console.log("out", out.echoed.message);
	75	+ return 42;
	76	+ `),
	77	+ createRuntimeAdapter(tools, outputEvents),
	78	+ );
	79	+
	80	+ expect(result.status).toBe("completed");
	81	+ expect(result.stdout).toContain("out hi");
	82	+ expect(result.stdout).toContain("result: 42");
	83	+ expect(outputEvents.some((event) => event.stream === "stdout")).toBe(true);
	84	+});
	85	+
	86	+test("returns denied when adapter marks tool call denied", async () => {
	87	+ const adapter: ExecutionAdapter = {
	88	+ async invokeTool() {
	89	+ return {
	90	+ ok: false,
	91	+ denied: true,
	92	+ error: "policy denied",
	93	+ };
	94	+ },
	95	+ emitOutput() {},
	96	+ };
	97	+
	98	+ const result = await runCodeWithAdapter(
	99	+ request(`
	100	+ await tools.admin.delete_data({ id: "x" });
	101	+ `),
	102	+ adapter,
	103	+ );
	104	+
	105	+ expect(result.status).toBe("denied");
	106	+ expect(result.error).toBe("policy denied");
	107	+ expect(result.stderr).toContain("policy denied");
	108	+});
	109	+
	110	+test("times out long-running code", async () => {
	111	+ const adapter: ExecutionAdapter = {
	112	+ async invokeTool() {
	113	+ return { ok: true, value: null };
	114	+ },
	115	+ emitOutput() {},
	116	+ };
	117	+
	118	+ const result = await runCodeWithAdapter(
	119	+ request(
	120	+ `
	121	+ await new Promise(() => {});
	122	+ `,
	123	+ 25,
	124	+ ),
	125	+ adapter,
	126	+ );
	127	+
	128	+ expect(result.status).toBe("timed_out");
	129	+ expect(result.error).toContain("timed out");
	130	+});
	131	+
	132	+test("sandboxed runner does not expose host globals", async () => {
	133	+ const outputEvents: RuntimeOutputEvent[] = [];
	134	+ const adapter = createRuntimeAdapter(new Map(), outputEvents);
	135	+
	136	+ const result = await runCodeWithAdapter(
	137	+ request(`
	138	+ const checks = {
	139	+ process: typeof process,
	140	+ bun: typeof Bun,
	141	+ fetch: typeof fetch,
	142	+ fsEscape: [].constructor.constructor("return typeof process")(),
	143	+ };
	144	+ console.log(JSON.stringify(checks));
	145	+ if (checks.process !== "undefined") throw new Error("process leaked");
	146	+ if (checks.bun !== "undefined") throw new Error("Bun leaked");
	147	+ if (checks.fetch !== "undefined") throw new Error("fetch leaked");
	148	+ if (checks.fsEscape !== "undefined") throw new Error("constructor escape leaked");
	149	+ return "ok";
	150	+ `),
	151	+ adapter,
	152	+ );
	153	+
	154	+ expect(result.status).toBe("completed");
	155	+ expect(result.stdout).toContain("\"process\":\"undefined\"");
	156	+ expect(result.stdout).toContain("\"bun\":\"undefined\"");
	157	+ expect(result.stdout).toContain("\"fetch\":\"undefined\"");
	158	+ expect(result.stdout).toContain("\"fsEscape\":\"undefined\"");
	159	+});
	160	+
	161	+test("runs openapi and graphql sourced tools through the runtime", async () => {
	162	+ const server = Bun.serve({
	163	+ port: 0,
	164	+ fetch: async (req) => {
	165	+ const url = new URL(req.url);
	166	+
	167	+ if (url.pathname === "/add" && req.method === "GET") {
	168	+ const a = Number(url.searchParams.get("a") ?? "0");
	169	+ const b = Number(url.searchParams.get("b") ?? "0");
	170	+ return Response.json({ sum: a + b });
	171	+ }
	172	+
	173	+ if (url.pathname === "/graphql" && req.method === "POST") {
	174	+ const body = (await req.json()) as { query?: string; variables?: Record<string, unknown> };
	175	+ const query = String(body.query ?? "");
	176	+
	177	+ if (query.includes("__schema")) {
	178	+ return Response.json({
	179	+ data: {
	180	+ __schema: {
	181	+ queryType: { name: "Query" },
	182	+ mutationType: { name: "Mutation" },
	183	+ types: [
	184	+ {
	185	+ kind: "OBJECT",
	186	+ name: "Query",
	187	+ fields: [
	188	+ {
	189	+ name: "hello",
	190	+ description: null,
	191	+ args: [],
	192	+ type: { kind: "SCALAR", name: "String", ofType: null },
	193	+ },
	194	+ ],
	195	+ inputFields: null,
	196	+ enumValues: null,
	197	+ },
	198	+ {
	199	+ kind: "OBJECT",
	200	+ name: "Mutation",
	201	+ fields: [
	202	+ {
	203	+ name: "increment",
	204	+ description: null,
	205	+ args: [
	206	+ {
	207	+ name: "value",
	208	+ description: null,
	209	+ defaultValue: null,
	210	+ type: {
	211	+ kind: "NON_NULL",
	212	+ name: null,
	213	+ ofType: { kind: "SCALAR", name: "Int", ofType: null },
	214	+ },
	215	+ },
	216	+ ],
	217	+ type: { kind: "SCALAR", name: "Int", ofType: null },
	218	+ },
	219	+ ],
	220	+ inputFields: null,
	221	+ enumValues: null,
	222	+ },
	223	+ { kind: "SCALAR", name: "String", fields: null, inputFields: null, enumValues: null },
	224	+ { kind: "SCALAR", name: "Int", fields: null, inputFields: null, enumValues: null },
	225	+ ],
	226	+ },
	227	+ },
	228	+ });
	229	+ }
	230	+
	231	+ if (query.includes("hello")) {
	232	+ return Response.json({ data: { hello: "world" } });
	233	+ }
	234	+
	235	+ if (query.includes("increment")) {
	236	+ const value = Number(body.variables?.value ?? 0);
	237	+ return Response.json({ data: { increment: value + 1 } });
	238	+ }
	239	+
	240	+ return Response.json({ errors: [{ message: "Unknown query" }] }, { status: 400 });
	241	+ }
	242	+
	243	+ return new Response("Not found", { status: 404 });
	244	+ },
	245	+ });
	246	+
	247	+ try {
	248	+ const baseUrl = `http://127.0.0.1:${server.port}`;
	249	+ const { tools } = await loadExternalTools([
	250	+ {
	251	+ type: "openapi",
	252	+ name: "calc",
	253	+ baseUrl,
	254	+ spec: {
	255	+ openapi: "3.0.3",
	256	+ info: { title: "Calc", version: "1.0.0" },
	257	+ paths: {
	258	+ "/add": {
	259	+ get: {
	260	+ operationId: "addNumbers",
	261	+ tags: ["math"],
	262	+ parameters: [
	263	+ { name: "a", in: "query", required: true, schema: { type: "number" } },
	264	+ { name: "b", in: "query", required: true, schema: { type: "number" } },
	265	+ ],
	266	+ responses: {
	267	+ "200": {
	268	+ description: "ok",
	269	+ content: {
	270	+ "application/json": {
	271	+ schema: {
	272	+ type: "object",
	273	+ properties: { sum: { type: "number" } },
	274	+ },
	275	+ },
	276	+ },
	277	+ },
	278	+ },
	279	+ },
	280	+ },
	281	+ },
	282	+ },
	283	+ },
	284	+ {
	285	+ type: "graphql",
	286	+ name: "gql",
	287	+ endpoint: `${baseUrl}/graphql`,
	288	+ },
	289	+ ]);
	290	+
	291	+ const toolMap = new Map(tools.map((tool) => [tool.path, tool]));
	292	+ const outputEvents: RuntimeOutputEvent[] = [];
	293	+
	294	+ const result = await runCodeWithAdapter(
	295	+ request(`
	296	+ const sum = await tools.calc.math.addnumbers({ a: 2, b: 5 });
	297	+ const hello = await tools.gql.query.hello({});
	298	+ const inc = await tools.gql.graphql({
	299	+ query: "mutation($value: Int!) { increment(value: $value) }",
	300	+ variables: { value: 3 },
	301	+ });
	302	+ console.log("sum", sum.sum);
	303	+ console.log("hello", hello.hello);
	304	+ console.log("inc", inc.increment);
	305	+ return sum.sum + inc.increment;
	306	+ `),
	307	+ createRuntimeAdapter(toolMap, outputEvents),
	308	+ );
	309	+
	310	+ expect(result.status).toBe("completed");
	311	+ expect(result.stdout).toContain("sum 7");
	312	+ expect(result.stdout).toContain("hello world");
	313	+ expect(result.stdout).toContain("inc 4");
	314	+ expect(result.stdout).toContain("result: 11");
	315	+ expect(outputEvents.length).toBeGreaterThan(0);
	316	+ } finally {
	317	+ server.stop(true);
	318	+ }
	319	+});

executor/apps/server/src/runtimes/runtime-core.ts+11−10

@@ -1,4 +1,5 @@
1	1	import { APPROVAL_DENIED_PREFIX, TASK_TIMEOUT_MARKER } from "../execution-constants";
	2	+import { Script, createContext } from "node:vm";
2	3	import type {
3	4	ExecutionAdapter,
4	5	SandboxExecutionRequest,
@@ -70,9 +71,6 @@ export async function runCodeWithAdapter( · 65 unmodified lines
70	71	const startedAt = Date.now();
71	72	const stdoutLines: string[] = [];
72	73	const stderrLines: string[] = [];
73		− const AsyncFunction = Object.getPrototypeOf(async function () {}).constructor as new (
74		− ...args: string[]
75		− ) => (...inputs: unknown[]) => Promise<unknown>;
76	74
77	75	const appendStdout = (line: string): void => {
78	76	stdoutLines.push(line);
@@ -106,11 +104,14 @@ export async function runCodeWithAdapter( · 27 unmodified lines
106	104	error: (...args: unknown[]) => appendStderr(formatArgs(args)),
107	105	};
108	106
109		− const runner = new AsyncFunction(
110		− "tools",
111		− "console",
112		− `"use strict";\n${request.code}`,
113		− );
	107	+ const context = createContext({
	108	+ tools,
	109	+ console: consoleProxy,
	110	+ setTimeout,
	111	+ clearTimeout,
	112	+ });
	113	+
	114	+ const runnerScript = new Script(`(async () => {\n"use strict";\n${request.code}\n})()`);
114	115
115	116	let timeoutHandle: ReturnType<typeof setTimeout> \| undefined;
116	117	const timeoutPromise = new Promise<never>((_resolve, reject) => {
@@ -121,7 +122,7 @@ export async function runCodeWithAdapter( · 4 unmodified lines
121	122
122	123	try {
123	124	const value = await Promise.race([
124		− runner(tools, consoleProxy),
	125	+ Promise.resolve(runnerScript.runInContext(context, { timeout: Math.max(1, request.timeoutMs) })),
125	126	timeoutPromise,
126	127	]);
127	128
@@ -138,7 +139,7 @@ export async function runCodeWithAdapter( · 10 unmodified lines
138	139	};
139	140	} catch (error) {
140	141	const message = error instanceof Error ? error.message : String(error);
141		− if (message === TASK_TIMEOUT_MARKER) {
	142	+ if (message === TASK_TIMEOUT_MARKER \|\| message.includes("Script execution timed out")) {
142	143	const timeoutMessage = `Execution timed out after ${request.timeoutMs}ms`;
143	144	appendStderr(timeoutMessage);
144	145	return {

executor/apps/server/src/runtimes/vercel-sandbox-runtime.ts+13−4

@@ -22,6 +22,7 @@ function buildRunnerScript(codeFilePath: string): string { · 21 unmodified lines
22	22	return `
23	23	import { readFile } from "node:fs/promises";
24	24	import { randomUUID } from "node:crypto";
	25	+import vm from "node:vm";
25	26
26	27	const RESULT_MARKER = ${JSON.stringify(RESULT_MARKER)};
27	28	const runId = process.env.EXECUTOR_RUN_ID;
@@ -145,8 +146,13 @@ const consoleProxy = { · 117 unmodified lines
145	146	error: (...args) => appendStderr(formatArgs(args)),
146	147	};
147	148
148		−const AsyncFunction = Object.getPrototypeOf(async function () {}).constructor;
149		−const runner = new AsyncFunction("tools", "console", '"use strict";\\n' + userCode);
	149	+const context = vm.createContext({
	150	+ tools,
	151	+ console: consoleProxy,
	152	+ setTimeout,
	153	+ clearTimeout,
	154	+});
	155	+const runnerScript = new vm.Script("(async () => {\\n\"use strict\";\\n" + userCode + "\\n})()");
150	156
151	157	const timeoutPromise = new Promise((_, reject) => {
152	158	setTimeout(() => reject(new Error("TASK_TIMEOUT")), requestTimeoutMs);
@@ -154,7 +160,10 @@ const timeoutPromise = new Promise((_, reject) => { · 1 unmodified line
154	160
155	161	let result;
156	162	try {
157		− const value = await Promise.race([runner(tools, consoleProxy), timeoutPromise]);
	163	+ const value = await Promise.race([
	164	+ Promise.resolve(runnerScript.runInContext(context, { timeout: Math.max(1, requestTimeoutMs) })),
	165	+ timeoutPromise,
	166	+ ]);
158	167	if (value !== undefined) {
159	168	appendStdout("result: " + formatArgs([value]));
160	169	}
@@ -169,7 +178,7 @@ try { · 8 unmodified lines
169	178	} catch (error) {
170	179	const message = error instanceof Error ? error.message : String(error);
171	180
172		− if (message === "TASK_TIMEOUT") {
	181	+ if (message === "TASK_TIMEOUT" \|\| message.includes("Script execution timed out")) {
173	182	const timeoutMessage = "Execution timed out after " + requestTimeoutMs + "ms";
174	183	appendStderr(timeoutMessage);
175	184	result = {

executor/apps/server/src/service.test.ts+108−9

@@ -21,6 +21,7 @@ class TestExecutorDatabase extends ExecutorDatabase { · 20 unmodified lines
21	21	private readonly tasks = new Map<string, TaskRecord>();
22	22	private readonly approvals = new Map<string, ApprovalRecord>();
23	23	private readonly events = new Map<string, TaskEventRecord[]>();
	24	+ private readonly policies = new Map<string, AccessPolicyRecord>();
24	25
25	26	constructor() {
26	27	super("http://127.0.0.1:3210");
@@ -217,7 +218,43 @@ class TestExecutorDatabase extends ExecutorDatabase { · 190 unmodified lines
217	218	}
218	219
219	220	async listAccessPolicies(_workspaceId: string): Promise<AccessPolicyRecord[]> {
220		− return [];
	221	+ return [...this.policies.values()]
	222	+ .filter((policy) => policy.workspaceId === _workspaceId)
	223	+ .sort((a, b) => {
	224	+ if (a.priority !== b.priority) {
	225	+ return b.priority - a.priority;
	226	+ }
	227	+ return a.createdAt - b.createdAt;
	228	+ });
	229	+ }
	230	+
	231	+ async upsertAccessPolicy(params: {
	232	+ id?: string;
	233	+ workspaceId: string;
	234	+ actorId?: string;
	235	+ clientId?: string;
	236	+ toolPathPattern: string;
	237	+ decision: "allow" \| "require_approval" \| "deny";
	238	+ priority?: number;
	239	+ }): Promise<AccessPolicyRecord> {
	240	+ const now = Date.now();
	241	+ const id = params.id ?? `policy_${crypto.randomUUID()}`;
	242	+ const existing = this.policies.get(id);
	243	+
	244	+ const record: AccessPolicyRecord = {
	245	+ id,
	246	+ workspaceId: params.workspaceId,
	247	+ actorId: params.actorId,
	248	+ clientId: params.clientId,
	249	+ toolPathPattern: params.toolPathPattern,
	250	+ decision: params.decision,
	251	+ priority: params.priority ?? 100,
	252	+ createdAt: existing?.createdAt ?? now,
	253	+ updatedAt: now,
	254	+ };
	255	+
	256	+ this.policies.set(id, record);
	257	+ return record;
221	258	}
222	259
223	260	async resolveCredential(): Promise<null> {
@@ -254,6 +291,11 @@ class InlineToolRuntime implements SandboxRuntime { · 30 unmodified lines
254	291	readonly label = "Inline";
255	292	readonly description = "Calls one tool then exits";
256	293
	294	+ constructor(
	295	+ private readonly toolPath = "admin.delete_data",
	296	+ private readonly input: unknown = { key: "abc" },
	297	+ ) {}
	298	+
257	299	async run(
258	300	request: SandboxExecutionRequest,
259	301	adapter: ExecutionAdapter,
@@ -262,8 +304,8 @@ class InlineToolRuntime implements SandboxRuntime { · 2 unmodified lines
262	304	const result = await adapter.invokeTool({
263	305	runId: request.taskId,
264	306	callId: "call_inline_1",
265		− toolPath: "admin.delete_data",
266		− input: { key: "abc" },
	307	+ toolPath: this.toolPath,
	308	+ input: this.input,
267	309	});
268	310
269	311	if (!result.ok) {
@@ -302,6 +344,21 @@ const tools: ToolDefinition[] = [ · 32 unmodified lines
302	344	},
303	345	];
304	346
	347	+async function waitForTaskStatus(
	348	+ service: ExecutorService,
	349	+ taskId: string,
	350	+ expected: TaskRecord["status"],
	351	+ timeoutMs = 2_000,
	352	+): Promise<TaskRecord \| null> {
	353	+ let task = await service.getTask(taskId);
	354	+ const waitUntil = Date.now() + timeoutMs;
	355	+ while (task?.status !== expected && Date.now() < waitUntil) {
	356	+ await Bun.sleep(25);
	357	+ task = await service.getTask(taskId);
	358	+ }
	359	+ return task;
	360	+}
	361	+
305	362	test("tool-level approval gates individual function call", async () => {
306	363	const service = new ExecutorService(
307	364	new TestExecutorDatabase(),
@@ -326,13 +383,55 @@ test("tool-level approval gates individual function call", async () => { · 18 unmodified lines
326	383	const resolved = await service.resolveApproval("ws_test", approvals[0]!.id, "approved", "test-user");
327	384	expect(resolved).toBeTruthy();
328	385
329		− let task = await service.getTask(created.task.id);
330		− const waitUntil = Date.now() + 2_000;
331		− while (task?.status !== "completed" && Date.now() < waitUntil) {
332		− await Bun.sleep(25);
333		− task = await service.getTask(created.task.id);
334		− }
	386	+ const task = await waitForTaskStatus(service, created.task.id, "completed");
335	387
336	388	expect(task?.status).toBe("completed");
337	389	expect((await service.listApprovals("ws_test", "approved")).length).toBe(1);
338	390	});
	391	+
	392	+test("policy deny blocks tool call without creating approval", async () => {
	393	+ const service = new ExecutorService(
	394	+ new TestExecutorDatabase(),
	395	+ new TaskEventHub(),
	396	+ [new InlineToolRuntime()],
	397	+ tools,
	398	+ );
	399	+
	400	+ await service.upsertAccessPolicy({
	401	+ workspaceId: "ws_deny",
	402	+ toolPathPattern: "admin.delete_data",
	403	+ decision: "deny",
	404	+ priority: 500,
	405	+ });
	406	+
	407	+ const created = await service.createTask({
	408	+ code: "unused",
	409	+ runtimeId: "inline",
	410	+ workspaceId: "ws_deny",
	411	+ actorId: "actor_test",
	412	+ });
	413	+
	414	+ const task = await waitForTaskStatus(service, created.task.id, "denied");
	415	+ expect(task?.status).toBe("denied");
	416	+ expect((await service.listPendingApprovals("ws_deny")).length).toBe(0);
	417	+});
	418	+
	419	+test("unknown tool path fails task execution", async () => {
	420	+ const service = new ExecutorService(
	421	+ new TestExecutorDatabase(),
	422	+ new TaskEventHub(),
	423	+ [new InlineToolRuntime("admin.missing_tool")],
	424	+ tools,
	425	+ );
	426	+
	427	+ const created = await service.createTask({
	428	+ code: "unused",
	429	+ runtimeId: "inline",
	430	+ workspaceId: "ws_unknown",
	431	+ actorId: "actor_test",
	432	+ });
	433	+
	434	+ const task = await waitForTaskStatus(service, created.task.id, "failed");
	435	+ expect(task?.status).toBe("failed");
	436	+ expect(task?.error).toContain("Unknown tool");
	437	+});

executor/bun.lock+3−0

@@ -9,6 +9,7 @@ · 8 unmodified lines
9	9	},
10	10	"devDependencies": {
11	11	"@types/bun": "latest",
	12	+ "convex-test": "^0.0.41",
12	13	"typescript": "latest",
13	14	},
14	15	},
@@ -738,6 +739,8 @@ · 723 unmodified lines
738	739
739	740	"convex": ["convex@1.31.7", "", { "dependencies": { "esbuild": "0.27.0", "prettier": "^3.0.0" }, "peerDependencies": { "@auth0/auth0-react": "^2.0.1", "@clerk/clerk-react": "^4.12.8 \|\| ^5.0.0", "react": "^18.0.0 \|\| ^19.0.0-0 \|\| ^19.0.0" }, "optionalPeers": ["@auth0/auth0-react", "@clerk/clerk-react", "react"], "bin": { "convex": "bin/main.js" } }, "sha512-PtNMe1mAIOvA8Yz100QTOaIdgt2rIuWqencVXrb4McdhxBHZ8IJ1eXTnrgCC9HydyilGT1pOn+KNqT14mqn9fQ=="],
740	741
	742	+ "convex-test": ["convex-test@0.0.41", "", { "peerDependencies": { "convex": "^1.16.4" } }, "sha512-GPHeYFOi70n7UtW0eCEQFVhzl/+m8PvbWkDCbKpHLybI1MrScf4sVpGeM0cC2qmtxiduxa2nLPbehPalhh9oyQ=="],
	743	+
741	744	"cookie": ["cookie@0.7.2", "", {}, "sha512-yki5XnKuf750l50uGTllt6kKILY4nQ1eNIQatoXEByZ5dWgnKqbnqmTrBE5B4N7lrMJKQ2ytWMiTO2o0v6Ew/w=="],
742	745
743	746	"cookie-signature": ["cookie-signature@1.2.2", "", {}, "sha512-D76uU73ulSXrD1UXF4KE2TMxVVwhsnCgfAyTg9k8P6KGZjlXKrOLe4dJQKI3Bxi5wjesZoFXJWElNWBjPZMbhg=="],

executor/convex/database.test.ts+138−0

@@ -0,0 +1,138 @@
	1	+import { expect, test } from "bun:test";
	2	+import { convexTest } from "convex-test";
	3	+import { api } from "./_generated/api";
	4	+import schema from "./schema";
	5	+
	6	+function setup() {
	7	+ return convexTest(schema, {
	8	+ "./database.ts": () => import("./database"),
	9	+ "./_generated/api.js": () => import("./_generated/api.js"),
	10	+ });
	11	+}
	12	+
	13	+test("task lifecycle supports queue, run, and complete", async () => {
	14	+ const t = setup();
	15	+
	16	+ const created = await t.mutation(api.database.createTask, {
	17	+ id: "task_1",
	18	+ code: "console.log('hello')",
	19	+ runtimeId: "local-bun",
	20	+ workspaceId: "ws_1",
	21	+ actorId: "actor_1",
	22	+ clientId: "web",
	23	+ });
	24	+
	25	+ expect(created.id).toBe("task_1");
	26	+ expect(created.status).toBe("queued");
	27	+
	28	+ const queued = await t.query(api.database.listQueuedTaskIds, { limit: 10 });
	29	+ expect(queued).toEqual(["task_1"]);
	30	+
	31	+ const running = await t.mutation(api.database.markTaskRunning, { taskId: "task_1" });
	32	+ expect(running?.status).toBe("running");
	33	+
	34	+ const secondRun = await t.mutation(api.database.markTaskRunning, { taskId: "task_1" });
	35	+ expect(secondRun).toBeNull();
	36	+
	37	+ const finished = await t.mutation(api.database.markTaskFinished, {
	38	+ taskId: "task_1",
	39	+ status: "completed",
	40	+ stdout: "ok",
	41	+ stderr: "",
	42	+ exitCode: 0,
	43	+ });
	44	+ expect(finished?.status).toBe("completed");
	45	+
	46	+ const queuedAfter = await t.query(api.database.listQueuedTaskIds, { limit: 10 });
	47	+ expect(queuedAfter).toEqual([]);
	48	+});
	49	+
	50	+test("approval lifecycle tracks pending and resolution", async () => {
	51	+ const t = setup();
	52	+
	53	+ await t.mutation(api.database.createTask, {
	54	+ id: "task_2",
	55	+ code: "await tools.admin.delete_data({ id: 'x' })",
	56	+ runtimeId: "local-bun",
	57	+ workspaceId: "ws_2",
	58	+ actorId: "actor_2",
	59	+ clientId: "web",
	60	+ });
	61	+
	62	+ const createdApproval = await t.mutation(api.database.createApproval, {
	63	+ id: "approval_1",
	64	+ taskId: "task_2",
	65	+ toolPath: "admin.delete_data",
	66	+ input: { id: "x" },
	67	+ });
	68	+ expect(createdApproval.status).toBe("pending");
	69	+
	70	+ const pending = await t.query(api.database.listPendingApprovals, { workspaceId: "ws_2" });
	71	+ expect(pending.length).toBe(1);
	72	+ expect(pending[0]?.task.id).toBe("task_2");
	73	+
	74	+ const resolved = await t.mutation(api.database.resolveApproval, {
	75	+ approvalId: "approval_1",
	76	+ decision: "approved",
	77	+ reviewerId: "reviewer_1",
	78	+ });
	79	+ expect(resolved?.status).toBe("approved");
	80	+
	81	+ const pendingAfter = await t.query(api.database.listPendingApprovals, { workspaceId: "ws_2" });
	82	+ expect(pendingAfter).toEqual([]);
	83	+});
	84	+
	85	+test("workspace tool inventory applies policy decisions by context", async () => {
	86	+ const t = setup();
	87	+
	88	+ await t.mutation(api.database.syncWorkspaceTools, {
	89	+ workspaceId: "ws_tools",
	90	+ tools: [
	91	+ {
	92	+ path: "utils.get_time",
	93	+ description: "Read current time",
	94	+ approval: "auto",
	95	+ },
	96	+ {
	97	+ path: "admin.delete_data",
	98	+ description: "Delete data",
	99	+ approval: "required",
	100	+ },
	101	+ ],
	102	+ });
	103	+
	104	+ await t.mutation(api.database.upsertAccessPolicy, {
	105	+ workspaceId: "ws_tools",
	106	+ toolPathPattern: "admin.*",
	107	+ decision: "deny",
	108	+ priority: 100,
	109	+ });
	110	+
	111	+ await t.mutation(api.database.upsertAccessPolicy, {
	112	+ workspaceId: "ws_tools",
	113	+ actorId: "actor_allow",
	114	+ toolPathPattern: "admin.delete_data",
	115	+ decision: "require_approval",
	116	+ priority: 200,
	117	+ });
	118	+
	119	+ const actorAllowTools = await t.query(api.database.listWorkspaceToolsForContext, {
	120	+ workspaceId: "ws_tools",
	121	+ actorId: "actor_allow",
	122	+ clientId: "web",
	123	+ });
	124	+
	125	+ const actorDenyTools = await t.query(api.database.listWorkspaceToolsForContext, {
	126	+ workspaceId: "ws_tools",
	127	+ actorId: "actor_other",
	128	+ clientId: "web",
	129	+ });
	130	+
	131	+ const actorAllow = actorAllowTools.filter((tool): tool is NonNullable<typeof tool> => tool !== null);
	132	+ const actorDeny = actorDenyTools.filter((tool): tool is NonNullable<typeof tool> => tool !== null);
	133	+
	134	+ expect(actorAllow.map((tool) => tool.path)).toEqual(["admin.delete_data", "utils.get_time"]);
	135	+ expect(actorAllow.find((tool) => tool.path === "admin.delete_data")?.approval).toBe("required");
	136	+
	137	+ expect(actorDeny.map((tool) => tool.path)).toEqual(["utils.get_time"]);
	138	+});

executor/package.json+1−0

@@ -22,6 +22,7 @@ · 21 unmodified lines
22	22	},
23	23	"devDependencies": {
24	24	"@types/bun": "latest",
	25	+ "convex-test": "^0.0.41",
25	26	"typescript": "latest"
26	27	}
27	28	}

Add convex-test dependency and implement access policy management

Keyboard shortcuts

General

Issues and pull requests

Issues, pull requests and Inbox

Repository code