- Added `convex-test` as a development dependency in `package.json` and `bun.lock`. - Introduced access policy management in `service.test.ts`, including methods for listing and upserting access policies. - Enhanced runtime execution handling in `runtime-core.ts` and `vercel-sandbox-runtime.ts` to support new policy checks and improved error handling for script execution timeouts.
Unified · Split
| @@ -0,0 +1,78 @@ | |||
| 1 | import { expect, test } from "bun:test"; | ||
| 2 | import { APPROVAL_DENIED_PREFIX } from "../execution-constants"; | ||
| 3 | import { InProcessExecutionAdapter } from "./in-process-execution-adapter"; | ||
| 4 | |||
| 5 | test("returns run mismatch without invoking tool", async () => { | ||
| 6 | let called = 0; | ||
| 7 | const adapter = new InProcessExecutionAdapter({ | ||
| 8 | runId: "run_expected", | ||
| 9 | invokeTool: async () => { | ||
| 10 | called += 1; | ||
| 11 | return { ok: true }; | ||
| 12 | }, | ||
| 13 | emitOutput: () => {}, | ||
| 14 | }); | ||
| 15 | |||
| 16 | const result = await adapter.invokeTool({ | ||
| 17 | runId: "run_other", | ||
| 18 | callId: "call_1", | ||
| 19 | toolPath: "utils.echo", | ||
| 20 | input: {}, | ||
| 21 | }); | ||
| 22 | |||
| 23 | expect(result.ok).toBe(false); | ||
| 24 | if (!result.ok) { | ||
| 25 | expect(result.error).toContain("Run mismatch"); | ||
| 26 | } | ||
| 27 | expect(called).toBe(0); | ||
| 28 | }); | ||
| 29 | |||
| 30 | test("maps approval denied errors to denied result", async () => { | ||
| 31 | const adapter = new InProcessExecutionAdapter({ | ||
| 32 | runId: "run_1", | ||
| 33 | invokeTool: async () => { | ||
| 34 | throw new Error(`${APPROVAL_DENIED_PREFIX}approval required`); | ||
| 35 | }, | ||
| 36 | emitOutput: () => {}, | ||
| 37 | }); | ||
| 38 | |||
| 39 | const result = await adapter.invokeTool({ | ||
| 40 | runId: "run_1", | ||
| 41 | callId: "call_1", | ||
| 42 | toolPath: "admin.delete_data", | ||
| 43 | input: { id: "abc" }, | ||
| 44 | }); | ||
| 45 | |||
| 46 | expect(result).toEqual({ | ||
| 47 | ok: false, | ||
| 48 | denied: true, | ||
| 49 | error: "approval required", | ||
| 50 | }); | ||
| 51 | }); | ||
| 52 | |||
| 53 | test("emits output only for matching run id", () => { | ||
| 54 | const lines: string[] = []; | ||
| 55 | const adapter = new InProcessExecutionAdapter({ | ||
| 56 | runId: "run_1", | ||
| 57 | invokeTool: async () => null, | ||
| 58 | emitOutput: (event) => { | ||
| 59 | lines.push(`${event.stream}:${event.line}`); | ||
| 60 | }, | ||
| 61 | }); | ||
| 62 | |||
| 63 | adapter.emitOutput({ | ||
| 64 | runId: "run_other", | ||
| 65 | stream: "stdout", | ||
| 66 | line: "ignored", | ||
| 67 | timestamp: Date.now(), | ||
| 68 | }); | ||
| 69 | |||
| 70 | adapter.emitOutput({ | ||
| 71 | runId: "run_1", | ||
| 72 | stream: "stdout", | ||
| 73 | line: "accepted", | ||
| 74 | timestamp: Date.now(), | ||
| 75 | }); | ||
| 76 | |||
| 77 | expect(lines).toEqual(["stdout:accepted"]); | ||
| 78 | }); | ||
| @@ -0,0 +1,319 @@ | |||
| 1 | import { expect, test } from "bun:test"; | ||
| 2 | import { runCodeWithAdapter } from "./runtime-core"; | ||
| 3 | import { loadExternalTools } from "../tool-sources"; | ||
| 4 | import type { | ||
| 5 | ExecutionAdapter, | ||
| 6 | RuntimeOutputEvent, | ||
| 7 | SandboxExecutionRequest, | ||
| 8 | ToolDefinition, | ||
| 9 | ToolRunContext, | ||
| 10 | } from "../types"; | ||
| 11 | |||
| 12 | function request(code: string, timeoutMs = 1_000): SandboxExecutionRequest { | ||
| 13 | return { | ||
| 14 | taskId: `task_${crypto.randomUUID()}`, | ||
| 15 | code, | ||
| 16 | timeoutMs, | ||
| 17 | }; | ||
| 18 | } | ||
| 19 | |||
| 20 | function createRuntimeAdapter( | ||
| 21 | tools: Map<string, ToolDefinition>, | ||
| 22 | outputEvents: RuntimeOutputEvent[], | ||
| 23 | ): ExecutionAdapter { | ||
| 24 | return { | ||
| 25 | async invokeTool(call) { | ||
| 26 | const tool = tools.get(call.toolPath); | ||
| 27 | if (!tool) { | ||
| 28 | return { | ||
| 29 | ok: false, | ||
| 30 | error: `Tool not found: ${call.toolPath}`, | ||
| 31 | }; | ||
| 32 | } | ||
| 33 | |||
| 34 | try { | ||
| 35 | const context: ToolRunContext = { | ||
| 36 | taskId: call.runId, | ||
| 37 | workspaceId: "ws_test", | ||
| 38 | actorId: "actor_test", | ||
| 39 | clientId: "web", | ||
| 40 | isToolAllowed: () => true, | ||
| 41 | }; | ||
| 42 | const value = await tool.run(call.input, context); | ||
| 43 | return { ok: true, value }; | ||
| 44 | } catch (error) { | ||
| 45 | return { | ||
| 46 | ok: false, | ||
| 47 | error: error instanceof Error ? error.message : String(error), | ||
| 48 | }; | ||
| 49 | } | ||
| 50 | }, | ||
| 51 | emitOutput(event) { | ||
| 52 | outputEvents.push(event); | ||
| 53 | }, | ||
| 54 | }; | ||
| 55 | } | ||
| 56 | |||
| 57 | test("executes tool calls and captures output", async () => { | ||
| 58 | const outputEvents: RuntimeOutputEvent[] = []; | ||
| 59 | const tools = new Map<string, ToolDefinition>([ | ||
| 60 | [ | ||
| 61 | "utils.echo", | ||
| 62 | { | ||
| 63 | path: "utils.echo", | ||
| 64 | description: "Echo back input", | ||
| 65 | approval: "auto", | ||
| 66 | run: async (input) => ({ echoed: input }), | ||
| 67 | }, | ||
| 68 | ], | ||
| 69 | ]); | ||
| 70 | |||
| 71 | const result = await runCodeWithAdapter( | ||
| 72 | request(` | ||
| 73 | const out = await tools.utils.echo({ message: "hi" }); | ||
| 74 | console.log("out", out.echoed.message); | ||
| 75 | return 42; | ||
| 76 | `), | ||
| 77 | createRuntimeAdapter(tools, outputEvents), | ||
| 78 | ); | ||
| 79 | |||
| 80 | expect(result.status).toBe("completed"); | ||
| 81 | expect(result.stdout).toContain("out hi"); | ||
| 82 | expect(result.stdout).toContain("result: 42"); | ||
| 83 | expect(outputEvents.some((event) => event.stream === "stdout")).toBe(true); | ||
| 84 | }); | ||
| 85 | |||
| 86 | test("returns denied when adapter marks tool call denied", async () => { | ||
| 87 | const adapter: ExecutionAdapter = { | ||
| 88 | async invokeTool() { | ||
| 89 | return { | ||
| 90 | ok: false, | ||
| 91 | denied: true, | ||
| 92 | error: "policy denied", | ||
| 93 | }; | ||
| 94 | }, | ||
| 95 | emitOutput() {}, | ||
| 96 | }; | ||
| 97 | |||
| 98 | const result = await runCodeWithAdapter( | ||
| 99 | request(` | ||
| 100 | await tools.admin.delete_data({ id: "x" }); | ||
| 101 | `), | ||
| 102 | adapter, | ||
| 103 | ); | ||
| 104 | |||
| 105 | expect(result.status).toBe("denied"); | ||
| 106 | expect(result.error).toBe("policy denied"); | ||
| 107 | expect(result.stderr).toContain("policy denied"); | ||
| 108 | }); | ||
| 109 | |||
| 110 | test("times out long-running code", async () => { | ||
| 111 | const adapter: ExecutionAdapter = { | ||
| 112 | async invokeTool() { | ||
| 113 | return { ok: true, value: null }; | ||
| 114 | }, | ||
| 115 | emitOutput() {}, | ||
| 116 | }; | ||
| 117 | |||
| 118 | const result = await runCodeWithAdapter( | ||
| 119 | request( | ||
| 120 | ` | ||
| 121 | await new Promise(() => {}); | ||
| 122 | `, | ||
| 123 | 25, | ||
| 124 | ), | ||
| 125 | adapter, | ||
| 126 | ); | ||
| 127 | |||
| 128 | expect(result.status).toBe("timed_out"); | ||
| 129 | expect(result.error).toContain("timed out"); | ||
| 130 | }); | ||
| 131 | |||
| 132 | test("sandboxed runner does not expose host globals", async () => { | ||
| 133 | const outputEvents: RuntimeOutputEvent[] = []; | ||
| 134 | const adapter = createRuntimeAdapter(new Map(), outputEvents); | ||
| 135 | |||
| 136 | const result = await runCodeWithAdapter( | ||
| 137 | request(` | ||
| 138 | const checks = { | ||
| 139 | process: typeof process, | ||
| 140 | bun: typeof Bun, | ||
| 141 | fetch: typeof fetch, | ||
| 142 | fsEscape: [].constructor.constructor("return typeof process")(), | ||
| 143 | }; | ||
| 144 | console.log(JSON.stringify(checks)); | ||
| 145 | if (checks.process !== "undefined") throw new Error("process leaked"); | ||
| 146 | if (checks.bun !== "undefined") throw new Error("Bun leaked"); | ||
| 147 | if (checks.fetch !== "undefined") throw new Error("fetch leaked"); | ||
| 148 | if (checks.fsEscape !== "undefined") throw new Error("constructor escape leaked"); | ||
| 149 | return "ok"; | ||
| 150 | `), | ||
| 151 | adapter, | ||
| 152 | ); | ||
| 153 | |||
| 154 | expect(result.status).toBe("completed"); | ||
| 155 | expect(result.stdout).toContain("\"process\":\"undefined\""); | ||
| 156 | expect(result.stdout).toContain("\"bun\":\"undefined\""); | ||
| 157 | expect(result.stdout).toContain("\"fetch\":\"undefined\""); | ||
| 158 | expect(result.stdout).toContain("\"fsEscape\":\"undefined\""); | ||
| 159 | }); | ||
| 160 | |||
| 161 | test("runs openapi and graphql sourced tools through the runtime", async () => { | ||
| 162 | const server = Bun.serve({ | ||
| 163 | port: 0, | ||
| 164 | fetch: async (req) => { | ||
| 165 | const url = new URL(req.url); | ||
| 166 | |||
| 167 | if (url.pathname === "/add" && req.method === "GET") { | ||
| 168 | const a = Number(url.searchParams.get("a") ?? "0"); | ||
| 169 | const b = Number(url.searchParams.get("b") ?? "0"); | ||
| 170 | return Response.json({ sum: a + b }); | ||
| 171 | } | ||
| 172 | |||
| 173 | if (url.pathname === "/graphql" && req.method === "POST") { | ||
| 174 | const body = (await req.json()) as { query?: string; variables?: Record<string, unknown> }; | ||
| 175 | const query = String(body.query ?? ""); | ||
| 176 | |||
| 177 | if (query.includes("__schema")) { | ||
| 178 | return Response.json({ | ||
| 179 | data: { | ||
| 180 | __schema: { | ||
| 181 | queryType: { name: "Query" }, | ||
| 182 | mutationType: { name: "Mutation" }, | ||
| 183 | types: [ | ||
| 184 | { | ||
| 185 | kind: "OBJECT", | ||
| 186 | name: "Query", | ||
| 187 | fields: [ | ||
| 188 | { | ||
| 189 | name: "hello", | ||
| 190 | description: null, | ||
| 191 | args: [], | ||
| 192 | type: { kind: "SCALAR", name: "String", ofType: null }, | ||
| 193 | }, | ||
| 194 | ], | ||
| 195 | inputFields: null, | ||
| 196 | enumValues: null, | ||
| 197 | }, | ||
| 198 | { | ||
| 199 | kind: "OBJECT", | ||
| 200 | name: "Mutation", | ||
| 201 | fields: [ | ||
| 202 | { | ||
| 203 | name: "increment", | ||
| 204 | description: null, | ||
| 205 | args: [ | ||
| 206 | { | ||
| 207 | name: "value", | ||
| 208 | description: null, | ||
| 209 | defaultValue: null, | ||
| 210 | type: { | ||
| 211 | kind: "NON_NULL", | ||
| 212 | name: null, | ||
| 213 | ofType: { kind: "SCALAR", name: "Int", ofType: null }, | ||
| 214 | }, | ||
| 215 | }, | ||
| 216 | ], | ||
| 217 | type: { kind: "SCALAR", name: "Int", ofType: null }, | ||
| 218 | }, | ||
| 219 | ], | ||
| 220 | inputFields: null, | ||
| 221 | enumValues: null, | ||
| 222 | }, | ||
| 223 | { kind: "SCALAR", name: "String", fields: null, inputFields: null, enumValues: null }, | ||
| 224 | { kind: "SCALAR", name: "Int", fields: null, inputFields: null, enumValues: null }, | ||
| 225 | ], | ||
| 226 | }, | ||
| 227 | }, | ||
| 228 | }); | ||
| 229 | } | ||
| 230 | |||
| 231 | if (query.includes("hello")) { | ||
| 232 | return Response.json({ data: { hello: "world" } }); | ||
| 233 | } | ||
| 234 | |||
| 235 | if (query.includes("increment")) { | ||
| 236 | const value = Number(body.variables?.value ?? 0); | ||
| 237 | return Response.json({ data: { increment: value + 1 } }); | ||
| 238 | } | ||
| 239 | |||
| 240 | return Response.json({ errors: [{ message: "Unknown query" }] }, { status: 400 }); | ||
| 241 | } | ||
| 242 | |||
| 243 | return new Response("Not found", { status: 404 }); | ||
| 244 | }, | ||
| 245 | }); | ||
| 246 | |||
| 247 | try { | ||
| 248 | const baseUrl = `http://127.0.0.1:${server.port}`; | ||
| 249 | const { tools } = await loadExternalTools([ | ||
| 250 | { | ||
| 251 | type: "openapi", | ||
| 252 | name: "calc", | ||
| 253 | baseUrl, | ||
| 254 | spec: { | ||
| 255 | openapi: "3.0.3", | ||
| 256 | info: { title: "Calc", version: "1.0.0" }, | ||
| 257 | paths: { | ||
| 258 | "/add": { | ||
| 259 | get: { | ||
| 260 | operationId: "addNumbers", | ||
| 261 | tags: ["math"], | ||
| 262 | parameters: [ | ||
| 263 | { name: "a", in: "query", required: true, schema: { type: "number" } }, | ||
| 264 | { name: "b", in: "query", required: true, schema: { type: "number" } }, | ||
| 265 | ], | ||
| 266 | responses: { | ||
| 267 | "200": { | ||
| 268 | description: "ok", | ||
| 269 | content: { | ||
| 270 | "application/json": { | ||
| 271 | schema: { | ||
| 272 | type: "object", | ||
| 273 | properties: { sum: { type: "number" } }, | ||
| 274 | }, | ||
| 275 | }, | ||
| 276 | }, | ||
| 277 | }, | ||
| 278 | }, | ||
| 279 | }, | ||
| 280 | }, | ||
| 281 | }, | ||
| 282 | }, | ||
| 283 | }, | ||
| 284 | { | ||
| 285 | type: "graphql", | ||
| 286 | name: "gql", | ||
| 287 | endpoint: `${baseUrl}/graphql`, | ||
| 288 | }, | ||
| 289 | ]); | ||
| 290 | |||
| 291 | const toolMap = new Map(tools.map((tool) => [tool.path, tool])); | ||
| 292 | const outputEvents: RuntimeOutputEvent[] = []; | ||
| 293 | |||
| 294 | const result = await runCodeWithAdapter( | ||
| 295 | request(` | ||
| 296 | const sum = await tools.calc.math.addnumbers({ a: 2, b: 5 }); | ||
| 297 | const hello = await tools.gql.query.hello({}); | ||
| 298 | const inc = await tools.gql.graphql({ | ||
| 299 | query: "mutation($value: Int!) { increment(value: $value) }", | ||
| 300 | variables: { value: 3 }, | ||
| 301 | }); | ||
| 302 | console.log("sum", sum.sum); | ||
| 303 | console.log("hello", hello.hello); | ||
| 304 | console.log("inc", inc.increment); | ||
| 305 | return sum.sum + inc.increment; | ||
| 306 | `), | ||
| 307 | createRuntimeAdapter(toolMap, outputEvents), | ||
| 308 | ); | ||
| 309 | |||
| 310 | expect(result.status).toBe("completed"); | ||
| 311 | expect(result.stdout).toContain("sum 7"); | ||
| 312 | expect(result.stdout).toContain("hello world"); | ||
| 313 | expect(result.stdout).toContain("inc 4"); | ||
| 314 | expect(result.stdout).toContain("result: 11"); | ||
| 315 | expect(outputEvents.length).toBeGreaterThan(0); | ||
| 316 | } finally { | ||
| 317 | server.stop(true); | ||
| 318 | } | ||
| 319 | }); | ||
| @@ -1,4 +1,5 @@ | |||
| 1 | import { APPROVAL_DENIED_PREFIX, TASK_TIMEOUT_MARKER } from "../execution-constants"; | 1 | import { APPROVAL_DENIED_PREFIX, TASK_TIMEOUT_MARKER } from "../execution-constants"; |
| 2 | import { Script, createContext } from "node:vm"; | ||
| 2 | import type { | 3 | import type { |
| 3 | ExecutionAdapter, | 4 | ExecutionAdapter, |
| 4 | SandboxExecutionRequest, | 5 | SandboxExecutionRequest, |
| @@ -70,9 +71,6 @@ export async function runCodeWithAdapter( · 65 unmodified lines | |||
| 70 | const startedAt = Date.now(); | 71 | const startedAt = Date.now(); |
| 71 | const stdoutLines: string[] = []; | 72 | const stdoutLines: string[] = []; |
| 72 | const stderrLines: string[] = []; | 73 | const stderrLines: string[] = []; |
| 73 | const AsyncFunction = Object.getPrototypeOf(async function () {}).constructor as new ( | ||
| 74 | ...args: string[] | ||
| 75 | ) => (...inputs: unknown[]) => Promise<unknown>; | ||
| 76 | 74 | ||
| 77 | const appendStdout = (line: string): void => { | 75 | const appendStdout = (line: string): void => { |
| 78 | stdoutLines.push(line); | 76 | stdoutLines.push(line); |
| @@ -106,11 +104,14 @@ export async function runCodeWithAdapter( · 27 unmodified lines | |||
| 106 | error: (...args: unknown[]) => appendStderr(formatArgs(args)), | 104 | error: (...args: unknown[]) => appendStderr(formatArgs(args)), |
| 107 | }; | 105 | }; |
| 108 | 106 | ||
| 109 | const runner = new AsyncFunction( | 107 | const context = createContext({ |
| 110 | "tools", | 108 | tools, |
| 111 | "console", | 109 | console: consoleProxy, |
| 112 | `"use strict";\n${request.code}`, | 110 | setTimeout, |
| 113 | ); | 111 | clearTimeout, |
| 112 | }); | ||
| 113 | |||
| 114 | const runnerScript = new Script(`(async () => {\n"use strict";\n${request.code}\n})()`); | ||
| 114 | 115 | ||
| 115 | let timeoutHandle: ReturnType<typeof setTimeout> | undefined; | 116 | let timeoutHandle: ReturnType<typeof setTimeout> | undefined; |
| 116 | const timeoutPromise = new Promise<never>((_resolve, reject) => { | 117 | const timeoutPromise = new Promise<never>((_resolve, reject) => { |
| @@ -121,7 +122,7 @@ export async function runCodeWithAdapter( · 4 unmodified lines | |||
| 121 | 122 | ||
| 122 | try { | 123 | try { |
| 123 | const value = await Promise.race([ | 124 | const value = await Promise.race([ |
| 124 | runner(tools, consoleProxy), | 125 | Promise.resolve(runnerScript.runInContext(context, { timeout: Math.max(1, request.timeoutMs) })), |
| 125 | timeoutPromise, | 126 | timeoutPromise, |
| 126 | ]); | 127 | ]); |
| 127 | 128 | ||
| @@ -138,7 +139,7 @@ export async function runCodeWithAdapter( · 10 unmodified lines | |||
| 138 | }; | 139 | }; |
| 139 | } catch (error) { | 140 | } catch (error) { |
| 140 | const message = error instanceof Error ? error.message : String(error); | 141 | const message = error instanceof Error ? error.message : String(error); |
| 141 | if (message === TASK_TIMEOUT_MARKER) { | 142 | if (message === TASK_TIMEOUT_MARKER || message.includes("Script execution timed out")) { |
| 142 | const timeoutMessage = `Execution timed out after ${request.timeoutMs}ms`; | 143 | const timeoutMessage = `Execution timed out after ${request.timeoutMs}ms`; |
| 143 | appendStderr(timeoutMessage); | 144 | appendStderr(timeoutMessage); |
| 144 | return { | 145 | return { |
| @@ -22,6 +22,7 @@ function buildRunnerScript(codeFilePath: string): string { · 21 unmodified lines | |||
| 22 | return ` | 22 | return ` |
| 23 | import { readFile } from "node:fs/promises"; | 23 | import { readFile } from "node:fs/promises"; |
| 24 | import { randomUUID } from "node:crypto"; | 24 | import { randomUUID } from "node:crypto"; |
| 25 | import vm from "node:vm"; | ||
| 25 | 26 | ||
| 26 | const RESULT_MARKER = ${JSON.stringify(RESULT_MARKER)}; | 27 | const RESULT_MARKER = ${JSON.stringify(RESULT_MARKER)}; |
| 27 | const runId = process.env.EXECUTOR_RUN_ID; | 28 | const runId = process.env.EXECUTOR_RUN_ID; |
| @@ -145,8 +146,13 @@ const consoleProxy = { · 117 unmodified lines | |||
| 145 | error: (...args) => appendStderr(formatArgs(args)), | 146 | error: (...args) => appendStderr(formatArgs(args)), |
| 146 | }; | 147 | }; |
| 147 | 148 | ||
| 148 | const AsyncFunction = Object.getPrototypeOf(async function () {}).constructor; | 149 | const context = vm.createContext({ |
| 149 | const runner = new AsyncFunction("tools", "console", '"use strict";\\n' + userCode); | 150 | tools, |
| 151 | console: consoleProxy, | ||
| 152 | setTimeout, | ||
| 153 | clearTimeout, | ||
| 154 | }); | ||
| 155 | const runnerScript = new vm.Script("(async () => {\\n\"use strict\";\\n" + userCode + "\\n})()"); | ||
| 150 | 156 | ||
| 151 | const timeoutPromise = new Promise((_, reject) => { | 157 | const timeoutPromise = new Promise((_, reject) => { |
| 152 | setTimeout(() => reject(new Error("TASK_TIMEOUT")), requestTimeoutMs); | 158 | setTimeout(() => reject(new Error("TASK_TIMEOUT")), requestTimeoutMs); |
| @@ -154,7 +160,10 @@ const timeoutPromise = new Promise((_, reject) => { · 1 unmodified line | |||
| 154 | 160 | ||
| 155 | let result; | 161 | let result; |
| 156 | try { | 162 | try { |
| 157 | const value = await Promise.race([runner(tools, consoleProxy), timeoutPromise]); | 163 | const value = await Promise.race([ |
| 164 | Promise.resolve(runnerScript.runInContext(context, { timeout: Math.max(1, requestTimeoutMs) })), | ||
| 165 | timeoutPromise, | ||
| 166 | ]); | ||
| 158 | if (value !== undefined) { | 167 | if (value !== undefined) { |
| 159 | appendStdout("result: " + formatArgs([value])); | 168 | appendStdout("result: " + formatArgs([value])); |
| 160 | } | 169 | } |
| @@ -169,7 +178,7 @@ try { · 8 unmodified lines | |||
| 169 | } catch (error) { | 178 | } catch (error) { |
| 170 | const message = error instanceof Error ? error.message : String(error); | 179 | const message = error instanceof Error ? error.message : String(error); |
| 171 | 180 | ||
| 172 | if (message === "TASK_TIMEOUT") { | 181 | if (message === "TASK_TIMEOUT" || message.includes("Script execution timed out")) { |
| 173 | const timeoutMessage = "Execution timed out after " + requestTimeoutMs + "ms"; | 182 | const timeoutMessage = "Execution timed out after " + requestTimeoutMs + "ms"; |
| 174 | appendStderr(timeoutMessage); | 183 | appendStderr(timeoutMessage); |
| 175 | result = { | 184 | result = { |
| @@ -21,6 +21,7 @@ class TestExecutorDatabase extends ExecutorDatabase { · 20 unmodified lines | |||
| 21 | private readonly tasks = new Map<string, TaskRecord>(); | 21 | private readonly tasks = new Map<string, TaskRecord>(); |
| 22 | private readonly approvals = new Map<string, ApprovalRecord>(); | 22 | private readonly approvals = new Map<string, ApprovalRecord>(); |
| 23 | private readonly events = new Map<string, TaskEventRecord[]>(); | 23 | private readonly events = new Map<string, TaskEventRecord[]>(); |
| 24 | private readonly policies = new Map<string, AccessPolicyRecord>(); | ||
| 24 | 25 | ||
| 25 | constructor() { | 26 | constructor() { |
| 26 | super("http://127.0.0.1:3210"); | 27 | super("http://127.0.0.1:3210"); |
| @@ -217,7 +218,43 @@ class TestExecutorDatabase extends ExecutorDatabase { · 190 unmodified lines | |||
| 217 | } | 218 | } |
| 218 | 219 | ||
| 219 | async listAccessPolicies(_workspaceId: string): Promise<AccessPolicyRecord[]> { | 220 | async listAccessPolicies(_workspaceId: string): Promise<AccessPolicyRecord[]> { |
| 220 | return []; | 221 | return [...this.policies.values()] |
| 222 | .filter((policy) => policy.workspaceId === _workspaceId) | ||
| 223 | .sort((a, b) => { | ||
| 224 | if (a.priority !== b.priority) { | ||
| 225 | return b.priority - a.priority; | ||
| 226 | } | ||
| 227 | return a.createdAt - b.createdAt; | ||
| 228 | }); | ||
| 229 | } | ||
| 230 | |||
| 231 | async upsertAccessPolicy(params: { | ||
| 232 | id?: string; | ||
| 233 | workspaceId: string; | ||
| 234 | actorId?: string; | ||
| 235 | clientId?: string; | ||
| 236 | toolPathPattern: string; | ||
| 237 | decision: "allow" | "require_approval" | "deny"; | ||
| 238 | priority?: number; | ||
| 239 | }): Promise<AccessPolicyRecord> { | ||
| 240 | const now = Date.now(); | ||
| 241 | const id = params.id ?? `policy_${crypto.randomUUID()}`; | ||
| 242 | const existing = this.policies.get(id); | ||
| 243 | |||
| 244 | const record: AccessPolicyRecord = { | ||
| 245 | id, | ||
| 246 | workspaceId: params.workspaceId, | ||
| 247 | actorId: params.actorId, | ||
| 248 | clientId: params.clientId, | ||
| 249 | toolPathPattern: params.toolPathPattern, | ||
| 250 | decision: params.decision, | ||
| 251 | priority: params.priority ?? 100, | ||
| 252 | createdAt: existing?.createdAt ?? now, | ||
| 253 | updatedAt: now, | ||
| 254 | }; | ||
| 255 | |||
| 256 | this.policies.set(id, record); | ||
| 257 | return record; | ||
| 221 | } | 258 | } |
| 222 | 259 | ||
| 223 | async resolveCredential(): Promise<null> { | 260 | async resolveCredential(): Promise<null> { |
| @@ -254,6 +291,11 @@ class InlineToolRuntime implements SandboxRuntime { · 30 unmodified lines | |||
| 254 | readonly label = "Inline"; | 291 | readonly label = "Inline"; |
| 255 | readonly description = "Calls one tool then exits"; | 292 | readonly description = "Calls one tool then exits"; |
| 256 | 293 | ||
| 294 | constructor( | ||
| 295 | private readonly toolPath = "admin.delete_data", | ||
| 296 | private readonly input: unknown = { key: "abc" }, | ||
| 297 | ) {} | ||
| 298 | |||
| 257 | async run( | 299 | async run( |
| 258 | request: SandboxExecutionRequest, | 300 | request: SandboxExecutionRequest, |
| 259 | adapter: ExecutionAdapter, | 301 | adapter: ExecutionAdapter, |
| @@ -262,8 +304,8 @@ class InlineToolRuntime implements SandboxRuntime { · 2 unmodified lines | |||
| 262 | const result = await adapter.invokeTool({ | 304 | const result = await adapter.invokeTool({ |
| 263 | runId: request.taskId, | 305 | runId: request.taskId, |
| 264 | callId: "call_inline_1", | 306 | callId: "call_inline_1", |
| 265 | toolPath: "admin.delete_data", | 307 | toolPath: this.toolPath, |
| 266 | input: { key: "abc" }, | 308 | input: this.input, |
| 267 | }); | 309 | }); |
| 268 | 310 | ||
| 269 | if (!result.ok) { | 311 | if (!result.ok) { |
| @@ -302,6 +344,21 @@ const tools: ToolDefinition[] = [ · 32 unmodified lines | |||
| 302 | }, | 344 | }, |
| 303 | ]; | 345 | ]; |
| 304 | 346 | ||
| 347 | async function waitForTaskStatus( | ||
| 348 | service: ExecutorService, | ||
| 349 | taskId: string, | ||
| 350 | expected: TaskRecord["status"], | ||
| 351 | timeoutMs = 2_000, | ||
| 352 | ): Promise<TaskRecord | null> { | ||
| 353 | let task = await service.getTask(taskId); | ||
| 354 | const waitUntil = Date.now() + timeoutMs; | ||
| 355 | while (task?.status !== expected && Date.now() < waitUntil) { | ||
| 356 | await Bun.sleep(25); | ||
| 357 | task = await service.getTask(taskId); | ||
| 358 | } | ||
| 359 | return task; | ||
| 360 | } | ||
| 361 | |||
| 305 | test("tool-level approval gates individual function call", async () => { | 362 | test("tool-level approval gates individual function call", async () => { |
| 306 | const service = new ExecutorService( | 363 | const service = new ExecutorService( |
| 307 | new TestExecutorDatabase(), | 364 | new TestExecutorDatabase(), |
| @@ -326,13 +383,55 @@ test("tool-level approval gates individual function call", async () => { · 18 unmodified lines | |||
| 326 | const resolved = await service.resolveApproval("ws_test", approvals[0]!.id, "approved", "test-user"); | 383 | const resolved = await service.resolveApproval("ws_test", approvals[0]!.id, "approved", "test-user"); |
| 327 | expect(resolved).toBeTruthy(); | 384 | expect(resolved).toBeTruthy(); |
| 328 | 385 | ||
| 329 | let task = await service.getTask(created.task.id); | 386 | const task = await waitForTaskStatus(service, created.task.id, "completed"); |
| 330 | const waitUntil = Date.now() + 2_000; | ||
| 331 | while (task?.status !== "completed" && Date.now() < waitUntil) { | ||
| 332 | await Bun.sleep(25); | ||
| 333 | task = await service.getTask(created.task.id); | ||
| 334 | } | ||
| 335 | 387 | ||
| 336 | expect(task?.status).toBe("completed"); | 388 | expect(task?.status).toBe("completed"); |
| 337 | expect((await service.listApprovals("ws_test", "approved")).length).toBe(1); | 389 | expect((await service.listApprovals("ws_test", "approved")).length).toBe(1); |
| 338 | }); | 390 | }); |
| 391 | |||
| 392 | test("policy deny blocks tool call without creating approval", async () => { | ||
| 393 | const service = new ExecutorService( | ||
| 394 | new TestExecutorDatabase(), | ||
| 395 | new TaskEventHub(), | ||
| 396 | [new InlineToolRuntime()], | ||
| 397 | tools, | ||
| 398 | ); | ||
| 399 | |||
| 400 | await service.upsertAccessPolicy({ | ||
| 401 | workspaceId: "ws_deny", | ||
| 402 | toolPathPattern: "admin.delete_data", | ||
| 403 | decision: "deny", | ||
| 404 | priority: 500, | ||
| 405 | }); | ||
| 406 | |||
| 407 | const created = await service.createTask({ | ||
| 408 | code: "unused", | ||
| 409 | runtimeId: "inline", | ||
| 410 | workspaceId: "ws_deny", | ||
| 411 | actorId: "actor_test", | ||
| 412 | }); | ||
| 413 | |||
| 414 | const task = await waitForTaskStatus(service, created.task.id, "denied"); | ||
| 415 | expect(task?.status).toBe("denied"); | ||
| 416 | expect((await service.listPendingApprovals("ws_deny")).length).toBe(0); | ||
| 417 | }); | ||
| 418 | |||
| 419 | test("unknown tool path fails task execution", async () => { | ||
| 420 | const service = new ExecutorService( | ||
| 421 | new TestExecutorDatabase(), | ||
| 422 | new TaskEventHub(), | ||
| 423 | [new InlineToolRuntime("admin.missing_tool")], | ||
| 424 | tools, | ||
| 425 | ); | ||
| 426 | |||
| 427 | const created = await service.createTask({ | ||
| 428 | code: "unused", | ||
| 429 | runtimeId: "inline", | ||
| 430 | workspaceId: "ws_unknown", | ||
| 431 | actorId: "actor_test", | ||
| 432 | }); | ||
| 433 | |||
| 434 | const task = await waitForTaskStatus(service, created.task.id, "failed"); | ||
| 435 | expect(task?.status).toBe("failed"); | ||
| 436 | expect(task?.error).toContain("Unknown tool"); | ||
| 437 | }); | ||
| @@ -9,6 +9,7 @@ · 8 unmodified lines | |||
| 9 | }, | 9 | }, |
| 10 | "devDependencies": { | 10 | "devDependencies": { |
| 11 | "@types/bun": "latest", | 11 | "@types/bun": "latest", |
| 12 | "convex-test": "^0.0.41", | ||
| 12 | "typescript": "latest", | 13 | "typescript": "latest", |
| 13 | }, | 14 | }, |
| 14 | }, | 15 | }, |
| @@ -738,6 +739,8 @@ · 723 unmodified lines | |||
| 738 | 739 | ||
| 739 | "convex": ["convex@1.31.7", "", { "dependencies": { "esbuild": "0.27.0", "prettier": "^3.0.0" }, "peerDependencies": { "@auth0/auth0-react": "^2.0.1", "@clerk/clerk-react": "^4.12.8 || ^5.0.0", "react": "^18.0.0 || ^19.0.0-0 || ^19.0.0" }, "optionalPeers": ["@auth0/auth0-react", "@clerk/clerk-react", "react"], "bin": { "convex": "bin/main.js" } }, "sha512-PtNMe1mAIOvA8Yz100QTOaIdgt2rIuWqencVXrb4McdhxBHZ8IJ1eXTnrgCC9HydyilGT1pOn+KNqT14mqn9fQ=="], | 740 | "convex": ["convex@1.31.7", "", { "dependencies": { "esbuild": "0.27.0", "prettier": "^3.0.0" }, "peerDependencies": { "@auth0/auth0-react": "^2.0.1", "@clerk/clerk-react": "^4.12.8 || ^5.0.0", "react": "^18.0.0 || ^19.0.0-0 || ^19.0.0" }, "optionalPeers": ["@auth0/auth0-react", "@clerk/clerk-react", "react"], "bin": { "convex": "bin/main.js" } }, "sha512-PtNMe1mAIOvA8Yz100QTOaIdgt2rIuWqencVXrb4McdhxBHZ8IJ1eXTnrgCC9HydyilGT1pOn+KNqT14mqn9fQ=="], |
| 740 | 741 | ||
| 742 | "convex-test": ["convex-test@0.0.41", "", { "peerDependencies": { "convex": "^1.16.4" } }, "sha512-GPHeYFOi70n7UtW0eCEQFVhzl/+m8PvbWkDCbKpHLybI1MrScf4sVpGeM0cC2qmtxiduxa2nLPbehPalhh9oyQ=="], | ||
| 743 | |||
| 741 | "cookie": ["cookie@0.7.2", "", {}, "sha512-yki5XnKuf750l50uGTllt6kKILY4nQ1eNIQatoXEByZ5dWgnKqbnqmTrBE5B4N7lrMJKQ2ytWMiTO2o0v6Ew/w=="], | 744 | "cookie": ["cookie@0.7.2", "", {}, "sha512-yki5XnKuf750l50uGTllt6kKILY4nQ1eNIQatoXEByZ5dWgnKqbnqmTrBE5B4N7lrMJKQ2ytWMiTO2o0v6Ew/w=="], |
| 742 | 745 | ||
| 743 | "cookie-signature": ["cookie-signature@1.2.2", "", {}, "sha512-D76uU73ulSXrD1UXF4KE2TMxVVwhsnCgfAyTg9k8P6KGZjlXKrOLe4dJQKI3Bxi5wjesZoFXJWElNWBjPZMbhg=="], | 746 | "cookie-signature": ["cookie-signature@1.2.2", "", {}, "sha512-D76uU73ulSXrD1UXF4KE2TMxVVwhsnCgfAyTg9k8P6KGZjlXKrOLe4dJQKI3Bxi5wjesZoFXJWElNWBjPZMbhg=="], |
| @@ -0,0 +1,138 @@ | |||
| 1 | import { expect, test } from "bun:test"; | ||
| 2 | import { convexTest } from "convex-test"; | ||
| 3 | import { api } from "./_generated/api"; | ||
| 4 | import schema from "./schema"; | ||
| 5 | |||
| 6 | function setup() { | ||
| 7 | return convexTest(schema, { | ||
| 8 | "./database.ts": () => import("./database"), | ||
| 9 | "./_generated/api.js": () => import("./_generated/api.js"), | ||
| 10 | }); | ||
| 11 | } | ||
| 12 | |||
| 13 | test("task lifecycle supports queue, run, and complete", async () => { | ||
| 14 | const t = setup(); | ||
| 15 | |||
| 16 | const created = await t.mutation(api.database.createTask, { | ||
| 17 | id: "task_1", | ||
| 18 | code: "console.log('hello')", | ||
| 19 | runtimeId: "local-bun", | ||
| 20 | workspaceId: "ws_1", | ||
| 21 | actorId: "actor_1", | ||
| 22 | clientId: "web", | ||
| 23 | }); | ||
| 24 | |||
| 25 | expect(created.id).toBe("task_1"); | ||
| 26 | expect(created.status).toBe("queued"); | ||
| 27 | |||
| 28 | const queued = await t.query(api.database.listQueuedTaskIds, { limit: 10 }); | ||
| 29 | expect(queued).toEqual(["task_1"]); | ||
| 30 | |||
| 31 | const running = await t.mutation(api.database.markTaskRunning, { taskId: "task_1" }); | ||
| 32 | expect(running?.status).toBe("running"); | ||
| 33 | |||
| 34 | const secondRun = await t.mutation(api.database.markTaskRunning, { taskId: "task_1" }); | ||
| 35 | expect(secondRun).toBeNull(); | ||
| 36 | |||
| 37 | const finished = await t.mutation(api.database.markTaskFinished, { | ||
| 38 | taskId: "task_1", | ||
| 39 | status: "completed", | ||
| 40 | stdout: "ok", | ||
| 41 | stderr: "", | ||
| 42 | exitCode: 0, | ||
| 43 | }); | ||
| 44 | expect(finished?.status).toBe("completed"); | ||
| 45 | |||
| 46 | const queuedAfter = await t.query(api.database.listQueuedTaskIds, { limit: 10 }); | ||
| 47 | expect(queuedAfter).toEqual([]); | ||
| 48 | }); | ||
| 49 | |||
| 50 | test("approval lifecycle tracks pending and resolution", async () => { | ||
| 51 | const t = setup(); | ||
| 52 | |||
| 53 | await t.mutation(api.database.createTask, { | ||
| 54 | id: "task_2", | ||
| 55 | code: "await tools.admin.delete_data({ id: 'x' })", | ||
| 56 | runtimeId: "local-bun", | ||
| 57 | workspaceId: "ws_2", | ||
| 58 | actorId: "actor_2", | ||
| 59 | clientId: "web", | ||
| 60 | }); | ||
| 61 | |||
| 62 | const createdApproval = await t.mutation(api.database.createApproval, { | ||
| 63 | id: "approval_1", | ||
| 64 | taskId: "task_2", | ||
| 65 | toolPath: "admin.delete_data", | ||
| 66 | input: { id: "x" }, | ||
| 67 | }); | ||
| 68 | expect(createdApproval.status).toBe("pending"); | ||
| 69 | |||
| 70 | const pending = await t.query(api.database.listPendingApprovals, { workspaceId: "ws_2" }); | ||
| 71 | expect(pending.length).toBe(1); | ||
| 72 | expect(pending[0]?.task.id).toBe("task_2"); | ||
| 73 | |||
| 74 | const resolved = await t.mutation(api.database.resolveApproval, { | ||
| 75 | approvalId: "approval_1", | ||
| 76 | decision: "approved", | ||
| 77 | reviewerId: "reviewer_1", | ||
| 78 | }); | ||
| 79 | expect(resolved?.status).toBe("approved"); | ||
| 80 | |||
| 81 | const pendingAfter = await t.query(api.database.listPendingApprovals, { workspaceId: "ws_2" }); | ||
| 82 | expect(pendingAfter).toEqual([]); | ||
| 83 | }); | ||
| 84 | |||
| 85 | test("workspace tool inventory applies policy decisions by context", async () => { | ||
| 86 | const t = setup(); | ||
| 87 | |||
| 88 | await t.mutation(api.database.syncWorkspaceTools, { | ||
| 89 | workspaceId: "ws_tools", | ||
| 90 | tools: [ | ||
| 91 | { | ||
| 92 | path: "utils.get_time", | ||
| 93 | description: "Read current time", | ||
| 94 | approval: "auto", | ||
| 95 | }, | ||
| 96 | { | ||
| 97 | path: "admin.delete_data", | ||
| 98 | description: "Delete data", | ||
| 99 | approval: "required", | ||
| 100 | }, | ||
| 101 | ], | ||
| 102 | }); | ||
| 103 | |||
| 104 | await t.mutation(api.database.upsertAccessPolicy, { | ||
| 105 | workspaceId: "ws_tools", | ||
| 106 | toolPathPattern: "admin.*", | ||
| 107 | decision: "deny", | ||
| 108 | priority: 100, | ||
| 109 | }); | ||
| 110 | |||
| 111 | await t.mutation(api.database.upsertAccessPolicy, { | ||
| 112 | workspaceId: "ws_tools", | ||
| 113 | actorId: "actor_allow", | ||
| 114 | toolPathPattern: "admin.delete_data", | ||
| 115 | decision: "require_approval", | ||
| 116 | priority: 200, | ||
| 117 | }); | ||
| 118 | |||
| 119 | const actorAllowTools = await t.query(api.database.listWorkspaceToolsForContext, { | ||
| 120 | workspaceId: "ws_tools", | ||
| 121 | actorId: "actor_allow", | ||
| 122 | clientId: "web", | ||
| 123 | }); | ||
| 124 | |||
| 125 | const actorDenyTools = await t.query(api.database.listWorkspaceToolsForContext, { | ||
| 126 | workspaceId: "ws_tools", | ||
| 127 | actorId: "actor_other", | ||
| 128 | clientId: "web", | ||
| 129 | }); | ||
| 130 | |||
| 131 | const actorAllow = actorAllowTools.filter((tool): tool is NonNullable<typeof tool> => tool !== null); | ||
| 132 | const actorDeny = actorDenyTools.filter((tool): tool is NonNullable<typeof tool> => tool !== null); | ||
| 133 | |||
| 134 | expect(actorAllow.map((tool) => tool.path)).toEqual(["admin.delete_data", "utils.get_time"]); | ||
| 135 | expect(actorAllow.find((tool) => tool.path === "admin.delete_data")?.approval).toBe("required"); | ||
| 136 | |||
| 137 | expect(actorDeny.map((tool) => tool.path)).toEqual(["utils.get_time"]); | ||
| 138 | }); | ||
| @@ -22,6 +22,7 @@ · 21 unmodified lines | |||
| 22 | }, | 22 | }, |
| 23 | "devDependencies": { | 23 | "devDependencies": { |
| 24 | "@types/bun": "latest", | 24 | "@types/bun": "latest", |
| 25 | "convex-test": "^0.0.41", | ||
| 25 | "typescript": "latest" | 26 | "typescript": "latest" |
| 26 | } | 27 | } |
| 27 | } | 28 | } |