History

deno subprocess

Rhys Sullivan committed 8 months ago

Commit f6167a71 parent be73cb111 files · +964 −163Browse files

Unified · Split

11 files
  1. v2/packages/engine/README.md +2 −1
  2. v2/packages/engine/src/deno-subprocess-runner.test.ts +95 −0
  3. v2/packages/engine/src/deno-subprocess-runner.ts +459 −0
  4. v2/packages/engine/src/deno-subprocess-worker.mjs +143 −0
  5. v2/packages/engine/src/index.ts +7 −0
  6. v2/packages/engine/src/openapi-provider.ts +35 −76
  7. v2/packages/engine/src/tool-providers.test.ts +2 −0
  8. v2/packages/engine/src/tool-providers.ts +15 −27
  9. v2/packages/schema/src/index.ts +1 −0
  10. v2/packages/schema/src/models/runtime-tool.ts +180 −0
  11. v2/packages/source-manager/src/openapi-extraction.ts +25 −59
v2/packages/engine/README.md+2−1
@@ -6,5 +6,6 @@ Current scaffold includes: · 5 unmodified lines
6- provider contracts (`ToolProvider`) and canonical tool descriptor model6- provider contracts (`ToolProvider`) and canonical tool descriptor model
7- provider registry service (`ToolProviderRegistryService`) with `discover`/`invoke` routing7- provider registry service (`ToolProviderRegistryService`) with `discover`/`invoke` routing
8- OpenAPI provider invocation and manifest-to-descriptor conversion helpers8- OpenAPI provider invocation and manifest-to-descriptor conversion helpers
9- minimal local JavaScript runner with `tools.*` proxy dispatch into provider registry9- minimal in-process JavaScript runner with `tools.*` proxy dispatch into provider registry
10- Deno subprocess runner with line-delimited JSON IPC for proxied `tools.*` calls
10- vertical integration test covering OpenAPI extraction -> descriptor conversion -> code execution -> HTTP call11- vertical integration test covering OpenAPI extraction -> descriptor conversion -> code execution -> HTTP call
v2/packages/engine/src/deno-subprocess-runner.test.ts+95−0
@@ -0,0 +1,95 @@
1import {
2 CanonicalToolDescriptorSchema,
3 type CanonicalToolDescriptor,
4} from "@executor-v2/schema";
5import { describe, expect, test } from "bun:test";
6import * as Effect from "effect/Effect";
7import * as Schema from "effect/Schema";
8
9import {
10 DenoSubprocessRunnerError,
11 executeJavaScriptInDenoSubprocess,
12} from "./deno-subprocess-runner";
13import {
14 makeToolProviderRegistry,
15 ToolProviderRegistryService,
16 type ToolProvider,
17} from "./tool-providers";
18
19const decodeCanonicalToolDescriptor = Schema.decodeUnknownSync(
20 CanonicalToolDescriptorSchema,
21);
22
23const sumToolDescriptor: CanonicalToolDescriptor = decodeCanonicalToolDescriptor({
24 providerKind: "in_memory",
25 sourceId: null,
26 workspaceId: null,
27 toolId: "sum",
28 name: "sum",
29 description: null,
30 invocationMode: "in_memory",
31 availability: "local_only",
32 providerPayload: null,
33});
34
35describe("executeJavaScriptInDenoSubprocess", () => {
36 test("runs code in Deno subprocess and proxies tool calls", async () => {
37 const sumProvider: ToolProvider = {
38 kind: "in_memory",
39 invoke: (input) =>
40 Effect.gen(function* () {
41 if (input.tool.toolId !== "sum") {
42 return {
43 output: `unknown tool: ${input.tool.toolId}`,
44 isError: true,
45 } as const;
46 }
47
48 const args = input.args as { a: number; b: number };
49 return {
50 output: args.a + args.b,
51 isError: false,
52 } as const;
53 }),
54 };
55
56 const registry = makeToolProviderRegistry([sumProvider]);
57
58 const result = await Effect.runPromise(
59 executeJavaScriptInDenoSubprocess({
60 code: "console.log('hello'); return await tools.sum({ a: 2, b: 3 });",
61 tools: [
62 {
63 descriptor: sumToolDescriptor,
64 source: null,
65 },
66 ],
67 timeoutMs: 10_000,
68 }).pipe(Effect.provideService(ToolProviderRegistryService, registry)),
69 );
70
71 expect(result).toBe(5);
72 });
73
74 test("returns a typed error when Deno executable is missing", async () => {
75 const registry = makeToolProviderRegistry([]);
76
77 const result = await Effect.runPromise(
78 Effect.either(
79 executeJavaScriptInDenoSubprocess({
80 code: "return 1;",
81 tools: [],
82 denoExecutable: "/definitely-missing-deno-binary",
83 timeoutMs: 1_000,
84 }).pipe(Effect.provideService(ToolProviderRegistryService, registry)),
85 ),
86 );
87
88 if (result._tag === "Right") {
89 throw new Error("expected missing Deno executable to fail");
90 }
91
92 expect(result.left).toBeInstanceOf(DenoSubprocessRunnerError);
93 expect(result.left.operation).toBe("spawn");
94 });
95});
v2/packages/engine/src/deno-subprocess-runner.ts+459−0
@@ -0,0 +1,459 @@
1import { existsSync } from "node:fs";
2import { spawn } from "node:child_process";
3import { fileURLToPath } from "node:url";
4
5import type { Source } from "@executor-v2/schema";
6import * as Data from "effect/Data";
7import * as Effect from "effect/Effect";
8import * as Runtime from "effect/Runtime";
9import * as Schema from "effect/Schema";
10
11import {
12 ToolProviderRegistryError,
13 ToolProviderRegistryService,
14 type CanonicalToolDescriptor,
15 type ToolProviderError,
16} from "./tool-providers";
17
18export class DenoSubprocessRunnerError extends Data.TaggedError(
19 "DenoSubprocessRunnerError",
20)<{
21 operation: string;
22 message: string;
23 details: string | null;
24}> {}
25
26export type DenoRunnableTool = {
27 descriptor: CanonicalToolDescriptor;
28 source: Source | null;
29};
30
31export type ExecuteJavaScriptInDenoInput = {
32 code: string;
33 tools: ReadonlyArray<DenoRunnableTool>;
34 timeoutMs?: number;
35 denoExecutable?: string;
36};
37
38const IPC_PREFIX = "@@engine-ipc@@";
39
40const HostStartMessageSchema = Schema.Struct({
41 type: Schema.Literal("start"),
42 code: Schema.String,
43 toolIds: Schema.Array(Schema.String),
44});
45
46const HostToolResultMessageSchema = Schema.Struct({
47 type: Schema.Literal("tool_result"),
48 requestId: Schema.String,
49 ok: Schema.Boolean,
50 value: Schema.optional(Schema.Unknown),
51 error: Schema.optional(Schema.String),
52});
53
54const HostToWorkerMessageSchema = Schema.Union(
55 HostStartMessageSchema,
56 HostToolResultMessageSchema,
57);
58
59const WorkerToolCallMessageSchema = Schema.Struct({
60 type: Schema.Literal("tool_call"),
61 requestId: Schema.String,
62 toolId: Schema.String,
63 args: Schema.Unknown,
64});
65
66const WorkerCompletedMessageSchema = Schema.Struct({
67 type: Schema.Literal("completed"),
68 result: Schema.Unknown,
69});
70
71const WorkerFailedMessageSchema = Schema.Struct({
72 type: Schema.Literal("failed"),
73 error: Schema.String,
74});
75
76const WorkerToHostMessageSchema = Schema.Union(
77 WorkerToolCallMessageSchema,
78 WorkerCompletedMessageSchema,
79 WorkerFailedMessageSchema,
80);
81
82type HostToWorkerMessage = typeof HostToWorkerMessageSchema.Type;
83type WorkerToHostMessage = typeof WorkerToHostMessageSchema.Type;
84type WorkerToolCallMessage = typeof WorkerToolCallMessageSchema.Type;
85
86const decodeWorkerMessageLine = Schema.decodeUnknownSync(
87 Schema.parseJson(WorkerToHostMessageSchema),
88);
89const encodeHostMessage = Schema.encodeSync(Schema.parseJson(HostToWorkerMessageSchema));
90
91const duplicateToolIdError = (toolId: string): DenoSubprocessRunnerError =>
92 new DenoSubprocessRunnerError({
93 operation: "build_tools",
94 message: `Duplicate tool id in run context: ${toolId}`,
95 details: null,
96 });
97
98const missingToolBindingError = (toolId: string): DenoSubprocessRunnerError =>
99 new DenoSubprocessRunnerError({
100 operation: "invoke_tool",
101 message: `Worker requested unknown tool id: ${toolId}`,
102 details: null,
103 });
104
105const toolCallFailedError = (toolId: string): DenoSubprocessRunnerError =>
106 new DenoSubprocessRunnerError({
107 operation: "invoke_tool",
108 message: `Tool call returned error: ${toolId}`,
109 details: null,
110 });
111
112const parseWorkerMessageError = (
113 line: string,
114 cause: unknown,
115): DenoSubprocessRunnerError =>
116 new DenoSubprocessRunnerError({
117 operation: "decode_worker_message",
118 message: "Failed to decode worker message",
119 details: `${line}\n${String(cause)}`,
120 });
121
122const workerProcessError = (
123 operation: string,
124 message: string,
125 details: string | null,
126): DenoSubprocessRunnerError =>
127 new DenoSubprocessRunnerError({
128 operation,
129 message,
130 details,
131 });
132
133const buildToolBindings = (
134 tools: ReadonlyArray<DenoRunnableTool>,
135): Effect.Effect<Map<string, DenoRunnableTool>, DenoSubprocessRunnerError> =>
136 Effect.gen(function* () {
137 const byToolId = new Map<string, DenoRunnableTool>();
138
139 for (const tool of tools) {
140 const toolId = tool.descriptor.toolId;
141 if (byToolId.has(toolId)) {
142 return yield* duplicateToolIdError(toolId);
143 }
144 byToolId.set(toolId, tool);
145 }
146
147 return byToolId;
148 });
149
150const defaultDenoExecutable = (): string => {
151 const configured = process.env.DENO_BIN?.trim();
152 if (configured) {
153 return configured;
154 }
155
156 const home = process.env.HOME?.trim();
157 if (home) {
158 const installedPath = `${home}/.deno/bin/deno`;
159 if (existsSync(installedPath)) {
160 return installedPath;
161 }
162 }
163
164 return "deno";
165};
166
167const workerScriptPath = fileURLToPath(
168 new URL("./deno-subprocess-worker.mjs", import.meta.url),
169);
170
171const writeMessage = (
172 stdin: NodeJS.WritableStream,
173 message: HostToWorkerMessage,
174): Effect.Effect<void, DenoSubprocessRunnerError> =>
175 Effect.try({
176 try: () => {
177 stdin.write(`${encodeHostMessage(message)}\n`);
178 },
179 catch: (cause) =>
180 workerProcessError(
181 "write_message",
182 "Failed to write message to Deno subprocess",
183 String(cause),
184 ),
185 });
186
187const handleToolCall = (
188 message: WorkerToolCallMessage,
189 toolBindings: ReadonlyMap<string, DenoRunnableTool>,
190 runPromise: <A, E>(
191 effect: Effect.Effect<A, E, never>,
192 ) => Promise<A>,
193 invokeTool: (
194 input: {
195 source: Source | null;
196 tool: CanonicalToolDescriptor;
197 args: unknown;
198 },
199 ) => Effect.Effect<
200 { output: unknown; isError: boolean },
201 ToolProviderRegistryError | ToolProviderError
202 >,
203 stdin: NodeJS.WritableStream,
204): Effect.Effect<void, DenoSubprocessRunnerError> =>
205 Effect.gen(function* () {
206 const binding = toolBindings.get(message.toolId);
207 if (!binding) {
208 return yield* writeMessage(stdin, {
209 type: "tool_result",
210 requestId: message.requestId,
211 ok: false,
212 error: missingToolBindingError(message.toolId).message,
213 });
214 }
215
216 const invokeResult = yield* Effect.tryPromise({
217 try: () =>
218 runPromise(
219 invokeTool({
220 source: binding.source,
221 tool: binding.descriptor,
222 args: message.args,
223 }),
224 ),
225 catch: (cause) =>
226 workerProcessError(
227 "invoke_tool",
228 "Tool invocation threw while handling worker tool_call",
229 String(cause),
230 ),
231 });
232
233 if (invokeResult.isError) {
234 yield* writeMessage(stdin, {
235 type: "tool_result",
236 requestId: message.requestId,
237 ok: false,
238 error: toolCallFailedError(message.toolId).message,
239 });
240 return;
241 }
242
243 yield* writeMessage(stdin, {
244 type: "tool_result",
245 requestId: message.requestId,
246 ok: true,
247 value: invokeResult.output,
248 });
249 });
250
251export const isDenoSubprocessRuntimeAvailable = (
252 executable: string = defaultDenoExecutable(),
253): boolean => executable.includes("/") ? existsSync(executable) : true;
254
255export const executeJavaScriptInDenoSubprocess = (
256 input: ExecuteJavaScriptInDenoInput,
257): Effect.Effect<
258 unknown,
259 DenoSubprocessRunnerError | ToolProviderRegistryError | ToolProviderError,
260 ToolProviderRegistryService
261> =>
262 Effect.gen(function* () {
263 const registry = yield* ToolProviderRegistryService;
264 const runtime = yield* Effect.runtime<never>();
265 const runPromise = Runtime.runPromise(runtime);
266 const toolBindings = yield* buildToolBindings(input.tools);
267 const toolIds = Array.from(toolBindings.keys()).sort();
268
269 const denoExecutable = input.denoExecutable ?? defaultDenoExecutable();
270 const timeoutMs = Math.max(100, input.timeoutMs ?? 30_000);
271
272 return yield* Effect.tryPromise({
273 try: () =>
274 new Promise<unknown>((resolve, reject) => {
275 const child = spawn(
276 denoExecutable,
277 [
278 "run",
279 "--quiet",
280 "--no-prompt",
281 "--no-check",
282 workerScriptPath,
283 ],
284 {
285 stdio: ["pipe", "pipe", "pipe"],
286 },
287 );
288
289 let settled = false;
290 let stdoutBuffer = "";
291 let stderrBuffer = "";
292
293 const finish = (result: { ok: true; value: unknown } | { ok: false; error: Error }) => {
294 if (settled) {
295 return;
296 }
297
298 settled = true;
299 clearTimeout(timeout);
300 child.stdout?.removeAllListeners();
301 child.stderr?.removeAllListeners();
302 child.removeAllListeners();
303
304 if (!child.killed) {
305 child.kill("SIGKILL");
306 }
307
308 if (result.ok) {
309 resolve(result.value);
310 } else {
311 reject(result.error);
312 }
313 };
314
315 const fail = (error: DenoSubprocessRunnerError) => {
316 finish({ ok: false, error });
317 };
318
319 const timeout = setTimeout(() => {
320 fail(
321 workerProcessError(
322 "timeout",
323 `Deno subprocess execution timed out after ${timeoutMs}ms`,
324 stderrBuffer.length > 0 ? stderrBuffer : null,
325 ),
326 );
327 }, timeoutMs);
328
329 child.on("error", (cause) => {
330 fail(
331 workerProcessError(
332 "spawn",
333 "Failed to spawn Deno subprocess",
334 cause instanceof Error ? cause.message : String(cause),
335 ),
336 );
337 });
338
339 child.on("exit", (code, signal) => {
340 if (settled) {
341 return;
342 }
343
344 fail(
345 workerProcessError(
346 "process_exit",
347 "Deno subprocess exited before returning terminal message",
348 `code=${String(code)} signal=${String(signal)} stderr=${stderrBuffer}`,
349 ),
350 );
351 });
352
353 if (child.stderr) {
354 child.stderr.setEncoding("utf8");
355 child.stderr.on("data", (chunk: string) => {
356 stderrBuffer += chunk;
357 });
358 }
359
360 if (child.stdout) {
361 child.stdout.setEncoding("utf8");
362 child.stdout.on("data", (chunk: string) => {
363 stdoutBuffer += chunk;
364
365 while (true) {
366 const newlineIndex = stdoutBuffer.indexOf("\n");
367 if (newlineIndex === -1) {
368 break;
369 }
370
371 const line = stdoutBuffer.slice(0, newlineIndex).trim();
372 stdoutBuffer = stdoutBuffer.slice(newlineIndex + 1);
373
374 if (line.length === 0) {
375 continue;
376 }
377
378 if (!line.startsWith(IPC_PREFIX)) {
379 continue;
380 }
381
382 const payload = line.slice(IPC_PREFIX.length);
383
384 let message: WorkerToHostMessage;
385 try {
386 message = decodeWorkerMessageLine(payload);
387 } catch (cause) {
388 fail(parseWorkerMessageError(payload, cause));
389 return;
390 }
391
392 if (message.type === "tool_call") {
393 runPromise(
394 handleToolCall(
395 message,
396 toolBindings,
397 runPromise,
398 (invokeInput) => registry.invoke(invokeInput),
399 child.stdin,
400 ),
401 ).catch((cause) => {
402 fail(
403 cause instanceof DenoSubprocessRunnerError
404 ? cause
405 : workerProcessError(
406 "handle_tool_call",
407 "Failed handling worker tool_call",
408 String(cause),
409 ),
410 );
411 });
412 continue;
413 }
414
415 if (message.type === "completed") {
416 finish({ ok: true, value: message.result });
417 return;
418 }
419
420 fail(
421 workerProcessError(
422 "worker_failed",
423 "Deno subprocess returned failed terminal message",
424 message.error,
425 ),
426 );
427 return;
428 }
429 });
430 }
431
432 runPromise(
433 writeMessage(child.stdin, {
434 type: "start",
435 code: input.code,
436 toolIds,
437 }),
438 ).catch((cause) => {
439 fail(
440 cause instanceof DenoSubprocessRunnerError
441 ? cause
442 : workerProcessError(
443 "start_message",
444 "Failed sending start message to Deno subprocess",
445 String(cause),
446 ),
447 );
448 });
449 }),
450 catch: (cause) =>
451 cause instanceof DenoSubprocessRunnerError
452 ? cause
453 : workerProcessError(
454 "execute",
455 "Unexpected Deno subprocess execution failure",
456 String(cause),
457 ),
458 });
459 });
v2/packages/engine/src/deno-subprocess-worker.mjs+143−0
@@ -0,0 +1,143 @@
1const encoder = new TextEncoder();
2const IPC_PREFIX = "@@engine-ipc@@";
3
4const pendingToolCalls = new Map();
5let started = false;
6
7const writeMessage = (message) => {
8 const payload = `${IPC_PREFIX}${JSON.stringify(message)}\n`;
9 Deno.stdout.writeSync(encoder.encode(payload));
10};
11
12const toErrorMessage = (error) => {
13 if (error instanceof Error) {
14 return error.stack ?? error.message;
15 }
16
17 return String(error);
18};
19
20const createToolCaller = (toolId) => (args) =>
21 new Promise((resolve, reject) => {
22 const requestId = crypto.randomUUID();
23 pendingToolCalls.set(requestId, { resolve, reject });
24
25 writeMessage({
26 type: "tool_call",
27 requestId,
28 toolId,
29 args: args === undefined ? {} : args,
30 });
31 });
32
33const runUserCode = async (code, toolIds) => {
34 const tools = Object.create(null);
35 for (const toolId of toolIds) {
36 tools[toolId] = createToolCaller(toolId);
37 }
38
39 const execute = new Function(
40 "tools",
41 `"use strict"; return (async () => {\n${code}\n})();`,
42 );
43
44 return await execute(tools);
45};
46
47const handleStart = (message) => {
48 if (started) {
49 writeMessage({
50 type: "failed",
51 error: "start message already received",
52 });
53 return;
54 }
55
56 started = true;
57
58 runUserCode(message.code, message.toolIds)
59 .then((result) => {
60 writeMessage({
61 type: "completed",
62 result,
63 });
64 })
65 .catch((error) => {
66 writeMessage({
67 type: "failed",
68 error: toErrorMessage(error),
69 });
70 });
71};
72
73const handleToolResult = (message) => {
74 const pending = pendingToolCalls.get(message.requestId);
75 if (!pending) {
76 return;
77 }
78
79 pendingToolCalls.delete(message.requestId);
80
81 if (message.ok) {
82 pending.resolve(message.value);
83 return;
84 }
85
86 pending.reject(new Error(message.error));
87};
88
89const handleHostMessage = (message) => {
90 if (!message || typeof message !== "object") {
91 return;
92 }
93
94 if (message.type === "start") {
95 handleStart(message);
96 return;
97 }
98
99 if (message.type === "tool_result") {
100 handleToolResult(message);
101 }
102};
103
104const decodeLines = async () => {
105 const reader = Deno.stdin.readable.getReader();
106 const decoder = new TextDecoder();
107 let buffer = "";
108
109 while (true) {
110 const { value, done } = await reader.read();
111 if (done) {
112 break;
113 }
114
115 buffer += decoder.decode(value, { stream: true });
116
117 while (true) {
118 const lineEnd = buffer.indexOf("\n");
119 if (lineEnd === -1) {
120 break;
121 }
122
123 const line = buffer.slice(0, lineEnd).trim();
124 buffer = buffer.slice(lineEnd + 1);
125
126 if (line.length === 0) {
127 continue;
128 }
129
130 try {
131 const message = JSON.parse(line);
132 handleHostMessage(message);
133 } catch (error) {
134 writeMessage({
135 type: "failed",
136 error: `invalid host message: ${toErrorMessage(error)}`,
137 });
138 }
139 }
140 }
141};
142
143await decodeLines();
v2/packages/engine/src/index.ts+7−0
@@ -1,3 +1,10 @@
1export {
2 executeJavaScriptInDenoSubprocess,
3 isDenoSubprocessRuntimeAvailable,
4 DenoSubprocessRunnerError,
5 type DenoRunnableTool,
6 type ExecuteJavaScriptInDenoInput,
7} from "./deno-subprocess-runner";
1export {8export {
2 executeJavaScriptWithTools,9 executeJavaScriptWithTools,
3 LocalCodeRunnerError,10 LocalCodeRunnerError,
v2/packages/engine/src/openapi-provider.ts+35−76
@@ -1,67 +1,18 @@
1import type { Source } from "@executor-v2/schema";1import {
2 OpenApiInvocationPayloadSchema,
3 OpenApiToolManifestSchema,
4 type CanonicalToolDescriptor,
5 type OpenApiCanonicalToolDescriptor,
6 type OpenApiInvocationPayload,
7 type OpenApiToolManifest,
8 type Source,
9} from "@executor-v2/schema";
2import * as Effect from "effect/Effect";10import * as Effect from "effect/Effect";
3import { pipe } from "effect/Function";11import { pipe } from "effect/Function";
4import * as ParseResult from "effect/ParseResult";12import * as ParseResult from "effect/ParseResult";
5import * as Schema from "effect/Schema";13import * as Schema from "effect/Schema";
6 14
7import {15import { ToolProviderError, type ToolProvider } from "./tool-providers";
8 ToolProviderError,
9 type CanonicalToolDescriptor,
10 type ToolProvider,
11} from "./tool-providers";
12
13const OPEN_API_PARAMETER_LOCATIONS = [
14 "path",
15 "query",
16 "header",
17 "cookie",
18] as const;
19
20const OPEN_API_HTTP_METHODS = [
21 "get",
22 "put",
23 "post",
24 "delete",
25 "patch",
26 "head",
27 "options",
28 "trace",
29] as const;
30
31const OpenApiInvocationPayloadSchema = Schema.Struct({
32 method: Schema.Literal(...OPEN_API_HTTP_METHODS),
33 pathTemplate: Schema.String,
34 parameters: Schema.Array(
35 Schema.Struct({
36 name: Schema.String,
37 location: Schema.Literal(...OPEN_API_PARAMETER_LOCATIONS),
38 required: Schema.Boolean,
39 }),
40 ),
41 requestBody: Schema.NullOr(
42 Schema.Struct({
43 required: Schema.Boolean,
44 contentTypes: Schema.Array(Schema.String),
45 }),
46 ),
47});
48
49type OpenApiInvocationPayload = typeof OpenApiInvocationPayloadSchema.Type;
50
51const OpenApiManifestSchema = Schema.Struct({
52 version: Schema.Literal(1),
53 sourceHash: Schema.String,
54 tools: Schema.Array(
55 Schema.Struct({
56 toolId: Schema.String,
57 name: Schema.String,
58 description: Schema.NullOr(Schema.String),
59 invocation: OpenApiInvocationPayloadSchema,
60 }),
61 ),
62});
63
64type OpenApiManifest = typeof OpenApiManifestSchema.Type;
65 16
66const OpenApiToolArgsSchema = Schema.Record({17const OpenApiToolArgsSchema = Schema.Record({
67 key: Schema.String,18 key: Schema.String,
@@ -71,12 +22,13 @@ const OpenApiToolArgsSchema = Schema.Record({ · 3 unmodified lines
71type OpenApiToolArgs = typeof OpenApiToolArgsSchema.Type;22type OpenApiToolArgs = typeof OpenApiToolArgsSchema.Type;
72 23
73const decodeOpenApiManifestJson = Schema.decodeUnknown(24const decodeOpenApiManifestJson = Schema.decodeUnknown(
74 Schema.parseJson(OpenApiManifestSchema),25 Schema.parseJson(OpenApiToolManifestSchema),
75);26);
76const decodeOpenApiInvocationPayload = Schema.decodeUnknown(27const decodeOpenApiInvocationPayload = Schema.decodeUnknown(
77 OpenApiInvocationPayloadSchema,28 OpenApiInvocationPayloadSchema,
78);29);
79const decodeOpenApiToolArgs = Schema.decodeUnknown(OpenApiToolArgsSchema);30const decodeOpenApiToolArgs = Schema.decodeUnknown(OpenApiToolArgsSchema);
31const encodeUnknownToJson = Schema.encode(Schema.parseJson(Schema.Unknown));
80 32
81const toOpenApiProviderError = (33const toOpenApiProviderError = (
82 operation: string,34 operation: string,
@@ -108,7 +60,7 @@ const argsValueToString = (value: unknown): string => { · 25 unmodified lines
108 return String(value);60 return String(value);
109 }61 }
110 62
111 return JSON.stringify(value);63 return String(value);
112};64};
113 65
114const replacePathTemplate = (66const replacePathTemplate = (
@@ -214,7 +166,16 @@ const buildFetchRequest = ( · 99 unmodified lines
214 });166 });
215 }167 }
216 } else {168 } else {
217 body = JSON.stringify(args.body);169 body = yield* pipe(
170 encodeUnknownToJson(args.body),
171 Effect.mapError((cause) =>
172 toOpenApiProviderError(
173 "invoke.encode_body",
174 "Failed to encode request body as JSON",
175 cause,
176 ),
177 ),
178 );
218 179
219 const preferredContentType = payload.requestBody.contentTypes[0];180 const preferredContentType = payload.requestBody.contentTypes[0];
220 if (preferredContentType) {181 if (preferredContentType) {
@@ -269,20 +230,18 @@ export const openApiToolDescriptorsFromManifest = ( · 48 unmodified lines
269 cause,230 cause,
270 ),231 ),
271 ),232 ),
272 Effect.map((manifest: OpenApiManifest) =>233 Effect.map((manifest: OpenApiToolManifest) =>
273 manifest.tools.map(234 manifest.tools.map((tool): OpenApiCanonicalToolDescriptor => ({
274 (tool): CanonicalToolDescriptor => ({235 providerKind: "openapi",
275 providerKind: "openapi",236 sourceId: source.id,
276 sourceId: source.id,237 workspaceId: source.workspaceId,
277 workspaceId: source.workspaceId,238 toolId: tool.toolId,
278 toolId: tool.toolId,239 name: tool.name,
279 name: tool.name,240 description: tool.description,
280 description: tool.description,241 invocationMode: "http",
281 invocationMode: "http",242 availability: "remote_capable",
282 availability: "remote_capable",243 providerPayload: tool.invocation,
283 providerPayload: tool.invocation,244 })),
284 }),
285 ),
286 ),245 ),
287 );246 );
288 247
v2/packages/engine/src/tool-providers.test.ts+2−0
@@ -48,6 +48,8 @@ describe("makeToolProviderRegistry", () => { · 47 unmodified lines
48 providerPayload: {48 providerPayload: {
49 method: "get",49 method: "get",
50 pathTemplate: "/healthz",50 pathTemplate: "/healthz",
51 parameters: [],
52 requestBody: null,
51 },53 },
52 },54 },
53 ],55 ],
v2/packages/engine/src/tool-providers.ts+15−27
@@ -1,31 +1,22 @@
1import type { Source, SourceKind } from "@executor-v2/schema";1import type {
2 CanonicalToolDescriptor as SchemaCanonicalToolDescriptor,
3 Source,
4 ToolAvailability as SchemaToolAvailability,
5 ToolDiscoveryResult as SchemaToolDiscoveryResult,
6 ToolInvocationMode as SchemaToolInvocationMode,
7 ToolInvokeResult,
8 ToolProviderKind as SchemaToolProviderKind,
9} from "@executor-v2/schema";
2import * as Context from "effect/Context";10import * as Context from "effect/Context";
3import * as Data from "effect/Data";11import * as Data from "effect/Data";
4import * as Effect from "effect/Effect";12import * as Effect from "effect/Effect";
5import * as Layer from "effect/Layer";13import * as Layer from "effect/Layer";
6 14
7export type ToolProviderKind = SourceKind | "in_memory";15export type ToolAvailability = SchemaToolAvailability;
8 16export type ToolInvocationMode = SchemaToolInvocationMode;
9export type ToolInvocationMode = "http" | "mcp" | "graphql" | "in_memory";17export type ToolProviderKind = SchemaToolProviderKind;
10 18export type CanonicalToolDescriptor = SchemaCanonicalToolDescriptor;
11export type ToolAvailability = "local_only" | "remote_capable";19export type ToolDiscoveryResult = SchemaToolDiscoveryResult;
12
13export type CanonicalToolDescriptor = {
14 providerKind: ToolProviderKind;
15 sourceId: Source["id"] | null;
16 workspaceId: Source["workspaceId"] | null;
17 toolId: string;
18 name: string;
19 description: string | null;
20 invocationMode: ToolInvocationMode;
21 availability: ToolAvailability;
22 providerPayload: unknown;
23};
24
25export type ToolDiscoveryResult = {
26 sourceHash: string | null;
27 tools: ReadonlyArray<CanonicalToolDescriptor>;
28};
29 20
30export type InvokeToolInput = {21export type InvokeToolInput = {
31 source: Source | null;22 source: Source | null;
@@ -33,10 +24,7 @@ export type InvokeToolInput = { · 1 unmodified line
33 args: unknown;24 args: unknown;
34};25};
35 26
36export type InvokeToolResult = {27export type InvokeToolResult = ToolInvokeResult;
37 output: unknown;
38 isError: boolean;
39};
40 28
41export class ToolProviderError extends Data.TaggedError("ToolProviderError")<{29export class ToolProviderError extends Data.TaggedError("ToolProviderError")<{
42 operation: string;30 operation: string;
v2/packages/schema/src/index.ts+1−0
@@ -13,4 +13,5 @@ export * from "./models/approval"; · 12 unmodified lines
13export * from "./models/task-run";13export * from "./models/task-run";
14export * from "./models/sync-state";14export * from "./models/sync-state";
15export * from "./models/event-envelope";15export * from "./models/event-envelope";
16export * from "./models/runtime-tool";
16 17
v2/packages/schema/src/models/runtime-tool.ts+180−0
@@ -0,0 +1,180 @@
1import { Schema } from "effect";
2
3import { SourceKindSchema } from "../enums";
4import { SourceIdSchema, WorkspaceIdSchema } from "../ids";
5
6export const OPEN_API_HTTP_METHODS = [
7 "get",
8 "put",
9 "post",
10 "delete",
11 "patch",
12 "head",
13 "options",
14 "trace",
15] as const;
16
17export const OPEN_API_PARAMETER_LOCATIONS = [
18 "path",
19 "query",
20 "header",
21 "cookie",
22] as const;
23
24export const ToolProviderKindSchema = Schema.Union(
25 SourceKindSchema,
26 Schema.Literal("in_memory"),
27);
28
29export const ToolInvocationModeSchema = Schema.Literal(
30 "http",
31 "mcp",
32 "graphql",
33 "in_memory",
34);
35
36export const ToolAvailabilitySchema = Schema.Literal(
37 "local_only",
38 "remote_capable",
39);
40
41export const OpenApiHttpMethodSchema = Schema.Literal(...OPEN_API_HTTP_METHODS);
42
43export const OpenApiParameterLocationSchema = Schema.Literal(
44 ...OPEN_API_PARAMETER_LOCATIONS,
45);
46
47export const OpenApiToolParameterSchema = Schema.Struct({
48 name: Schema.String,
49 location: OpenApiParameterLocationSchema,
50 required: Schema.Boolean,
51});
52
53export const OpenApiToolRequestBodySchema = Schema.Struct({
54 required: Schema.Boolean,
55 contentTypes: Schema.Array(Schema.String),
56});
57
58export const OpenApiInvocationPayloadSchema = Schema.Struct({
59 method: OpenApiHttpMethodSchema,
60 pathTemplate: Schema.String,
61 parameters: Schema.Array(OpenApiToolParameterSchema),
62 requestBody: Schema.NullOr(OpenApiToolRequestBodySchema),
63});
64
65export const OpenApiExtractedToolSchema = Schema.Struct({
66 toolId: Schema.String,
67 name: Schema.String,
68 description: Schema.NullOr(Schema.String),
69 method: OpenApiHttpMethodSchema,
70 path: Schema.String,
71 invocation: OpenApiInvocationPayloadSchema,
72 operationHash: Schema.String,
73});
74
75export const OpenApiToolManifestSchema = Schema.Struct({
76 version: Schema.Literal(1),
77 sourceHash: Schema.String,
78 tools: Schema.Array(OpenApiExtractedToolSchema),
79});
80
81const SourceBackedCanonicalToolDescriptorBaseSchema = Schema.Struct({
82 sourceId: SourceIdSchema,
83 workspaceId: WorkspaceIdSchema,
84 toolId: Schema.String,
85 name: Schema.String,
86 description: Schema.NullOr(Schema.String),
87 availability: ToolAvailabilitySchema,
88});
89
90export const OpenApiCanonicalToolDescriptorSchema =
91 SourceBackedCanonicalToolDescriptorBaseSchema.pipe(
92 Schema.extend(
93 Schema.Struct({
94 providerKind: Schema.Literal("openapi"),
95 invocationMode: Schema.Literal("http"),
96 providerPayload: OpenApiInvocationPayloadSchema,
97 }),
98 ),
99 );
100
101export const McpCanonicalToolDescriptorSchema =
102 SourceBackedCanonicalToolDescriptorBaseSchema.pipe(
103 Schema.extend(
104 Schema.Struct({
105 providerKind: Schema.Literal("mcp"),
106 invocationMode: Schema.Literal("mcp"),
107 providerPayload: Schema.Unknown,
108 }),
109 ),
110 );
111
112export const GraphqlCanonicalToolDescriptorSchema =
113 SourceBackedCanonicalToolDescriptorBaseSchema.pipe(
114 Schema.extend(
115 Schema.Struct({
116 providerKind: Schema.Literal("graphql"),
117 invocationMode: Schema.Literal("graphql"),
118 providerPayload: Schema.Unknown,
119 }),
120 ),
121 );
122
123export const InternalCanonicalToolDescriptorSchema =
124 SourceBackedCanonicalToolDescriptorBaseSchema.pipe(
125 Schema.extend(
126 Schema.Struct({
127 providerKind: Schema.Literal("internal"),
128 invocationMode: Schema.Literal("in_memory"),
129 providerPayload: Schema.Unknown,
130 }),
131 ),
132 );
133
134export const InMemoryCanonicalToolDescriptorSchema = Schema.Struct({
135 providerKind: Schema.Literal("in_memory"),
136 sourceId: Schema.Null,
137 workspaceId: Schema.Null,
138 toolId: Schema.String,
139 name: Schema.String,
140 description: Schema.NullOr(Schema.String),
141 invocationMode: Schema.Literal("in_memory"),
142 availability: ToolAvailabilitySchema,
143 providerPayload: Schema.Unknown,
144});
145
146export const CanonicalToolDescriptorSchema = Schema.Union(
147 OpenApiCanonicalToolDescriptorSchema,
148 McpCanonicalToolDescriptorSchema,
149 GraphqlCanonicalToolDescriptorSchema,
150 InternalCanonicalToolDescriptorSchema,
151 InMemoryCanonicalToolDescriptorSchema,
152);
153
154export const ToolDiscoveryResultSchema = Schema.Struct({
155 sourceHash: Schema.NullOr(Schema.String),
156 tools: Schema.Array(CanonicalToolDescriptorSchema),
157});
158
159export const ToolInvokeResultSchema = Schema.Struct({
160 output: Schema.Unknown,
161 isError: Schema.Boolean,
162});
163
164export type ToolProviderKind = typeof ToolProviderKindSchema.Type;
165export type ToolInvocationMode = typeof ToolInvocationModeSchema.Type;
166export type ToolAvailability = typeof ToolAvailabilitySchema.Type;
167
168export type OpenApiHttpMethod = typeof OpenApiHttpMethodSchema.Type;
169export type OpenApiParameterLocation = typeof OpenApiParameterLocationSchema.Type;
170export type OpenApiToolParameter = typeof OpenApiToolParameterSchema.Type;
171export type OpenApiToolRequestBody = typeof OpenApiToolRequestBodySchema.Type;
172export type OpenApiInvocationPayload = typeof OpenApiInvocationPayloadSchema.Type;
173export type OpenApiExtractedTool = typeof OpenApiExtractedToolSchema.Type;
174export type OpenApiToolManifest = typeof OpenApiToolManifestSchema.Type;
175
176export type OpenApiCanonicalToolDescriptor =
177 typeof OpenApiCanonicalToolDescriptorSchema.Type;
178export type CanonicalToolDescriptor = typeof CanonicalToolDescriptorSchema.Type;
179export type ToolDiscoveryResult = typeof ToolDiscoveryResultSchema.Type;
180export type ToolInvokeResult = typeof ToolInvokeResultSchema.Type;
v2/packages/source-manager/src/openapi-extraction.ts+25−59
@@ -5,7 +5,20 @@ import { · 4 unmodified lines
5 type ToolArtifactStore,5 type ToolArtifactStore,
6} from "@executor-v2/persistence-ports";6} from "@executor-v2/persistence-ports";
7import {7import {
8 OPEN_API_HTTP_METHODS,
9 OPEN_API_PARAMETER_LOCATIONS,
10 OpenApiExtractedToolSchema,
11 OpenApiInvocationPayloadSchema,
12 OpenApiToolManifestSchema,
13 OpenApiToolParameterSchema,
14 OpenApiToolRequestBodySchema,
8 ToolArtifactIdSchema,15 ToolArtifactIdSchema,
16 type OpenApiExtractedTool,
17 type OpenApiHttpMethod,
18 type OpenApiInvocationPayload,
19 type OpenApiToolManifest,
20 type OpenApiToolParameter,
21 type OpenApiToolRequestBody,
9 type Source,22 type Source,
10 type ToolArtifact,23 type ToolArtifact,
11} from "@executor-v2/schema";24} from "@executor-v2/schema";
@@ -16,27 +29,9 @@ import * as Option from "effect/Option"; · 4 unmodified lines
16import * as ParseResult from "effect/ParseResult";29import * as ParseResult from "effect/ParseResult";
17import * as Schema from "effect/Schema";30import * as Schema from "effect/Schema";
18 31
19const HTTP_METHODS = [32const HTTP_METHODS = OPEN_API_HTTP_METHODS;
20 "get",
21 "put",
22 "post",
23 "delete",
24 "patch",
25 "head",
26 "options",
27 "trace",
28] as const;
29 33
30type HttpMethod = (typeof HTTP_METHODS)[number];34type HttpMethod = OpenApiHttpMethod;
31
32const OPEN_API_PARAMETER_LOCATIONS = [
33 "path",
34 "query",
35 "header",
36 "cookie",
37] as const;
38
39type OpenApiParameterLocation = (typeof OPEN_API_PARAMETER_LOCATIONS)[number];
40 35
41type OpenApiExtractionStage =36type OpenApiExtractionStage =
42 | "validate"37 | "validate"
@@ -50,51 +45,22 @@ export class OpenApiExtractionError extends Data.TaggedError("OpenApiExtractionE · 7 unmodified lines
50 details: string | null;45 details: string | null;
51}> {}46}> {}
52 47
53export const ExtractedToolParameterSchema = Schema.Struct({48export const ExtractedToolParameterSchema = OpenApiToolParameterSchema;
54 name: Schema.String,49export const ExtractedToolRequestBodySchema = OpenApiToolRequestBodySchema;
55 location: Schema.Literal(...OPEN_API_PARAMETER_LOCATIONS),50export const ExtractedToolInvocationSchema = OpenApiInvocationPayloadSchema;
56 required: Schema.Boolean,51export const ExtractedToolSchema = OpenApiExtractedToolSchema;
57});52export const ToolManifestSchema = OpenApiToolManifestSchema;
58
59export const ExtractedToolRequestBodySchema = Schema.Struct({
60 required: Schema.Boolean,
61 contentTypes: Schema.Array(Schema.String),
62});
63
64export const ExtractedToolInvocationSchema = Schema.Struct({
65 method: Schema.String,
66 pathTemplate: Schema.String,
67 parameters: Schema.Array(ExtractedToolParameterSchema),
68 requestBody: Schema.NullOr(ExtractedToolRequestBodySchema),
69});
70
71export const ExtractedToolSchema = Schema.Struct({
72 toolId: Schema.String,
73 name: Schema.String,
74 description: Schema.NullOr(Schema.String),
75 method: Schema.String,
76 path: Schema.String,
77 invocation: ExtractedToolInvocationSchema,
78 operationHash: Schema.String,
79});
80 53
81export type ExtractedToolParameter = typeof ExtractedToolParameterSchema.Type;54export type ExtractedToolParameter = OpenApiToolParameter;
82export type ExtractedToolRequestBody = typeof ExtractedToolRequestBodySchema.Type;55export type ExtractedToolRequestBody = OpenApiToolRequestBody;
83export type ExtractedToolInvocation = typeof ExtractedToolInvocationSchema.Type;56export type ExtractedToolInvocation = OpenApiInvocationPayload;
84export type ExtractedTool = typeof ExtractedToolSchema.Type;57export type ExtractedTool = OpenApiExtractedTool;
85 58export type ToolManifest = OpenApiToolManifest;
86export const ToolManifestSchema = Schema.Struct({
87 version: Schema.Literal(1),
88 sourceHash: Schema.String,
89 tools: Schema.Array(ExtractedToolSchema),
90});
91 59
92const ToolManifestFromJsonSchema = Schema.parseJson(ToolManifestSchema);60const ToolManifestFromJsonSchema = Schema.parseJson(ToolManifestSchema);
93const encodeManifestToJson = Schema.encode(ToolManifestFromJsonSchema);61const encodeManifestToJson = Schema.encode(ToolManifestFromJsonSchema);
94const decodeToolArtifactId = Schema.decodeUnknownSync(ToolArtifactIdSchema);62const decodeToolArtifactId = Schema.decodeUnknownSync(ToolArtifactIdSchema);
95 63
96export type ToolManifest = typeof ToolManifestSchema.Type;
97
98export type ToolManifestDiff = {64export type ToolManifestDiff = {
99 added: Array<string>;65 added: Array<string>;
100 changed: Array<string>;66 changed: Array<string>;