History

deno subprocess

Rhys Sullivan committed 8 months ago

Commit f6167a71 parent be73cb111 files · +964 −163Browse files

Unified · Split

11 files
  1. v2/packages/engine/README.md +2 −1
  2. v2/packages/engine/src/deno-subprocess-runner.test.ts +95 −0
  3. v2/packages/engine/src/deno-subprocess-runner.ts +459 −0
  4. v2/packages/engine/src/deno-subprocess-worker.mjs +143 −0
  5. v2/packages/engine/src/index.ts +7 −0
  6. v2/packages/engine/src/openapi-provider.ts +35 −76
  7. v2/packages/engine/src/tool-providers.test.ts +2 −0
  8. v2/packages/engine/src/tool-providers.ts +15 −27
  9. v2/packages/schema/src/index.ts +1 −0
  10. v2/packages/schema/src/models/runtime-tool.ts +180 −0
  11. v2/packages/source-manager/src/openapi-extraction.ts +25 −59
v2/packages/engine/README.md+2−1
@@ -6,5 +6,6 @@ Current scaffold includes: · 5 unmodified lines
66 - provider contracts (`ToolProvider`) and canonical tool descriptor model
77 - provider registry service (`ToolProviderRegistryService`) with `discover`/`invoke` routing
88 - OpenAPI provider invocation and manifest-to-descriptor conversion helpers
9−- minimal local JavaScript runner with `tools.*` proxy dispatch into provider registry
9+- minimal in-process JavaScript runner with `tools.*` proxy dispatch into provider registry
10+- Deno subprocess runner with line-delimited JSON IPC for proxied `tools.*` calls
1011 - vertical integration test covering OpenAPI extraction -> descriptor conversion -> code execution -> HTTP call
v2/packages/engine/src/deno-subprocess-runner.test.ts+95−0
@@ -0,0 +1,95 @@
1+import {
2+ CanonicalToolDescriptorSchema,
3+ type CanonicalToolDescriptor,
4+} from "@executor-v2/schema";
5+import { describe, expect, test } from "bun:test";
6+import * as Effect from "effect/Effect";
7+import * as Schema from "effect/Schema";
8+
9+import {
10+ DenoSubprocessRunnerError,
11+ executeJavaScriptInDenoSubprocess,
12+} from "./deno-subprocess-runner";
13+import {
14+ makeToolProviderRegistry,
15+ ToolProviderRegistryService,
16+ type ToolProvider,
17+} from "./tool-providers";
18+
19+const decodeCanonicalToolDescriptor = Schema.decodeUnknownSync(
20+ CanonicalToolDescriptorSchema,
21+);
22+
23+const sumToolDescriptor: CanonicalToolDescriptor = decodeCanonicalToolDescriptor({
24+ providerKind: "in_memory",
25+ sourceId: null,
26+ workspaceId: null,
27+ toolId: "sum",
28+ name: "sum",
29+ description: null,
30+ invocationMode: "in_memory",
31+ availability: "local_only",
32+ providerPayload: null,
33+});
34+
35+describe("executeJavaScriptInDenoSubprocess", () => {
36+ test("runs code in Deno subprocess and proxies tool calls", async () => {
37+ const sumProvider: ToolProvider = {
38+ kind: "in_memory",
39+ invoke: (input) =>
40+ Effect.gen(function* () {
41+ if (input.tool.toolId !== "sum") {
42+ return {
43+ output: `unknown tool: ${input.tool.toolId}`,
44+ isError: true,
45+ } as const;
46+ }
47+
48+ const args = input.args as { a: number; b: number };
49+ return {
50+ output: args.a + args.b,
51+ isError: false,
52+ } as const;
53+ }),
54+ };
55+
56+ const registry = makeToolProviderRegistry([sumProvider]);
57+
58+ const result = await Effect.runPromise(
59+ executeJavaScriptInDenoSubprocess({
60+ code: "console.log('hello'); return await tools.sum({ a: 2, b: 3 });",
61+ tools: [
62+ {
63+ descriptor: sumToolDescriptor,
64+ source: null,
65+ },
66+ ],
67+ timeoutMs: 10_000,
68+ }).pipe(Effect.provideService(ToolProviderRegistryService, registry)),
69+ );
70+
71+ expect(result).toBe(5);
72+ });
73+
74+ test("returns a typed error when Deno executable is missing", async () => {
75+ const registry = makeToolProviderRegistry([]);
76+
77+ const result = await Effect.runPromise(
78+ Effect.either(
79+ executeJavaScriptInDenoSubprocess({
80+ code: "return 1;",
81+ tools: [],
82+ denoExecutable: "/definitely-missing-deno-binary",
83+ timeoutMs: 1_000,
84+ }).pipe(Effect.provideService(ToolProviderRegistryService, registry)),
85+ ),
86+ );
87+
88+ if (result._tag === "Right") {
89+ throw new Error("expected missing Deno executable to fail");
90+ }
91+
92+ expect(result.left).toBeInstanceOf(DenoSubprocessRunnerError);
93+ expect(result.left.operation).toBe("spawn");
94+ });
95+});
v2/packages/engine/src/deno-subprocess-runner.ts+459−0
@@ -0,0 +1,459 @@
1+import { existsSync } from "node:fs";
2+import { spawn } from "node:child_process";
3+import { fileURLToPath } from "node:url";
4+
5+import type { Source } from "@executor-v2/schema";
6+import * as Data from "effect/Data";
7+import * as Effect from "effect/Effect";
8+import * as Runtime from "effect/Runtime";
9+import * as Schema from "effect/Schema";
10+
11+import {
12+ ToolProviderRegistryError,
13+ ToolProviderRegistryService,
14+ type CanonicalToolDescriptor,
15+ type ToolProviderError,
16+} from "./tool-providers";
17+
18+export class DenoSubprocessRunnerError extends Data.TaggedError(
19+ "DenoSubprocessRunnerError",
20+)<{
21+ operation: string;
22+ message: string;
23+ details: string | null;
24+}> {}
25+
26+export type DenoRunnableTool = {
27+ descriptor: CanonicalToolDescriptor;
28+ source: Source | null;
29+};
30+
31+export type ExecuteJavaScriptInDenoInput = {
32+ code: string;
33+ tools: ReadonlyArray<DenoRunnableTool>;
34+ timeoutMs?: number;
35+ denoExecutable?: string;
36+};
37+
38+const IPC_PREFIX = "@@engine-ipc@@";
39+
40+const HostStartMessageSchema = Schema.Struct({
41+ type: Schema.Literal("start"),
42+ code: Schema.String,
43+ toolIds: Schema.Array(Schema.String),
44+});
45+
46+const HostToolResultMessageSchema = Schema.Struct({
47+ type: Schema.Literal("tool_result"),
48+ requestId: Schema.String,
49+ ok: Schema.Boolean,
50+ value: Schema.optional(Schema.Unknown),
51+ error: Schema.optional(Schema.String),
52+});
53+
54+const HostToWorkerMessageSchema = Schema.Union(
55+ HostStartMessageSchema,
56+ HostToolResultMessageSchema,
57+);
58+
59+const WorkerToolCallMessageSchema = Schema.Struct({
60+ type: Schema.Literal("tool_call"),
61+ requestId: Schema.String,
62+ toolId: Schema.String,
63+ args: Schema.Unknown,
64+});
65+
66+const WorkerCompletedMessageSchema = Schema.Struct({
67+ type: Schema.Literal("completed"),
68+ result: Schema.Unknown,
69+});
70+
71+const WorkerFailedMessageSchema = Schema.Struct({
72+ type: Schema.Literal("failed"),
73+ error: Schema.String,
74+});
75+
76+const WorkerToHostMessageSchema = Schema.Union(
77+ WorkerToolCallMessageSchema,
78+ WorkerCompletedMessageSchema,
79+ WorkerFailedMessageSchema,
80+);
81+
82+type HostToWorkerMessage = typeof HostToWorkerMessageSchema.Type;
83+type WorkerToHostMessage = typeof WorkerToHostMessageSchema.Type;
84+type WorkerToolCallMessage = typeof WorkerToolCallMessageSchema.Type;
85+
86+const decodeWorkerMessageLine = Schema.decodeUnknownSync(
87+ Schema.parseJson(WorkerToHostMessageSchema),
88+);
89+const encodeHostMessage = Schema.encodeSync(Schema.parseJson(HostToWorkerMessageSchema));
90+
91+const duplicateToolIdError = (toolId: string): DenoSubprocessRunnerError =>
92+ new DenoSubprocessRunnerError({
93+ operation: "build_tools",
94+ message: `Duplicate tool id in run context: ${toolId}`,
95+ details: null,
96+ });
97+
98+const missingToolBindingError = (toolId: string): DenoSubprocessRunnerError =>
99+ new DenoSubprocessRunnerError({
100+ operation: "invoke_tool",
101+ message: `Worker requested unknown tool id: ${toolId}`,
102+ details: null,
103+ });
104+
105+const toolCallFailedError = (toolId: string): DenoSubprocessRunnerError =>
106+ new DenoSubprocessRunnerError({
107+ operation: "invoke_tool",
108+ message: `Tool call returned error: ${toolId}`,
109+ details: null,
110+ });
111+
112+const parseWorkerMessageError = (
113+ line: string,
114+ cause: unknown,
115+): DenoSubprocessRunnerError =>
116+ new DenoSubprocessRunnerError({
117+ operation: "decode_worker_message",
118+ message: "Failed to decode worker message",
119+ details: `${line}\n${String(cause)}`,
120+ });
121+
122+const workerProcessError = (
123+ operation: string,
124+ message: string,
125+ details: string | null,
126+): DenoSubprocessRunnerError =>
127+ new DenoSubprocessRunnerError({
128+ operation,
129+ message,
130+ details,
131+ });
132+
133+const buildToolBindings = (
134+ tools: ReadonlyArray<DenoRunnableTool>,
135+): Effect.Effect<Map<string, DenoRunnableTool>, DenoSubprocessRunnerError> =>
136+ Effect.gen(function* () {
137+ const byToolId = new Map<string, DenoRunnableTool>();
138+
139+ for (const tool of tools) {
140+ const toolId = tool.descriptor.toolId;
141+ if (byToolId.has(toolId)) {
142+ return yield* duplicateToolIdError(toolId);
143+ }
144+ byToolId.set(toolId, tool);
145+ }
146+
147+ return byToolId;
148+ });
149+
150+const defaultDenoExecutable = (): string => {
151+ const configured = process.env.DENO_BIN?.trim();
152+ if (configured) {
153+ return configured;
154+ }
155+
156+ const home = process.env.HOME?.trim();
157+ if (home) {
158+ const installedPath = `${home}/.deno/bin/deno`;
159+ if (existsSync(installedPath)) {
160+ return installedPath;
161+ }
162+ }
163+
164+ return "deno";
165+};
166+
167+const workerScriptPath = fileURLToPath(
168+ new URL("./deno-subprocess-worker.mjs", import.meta.url),
169+);
170+
171+const writeMessage = (
172+ stdin: NodeJS.WritableStream,
173+ message: HostToWorkerMessage,
174+): Effect.Effect<void, DenoSubprocessRunnerError> =>
175+ Effect.try({
176+ try: () => {
177+ stdin.write(`${encodeHostMessage(message)}\n`);
178+ },
179+ catch: (cause) =>
180+ workerProcessError(
181+ "write_message",
182+ "Failed to write message to Deno subprocess",
183+ String(cause),
184+ ),
185+ });
186+
187+const handleToolCall = (
188+ message: WorkerToolCallMessage,
189+ toolBindings: ReadonlyMap<string, DenoRunnableTool>,
190+ runPromise: <A, E>(
191+ effect: Effect.Effect<A, E, never>,
192+ ) => Promise<A>,
193+ invokeTool: (
194+ input: {
195+ source: Source | null;
196+ tool: CanonicalToolDescriptor;
197+ args: unknown;
198+ },
199+ ) => Effect.Effect<
200+ { output: unknown; isError: boolean },
201+ ToolProviderRegistryError | ToolProviderError
202+ >,
203+ stdin: NodeJS.WritableStream,
204+): Effect.Effect<void, DenoSubprocessRunnerError> =>
205+ Effect.gen(function* () {
206+ const binding = toolBindings.get(message.toolId);
207+ if (!binding) {
208+ return yield* writeMessage(stdin, {
209+ type: "tool_result",
210+ requestId: message.requestId,
211+ ok: false,
212+ error: missingToolBindingError(message.toolId).message,
213+ });
214+ }
215+
216+ const invokeResult = yield* Effect.tryPromise({
217+ try: () =>
218+ runPromise(
219+ invokeTool({
220+ source: binding.source,
221+ tool: binding.descriptor,
222+ args: message.args,
223+ }),
224+ ),
225+ catch: (cause) =>
226+ workerProcessError(
227+ "invoke_tool",
228+ "Tool invocation threw while handling worker tool_call",
229+ String(cause),
230+ ),
231+ });
232+
233+ if (invokeResult.isError) {
234+ yield* writeMessage(stdin, {
235+ type: "tool_result",
236+ requestId: message.requestId,
237+ ok: false,
238+ error: toolCallFailedError(message.toolId).message,
239+ });
240+ return;
241+ }
242+
243+ yield* writeMessage(stdin, {
244+ type: "tool_result",
245+ requestId: message.requestId,
246+ ok: true,
247+ value: invokeResult.output,
248+ });
249+ });
250+
251+export const isDenoSubprocessRuntimeAvailable = (
252+ executable: string = defaultDenoExecutable(),
253+): boolean => executable.includes("/") ? existsSync(executable) : true;
254+
255+export const executeJavaScriptInDenoSubprocess = (
256+ input: ExecuteJavaScriptInDenoInput,
257+): Effect.Effect<
258+ unknown,
259+ DenoSubprocessRunnerError | ToolProviderRegistryError | ToolProviderError,
260+ ToolProviderRegistryService
261+> =>
262+ Effect.gen(function* () {
263+ const registry = yield* ToolProviderRegistryService;
264+ const runtime = yield* Effect.runtime<never>();
265+ const runPromise = Runtime.runPromise(runtime);
266+ const toolBindings = yield* buildToolBindings(input.tools);
267+ const toolIds = Array.from(toolBindings.keys()).sort();
268+
269+ const denoExecutable = input.denoExecutable ?? defaultDenoExecutable();
270+ const timeoutMs = Math.max(100, input.timeoutMs ?? 30_000);
271+
272+ return yield* Effect.tryPromise({
273+ try: () =>
274+ new Promise<unknown>((resolve, reject) => {
275+ const child = spawn(
276+ denoExecutable,
277+ [
278+ "run",
279+ "--quiet",
280+ "--no-prompt",
281+ "--no-check",
282+ workerScriptPath,
283+ ],
284+ {
285+ stdio: ["pipe", "pipe", "pipe"],
286+ },
287+ );
288+
289+ let settled = false;
290+ let stdoutBuffer = "";
291+ let stderrBuffer = "";
292+
293+ const finish = (result: { ok: true; value: unknown } | { ok: false; error: Error }) => {
294+ if (settled) {
295+ return;
296+ }
297+
298+ settled = true;
299+ clearTimeout(timeout);
300+ child.stdout?.removeAllListeners();
301+ child.stderr?.removeAllListeners();
302+ child.removeAllListeners();
303+
304+ if (!child.killed) {
305+ child.kill("SIGKILL");
306+ }
307+
308+ if (result.ok) {
309+ resolve(result.value);
310+ } else {
311+ reject(result.error);
312+ }
313+ };
314+
315+ const fail = (error: DenoSubprocessRunnerError) => {
316+ finish({ ok: false, error });
317+ };
318+
319+ const timeout = setTimeout(() => {
320+ fail(
321+ workerProcessError(
322+ "timeout",
323+ `Deno subprocess execution timed out after ${timeoutMs}ms`,
324+ stderrBuffer.length > 0 ? stderrBuffer : null,
325+ ),
326+ );
327+ }, timeoutMs);
328+
329+ child.on("error", (cause) => {
330+ fail(
331+ workerProcessError(
332+ "spawn",
333+ "Failed to spawn Deno subprocess",
334+ cause instanceof Error ? cause.message : String(cause),
335+ ),
336+ );
337+ });
338+
339+ child.on("exit", (code, signal) => {
340+ if (settled) {
341+ return;
342+ }
343+
344+ fail(
345+ workerProcessError(
346+ "process_exit",
347+ "Deno subprocess exited before returning terminal message",
348+ `code=${String(code)} signal=${String(signal)} stderr=${stderrBuffer}`,
349+ ),
350+ );
351+ });
352+
353+ if (child.stderr) {
354+ child.stderr.setEncoding("utf8");
355+ child.stderr.on("data", (chunk: string) => {
356+ stderrBuffer += chunk;
357+ });
358+ }
359+
360+ if (child.stdout) {
361+ child.stdout.setEncoding("utf8");
362+ child.stdout.on("data", (chunk: string) => {
363+ stdoutBuffer += chunk;
364+
365+ while (true) {
366+ const newlineIndex = stdoutBuffer.indexOf("\n");
367+ if (newlineIndex === -1) {
368+ break;
369+ }
370+
371+ const line = stdoutBuffer.slice(0, newlineIndex).trim();
372+ stdoutBuffer = stdoutBuffer.slice(newlineIndex + 1);
373+
374+ if (line.length === 0) {
375+ continue;
376+ }
377+
378+ if (!line.startsWith(IPC_PREFIX)) {
379+ continue;
380+ }
381+
382+ const payload = line.slice(IPC_PREFIX.length);
383+
384+ let message: WorkerToHostMessage;
385+ try {
386+ message = decodeWorkerMessageLine(payload);
387+ } catch (cause) {
388+ fail(parseWorkerMessageError(payload, cause));
389+ return;
390+ }
391+
392+ if (message.type === "tool_call") {
393+ runPromise(
394+ handleToolCall(
395+ message,
396+ toolBindings,
397+ runPromise,
398+ (invokeInput) => registry.invoke(invokeInput),
399+ child.stdin,
400+ ),
401+ ).catch((cause) => {
402+ fail(
403+ cause instanceof DenoSubprocessRunnerError
404+ ? cause
405+ : workerProcessError(
406+ "handle_tool_call",
407+ "Failed handling worker tool_call",
408+ String(cause),
409+ ),
410+ );
411+ });
412+ continue;
413+ }
414+
415+ if (message.type === "completed") {
416+ finish({ ok: true, value: message.result });
417+ return;
418+ }
419+
420+ fail(
421+ workerProcessError(
422+ "worker_failed",
423+ "Deno subprocess returned failed terminal message",
424+ message.error,
425+ ),
426+ );
427+ return;
428+ }
429+ });
430+ }
431+
432+ runPromise(
433+ writeMessage(child.stdin, {
434+ type: "start",
435+ code: input.code,
436+ toolIds,
437+ }),
438+ ).catch((cause) => {
439+ fail(
440+ cause instanceof DenoSubprocessRunnerError
441+ ? cause
442+ : workerProcessError(
443+ "start_message",
444+ "Failed sending start message to Deno subprocess",
445+ String(cause),
446+ ),
447+ );
448+ });
449+ }),
450+ catch: (cause) =>
451+ cause instanceof DenoSubprocessRunnerError
452+ ? cause
453+ : workerProcessError(
454+ "execute",
455+ "Unexpected Deno subprocess execution failure",
456+ String(cause),
457+ ),
458+ });
459+ });
v2/packages/engine/src/deno-subprocess-worker.mjs+143−0
@@ -0,0 +1,143 @@
1+const encoder = new TextEncoder();
2+const IPC_PREFIX = "@@engine-ipc@@";
3+
4+const pendingToolCalls = new Map();
5+let started = false;
6+
7+const writeMessage = (message) => {
8+ const payload = `${IPC_PREFIX}${JSON.stringify(message)}\n`;
9+ Deno.stdout.writeSync(encoder.encode(payload));
10+};
11+
12+const toErrorMessage = (error) => {
13+ if (error instanceof Error) {
14+ return error.stack ?? error.message;
15+ }
16+
17+ return String(error);
18+};
19+
20+const createToolCaller = (toolId) => (args) =>
21+ new Promise((resolve, reject) => {
22+ const requestId = crypto.randomUUID();
23+ pendingToolCalls.set(requestId, { resolve, reject });
24+
25+ writeMessage({
26+ type: "tool_call",
27+ requestId,
28+ toolId,
29+ args: args === undefined ? {} : args,
30+ });
31+ });
32+
33+const runUserCode = async (code, toolIds) => {
34+ const tools = Object.create(null);
35+ for (const toolId of toolIds) {
36+ tools[toolId] = createToolCaller(toolId);
37+ }
38+
39+ const execute = new Function(
40+ "tools",
41+ `"use strict"; return (async () => {\n${code}\n})();`,
42+ );
43+
44+ return await execute(tools);
45+};
46+
47+const handleStart = (message) => {
48+ if (started) {
49+ writeMessage({
50+ type: "failed",
51+ error: "start message already received",
52+ });
53+ return;
54+ }
55+
56+ started = true;
57+
58+ runUserCode(message.code, message.toolIds)
59+ .then((result) => {
60+ writeMessage({
61+ type: "completed",
62+ result,
63+ });
64+ })
65+ .catch((error) => {
66+ writeMessage({
67+ type: "failed",
68+ error: toErrorMessage(error),
69+ });
70+ });
71+};
72+
73+const handleToolResult = (message) => {
74+ const pending = pendingToolCalls.get(message.requestId);
75+ if (!pending) {
76+ return;
77+ }
78+
79+ pendingToolCalls.delete(message.requestId);
80+
81+ if (message.ok) {
82+ pending.resolve(message.value);
83+ return;
84+ }
85+
86+ pending.reject(new Error(message.error));
87+};
88+
89+const handleHostMessage = (message) => {
90+ if (!message || typeof message !== "object") {
91+ return;
92+ }
93+
94+ if (message.type === "start") {
95+ handleStart(message);
96+ return;
97+ }
98+
99+ if (message.type === "tool_result") {
100+ handleToolResult(message);
101+ }
102+};
103+
104+const decodeLines = async () => {
105+ const reader = Deno.stdin.readable.getReader();
106+ const decoder = new TextDecoder();
107+ let buffer = "";
108+
109+ while (true) {
110+ const { value, done } = await reader.read();
111+ if (done) {
112+ break;
113+ }
114+
115+ buffer += decoder.decode(value, { stream: true });
116+
117+ while (true) {
118+ const lineEnd = buffer.indexOf("\n");
119+ if (lineEnd === -1) {
120+ break;
121+ }
122+
123+ const line = buffer.slice(0, lineEnd).trim();
124+ buffer = buffer.slice(lineEnd + 1);
125+
126+ if (line.length === 0) {
127+ continue;
128+ }
129+
130+ try {
131+ const message = JSON.parse(line);
132+ handleHostMessage(message);
133+ } catch (error) {
134+ writeMessage({
135+ type: "failed",
136+ error: `invalid host message: ${toErrorMessage(error)}`,
137+ });
138+ }
139+ }
140+ }
141+};
142+
143+await decodeLines();
v2/packages/engine/src/index.ts+7−0
@@ -1,3 +1,10 @@
1+export {
2+ executeJavaScriptInDenoSubprocess,
3+ isDenoSubprocessRuntimeAvailable,
4+ DenoSubprocessRunnerError,
5+ type DenoRunnableTool,
6+ type ExecuteJavaScriptInDenoInput,
7+} from "./deno-subprocess-runner";
18 export {
29 executeJavaScriptWithTools,
310 LocalCodeRunnerError,
v2/packages/engine/src/openapi-provider.ts+35−76
@@ -1,67 +1,18 @@
1−import type { Source } from "@executor-v2/schema";
1+import {
2+ OpenApiInvocationPayloadSchema,
3+ OpenApiToolManifestSchema,
4+ type CanonicalToolDescriptor,
5+ type OpenApiCanonicalToolDescriptor,
6+ type OpenApiInvocationPayload,
7+ type OpenApiToolManifest,
8+ type Source,
9+} from "@executor-v2/schema";
210 import * as Effect from "effect/Effect";
311 import { pipe } from "effect/Function";
412 import * as ParseResult from "effect/ParseResult";
513 import * as Schema from "effect/Schema";
614
7−import {
8− ToolProviderError,
9− type CanonicalToolDescriptor,
10− type ToolProvider,
11−} from "./tool-providers";
12−
13−const OPEN_API_PARAMETER_LOCATIONS = [
14− "path",
15− "query",
16− "header",
17− "cookie",
18−] as const;
19−
20−const OPEN_API_HTTP_METHODS = [
21− "get",
22− "put",
23− "post",
24− "delete",
25− "patch",
26− "head",
27− "options",
28− "trace",
29−] as const;
30−
31−const OpenApiInvocationPayloadSchema = Schema.Struct({
32− method: Schema.Literal(...OPEN_API_HTTP_METHODS),
33− pathTemplate: Schema.String,
34− parameters: Schema.Array(
35− Schema.Struct({
36− name: Schema.String,
37− location: Schema.Literal(...OPEN_API_PARAMETER_LOCATIONS),
38− required: Schema.Boolean,
39− }),
40− ),
41− requestBody: Schema.NullOr(
42− Schema.Struct({
43− required: Schema.Boolean,
44− contentTypes: Schema.Array(Schema.String),
45− }),
46− ),
47−});
48−
49−type OpenApiInvocationPayload = typeof OpenApiInvocationPayloadSchema.Type;
50−
51−const OpenApiManifestSchema = Schema.Struct({
52− version: Schema.Literal(1),
53− sourceHash: Schema.String,
54− tools: Schema.Array(
55− Schema.Struct({
56− toolId: Schema.String,
57− name: Schema.String,
58− description: Schema.NullOr(Schema.String),
59− invocation: OpenApiInvocationPayloadSchema,
60− }),
61− ),
62−});
63−
64−type OpenApiManifest = typeof OpenApiManifestSchema.Type;
15+import { ToolProviderError, type ToolProvider } from "./tool-providers";
6516
6617 const OpenApiToolArgsSchema = Schema.Record({
6718 key: Schema.String,
@@ -71,12 +22,13 @@ const OpenApiToolArgsSchema = Schema.Record({ · 3 unmodified lines
7122 type OpenApiToolArgs = typeof OpenApiToolArgsSchema.Type;
7223
7324 const decodeOpenApiManifestJson = Schema.decodeUnknown(
74− Schema.parseJson(OpenApiManifestSchema),
25+ Schema.parseJson(OpenApiToolManifestSchema),
7526 );
7627 const decodeOpenApiInvocationPayload = Schema.decodeUnknown(
7728 OpenApiInvocationPayloadSchema,
7829 );
7930 const decodeOpenApiToolArgs = Schema.decodeUnknown(OpenApiToolArgsSchema);
31+const encodeUnknownToJson = Schema.encode(Schema.parseJson(Schema.Unknown));
8032
8133 const toOpenApiProviderError = (
8234 operation: string,
@@ -108,7 +60,7 @@ const argsValueToString = (value: unknown): string => { · 25 unmodified lines
10860 return String(value);
10961 }
11062
111− return JSON.stringify(value);
63+ return String(value);
11264 };
11365
11466 const replacePathTemplate = (
@@ -214,7 +166,16 @@ const buildFetchRequest = ( · 99 unmodified lines
214166 });
215167 }
216168 } else {
217− body = JSON.stringify(args.body);
169+ body = yield* pipe(
170+ encodeUnknownToJson(args.body),
171+ Effect.mapError((cause) =>
172+ toOpenApiProviderError(
173+ "invoke.encode_body",
174+ "Failed to encode request body as JSON",
175+ cause,
176+ ),
177+ ),
178+ );
218179
219180 const preferredContentType = payload.requestBody.contentTypes[0];
220181 if (preferredContentType) {
@@ -269,20 +230,18 @@ export const openApiToolDescriptorsFromManifest = ( · 48 unmodified lines
269230 cause,
270231 ),
271232 ),
272− Effect.map((manifest: OpenApiManifest) =>
273− manifest.tools.map(
274− (tool): CanonicalToolDescriptor => ({
275− providerKind: "openapi",
276− sourceId: source.id,
277− workspaceId: source.workspaceId,
278− toolId: tool.toolId,
279− name: tool.name,
280− description: tool.description,
281− invocationMode: "http",
282− availability: "remote_capable",
283− providerPayload: tool.invocation,
284− }),
285− ),
233+ Effect.map((manifest: OpenApiToolManifest) =>
234+ manifest.tools.map((tool): OpenApiCanonicalToolDescriptor => ({
235+ providerKind: "openapi",
236+ sourceId: source.id,
237+ workspaceId: source.workspaceId,
238+ toolId: tool.toolId,
239+ name: tool.name,
240+ description: tool.description,
241+ invocationMode: "http",
242+ availability: "remote_capable",
243+ providerPayload: tool.invocation,
244+ })),
286245 ),
287246 );
288247
v2/packages/engine/src/tool-providers.test.ts+2−0
@@ -48,6 +48,8 @@ describe("makeToolProviderRegistry", () => { · 47 unmodified lines
4848 providerPayload: {
4949 method: "get",
5050 pathTemplate: "/healthz",
51+ parameters: [],
52+ requestBody: null,
5153 },
5254 },
5355 ],
v2/packages/engine/src/tool-providers.ts+15−27
@@ -1,31 +1,22 @@
1−import type { Source, SourceKind } from "@executor-v2/schema";
1+import type {
2+ CanonicalToolDescriptor as SchemaCanonicalToolDescriptor,
3+ Source,
4+ ToolAvailability as SchemaToolAvailability,
5+ ToolDiscoveryResult as SchemaToolDiscoveryResult,
6+ ToolInvocationMode as SchemaToolInvocationMode,
7+ ToolInvokeResult,
8+ ToolProviderKind as SchemaToolProviderKind,
9+} from "@executor-v2/schema";
210 import * as Context from "effect/Context";
311 import * as Data from "effect/Data";
412 import * as Effect from "effect/Effect";
513 import * as Layer from "effect/Layer";
614
7−export type ToolProviderKind = SourceKind | "in_memory";
8−
9−export type ToolInvocationMode = "http" | "mcp" | "graphql" | "in_memory";
10−
11−export type ToolAvailability = "local_only" | "remote_capable";
12−
13−export type CanonicalToolDescriptor = {
14− providerKind: ToolProviderKind;
15− sourceId: Source["id"] | null;
16− workspaceId: Source["workspaceId"] | null;
17− toolId: string;
18− name: string;
19− description: string | null;
20− invocationMode: ToolInvocationMode;
21− availability: ToolAvailability;
22− providerPayload: unknown;
23−};
24−
25−export type ToolDiscoveryResult = {
26− sourceHash: string | null;
27− tools: ReadonlyArray<CanonicalToolDescriptor>;
28−};
15+export type ToolAvailability = SchemaToolAvailability;
16+export type ToolInvocationMode = SchemaToolInvocationMode;
17+export type ToolProviderKind = SchemaToolProviderKind;
18+export type CanonicalToolDescriptor = SchemaCanonicalToolDescriptor;
19+export type ToolDiscoveryResult = SchemaToolDiscoveryResult;
2920
3021 export type InvokeToolInput = {
3122 source: Source | null;
@@ -33,10 +24,7 @@ export type InvokeToolInput = { · 1 unmodified line
3324 args: unknown;
3425 };
3526
36−export type InvokeToolResult = {
37− output: unknown;
38− isError: boolean;
39−};
27+export type InvokeToolResult = ToolInvokeResult;
4028
4129 export class ToolProviderError extends Data.TaggedError("ToolProviderError")<{
4230 operation: string;
v2/packages/schema/src/index.ts+1−0
@@ -13,4 +13,5 @@ export * from "./models/approval"; · 12 unmodified lines
1313 export * from "./models/task-run";
1414 export * from "./models/sync-state";
1515 export * from "./models/event-envelope";
16+export * from "./models/runtime-tool";
1617
v2/packages/schema/src/models/runtime-tool.ts+180−0
@@ -0,0 +1,180 @@
1+import { Schema } from "effect";
2+
3+import { SourceKindSchema } from "../enums";
4+import { SourceIdSchema, WorkspaceIdSchema } from "../ids";
5+
6+export const OPEN_API_HTTP_METHODS = [
7+ "get",
8+ "put",
9+ "post",
10+ "delete",
11+ "patch",
12+ "head",
13+ "options",
14+ "trace",
15+] as const;
16+
17+export const OPEN_API_PARAMETER_LOCATIONS = [
18+ "path",
19+ "query",
20+ "header",
21+ "cookie",
22+] as const;
23+
24+export const ToolProviderKindSchema = Schema.Union(
25+ SourceKindSchema,
26+ Schema.Literal("in_memory"),
27+);
28+
29+export const ToolInvocationModeSchema = Schema.Literal(
30+ "http",
31+ "mcp",
32+ "graphql",
33+ "in_memory",
34+);
35+
36+export const ToolAvailabilitySchema = Schema.Literal(
37+ "local_only",
38+ "remote_capable",
39+);
40+
41+export const OpenApiHttpMethodSchema = Schema.Literal(...OPEN_API_HTTP_METHODS);
42+
43+export const OpenApiParameterLocationSchema = Schema.Literal(
44+ ...OPEN_API_PARAMETER_LOCATIONS,
45+);
46+
47+export const OpenApiToolParameterSchema = Schema.Struct({
48+ name: Schema.String,
49+ location: OpenApiParameterLocationSchema,
50+ required: Schema.Boolean,
51+});
52+
53+export const OpenApiToolRequestBodySchema = Schema.Struct({
54+ required: Schema.Boolean,
55+ contentTypes: Schema.Array(Schema.String),
56+});
57+
58+export const OpenApiInvocationPayloadSchema = Schema.Struct({
59+ method: OpenApiHttpMethodSchema,
60+ pathTemplate: Schema.String,
61+ parameters: Schema.Array(OpenApiToolParameterSchema),
62+ requestBody: Schema.NullOr(OpenApiToolRequestBodySchema),
63+});
64+
65+export const OpenApiExtractedToolSchema = Schema.Struct({
66+ toolId: Schema.String,
67+ name: Schema.String,
68+ description: Schema.NullOr(Schema.String),
69+ method: OpenApiHttpMethodSchema,
70+ path: Schema.String,
71+ invocation: OpenApiInvocationPayloadSchema,
72+ operationHash: Schema.String,
73+});
74+
75+export const OpenApiToolManifestSchema = Schema.Struct({
76+ version: Schema.Literal(1),
77+ sourceHash: Schema.String,
78+ tools: Schema.Array(OpenApiExtractedToolSchema),
79+});
80+
81+const SourceBackedCanonicalToolDescriptorBaseSchema = Schema.Struct({
82+ sourceId: SourceIdSchema,
83+ workspaceId: WorkspaceIdSchema,
84+ toolId: Schema.String,
85+ name: Schema.String,
86+ description: Schema.NullOr(Schema.String),
87+ availability: ToolAvailabilitySchema,
88+});
89+
90+export const OpenApiCanonicalToolDescriptorSchema =
91+ SourceBackedCanonicalToolDescriptorBaseSchema.pipe(
92+ Schema.extend(
93+ Schema.Struct({
94+ providerKind: Schema.Literal("openapi"),
95+ invocationMode: Schema.Literal("http"),
96+ providerPayload: OpenApiInvocationPayloadSchema,
97+ }),
98+ ),
99+ );
100+
101+export const McpCanonicalToolDescriptorSchema =
102+ SourceBackedCanonicalToolDescriptorBaseSchema.pipe(
103+ Schema.extend(
104+ Schema.Struct({
105+ providerKind: Schema.Literal("mcp"),
106+ invocationMode: Schema.Literal("mcp"),
107+ providerPayload: Schema.Unknown,
108+ }),
109+ ),
110+ );
111+
112+export const GraphqlCanonicalToolDescriptorSchema =
113+ SourceBackedCanonicalToolDescriptorBaseSchema.pipe(
114+ Schema.extend(
115+ Schema.Struct({
116+ providerKind: Schema.Literal("graphql"),
117+ invocationMode: Schema.Literal("graphql"),
118+ providerPayload: Schema.Unknown,
119+ }),
120+ ),
121+ );
122+
123+export const InternalCanonicalToolDescriptorSchema =
124+ SourceBackedCanonicalToolDescriptorBaseSchema.pipe(
125+ Schema.extend(
126+ Schema.Struct({
127+ providerKind: Schema.Literal("internal"),
128+ invocationMode: Schema.Literal("in_memory"),
129+ providerPayload: Schema.Unknown,
130+ }),
131+ ),
132+ );
133+
134+export const InMemoryCanonicalToolDescriptorSchema = Schema.Struct({
135+ providerKind: Schema.Literal("in_memory"),
136+ sourceId: Schema.Null,
137+ workspaceId: Schema.Null,
138+ toolId: Schema.String,
139+ name: Schema.String,
140+ description: Schema.NullOr(Schema.String),
141+ invocationMode: Schema.Literal("in_memory"),
142+ availability: ToolAvailabilitySchema,
143+ providerPayload: Schema.Unknown,
144+});
145+
146+export const CanonicalToolDescriptorSchema = Schema.Union(
147+ OpenApiCanonicalToolDescriptorSchema,
148+ McpCanonicalToolDescriptorSchema,
149+ GraphqlCanonicalToolDescriptorSchema,
150+ InternalCanonicalToolDescriptorSchema,
151+ InMemoryCanonicalToolDescriptorSchema,
152+);
153+
154+export const ToolDiscoveryResultSchema = Schema.Struct({
155+ sourceHash: Schema.NullOr(Schema.String),
156+ tools: Schema.Array(CanonicalToolDescriptorSchema),
157+});
158+
159+export const ToolInvokeResultSchema = Schema.Struct({
160+ output: Schema.Unknown,
161+ isError: Schema.Boolean,
162+});
163+
164+export type ToolProviderKind = typeof ToolProviderKindSchema.Type;
165+export type ToolInvocationMode = typeof ToolInvocationModeSchema.Type;
166+export type ToolAvailability = typeof ToolAvailabilitySchema.Type;
167+
168+export type OpenApiHttpMethod = typeof OpenApiHttpMethodSchema.Type;
169+export type OpenApiParameterLocation = typeof OpenApiParameterLocationSchema.Type;
170+export type OpenApiToolParameter = typeof OpenApiToolParameterSchema.Type;
171+export type OpenApiToolRequestBody = typeof OpenApiToolRequestBodySchema.Type;
172+export type OpenApiInvocationPayload = typeof OpenApiInvocationPayloadSchema.Type;
173+export type OpenApiExtractedTool = typeof OpenApiExtractedToolSchema.Type;
174+export type OpenApiToolManifest = typeof OpenApiToolManifestSchema.Type;
175+
176+export type OpenApiCanonicalToolDescriptor =
177+ typeof OpenApiCanonicalToolDescriptorSchema.Type;
178+export type CanonicalToolDescriptor = typeof CanonicalToolDescriptorSchema.Type;
179+export type ToolDiscoveryResult = typeof ToolDiscoveryResultSchema.Type;
180+export type ToolInvokeResult = typeof ToolInvokeResultSchema.Type;
v2/packages/source-manager/src/openapi-extraction.ts+25−59
@@ -5,7 +5,20 @@ import { · 4 unmodified lines
55 type ToolArtifactStore,
66 } from "@executor-v2/persistence-ports";
77 import {
8+ OPEN_API_HTTP_METHODS,
9+ OPEN_API_PARAMETER_LOCATIONS,
10+ OpenApiExtractedToolSchema,
11+ OpenApiInvocationPayloadSchema,
12+ OpenApiToolManifestSchema,
13+ OpenApiToolParameterSchema,
14+ OpenApiToolRequestBodySchema,
815 ToolArtifactIdSchema,
16+ type OpenApiExtractedTool,
17+ type OpenApiHttpMethod,
18+ type OpenApiInvocationPayload,
19+ type OpenApiToolManifest,
20+ type OpenApiToolParameter,
21+ type OpenApiToolRequestBody,
922 type Source,
1023 type ToolArtifact,
1124 } from "@executor-v2/schema";
@@ -16,27 +29,9 @@ import * as Option from "effect/Option"; · 4 unmodified lines
1629 import * as ParseResult from "effect/ParseResult";
1730 import * as Schema from "effect/Schema";
1831
19−const HTTP_METHODS = [
20− "get",
21− "put",
22− "post",
23− "delete",
24− "patch",
25− "head",
26− "options",
27− "trace",
28−] as const;
32+const HTTP_METHODS = OPEN_API_HTTP_METHODS;
2933
30−type HttpMethod = (typeof HTTP_METHODS)[number];
31−
32−const OPEN_API_PARAMETER_LOCATIONS = [
33− "path",
34− "query",
35− "header",
36− "cookie",
37−] as const;
38−
39−type OpenApiParameterLocation = (typeof OPEN_API_PARAMETER_LOCATIONS)[number];
34+type HttpMethod = OpenApiHttpMethod;
4035
4136 type OpenApiExtractionStage =
4237 | "validate"
@@ -50,51 +45,22 @@ export class OpenApiExtractionError extends Data.TaggedError("OpenApiExtractionE · 7 unmodified lines
5045 details: string | null;
5146 }> {}
5247
53−export const ExtractedToolParameterSchema = Schema.Struct({
54− name: Schema.String,
55− location: Schema.Literal(...OPEN_API_PARAMETER_LOCATIONS),
56− required: Schema.Boolean,
57−});
58−
59−export const ExtractedToolRequestBodySchema = Schema.Struct({
60− required: Schema.Boolean,
61− contentTypes: Schema.Array(Schema.String),
62−});
63−
64−export const ExtractedToolInvocationSchema = Schema.Struct({
65− method: Schema.String,
66− pathTemplate: Schema.String,
67− parameters: Schema.Array(ExtractedToolParameterSchema),
68− requestBody: Schema.NullOr(ExtractedToolRequestBodySchema),
69−});
70−
71−export const ExtractedToolSchema = Schema.Struct({
72− toolId: Schema.String,
73− name: Schema.String,
74− description: Schema.NullOr(Schema.String),
75− method: Schema.String,
76− path: Schema.String,
77− invocation: ExtractedToolInvocationSchema,
78− operationHash: Schema.String,
79−});
48+export const ExtractedToolParameterSchema = OpenApiToolParameterSchema;
49+export const ExtractedToolRequestBodySchema = OpenApiToolRequestBodySchema;
50+export const ExtractedToolInvocationSchema = OpenApiInvocationPayloadSchema;
51+export const ExtractedToolSchema = OpenApiExtractedToolSchema;
52+export const ToolManifestSchema = OpenApiToolManifestSchema;
8053
81−export type ExtractedToolParameter = typeof ExtractedToolParameterSchema.Type;
82−export type ExtractedToolRequestBody = typeof ExtractedToolRequestBodySchema.Type;
83−export type ExtractedToolInvocation = typeof ExtractedToolInvocationSchema.Type;
84−export type ExtractedTool = typeof ExtractedToolSchema.Type;
85−
86−export const ToolManifestSchema = Schema.Struct({
87− version: Schema.Literal(1),
88− sourceHash: Schema.String,
89− tools: Schema.Array(ExtractedToolSchema),
90−});
54+export type ExtractedToolParameter = OpenApiToolParameter;
55+export type ExtractedToolRequestBody = OpenApiToolRequestBody;
56+export type ExtractedToolInvocation = OpenApiInvocationPayload;
57+export type ExtractedTool = OpenApiExtractedTool;
58+export type ToolManifest = OpenApiToolManifest;
9159
9260 const ToolManifestFromJsonSchema = Schema.parseJson(ToolManifestSchema);
9361 const encodeManifestToJson = Schema.encode(ToolManifestFromJsonSchema);
9462 const decodeToolArtifactId = Schema.decodeUnknownSync(ToolArtifactIdSchema);
9563
96−export type ToolManifest = typeof ToolManifestSchema.Type;
97−
9864 export type ToolManifestDiff = {
9965 added: Array<string>;
10066 changed: Array<string>;