History

Refine executor app structure and openapi flow

Rhys Sullivan committed 8 months ago

Commit c98fa621 parent 93ee28f14 files · +792 −301Browse files

Unified · Split

14 files
  1. apps/executor/bin/executor +1 −1
  2. apps/executor/package.json +2 −1
  3. apps/executor/src/cli/dev.ts +200 −0
  4. apps/executor/src/cli/main.ts +92 −6
  5. apps/executor/src/control-plane-execution-resolver.ts +0 −182
  6. apps/executor/src/dev.ts +0 −105
  7. apps/executor/src/index.ts +0 −2
  8. apps/executor/src/server/config.ts +0 −0
  9. apps/executor/src/server/control-plane-execution-resolver.ts +281 −0
  10. apps/executor/src/server/index.ts +0 −0
  11. apps/executor/src/server/server.test.ts +144 −2
  12. bun.lock +1 −0
  13. packages/codemode-openapi/src/openapi-tools.test.ts +49 −0
  14. packages/codemode-openapi/src/openapi-tools.ts +22 −2
apps/executor/bin/executor+1−1
@@ -1,4 +1,4 @@
1#!/usr/bin/env bash1#!/usr/bin/env bash
2set -euo pipefail2set -euo pipefail
3SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"3SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
4exec bun "$SCRIPT_DIR/../src/main.ts" "$@"4exec bun "$SCRIPT_DIR/../src/cli/main.ts" "$@"
apps/executor/package.json+2−1
@@ -9,7 +9,7 @@ · 8 unmodified lines
9 "typecheck": "bunx tsc --noEmit -p tsconfig.json",9 "typecheck": "bunx tsc --noEmit -p tsconfig.json",
10 "test": "vitest run",10 "test": "vitest run",
11 "test:watch": "vitest",11 "test:watch": "vitest",
12 "start": "bun src/main.ts"12 "start": "bun src/cli/main.ts"
13 },13 },
14 "dependencies": {14 "dependencies": {
15 "@effect/cli": "catalog:",15 "@effect/cli": "catalog:",
@@ -17,6 +17,7 @@ · 1 unmodified line
17 "@effect/platform-node": "catalog:",17 "@effect/platform-node": "catalog:",
18 "@executor-v3/codemode-core": "workspace:*",18 "@executor-v3/codemode-core": "workspace:*",
19 "@executor-v3/codemode-mcp": "workspace:*",19 "@executor-v3/codemode-mcp": "workspace:*",
20 "@executor-v3/codemode-openapi": "workspace:*",
20 "@executor-v3/control-plane": "workspace:*",21 "@executor-v3/control-plane": "workspace:*",
21 "@executor-v3/runtime-local-inproc": "workspace:*",22 "@executor-v3/runtime-local-inproc": "workspace:*",
22 "effect": "catalog:"23 "effect": "catalog:"
apps/executor/src/cli/dev.ts+200−0
@@ -0,0 +1,200 @@
1import {
2 type ControlPlaneClient,
3 WorkspaceIdSchema,
4} from "@executor-v3/control-plane";
5import * as Effect from "effect/Effect";
6
7export type SeedDemoMcpSourceInput = {
8 client: ControlPlaneClient;
9 workspaceId: string;
10 endpoint: string;
11 name: string;
12 namespace: string;
13};
14
15export type SeedDemoMcpSourceResult =
16 | {
17 action: "noop";
18 sourceId: string;
19 workspaceId: string;
20 endpoint: string;
21 }
22 | {
23 action: "updated" | "created";
24 sourceId: string;
25 workspaceId: string;
26 endpoint: string;
27 };
28
29export type SeedGithubOpenApiSourceInput = {
30 client: ControlPlaneClient;
31 workspaceId: string;
32 endpoint: string;
33 specUrl: string;
34 name: string;
35 namespace: string;
36 credentialEnvVar?: string;
37};
38
39export const seedDemoMcpSourceInWorkspace = (
40 input: SeedDemoMcpSourceInput,
41): Effect.Effect<SeedDemoMcpSourceResult, unknown, never> =>
42 Effect.gen(function* () {
43 const workspaceId = WorkspaceIdSchema.make(input.workspaceId);
44
45 const existing = yield* input.client.sources.list({
46 path: {
47 workspaceId,
48 },
49 });
50
51 const existingByName = existing.find(
52 (source) => source.kind === "mcp" && source.name === input.name,
53 );
54
55 if (
56 existingByName !== undefined
57 && existingByName.endpoint === input.endpoint
58 && existingByName.configJson
59 === JSON.stringify({
60 namespace: input.namespace,
61 transport: "streamable-http",
62 })
63 ) {
64 return {
65 action: "noop",
66 sourceId: existingByName.id,
67 workspaceId: input.workspaceId,
68 endpoint: existingByName.endpoint,
69 };
70 }
71
72 if (existingByName !== undefined) {
73 const updated = yield* input.client.sources.update({
74 path: {
75 workspaceId,
76 sourceId: existingByName.id,
77 },
78 payload: {
79 endpoint: input.endpoint,
80 status: "connected",
81 enabled: true,
82 configJson: JSON.stringify({
83 namespace: input.namespace,
84 transport: "streamable-http",
85 }),
86 },
87 });
88
89 return {
90 action: "updated",
91 sourceId: updated.id,
92 workspaceId: input.workspaceId,
93 endpoint: updated.endpoint,
94 };
95 }
96
97 const created = yield* input.client.sources.create({
98 path: {
99 workspaceId,
100 },
101 payload: {
102 name: input.name,
103 kind: "mcp",
104 endpoint: input.endpoint,
105 status: "connected",
106 enabled: true,
107 configJson: JSON.stringify({
108 namespace: input.namespace,
109 transport: "streamable-http",
110 }),
111 },
112 });
113
114 return {
115 action: "created",
116 sourceId: created.id,
117 workspaceId: input.workspaceId,
118 endpoint: created.endpoint,
119 };
120 });
121
122export const seedGithubOpenApiSourceInWorkspace = (
123 input: SeedGithubOpenApiSourceInput,
124): Effect.Effect<SeedDemoMcpSourceResult, unknown, never> =>
125 Effect.gen(function* () {
126 const workspaceId = WorkspaceIdSchema.make(input.workspaceId);
127 const configJson = JSON.stringify({
128 namespace: input.namespace,
129 specUrl: input.specUrl,
130 credentialEnvVar: input.credentialEnvVar ?? "GITHUB_TOKEN",
131 credentialHeader: "Authorization",
132 credentialPrefix: "Bearer ",
133 });
134
135 const existing = yield* input.client.sources.list({
136 path: {
137 workspaceId,
138 },
139 });
140
141 const existingByName = existing.find(
142 (source) => source.kind === "openapi" && source.name === input.name,
143 );
144
145 if (
146 existingByName !== undefined
147 && existingByName.endpoint === input.endpoint
148 && existingByName.configJson === configJson
149 ) {
150 return {
151 action: "noop",
152 sourceId: existingByName.id,
153 workspaceId: input.workspaceId,
154 endpoint: existingByName.endpoint,
155 };
156 }
157
158 if (existingByName !== undefined) {
159 const updated = yield* input.client.sources.update({
160 path: {
161 workspaceId,
162 sourceId: existingByName.id,
163 },
164 payload: {
165 endpoint: input.endpoint,
166 status: "connected",
167 enabled: true,
168 configJson,
169 },
170 });
171
172 return {
173 action: "updated",
174 sourceId: updated.id,
175 workspaceId: input.workspaceId,
176 endpoint: updated.endpoint,
177 };
178 }
179
180 const created = yield* input.client.sources.create({
181 path: {
182 workspaceId,
183 },
184 payload: {
185 name: input.name,
186 kind: "openapi",
187 endpoint: input.endpoint,
188 status: "connected",
189 enabled: true,
190 configJson,
191 },
192 });
193
194 return {
195 action: "created",
196 sourceId: created.id,
197 workspaceId: input.workspaceId,
198 endpoint: created.endpoint,
199 };
200 });
apps/executor/src/cli/main.tsrenamed from apps/executor/src/main.ts+92−6
@@ -25,9 +25,12 @@ import { · 24 unmodified lines
25 DEFAULT_SERVER_PORT,25 DEFAULT_SERVER_PORT,
26 SERVER_POLL_INTERVAL_MS,26 SERVER_POLL_INTERVAL_MS,
27 SERVER_START_TIMEOUT_MS,27 SERVER_START_TIMEOUT_MS,
28} from "./config";28} from "../server/config";
29import { seedDemoMcpSourceInWorkspace } from "./dev";29import {
30import { runLocalExecutorServer } from "./server";30 seedDemoMcpSourceInWorkspace,
31 seedGithubOpenApiSourceInWorkspace,
32} from "./dev";
33import { runLocalExecutorServer } from "../server";
31 34
32const toError = (cause: unknown): Error =>35const toError = (cause: unknown): Error =>
33 cause instanceof Error ? cause : new Error(String(cause));36 cause instanceof Error ? cause : new Error(String(cause));
@@ -52,9 +55,23 @@ const promptLine = (prompt: string): Effect.Effect<string, Error, never> => · 18 unmodified lines
52 catch: toError,55 catch: toError,
53 });56 });
54 57
58const readStdin = (): Effect.Effect<string, Error, never> =>
59 Effect.tryPromise({
60 try: async () => {
61 let contents = "";
62 process.stdin.setEncoding("utf8");
63 for await (const chunk of process.stdin) {
64 contents += chunk;
65 }
66 return contents;
67 },
68 catch: toError,
69 });
70
55const readCode = (input: {71const readCode = (input: {
56 code?: string;72 code?: string;
57 file?: string;73 file?: string;
74 stdin?: boolean;
58}): Effect.Effect<string, Error, never> =>75}): Effect.Effect<string, Error, never> =>
59 Effect.gen(function* () {76 Effect.gen(function* () {
60 if (input.code && input.code.trim().length > 0) {77 if (input.code && input.code.trim().length > 0) {
@@ -71,7 +88,15 @@ const readCode = (input: { · 10 unmodified lines
71 }88 }
72 }89 }
73 90
74 return yield* Effect.fail(new Error("Provide --code or --file."));91 const shouldReadStdin = input.stdin === true || !process.stdin.isTTY;
92 if (shouldReadStdin) {
93 const contents = yield* readStdin();
94 if (contents.trim().length > 0) {
95 return contents;
96 }
97 }
98
99 return yield* Effect.fail(new Error("Provide --code, --file, or pipe code over stdin."));
75 });100 });
76 101
77const getBootstrapClient = (baseUrl: string = DEFAULT_SERVER_BASE_URL) =>102const getBootstrapClient = (baseUrl: string = DEFAULT_SERVER_BASE_URL) =>
@@ -248,6 +273,32 @@ const seedDemoMcpSource = (input: { · 170 unmodified lines
248 });273 });
249 });274 });
250 275
276const seedGithubOpenApiSource = (input: {
277 baseUrl: string;
278 endpoint: string;
279 specUrl: string;
280 name: string;
281 namespace: string;
282 credentialEnvVar?: string;
283}) =>
284 Effect.gen(function* () {
285 yield* ensureServer(input.baseUrl);
286 const { installation, client } = yield* getLocalAuthedClient(input.baseUrl);
287 const result = yield* seedGithubOpenApiSourceInWorkspace({
288 client,
289 workspaceId: installation.workspaceId,
290 endpoint: input.endpoint,
291 specUrl: input.specUrl,
292 name: input.name,
293 namespace: input.namespace,
294 credentialEnvVar: input.credentialEnvVar,
295 });
296
297 yield* Effect.sync(() => {
298 console.log(JSON.stringify(result));
299 });
300 });
301
251const driveExecution = (input: {302const driveExecution = (input: {
252 client: ControlPlaneClient;303 client: ControlPlaneClient;
253 workspaceId: ExecutionEnvelope["execution"]["workspaceId"];304 workspaceId: ExecutionEnvelope["execution"]["workspaceId"];
@@ -310,13 +361,15 @@ const runCommand = Command.make( · 56 unmodified lines
310 {361 {
311 code: Options.text("code").pipe(Options.optional),362 code: Options.text("code").pipe(Options.optional),
312 file: Options.text("file").pipe(Options.optional),363 file: Options.text("file").pipe(Options.optional),
364 stdin: Options.boolean("stdin").pipe(Options.withDefault(false)),
313 baseUrl: Options.text("base-url").pipe(Options.withDefault(DEFAULT_SERVER_BASE_URL)),365 baseUrl: Options.text("base-url").pipe(Options.withDefault(DEFAULT_SERVER_BASE_URL)),
314 },366 },
315 ({ code, file, baseUrl }) =>367 ({ code, file, stdin, baseUrl }) =>
316 Effect.gen(function* () {368 Effect.gen(function* () {
317 const resolvedCode = yield* readCode({369 const resolvedCode = yield* readCode({
318 code: Option.getOrUndefined(code),370 code: Option.getOrUndefined(code),
319 file: Option.getOrUndefined(file),371 file: Option.getOrUndefined(file),
372 stdin,
320 });373 });
321 374
322 yield* ensureServer(baseUrl);375 yield* ensureServer(baseUrl);
@@ -393,8 +446,41 @@ const devSeedMcpDemoCommand = Command.make( · 70 unmodified lines
393 ),446 ),
394);447);
395 448
449const devSeedGithubCommand = Command.make(
450 "seed-github",
451 {
452 baseUrl: Options.text("base-url").pipe(Options.withDefault(DEFAULT_SERVER_BASE_URL)),
453 endpoint: Options.text("endpoint").pipe(
454 Options.withDefault("https://api.github.com"),
455 ),
456 specUrl: Options.text("spec-url").pipe(
457 Options.withDefault(
458 "https://raw.githubusercontent.com/github/rest-api-description/main/descriptions/api.github.com/api.github.com.json",
459 ),
460 ),
461 name: Options.text("name").pipe(Options.withDefault("GitHub")),
462 namespace: Options.text("namespace").pipe(Options.withDefault("github")),
463 credentialEnvVar: Options.text("credential-env-var").pipe(
464 Options.withDefault("GITHUB_TOKEN"),
465 ),
466 },
467 ({ baseUrl, endpoint, specUrl, name, namespace, credentialEnvVar }) =>
468 seedGithubOpenApiSource({
469 baseUrl,
470 endpoint,
471 specUrl,
472 name,
473 namespace,
474 credentialEnvVar,
475 }),
476).pipe(
477 Command.withDescription(
478 "Seed a GitHub OpenAPI source into the default workspace",
479 ),
480);
481
396const devCommand = Command.make("dev").pipe(482const devCommand = Command.make("dev").pipe(
397 Command.withSubcommands([devSeedMcpDemoCommand] as any),483 Command.withSubcommands([devSeedMcpDemoCommand, devSeedGithubCommand] as any),
398 Command.withDescription("Development helpers"),484 Command.withDescription("Development helpers"),
399);485);
400 486
apps/executor/src/control-plane-execution-resolver.ts+0−182
@@ -1,182 +0,0 @@
1import {
2 type Source,
3 type SqlControlPlaneRuntime,
4 type ResolveExecutionEnvironment,
5} from "@executor-v3/control-plane";
6import {
7 createToolCatalogFromTools,
8 createSystemToolMap,
9 makeToolInvokerFromTools,
10 mergeToolMaps,
11} from "@executor-v3/codemode-core";
12import { createSdkMcpConnector, discoverMcpToolsFromConnector } from "@executor-v3/codemode-mcp";
13import { makeInProcessExecutor } from "@executor-v3/runtime-local-inproc";
14import * as Effect from "effect/Effect";
15import * as Schema from "effect/Schema";
16
17const McpSourceConfigSchema = Schema.Struct({
18 namespace: Schema.optional(Schema.String),
19 transport: Schema.optional(Schema.Literal("auto", "streamable-http", "sse")),
20 queryParams: Schema.optional(Schema.Record({ key: Schema.String, value: Schema.String })),
21 headers: Schema.optional(Schema.Record({ key: Schema.String, value: Schema.String })),
22}).pipe(Schema.partialWith({ exact: true }));
23
24const decodeMcpSourceConfig = Schema.decodeUnknown(McpSourceConfigSchema);
25
26const namespaceFromSourceName = (name: string): string => {
27 const normalized = name
28 .trim()
29 .toLowerCase()
30 .replace(/[^a-z0-9]+/g, ".")
31 .replace(/^\.+|\.+$/g, "");
32
33 return normalized.length > 0 ? normalized : "source";
34};
35
36const loadWorkspaceMcpTools = (input: {
37 runtime: SqlControlPlaneRuntime;
38 workspaceId: Parameters<SqlControlPlaneRuntime["persistence"]["rows"]["sources"]["listByWorkspaceId"]>[0];
39}): Effect.Effect<
40 {
41 tools: ReturnType<typeof mergeToolMaps>;
42 catalog: ReturnType<typeof createToolCatalogFromTools>;
43 },
44 Error,
45 never
46> =>
47 Effect.gen(function* () {
48 const sources: ReadonlyArray<Source> = yield* input.runtime.persistence.rows.sources
49 .listByWorkspaceId(input.workspaceId)
50 .pipe(
51 Effect.mapError((cause) =>
52 cause instanceof Error ? cause : new Error(String(cause)),
53 ),
54 );
55
56 const enabledMcpSources = sources.filter(
57 (source) => source.enabled && source.kind === "mcp",
58 );
59
60 const discovered = yield* Effect.forEach(
61 enabledMcpSources,
62 (source: Source) =>
63 Effect.gen(function* () {
64 const rawConfig = yield* Effect.try({
65 try: () => (source.configJson.length > 0 ? JSON.parse(source.configJson) : {}),
66 catch: (cause) =>
67 new Error(
68 `Invalid JSON config for source ${source.id}: ${
69 cause instanceof Error ? cause.message : String(cause)
70 }`,
71 ),
72 });
73
74 const config = yield* decodeMcpSourceConfig(rawConfig).pipe(
75 Effect.mapError(
76 (cause) =>
77 new Error(
78 `Invalid MCP source config for ${source.id}: ${
79 cause instanceof Error ? cause.message : String(cause)
80 }`,
81 ),
82 ),
83 );
84
85 const connector = yield* Effect.try({
86 try: () =>
87 createSdkMcpConnector({
88 endpoint: source.endpoint,
89 transport: config.transport,
90 queryParams: config.queryParams,
91 headers: config.headers,
92 }),
93 catch: (cause) =>
94 new Error(
95 `Failed creating MCP connector for ${source.id}: ${
96 cause instanceof Error ? cause.message : String(cause)
97 }`,
98 ),
99 });
100
101 return yield* discoverMcpToolsFromConnector({
102 connect: connector,
103 namespace: config.namespace ?? namespaceFromSourceName(source.name),
104 sourceKey: source.id,
105 }).pipe(
106 Effect.mapError(
107 (cause) =>
108 new Error(
109 `Failed discovering MCP tools for ${source.id}: ${
110 cause instanceof Error ? cause.message : String(cause)
111 }`,
112 ),
113 ),
114 );
115 }),
116 { concurrency: "unbounded" },
117 );
118
119 const sourceTools = yield* Effect.try({
120 try: () =>
121 mergeToolMaps(discovered.map((item) => item.tools), {
122 conflictMode: "throw",
123 }),
124 catch: (cause) =>
125 new Error(
126 `Failed merging discovered MCP tools: ${
127 cause instanceof Error ? cause.message : String(cause)
128 }`,
129 ),
130 });
131 const catalog = yield* Effect.try({
132 try: () => createToolCatalogFromTools({ tools: sourceTools }),
133 catch: (cause) =>
134 new Error(
135 `Failed creating tool catalog from MCP tools: ${
136 cause instanceof Error ? cause.message : String(cause)
137 }`,
138 ),
139 });
140 const allTools = yield* Effect.try({
141 try: () =>
142 mergeToolMaps([sourceTools, createSystemToolMap({ catalog })]),
143 catch: (cause) =>
144 new Error(
145 `Failed creating MCP execution tool map: ${
146 cause instanceof Error ? cause.message : String(cause)
147 }`,
148 ),
149 });
150
151 return {
152 tools: allTools,
153 catalog,
154 };
155 });
156
157export const makeControlPlaneExecutionResolver = (
158 getRuntime: () => SqlControlPlaneRuntime | null,
159): ResolveExecutionEnvironment =>
160 (input) =>
161 Effect.gen(function* () {
162 const runtime = getRuntime();
163 if (runtime === null) {
164 return yield* Effect.fail(
165 new Error("Control-plane runtime is not ready"),
166 );
167 }
168
169 const loaded = yield* loadWorkspaceMcpTools({
170 runtime,
171 workspaceId: input.workspaceId,
172 });
173
174 return {
175 executor: makeInProcessExecutor(),
176 toolInvoker: makeToolInvokerFromTools({
177 tools: loaded.tools,
178 onElicitation: input.onElicitation,
179 }),
180 catalog: loaded.catalog,
181 };
182 });
apps/executor/src/dev.ts+0−105
@@ -1,105 +0,0 @@
1import type { ControlPlaneClient } from "@executor-v3/control-plane";
2import * as Effect from "effect/Effect";
3
4export type SeedDemoMcpSourceInput = {
5 client: ControlPlaneClient;
6 workspaceId: string;
7 endpoint: string;
8 name: string;
9 namespace: string;
10};
11
12export type SeedDemoMcpSourceResult =
13 | {
14 action: "noop";
15 sourceId: string;
16 workspaceId: string;
17 endpoint: string;
18 }
19 | {
20 action: "updated" | "created";
21 sourceId: string;
22 workspaceId: string;
23 endpoint: string;
24 };
25
26export const seedDemoMcpSourceInWorkspace = (
27 input: SeedDemoMcpSourceInput,
28): Effect.Effect<SeedDemoMcpSourceResult, unknown, never> =>
29 Effect.gen(function* () {
30 const existing = yield* input.client.sources.list({
31 path: {
32 workspaceId: input.workspaceId as never,
33 },
34 });
35
36 const existingByName = existing.find(
37 (source) => source.kind === "mcp" && source.name === input.name,
38 );
39
40 if (
41 existingByName !== undefined
42 && existingByName.endpoint === input.endpoint
43 && existingByName.configJson
44 === JSON.stringify({
45 namespace: input.namespace,
46 transport: "streamable-http",
47 })
48 ) {
49 return {
50 action: "noop",
51 sourceId: existingByName.id,
52 workspaceId: input.workspaceId,
53 endpoint: existingByName.endpoint,
54 };
55 }
56
57 if (existingByName !== undefined) {
58 const updated = yield* input.client.sources.update({
59 path: {
60 workspaceId: input.workspaceId as never,
61 sourceId: existingByName.id,
62 },
63 payload: {
64 endpoint: input.endpoint,
65 status: "connected",
66 enabled: true,
67 configJson: JSON.stringify({
68 namespace: input.namespace,
69 transport: "streamable-http",
70 }),
71 },
72 });
73
74 return {
75 action: "updated",
76 sourceId: updated.id,
77 workspaceId: input.workspaceId,
78 endpoint: updated.endpoint,
79 };
80 }
81
82 const created = yield* input.client.sources.create({
83 path: {
84 workspaceId: input.workspaceId as never,
85 },
86 payload: {
87 name: input.name,
88 kind: "mcp",
89 endpoint: input.endpoint,
90 status: "connected",
91 enabled: true,
92 configJson: JSON.stringify({
93 namespace: input.namespace,
94 transport: "streamable-http",
95 }),
96 },
97 });
98
99 return {
100 action: "created",
101 sourceId: created.id,
102 workspaceId: input.workspaceId,
103 endpoint: created.endpoint,
104 };
105 });
apps/executor/src/index.ts+0−2
@@ -1,2 +0,0 @@
1export * from "./config";
2export * from "./server";
apps/executor/src/server/config.tsrenamed from apps/executor/src/config.ts+0−0

File renamed without changes.

apps/executor/src/server/control-plane-execution-resolver.ts+281−0
@@ -0,0 +1,281 @@
1import {
2 type Source,
3 type SqlControlPlaneRuntime,
4 type ResolveExecutionEnvironment,
5} from "@executor-v3/control-plane";
6import {
7 type ToolMap,
8 createToolCatalogFromTools,
9 createSystemToolMap,
10 makeToolInvokerFromTools,
11 mergeToolMaps,
12} from "@executor-v3/codemode-core";
13import { createSdkMcpConnector, discoverMcpToolsFromConnector } from "@executor-v3/codemode-mcp";
14import {
15 createOpenApiToolsFromSpec,
16 fetchOpenApiDocument,
17} from "@executor-v3/codemode-openapi";
18import { makeInProcessExecutor } from "@executor-v3/runtime-local-inproc";
19import * as Effect from "effect/Effect";
20import * as Schema from "effect/Schema";
21
22const McpSourceConfigSchema = Schema.Struct({
23 namespace: Schema.optional(Schema.String),
24 transport: Schema.optional(Schema.Literal("auto", "streamable-http", "sse")),
25 queryParams: Schema.optional(Schema.Record({ key: Schema.String, value: Schema.String })),
26 headers: Schema.optional(Schema.Record({ key: Schema.String, value: Schema.String })),
27}).pipe(Schema.partialWith({ exact: true }));
28
29const decodeMcpSourceConfig = Schema.decodeUnknown(McpSourceConfigSchema);
30
31const OpenApiSourceConfigSchema = Schema.Struct({
32 namespace: Schema.optional(Schema.String),
33 specUrl: Schema.optional(Schema.String),
34 credentialEnvVar: Schema.optional(Schema.String),
35 credentialHeader: Schema.optional(Schema.String),
36 credentialPrefix: Schema.optional(Schema.String),
37 defaultHeaders: Schema.optional(Schema.Record({ key: Schema.String, value: Schema.String })),
38}).pipe(Schema.partialWith({ exact: true }));
39
40const decodeOpenApiSourceConfig = Schema.decodeUnknown(OpenApiSourceConfigSchema);
41
42const namespaceFromSourceName = (name: string): string => {
43 const normalized = name
44 .trim()
45 .toLowerCase()
46 .replace(/[^a-z0-9]+/g, ".")
47 .replace(/^\.+|\.+$/g, "");
48
49 return normalized.length > 0 ? normalized : "source";
50};
51
52const loadWorkspaceSourceTools = (input: {
53 runtime: SqlControlPlaneRuntime;
54 workspaceId: Parameters<SqlControlPlaneRuntime["persistence"]["rows"]["sources"]["listByWorkspaceId"]>[0];
55}): Effect.Effect<
56 {
57 tools: ToolMap;
58 catalog: ReturnType<typeof createToolCatalogFromTools>;
59 },
60 Error,
61 never
62> =>
63 Effect.gen(function* () {
64 const sources: ReadonlyArray<Source> = yield* input.runtime.persistence.rows.sources
65 .listByWorkspaceId(input.workspaceId)
66 .pipe(
67 Effect.mapError((cause) =>
68 cause instanceof Error ? cause : new Error(String(cause)),
69 ),
70 );
71
72 const enabledSources = sources.filter((source) => source.enabled);
73
74 const discoveredToolMaps = yield* Effect.forEach(
75 enabledSources,
76 (source: Source) =>
77 Effect.gen(function* () {
78 if (source.kind === "mcp") {
79 const rawConfig = yield* Effect.try({
80 try: () => (source.configJson.length > 0 ? JSON.parse(source.configJson) : {}),
81 catch: (cause) =>
82 new Error(
83 `Invalid JSON config for source ${source.id}: ${
84 cause instanceof Error ? cause.message : String(cause)
85 }`,
86 ),
87 });
88
89 const config = yield* decodeMcpSourceConfig(rawConfig).pipe(
90 Effect.mapError(
91 (cause) =>
92 new Error(
93 `Invalid MCP source config for ${source.id}: ${
94 cause instanceof Error ? cause.message : String(cause)
95 }`,
96 ),
97 ),
98 );
99
100 const connector = yield* Effect.try({
101 try: () =>
102 createSdkMcpConnector({
103 endpoint: source.endpoint,
104 transport: config.transport,
105 queryParams: config.queryParams,
106 headers: config.headers,
107 }),
108 catch: (cause) =>
109 new Error(
110 `Failed creating MCP connector for ${source.id}: ${
111 cause instanceof Error ? cause.message : String(cause)
112 }`,
113 ),
114 });
115
116 const discovered = yield* discoverMcpToolsFromConnector({
117 connect: connector,
118 namespace: config.namespace ?? namespaceFromSourceName(source.name),
119 sourceKey: source.id,
120 }).pipe(
121 Effect.mapError(
122 (cause) =>
123 new Error(
124 `Failed discovering MCP tools for ${source.id}: ${
125 cause instanceof Error ? cause.message : String(cause)
126 }`,
127 ),
128 ),
129 );
130
131 return discovered.tools;
132 }
133
134 if (source.kind === "openapi") {
135 const rawConfig = yield* Effect.try({
136 try: () => (source.configJson.length > 0 ? JSON.parse(source.configJson) : {}),
137 catch: (cause) =>
138 new Error(
139 `Invalid JSON config for source ${source.id}: ${
140 cause instanceof Error ? cause.message : String(cause)
141 }`,
142 ),
143 });
144
145 const config = yield* decodeOpenApiSourceConfig(rawConfig).pipe(
146 Effect.mapError(
147 (cause) =>
148 new Error(
149 `Invalid OpenAPI source config for ${source.id}: ${
150 cause instanceof Error ? cause.message : String(cause)
151 }`,
152 ),
153 ),
154 );
155
156 const specUrl = config.specUrl?.trim();
157 if (!specUrl) {
158 return yield* Effect.fail(
159 new Error(`Missing OpenAPI specUrl for source ${source.id}`),
160 );
161 }
162
163 const openApiDocument = yield* Effect.tryPromise({
164 try: () => fetchOpenApiDocument(specUrl),
165 catch: (cause) =>
166 new Error(
167 `Failed fetching OpenAPI spec for ${source.id}: ${
168 cause instanceof Error ? cause.message : String(cause)
169 }`,
170 ),
171 });
172
173 const defaultHeaders = config.defaultHeaders ?? {};
174 const credentialHeader = config.credentialHeader?.trim() || "Authorization";
175 const credentialPrefix = config.credentialPrefix ?? "Bearer ";
176 const credentialEnvVar = config.credentialEnvVar?.trim();
177 const credentialValue = credentialEnvVar
178 ? process.env[credentialEnvVar]?.trim()
179 : undefined;
180 const credentialHeaders =
181 credentialEnvVar && credentialValue
182 ? {
183 [credentialHeader]: `${credentialPrefix}${credentialValue}`,
184 }
185 : {};
186
187 const extracted = yield* createOpenApiToolsFromSpec({
188 sourceName: source.name,
189 openApiSpec: openApiDocument,
190 baseUrl: source.endpoint,
191 namespace: config.namespace ?? namespaceFromSourceName(source.name),
192 sourceKey: source.id,
193 defaultHeaders,
194 credentialHeaders,
195 }).pipe(
196 Effect.mapError(
197 (cause) =>
198 new Error(
199 `Failed loading OpenAPI tools for ${source.id}: ${
200 cause instanceof Error ? cause.message : String(cause)
201 }`,
202 ),
203 ),
204 );
205
206 return extracted.tools;
207 }
208
209 return {};
210 }),
211 { concurrency: "unbounded" },
212 );
213
214 const sourceTools = yield* Effect.try({
215 try: () =>
216 mergeToolMaps(discoveredToolMaps, {
217 conflictMode: "throw",
218 }),
219 catch: (cause) =>
220 new Error(
221 `Failed merging discovered source tools: ${
222 cause instanceof Error ? cause.message : String(cause)
223 }`,
224 ),
225 });
226 const catalog = yield* Effect.try({
227 try: () => createToolCatalogFromTools({ tools: sourceTools }),
228 catch: (cause) =>
229 new Error(
230 `Failed creating tool catalog from source tools: ${
231 cause instanceof Error ? cause.message : String(cause)
232 }`,
233 ),
234 });
235 const allTools = yield* Effect.try({
236 try: () =>
237 mergeToolMaps([sourceTools, createSystemToolMap({ catalog })]),
238 catch: (cause) =>
239 new Error(
240 `Failed creating source execution tool map: ${
241 cause instanceof Error ? cause.message : String(cause)
242 }`,
243 ),
244 });
245
246 return {
247 tools: allTools,
248 catalog,
249 };
250 }).pipe(
251 Effect.mapError((cause) =>
252 cause instanceof Error ? cause : new Error(String(cause)),
253 ),
254 );
255
256export const makeControlPlaneExecutionResolver = (
257 getRuntime: () => SqlControlPlaneRuntime | null,
258): ResolveExecutionEnvironment =>
259 (input) =>
260 Effect.gen(function* () {
261 const runtime = getRuntime();
262 if (runtime === null) {
263 return yield* Effect.fail(
264 new Error("Control-plane runtime is not ready"),
265 );
266 }
267
268 const loaded = yield* loadWorkspaceSourceTools({
269 runtime,
270 workspaceId: input.workspaceId,
271 });
272
273 return {
274 executor: makeInProcessExecutor(),
275 toolInvoker: makeToolInvokerFromTools({
276 tools: loaded.tools,
277 onElicitation: input.onElicitation,
278 }),
279 catalog: loaded.catalog,
280 };
281 });
apps/executor/src/server/index.tsrenamed from apps/executor/src/server.ts+0−0

File renamed without changes.

apps/executor/src/server/server.test.tsrenamed from apps/executor/src/server.test.ts+144−2
@@ -1,3 +1,12 @@
1import { createServer, type IncomingMessage, type ServerResponse } from "node:http";
2
3import {
4 HttpApi,
5 HttpApiEndpoint,
6 HttpApiGroup,
7 HttpApiSchema,
8 OpenApi,
9} from "@effect/platform";
1import { describe, expect, it } from "@effect/vitest";10import { describe, expect, it } from "@effect/vitest";
2import * as Effect from "effect/Effect";11import * as Effect from "effect/Effect";
3import * as Schema from "effect/Schema";12import * as Schema from "effect/Schema";
@@ -11,8 +20,11 @@ import { · 7 unmodified lines
11} from "@executor-v3/control-plane";20} from "@executor-v3/control-plane";
12import { makeInProcessExecutor } from "@executor-v3/runtime-local-inproc";21import { makeInProcessExecutor } from "@executor-v3/runtime-local-inproc";
13 22
14import { seedDemoMcpSourceInWorkspace } from "./dev";23import {
15import { makeLocalExecutorServer } from "./server";24 seedDemoMcpSourceInWorkspace,
25 seedGithubOpenApiSourceInWorkspace,
26} from "../cli/dev";
27import { makeLocalExecutorServer } from ".";
16 28
17const executionResolver: ResolveExecutionEnvironment = () =>29const executionResolver: ResolveExecutionEnvironment = () =>
18 Effect.succeed({30 Effect.succeed({
@@ -39,6 +51,77 @@ const makeServer = makeLocalExecutorServer({ · 20 unmodified lines
39 executionResolver,51 executionResolver,
40});52});
41 53
54const ownerParam = HttpApiSchema.param("owner", Schema.String);
55const repoParam = HttpApiSchema.param("repo", Schema.String);
56
57class ExecutorDemoReposApi extends HttpApiGroup.make("repos")
58 .add(
59 HttpApiEndpoint.get("getRepo")`/repos/${ownerParam}/${repoParam}`
60 .addSuccess(
61 Schema.Struct({
62 full_name: Schema.String,
63 private: Schema.Boolean,
64 }),
65 ),
66 )
67{}
68
69class ExecutorDemoApi extends HttpApi.make("executorDemo").add(ExecutorDemoReposApi) {}
70
71const executorDemoOpenApiSpec = OpenApi.fromApi(ExecutorDemoApi);
72
73const startOpenApiDemoServer = async () => {
74 const seenAuthHeaders: Array<string | null> = [];
75
76 const handler = (req: IncomingMessage, res: ServerResponse) => {
77 if (req.url === "/openapi.json") {
78 res.statusCode = 200;
79 res.setHeader("content-type", "application/json");
80 res.end(JSON.stringify(executorDemoOpenApiSpec));
81 return;
82 }
83
84 const match = req.url?.match(/^\/repos\/([^/]+)\/([^/]+)$/);
85 if (req.method === "GET" && match) {
86 seenAuthHeaders.push(
87 typeof req.headers.authorization === "string" ? req.headers.authorization : null,
88 );
89 res.statusCode = 200;
90 res.setHeader("content-type", "application/json");
91 res.end(
92 JSON.stringify({
93 full_name: `${decodeURIComponent(match[1] ?? "")}/${decodeURIComponent(match[2] ?? "")}`,
94 private: false,
95 }),
96 );
97 return;
98 }
99
100 res.statusCode = 404;
101 res.end("not found");
102 };
103
104 const server = createServer(handler);
105 await new Promise<void>((resolve, reject) => {
106 server.once("error", reject);
107 server.listen(0, "127.0.0.1", () => resolve());
108 });
109 const address = server.address();
110 if (!address || typeof address === "string") {
111 throw new Error("Failed to bind OpenAPI demo server");
112 }
113
114 return {
115 baseUrl: `http://127.0.0.1:${address.port}`,
116 specUrl: `http://127.0.0.1:${address.port}/openapi.json`,
117 seenAuthHeaders,
118 close: () =>
119 new Promise<void>((resolve, reject) => {
120 server.close((error) => (error ? reject(error) : resolve()));
121 }),
122 };
123};
124
42describe("local-executor-server", () => {125describe("local-executor-server", () => {
43 it.scoped("serves the control-plane API and executes code", () =>126 it.scoped("serves the control-plane API and executes code", () =>
44 Effect.gen(function* () {127 Effect.gen(function* () {
@@ -268,6 +351,65 @@ describe("local-executor-server", () => { · 223 unmodified lines
268 }),351 }),
269 );352 );
270 353
354 it.scoped("loads OpenAPI sources from control-plane state and calls them", () =>
355 Effect.gen(function* () {
356 const openApiServer = yield* Effect.acquireRelease(
357 Effect.promise(() => startOpenApiDemoServer()),
358 (server) => Effect.promise(() => server.close()).pipe(Effect.orDie),
359 );
360
361 const previousGithubToken = process.env.GITHUB_TOKEN;
362 process.env.GITHUB_TOKEN = "ghp_test_executor";
363
364 const server = yield* makeLocalExecutorServer({
365 port: 0,
366 localDataDir: ":memory:",
367 });
368
369 const bootstrapClient = yield* makeControlPlaneClient({
370 baseUrl: server.baseUrl,
371 });
372 const installation = yield* bootstrapClient.local.installation({});
373 const client = yield* makeControlPlaneClient({
374 baseUrl: server.baseUrl,
375 accountId: installation.accountId,
376 });
377
378 yield* seedGithubOpenApiSourceInWorkspace({
379 client,
380 workspaceId: installation.workspaceId,
381 endpoint: openApiServer.baseUrl,
382 specUrl: openApiServer.specUrl,
383 name: "GitHub",
384 namespace: "github",
385 });
386
387 const execution = yield* client.executions.create({
388 path: {
389 workspaceId: installation.workspaceId,
390 },
391 payload: {
392 code: 'return await tools.github.repos.getRepo({ owner: "vercel", repo: "ai" });',
393 },
394 }).pipe(
395 Effect.ensuring(
396 Effect.sync(() => {
397 if (previousGithubToken === undefined) {
398 delete process.env.GITHUB_TOKEN;
399 } else {
400 process.env.GITHUB_TOKEN = previousGithubToken;
401 }
402 }),
403 ),
404 );
405
406 expect(execution.execution.status).toBe("completed");
407 expect(execution.pendingInteraction).toBeNull();
408 expect(execution.execution.resultJson).toContain("\"full_name\":\"vercel/ai\"");
409 expect(openApiServer.seenAuthHeaders).toEqual(["Bearer ghp_test_executor"]);
410 }),
411 );
412
271 it.scoped("returns a typed storage error when a configured MCP endpoint is invalid", () =>413 it.scoped("returns a typed storage error when a configured MCP endpoint is invalid", () =>
272 Effect.gen(function* () {414 Effect.gen(function* () {
273 const server = yield* makeLocalExecutorServer({415 const server = yield* makeLocalExecutorServer({
bun.lock+1−0
@@ -19,6 +19,7 @@ · 18 unmodified lines
19 "@effect/platform-node": "catalog:",19 "@effect/platform-node": "catalog:",
20 "@executor-v3/codemode-core": "workspace:*",20 "@executor-v3/codemode-core": "workspace:*",
21 "@executor-v3/codemode-mcp": "workspace:*",21 "@executor-v3/codemode-mcp": "workspace:*",
22 "@executor-v3/codemode-openapi": "workspace:*",
22 "@executor-v3/control-plane": "workspace:*",23 "@executor-v3/control-plane": "workspace:*",
23 "@executor-v3/runtime-local-inproc": "workspace:*",24 "@executor-v3/runtime-local-inproc": "workspace:*",
24 "effect": "catalog:",25 "effect": "catalog:",
packages/codemode-openapi/src/openapi-tools.test.ts+49−0
@@ -388,4 +388,53 @@ describe("openapi-tools", () => { · 387 unmodified lines
388 expect(server.requests[0]?.query).toContain("include=all");388 expect(server.requests[0]?.query).toContain("include=all");
389 }),389 }),
390 );390 );
391
392 it.scoped("falls back to path-template args when extracted path parameters are missing", () =>
393 Effect.gen(function* () {
394 const server = yield* makeTestServer;
395
396 const extracted = yield* createOpenApiToolsFromSpec({
397 sourceName: "generated",
398 openApiSpec: generatedOpenApiSpec,
399 baseUrl: server.baseUrl,
400 namespace: "source.generated",
401 });
402
403 const manifestWithoutPathParameters = {
404 ...extracted.manifest,
405 tools: extracted.manifest.tools.map((tool) =>
406 tool.toolId === "repos/getRepo"
407 ? {
408 ...tool,
409 invocation: {
410 ...tool.invocation,
411 parameters: [],
412 },
413 }
414 : tool),
415 };
416
417 const tools = createOpenApiToolsFromManifest({
418 manifest: manifestWithoutPathParameters,
419 baseUrl: server.baseUrl,
420 namespace: "source.generated",
421 });
422
423 const getRepo = resolveToolExecutor(tools, "source.generated.repos.getRepo");
424 const result = yield* Effect.promise(() =>
425 getRepo({ owner: "octocat", repo: "hello-world" }),
426 );
427
428 expect(result).toEqual({
429 status: 200,
430 headers: expect.any(Object),
431 body: {
432 full_name: "octocat/hello-world",
433 include: null,
434 },
435 });
436 expect(server.requests).toHaveLength(1);
437 expect(server.requests[0]?.path).toBe("/repos/octocat/hello-world");
438 }),
439 );
391});440});
packages/codemode-openapi/src/openapi-tools.ts+22−2
@@ -129,8 +129,28 @@ const replacePathTemplate = ( · 128 unmodified lines
129 .map((match) => match[1])129 .map((match) => match[1])
130 .filter((value): value is string => typeof value === "string" && value.length > 0);130 .filter((value): value is string => typeof value === "string" && value.length > 0);
131 131
132 if (unresolvedPathParameters.length > 0) {132 for (const parameterName of unresolvedPathParameters) {
133 const names = [...new Set(unresolvedPathParameters)].sort().join(", ");133 const parameterValue = args[parameterName]
134 ?? (isRecord(args.path) ? args.path[parameterName] : undefined)
135 ?? (isRecord(args.pathParams) ? args.pathParams[parameterName] : undefined)
136 ?? (isRecord(args.params) ? args.params[parameterName] : undefined);
137
138 if (parameterValue === undefined || parameterValue === null) {
139 continue;
140 }
141
142 resolvedPath = resolvedPath.replaceAll(
143 `{${parameterName}}`,
144 encodeURIComponent(String(parameterValue)),
145 );
146 }
147
148 const stillUnresolvedPathParameters = [...resolvedPath.matchAll(/\{([^{}]+)\}/g)]
149 .map((match) => match[1])
150 .filter((value): value is string => typeof value === "string" && value.length > 0);
151
152 if (stillUnresolvedPathParameters.length > 0) {
153 const names = [...new Set(stillUnresolvedPathParameters)].sort().join(", ");
134 throw new OpenApiToolInvocationError({154 throw new OpenApiToolInvocationError({
135 operation: "resolve_path",155 operation: "resolve_path",
136 message: `Unresolved path parameters after substitution: ${names}`,156 message: `Unresolved path parameters after substitution: ${names}`,

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t