History

Add GitHub sync and self-host custom tools wiring

Rhys Sullivan committed 4 months ago

Commit 275446e1 parent 41eb3ae19 files · +1665 −40Browse files

Showing the changes to apps (8 of 19 files). Show all

Unified · Split

8 files
  1. apps/host-selfhost/executor.config.ts +2 −0
  2. apps/host-selfhost/package.json +2 −0
  3. apps/host-selfhost/src/app.ts +4 −0
  4. apps/host-selfhost/src/apps-resolver.ts +130 −0
  5. apps/host-selfhost/src/apps-route.ts +91 −0
  6. apps/host-selfhost/src/apps-wire.node.test.ts +549 −0
  7. apps/host-selfhost/src/apps.ts +71 −0
  8. apps/host-selfhost/src/testing/test-app.ts +4 −0
apps/host-selfhost/executor.config.ts+2−0
@@ -8,6 +8,7 @@ import { encryptedSecretsPlugin } from "@executor-js/plugin-encrypted-secrets"; · 7 unmodified lines
8import { toolkitsPlugin } from "@executor-js/plugin-toolkits/server";8import { toolkitsPlugin } from "@executor-js/plugin-toolkits/server";
9 9
10import { resolveSecretKey } from "./src/config";10import { resolveSecretKey } from "./src/config";
11import { getSelfHostAppsSubsystem } from "./src/apps";
11 12
12// ---------------------------------------------------------------------------13// ---------------------------------------------------------------------------
13// Single source of truth for the self-hosted app's plugin list.14// Single source of truth for the self-hosted app's plugin list.
@@ -36,6 +37,7 @@ export default defineExecutorConfig({ · 22 unmodified lines
36 mcpHttpPlugin({ dangerouslyAllowStdioMCP: false }),37 mcpHttpPlugin({ dangerouslyAllowStdioMCP: false }),
37 graphqlHttpPlugin(),38 graphqlHttpPlugin(),
38 toolkitsPlugin({ activeToolkitSlug }),39 toolkitsPlugin({ activeToolkitSlug }),
40 getSelfHostAppsSubsystem().plugin,
39 // First writable secret provider -> the default for `secrets.set`.41 // First writable secret provider -> the default for `secrets.set`.
40 encryptedSecretsPlugin({ key: resolveSecretKey() }),42 encryptedSecretsPlugin({ key: resolveSecretKey() }),
41 ] as const,43 ] as const,
apps/host-selfhost/package.json+2−0
@@ -26,6 +26,7 @@ · 25 unmodified lines
26 "@executor-js/execution": "workspace:*",26 "@executor-js/execution": "workspace:*",
27 "@executor-js/fumadb": "workspace:*",27 "@executor-js/fumadb": "workspace:*",
28 "@executor-js/host-mcp": "workspace:*",28 "@executor-js/host-mcp": "workspace:*",
29 "@executor-js/plugin-apps": "workspace:*",
29 "@executor-js/plugin-encrypted-secrets": "workspace:*",30 "@executor-js/plugin-encrypted-secrets": "workspace:*",
30 "@executor-js/plugin-google": "workspace:*",31 "@executor-js/plugin-google": "workspace:*",
31 "@executor-js/plugin-graphql": "workspace:*",32 "@executor-js/plugin-graphql": "workspace:*",
@@ -48,6 +49,7 @@ · 16 unmodified lines
48 },49 },
49 "devDependencies": {50 "devDependencies": {
50 "@effect/vitest": "catalog:",51 "@effect/vitest": "catalog:",
52 "@executor-js/emulate": "^0.13.2",
51 "@executor-js/vite-plugin": "workspace:*",53 "@executor-js/vite-plugin": "workspace:*",
52 "@tailwindcss/vite": "catalog:",54 "@tailwindcss/vite": "catalog:",
53 "@tanstack/router-plugin": "^1.167.12",55 "@tanstack/router-plugin": "^1.167.12",
apps/host-selfhost/src/app.ts+4−0
@@ -22,6 +22,8 @@ import { makeSelfHostMcpSeams } from "./mcp"; · 21 unmodified lines
22import { selfHostPlugins } from "./plugins";22import { selfHostPlugins } from "./plugins";
23import { ErrorCaptureLive } from "./observability";23import { ErrorCaptureLive } from "./observability";
24import { oauthCallbackSignInRedirectLocation } from "./auth/oauth-callback-login";24import { oauthCallbackSignInRedirectLocation } from "./auth/oauth-callback-login";
25import { closeSelfHostAppsSubsystem } from "./apps";
26import { makeSelfHostAppsSyncRoute } from "./apps-route";
25 27
26// ===========================================================================28// ===========================================================================
27// The self-hosted Executor app, as ONE `ExecutorApp.make` call.29// The self-hosted Executor app, as ONE `ExecutorApp.make` call.
@@ -119,6 +121,7 @@ export const makeSelfHostApp = async (options: MakeSelfHostAppOptions = {}) => { · 91 unmodified lines
119 makeSelfHostAdminApiLayer({ betterAuth, db: dbHandle, mountPrefix: "/api" }),121 makeSelfHostAdminApiLayer({ betterAuth, db: dbHandle, mountPrefix: "/api" }),
120 // Public system API: /api/health + /api/setup-status (unauthenticated).122 // Public system API: /api/health + /api/setup-status (unauthenticated).
121 makeSelfHostSystemApiLayer({ betterAuth, db: dbHandle, mountPrefix: "/api" }),123 makeSelfHostSystemApiLayer({ betterAuth, db: dbHandle, mountPrefix: "/api" }),
124 makeSelfHostAppsSyncRoute({ identity: identityLayer, db: dbHandle }),
122 // Swagger UI at /docs, over the /api-prefixed spec (matches the served paths).125 // Swagger UI at /docs, over the /api-prefixed spec (matches the served paths).
123 HttpApiSwagger.layer(composePluginApi(selfHostPlugins).prefix("/api"), { path: "/docs" }),126 HttpApiSwagger.layer(composePluginApi(selfHostPlugins).prefix("/api"), { path: "/docs" }),
124 ],127 ],
@@ -139,6 +142,7 @@ export const makeSelfHostApp = async (options: MakeSelfHostAppOptions = {}) => { · 14 unmodified lines
139 toWebHandler,142 toWebHandler,
140 betterAuth,143 betterAuth,
141 closeDb: async () => {144 closeDb: async () => {
145 await closeSelfHostAppsSubsystem();
142 await mcp.close();146 await mcp.close();
143 await dbHandle.close();147 await dbHandle.close();
144 },148 },
apps/host-selfhost/src/apps-resolver.ts+130−0
@@ -0,0 +1,130 @@
1import { Effect } from "effect";
2
3import {
4 ConnectionName,
5 IntegrationSlug,
6 ToolAddress,
7 isToolResult,
8 type ConnectionRef,
9 type ExecuteError,
10 type Owner,
11 type PluginCtx,
12} from "@executor-js/sdk";
13import {
14 BindingError,
15 type ClientResolver,
16 type ConnectionCandidate,
17} from "@executor-js/plugin-apps/api";
18
19const parseConnectionAddress = (address: string): ConnectionRef | null => {
20 const parts = address.split(".");
21 if (parts.length !== 4 || parts[0] !== "tools") return null;
22 const [, integration, owner, name] = parts;
23 if (!integration || !name) return null;
24 if (owner !== "org" && owner !== "user") return null;
25 return {
26 owner: owner as Owner,
27 integration: IntegrationSlug.make(integration),
28 name: ConnectionName.make(name),
29 };
30};
31
32const bindingError = (input: {
33 readonly integration: string;
34 readonly connection?: string;
35 readonly message: string;
36 readonly cause?: unknown;
37}): BindingError =>
38 new BindingError({
39 role: input.integration,
40 integration: input.integration,
41 requestedConnection: input.connection,
42 message: input.message,
43 });
44
45const executeErrorMessage = (cause: ExecuteError): string =>
46 typeof cause.message === "string" && cause.message.length > 0 ? cause.message : cause._tag;
47
48const toCandidate = (connection: {
49 readonly address: unknown;
50 readonly integration: unknown;
51 readonly name?: unknown;
52 readonly owner?: unknown;
53}): ConnectionCandidate => ({
54 address: String(connection.address),
55 integration: String(connection.integration),
56 ...(connection.name !== undefined ? { name: String(connection.name) } : {}),
57 ...(connection.owner !== undefined ? { owner: String(connection.owner) } : {}),
58});
59
60export const makeSelfHostAppsResolver = (input: { readonly ctx: PluginCtx }): ClientResolver => ({
61 listConnections: ({ integration }) =>
62 input.ctx.connections.list({ integration: IntegrationSlug.make(integration) }).pipe(
63 Effect.map((connections) => connections.map(toCandidate)),
64 Effect.mapError((cause) =>
65 bindingError({
66 integration,
67 message: `failed to list ${integration} connections`,
68 cause,
69 }),
70 ),
71 ),
72
73 resolveConnection: ({ connection }) =>
74 Effect.gen(function* () {
75 const ref = parseConnectionAddress(connection);
76 if (!ref) return null;
77 const row = yield* input.ctx.connections.get(ref).pipe(
78 Effect.mapError((cause) =>
79 bindingError({
80 integration: String(ref.integration),
81 connection,
82 message: `failed to resolve connection ${connection}`,
83 cause,
84 }),
85 ),
86 );
87 return row ? toCandidate(row) : null;
88 }),
89
90 call: ({ integration, connection, path, args }) =>
91 Effect.gen(function* () {
92 const ref = parseConnectionAddress(connection);
93 if (!ref) {
94 return yield* bindingError({
95 integration,
96 connection,
97 message: `invalid connection address ${connection}`,
98 });
99 }
100 if (String(ref.integration) !== integration) {
101 return yield* bindingError({
102 integration,
103 connection,
104 message: `connection "${connection}" belongs to integration "${ref.integration}", not "${integration}"`,
105 });
106 }
107 const tool = path.join(".");
108 const address = ToolAddress.make(`${connection}.${tool}`);
109 const payload = args[0] ?? {};
110 const result = yield* input.ctx.execute(address, payload).pipe(
111 Effect.mapError((cause) =>
112 bindingError({
113 integration,
114 connection,
115 message: executeErrorMessage(cause),
116 cause,
117 }),
118 ),
119 );
120 if (isToolResult(result)) {
121 if (result.ok) return result.data;
122 return yield* bindingError({
123 integration,
124 connection,
125 message: result.error.message,
126 });
127 }
128 return result;
129 }),
130});
apps/host-selfhost/src/apps-route.ts+91−0
@@ -0,0 +1,91 @@
1import { HttpEffect, HttpRouter } from "effect/unstable/http";
2import { Data, Effect, Layer } from "effect";
3
4import {
5 IdentityProvider,
6 RequestOrgSlug,
7 RequestWebOrigin,
8 makeScopedExecutor,
9 type Principal,
10} from "@executor-js/api/server";
11import { ToolAddress } from "@executor-js/sdk";
12
13import { SelfHostScopedExecutorSeams } from "./execution";
14import { SelfHostDb, type SelfHostDbHandle } from "./db/self-host-db";
15import type { SelfHostPlugins } from "./plugins";
16
17class AppsSyncRouteError extends Data.TaggedError("AppsSyncRouteError")<{
18 readonly status: number;
19 readonly message: string;
20}> {}
21
22const jsonResponse = (body: unknown, status = 200): Response =>
23 new Response(JSON.stringify(body), {
24 status,
25 headers: { "content-type": "application/json" },
26 });
27
28const parseJson = (request: Request): Effect.Effect<unknown, AppsSyncRouteError> =>
29 Effect.tryPromise({
30 try: () => request.json() as Promise<unknown>,
31 catch: () =>
32 new AppsSyncRouteError({
33 status: 400,
34 message: "Invalid JSON body",
35 }),
36 });
37
38const buildExecutor = (principal: Principal, request: Request) => {
39 const withOrigin = makeScopedExecutor<SelfHostPlugins>(
40 principal.accountId,
41 principal.organizationId,
42 principal.organizationName,
43 ).pipe(Effect.provideService(RequestWebOrigin, { origin: new URL(request.url).origin }));
44 return principal.organizationSlug
45 ? withOrigin.pipe(Effect.provideService(RequestOrgSlug, { slug: principal.organizationSlug }))
46 : withOrigin;
47};
48
49export interface SelfHostAppsSyncRouteDeps {
50 readonly identity: Layer.Layer<IdentityProvider>;
51 readonly db: SelfHostDbHandle;
52}
53
54export const makeSelfHostAppsSyncRoute = ({ identity, db }: SelfHostAppsSyncRouteDeps) => {
55 const services = Layer.merge(
56 SelfHostScopedExecutorSeams.pipe(Layer.provide(Layer.succeed(SelfHostDb)(db))),
57 identity,
58 );
59 return HttpRouter.add(
60 "POST",
61 "/api/apps/sources/github/sync",
62 HttpEffect.fromWebHandler(
63 (request): Promise<Response> =>
64 Effect.runPromise(
65 Effect.gen(function* () {
66 const identityProvider = yield* IdentityProvider;
67 const principal = yield* identityProvider.authenticate(request);
68 const payload = yield* parseJson(request);
69 const executor = yield* buildExecutor(principal, request);
70 const result = yield* executor.execute(
71 ToolAddress.make("executor.apps.sync_github_source"),
72 payload,
73 );
74 return jsonResponse(result);
75 }).pipe(
76 Effect.catchTags({
77 Unauthorized: () => Effect.succeed(new Response("Unauthorized", { status: 401 })),
78 NoOrganization: () => Effect.succeed(new Response("Forbidden", { status: 403 })),
79 Unavailable: () => Effect.succeed(new Response("Unavailable", { status: 503 })),
80 AppsSyncRouteError: (error) =>
81 Effect.succeed(jsonResponse({ error: error.message }, error.status)),
82 }),
83 Effect.catchCause(() =>
84 Effect.succeed(new Response("Internal Server Error", { status: 500 })),
85 ),
86 Effect.provide(services),
87 ),
88 ),
89 ),
90 );
91};
apps/host-selfhost/src/apps-wire.node.test.ts+549−0
@@ -0,0 +1,549 @@
1import { randomBytes } from "node:crypto";
2import { mkdtempSync } from "node:fs";
3import { createServer, type IncomingMessage, type ServerResponse } from "node:http";
4import { createServer as createNetServer } from "node:net";
5import { tmpdir } from "node:os";
6import { join } from "node:path";
7
8import { afterAll, beforeAll, expect, test } from "@effect/vitest";
9import { createEmulator, type Emulator } from "@executor-js/emulate";
10
11const dataDir = mkdtempSync(join(tmpdir(), "eh-apps-wire-"));
12const dbPath = join(dataDir, "data.db");
13process.env.EXECUTOR_DATA_DIR = dataDir;
14process.env.EXECUTOR_SECRET_KEY = "apps-wire-secret-key";
15process.env.EXECUTOR_ALLOW_LOCAL_NETWORK = "true";
16
17const TEST_USER = "apps-user";
18const TEST_ORG = "apps-org";
19const OWNER = "syncer";
20const REPO = "custom-tools";
21const REPO_FULL_NAME = `${OWNER}/${REPO}`;
22const GITHUB_CONNECTION = "tools.github.user.main";
23
24interface TestHttpServer {
25 readonly baseUrl: string;
26 readonly close: () => Promise<void>;
27}
28
29interface SyncResult {
30 readonly status: "published" | "up-to-date" | "failed";
31 readonly snapshotId?: string;
32 readonly upstreamSha?: string;
33 readonly tools: readonly string[];
34 readonly skipped: readonly { readonly path: string; readonly reason: string }[];
35 readonly errors?: readonly unknown[];
36}
37
38interface ToolRow {
39 readonly address: string;
40 readonly name: string;
41 readonly integration: string;
42}
43
44interface ExecuteResponse {
45 readonly status: "completed" | "paused";
46 readonly text: string;
47 readonly structured: {
48 readonly status?: string;
49 readonly result?: unknown;
50 readonly error?: unknown;
51 };
52 readonly isError?: boolean;
53}
54
55const bearerTemplate = {
56 slug: "bearer",
57 type: "apiKey",
58 label: "Bearer token",
59 headers: { Authorization: ["Bearer ", { type: "variable", name: "token" }] },
60};
61
62const jsonHeaders = {
63 "content-type": "application/json",
64 "x-test-user": TEST_USER,
65 "x-test-org": TEST_ORG,
66};
67
68const findFreePort = (): Promise<number> =>
69 new Promise((resolve, reject) => {
70 const probe = createNetServer();
71 probe.once("error", reject);
72 probe.listen(0, "127.0.0.1", () => {
73 const address = probe.address();
74 const port = typeof address === "object" && address ? address.port : 0;
75 probe.close((error) => (error ? reject(error) : resolve(port)));
76 });
77 });
78
79const readRequestBody = async (request: IncomingMessage): Promise<Buffer | undefined> => {
80 if (request.method === "GET" || request.method === "HEAD") return undefined;
81 const chunks: Buffer[] = [];
82 for await (const chunk of request) {
83 chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk));
84 }
85 const body = Buffer.concat(chunks);
86 return body.length > 0 ? body : undefined;
87};
88
89const webRequestFromNode = async (request: IncomingMessage, port: number): Promise<Request> => {
90 const headers = new Headers();
91 for (const [name, raw] of Object.entries(request.headers)) {
92 if (Array.isArray(raw)) {
93 for (const value of raw) headers.append(name, value);
94 } else if (raw !== undefined) {
95 headers.set(name, raw);
96 }
97 }
98 const body = await readRequestBody(request);
99 return new Request(`http://127.0.0.1:${port}${request.url ?? "/"}`, {
100 method: request.method,
101 headers,
102 body: body ? new Uint8Array(body) : undefined,
103 });
104};
105
106const sendWebResponse = async (webResponse: Response, response: ServerResponse): Promise<void> => {
107 response.statusCode = webResponse.status;
108 webResponse.headers.forEach((value, name) => response.setHeader(name, value));
109 response.end(Buffer.from(await webResponse.arrayBuffer()));
110};
111
112const startHttpServer = async (
113 handler: (request: Request) => Promise<Response>,
114): Promise<TestHttpServer> => {
115 let port = 0;
116 const server = createServer((request, response) => {
117 void (async () => {
118 try {
119 await sendWebResponse(await handler(await webRequestFromNode(request, port)), response);
120 } catch (error) {
121 response.statusCode = 500;
122 response.end(error instanceof Error ? error.message : String(error));
123 }
124 })();
125 });
126 await new Promise<void>((resolve, reject) => {
127 server.once("error", reject);
128 server.listen(0, "127.0.0.1", () => {
129 const address = server.address();
130 port = typeof address === "object" && address ? address.port : 0;
131 resolve();
132 });
133 });
134 return {
135 baseUrl: `http://127.0.0.1:${port}`,
136 close: () =>
137 new Promise((resolve, reject) => {
138 server.close((error) => (error ? reject(error) : resolve()));
139 }),
140 };
141};
142
143const requestJson = async <T>(
144 path: string,
145 init: RequestInit = {},
146 expectedStatus = 200,
147): Promise<T> => {
148 const response = await fetch(`${server.baseUrl}${path}`, init);
149 const text = await response.text();
150 expect(response.status).toBe(expectedStatus);
151 return (text.length > 0 ? JSON.parse(text) : null) as T;
152};
153
154const postJson = <T>(path: string, body: unknown, expectedStatus = 200): Promise<T> =>
155 requestJson<T>(
156 path,
157 {
158 method: "POST",
159 headers: jsonHeaders,
160 body: JSON.stringify(body),
161 },
162 expectedStatus,
163 );
164
165const githubFetch = async <T>(
166 emulator: Emulator,
167 token: string,
168 path: string,
169 init: RequestInit = {},
170): Promise<T> => {
171 const headers = new Headers(init.headers);
172 headers.set("authorization", `Bearer ${token}`);
173 headers.set("accept", "application/vnd.github+json");
174 if (init.body !== undefined) headers.set("content-type", "application/json");
175 const response = await fetch(`${emulator.url}${path}`, { ...init, headers });
176 const text = await response.text();
177 expect(response.ok).toBe(true);
178 return (text.length > 0 ? JSON.parse(text) : null) as T;
179};
180
181const createIssue = (emulator: Emulator, token: string, title: string): Promise<unknown> =>
182 githubFetch(emulator, token, `/repos/${OWNER}/${REPO}/issues`, {
183 method: "POST",
184 body: JSON.stringify({ title }),
185 });
186
187const putRepoFiles = async (
188 emulator: Emulator,
189 token: string,
190 files: Readonly<Record<string, string>>,
191): Promise<string> => {
192 const ref = await githubFetch<{ object: { sha: string } }>(
193 emulator,
194 token,
195 `/repos/${OWNER}/${REPO}/git/ref/heads/main`,
196 );
197 const parentSha = ref.object.sha;
198 const tree = await githubFetch<{ sha: string }>(
199 emulator,
200 token,
201 `/repos/${OWNER}/${REPO}/git/trees`,
202 {
203 method: "POST",
204 body: JSON.stringify({
205 tree: Object.entries(files).map(([path, content]) => ({
206 path,
207 mode: "100644",
208 type: "blob",
209 content,
210 })),
211 }),
212 },
213 );
214 const commit = await githubFetch<{ sha: string }>(
215 emulator,
216 token,
217 `/repos/${OWNER}/${REPO}/git/commits`,
218 {
219 method: "POST",
220 body: JSON.stringify({
221 message: `Update custom tools ${randomBytes(3).toString("hex")}`,
222 tree: tree.sha,
223 parents: [parentSha],
224 }),
225 },
226 );
227 await githubFetch(emulator, token, `/repos/${OWNER}/${REPO}/git/refs/heads/main`, {
228 method: "PATCH",
229 body: JSON.stringify({ sha: commit.sha }),
230 });
231 return commit.sha;
232};
233
234const executorJson = JSON.stringify(
235 {
236 $schema: "https://executor.sh/schemas/executor.json",
237 description: "Curated tools wrapping our raw integration surface.",
238 },
239 null,
240 2,
241);
242
243const dealPipelineSyncSource = `import { z } from "zod";
244import { defineTool, integration } from "executor:app";
245
246/**
247 * The deterministic-program case: pull GitHub issues into the scope database
248 * so agents query a table instead of paging the raw API.
249 */
250export default defineTool({
251 description:
252 "Refresh the deals table from GitHub issues. Syncs issue numbers and titles " +
253 "for pipeline questions.",
254
255 integrations: {
256 github: integration("github"),
257 },
258
259 input: z.object({
260 owner: z.string(),
261 repo: z.string(),
262 }),
263
264 output: z.object({ synced: z.number() }),
265
266 annotations: { readOnly: false, destructive: false },
267
268 async handler({ owner, repo }, { github, db }) {
269 await db.sql\`
270 CREATE TABLE IF NOT EXISTS deals (
271 id TEXT PRIMARY KEY,
272 name TEXT NOT NULL,
273 stage TEXT NOT NULL
274 )\`;
275
276 const issues = await github.repos.listIssues({ owner, repo });
277
278 let synced = 0;
279 for (const issue of issues) {
280 await db.sql\`
281 INSERT INTO deals (id, name, stage)
282 VALUES (\${String(issue.number)}, \${issue.title}, 'open')
283 ON CONFLICT (id) DO UPDATE SET
284 name = excluded.name,
285 stage = excluded.stage\`;
286 synced++;
287 }
288
289 return { synced };
290 },
291});
292`;
293
294const findDealDocsSource = `import { z } from "zod";
295import { defineTool, integration } from "executor:app";
296
297/**
298 * The curation case: a read-only query over the scope database populated by
299 * the sync tool.
300 */
301export default defineTool({
302 description:
303 "Find documents for a deal across synced pipeline records. Searches " +
304 "stored issue titles.",
305
306 integrations: {
307 github: integration("github"),
308 },
309
310 input: z.object({
311 limit: z.number().int().max(50).default(20),
312 }),
313
314 output: z.object({
315 documents: z.array(
316 z.object({
317 name: z.string(),
318 stage: z.string(),
319 }),
320 ),
321 }),
322
323 annotations: { readOnly: true, destructive: false },
324
325 async handler({ limit }, { db }) {
326 await db.sql\`
327 CREATE TABLE IF NOT EXISTS deals (
328 id TEXT PRIMARY KEY,
329 name TEXT NOT NULL,
330 stage TEXT NOT NULL
331 )\`;
332 const rows = await db.sql\`
333 SELECT name, stage FROM deals ORDER BY id LIMIT \${limit}\`;
334
335 return {
336 documents: rows.map((row) => ({
337 name: String(row.name),
338 stage: String(row.stage),
339 })),
340 };
341 },
342});
343`;
344
345const extraToolSource = `import { z } from "zod";
346import { defineTool } from "executor:app";
347
348export default defineTool({
349 description: "Return a static custom-tools health marker.",
350 input: z.object({}),
351 output: z.object({ ok: z.boolean() }),
352 annotations: { readOnly: true, destructive: false },
353 async handler() {
354 return { ok: true };
355 },
356});
357`;
358
359const initialFiles = (): Record<string, string> => ({
360 "executor.json": executorJson,
361 "tools/deal-pipeline-sync.ts": dealPipelineSyncSource,
362 "tools/find-deal-docs.ts": findDealDocsSource,
363});
364
365const registerGitHubIntegration = async (emulator: Emulator, token: string): Promise<void> => {
366 await postJson("/api/openapi/specs", {
367 spec: { kind: "url", url: emulator.openapiUrl },
368 slug: "github",
369 baseUrl: emulator.url,
370 authenticationTemplate: [bearerTemplate],
371 });
372 const created = await postJson<{ address: string }>("/api/connections", {
373 owner: "user",
374 name: "main",
375 integration: "github",
376 template: "bearer",
377 value: token,
378 });
379 expect(created.address).toBe(GITHUB_CONNECTION);
380};
381
382const syncSource = (repo = REPO_FULL_NAME): Promise<SyncResult> =>
383 postJson<SyncResult>("/api/apps/sources/github/sync", {
384 repo,
385 connection: GITHUB_CONNECTION,
386 });
387
388const listAppTools = (): Promise<readonly ToolRow[]> =>
389 requestJson<readonly ToolRow[]>("/api/tools?integration=apps", {
390 headers: jsonHeaders,
391 });
392
393const execute = (code: string): Promise<ExecuteResponse> =>
394 postJson<ExecuteResponse>("/api/executions", { code, autoApprove: true });
395
396const executeResult = async (code: string): Promise<unknown> => {
397 const response = await execute(code);
398 expect(response.status).toBe("completed");
399 expect(response.isError).toBe(false);
400 return response.structured.result;
401};
402
403const callAppToolCode = (toolName: string, args: unknown): string => `
404const found = await tools.search({ namespace: "apps", query: ${JSON.stringify(toolName)}, limit: 20 });
405const item = found.items.find((candidate) => candidate.path.endsWith(${JSON.stringify(toolName)}));
406if (!item) return { ok: false, missing: ${JSON.stringify(toolName)}, found };
407let fn = tools;
408for (const segment of item.path.split(".")) fn = fn[segment];
409const result = await fn(${JSON.stringify(args)});
410return { path: item.path, result };
411`;
412
413let github!: Emulator;
414let server!: TestHttpServer;
415let disposeApp: () => Promise<void> = async () => {};
416
417beforeAll(async () => {
418 const emulatorPort = await findFreePort();
419 github = await createEmulator({
420 service: "github",
421 port: emulatorPort,
422 seed: {
423 github: {
424 users: [{ login: OWNER, name: "Syncer" }],
425 repos: [{ owner: OWNER, name: REPO, auto_init: true }],
426 },
427 },
428 });
429 const { makeSelfHostTestApp, headerIdentityLayer } = await import("./testing/test-app");
430 const app = await makeSelfHostTestApp({ identity: headerIdentityLayer, dbPath });
431 disposeApp = app.dispose;
432 server = await startHttpServer(app.handler);
433});
434
435afterAll(async () => {
436 await server?.close();
437 await disposeApp();
438 await github?.close();
439});
440
441test("GitHub source sync publishes and invokes custom tools through self-host HTTP", async () => {
442 const credential = await github.credentials.mint({
443 type: "api-key",
444 login: OWNER,
445 scopes: ["repo", "user"],
446 });
447 const token = credential.token;
448 expect(token).toBeTruthy();
449 await registerGitHubIntegration(github, token!);
450 await createIssue(github, token!, "Acme renewal diligence");
451 await createIssue(github, token!, "Beta pipeline memo");
452 await putRepoFiles(github, token!, initialFiles());
453
454 const unauthorized = await fetch(`${server.baseUrl}/api/apps/sources/github/sync`, {
455 method: "POST",
456 headers: { "content-type": "application/json" },
457 body: JSON.stringify({ repo: REPO_FULL_NAME, connection: GITHUB_CONNECTION }),
458 });
459 expect(unauthorized.status).toBe(401);
460
461 const published = await syncSource();
462 expect(published.status).toBe("published");
463 expect(published.tools).toEqual(["deal-pipeline-sync", "find-deal-docs"]);
464 expect(published.skipped).toEqual([]);
465 const firstSnapshot = published.snapshotId;
466 expect(firstSnapshot).toBeTruthy();
467
468 const listed = await listAppTools();
469 expect(listed.map((tool) => tool.name).sort()).toEqual(["deal-pipeline-sync", "find-deal-docs"]);
470
471 const searchResult = await executeResult(
472 'const found = await tools.search({ namespace: "apps", query: "pipeline", limit: 10 }); return found.items.map((item) => item.path);',
473 );
474 expect(JSON.stringify(searchResult)).toContain("deal-pipeline-sync");
475
476 const schema = await requestJson<{ inputSchema: { properties?: Record<string, unknown> } }>(
477 `/api/tools/schema?address=${encodeURIComponent(listed.find((tool) => tool.name === "deal-pipeline-sync")!.address)}`,
478 { headers: jsonHeaders },
479 );
480 expect(schema.inputSchema.properties?.github).toMatchObject({
481 type: "string",
482 enum: [GITHUB_CONNECTION],
483 });
484
485 const invoke = (await executeResult(
486 callAppToolCode("deal-pipeline-sync", {
487 github: GITHUB_CONNECTION,
488 owner: OWNER,
489 repo: REPO,
490 }),
491 )) as { result?: { ok?: boolean; data?: { synced?: number }; error?: unknown } };
492 expect(invoke.result?.ok).toBe(true);
493 expect(invoke.result?.data?.synced).toBe(2);
494
495 const ledger = await github.ledger.list();
496 const issueList = ledger.find(
497 (entry) =>
498 entry.operationId === "issues/listForRepo" && entry.path === `/repos/${OWNER}/${REPO}/issues`,
499 );
500 expect(issueList?.identity.user?.login).toBe(OWNER);
501
502 const readDb = (await executeResult(callAppToolCode("find-deal-docs", { limit: 10 }))) as {
503 result?: { ok?: boolean; data?: { documents?: readonly { name: string }[] } };
504 };
505 expect(readDb.result?.ok).toBe(true);
506 expect(readDb.result?.data?.documents?.map((row) => row.name).sort()).toEqual([
507 "Acme renewal diligence",
508 "Beta pipeline memo",
509 ]);
510
511 const upToDate = await syncSource();
512 expect(upToDate.status).toBe("up-to-date");
513 expect("snapshotId" in upToDate).toBe(false);
514
515 await putRepoFiles(github, token!, {
516 ...initialFiles(),
517 "tools/extra-tool.ts": extraToolSource,
518 });
519 const withExtra = await syncSource();
520 expect(withExtra.status).toBe("published");
521 expect(withExtra.snapshotId).not.toBe(firstSnapshot);
522 expect((await listAppTools()).map((tool) => tool.name).sort()).toEqual([
523 "deal-pipeline-sync",
524 "extra-tool",
525 "find-deal-docs",
526 ]);
527
528 await putRepoFiles(github, token!, initialFiles());
529 const removed = await syncSource();
530 expect(removed.status).toBe("published");
531 expect((await listAppTools()).map((tool) => tool.name).sort()).toEqual([
532 "deal-pipeline-sync",
533 "find-deal-docs",
534 ]);
535
536 await putRepoFiles(github, token!, {
537 ...initialFiles(),
538 "workflows/x.ts": "export default {};",
539 });
540 const skipped = await syncSource();
541 expect(skipped.status).toBe("published");
542 expect(skipped.skipped).toEqual([{ path: "workflows/x.ts", reason: "not supported yet" }]);
543 const afterSkipped = await listAppTools();
544 expect(afterSkipped.map((tool) => tool.name).sort()).toEqual([
545 "deal-pipeline-sync",
546 "find-deal-docs",
547 ]);
548 expect(afterSkipped.some((tool) => tool.address.includes("workflow"))).toBe(false);
549});
apps/host-selfhost/src/apps.ts+71−0
@@ -0,0 +1,71 @@
1import { join } from "node:path";
2
3import { Effect } from "effect";
4
5import {
6 appsPlugin,
7 BindingError,
8 makeSelfHostAppsRuntime,
9 makeSqliteAppsStore,
10 type AppsRuntime,
11 type ClientResolver,
12} from "@executor-js/plugin-apps/api";
13import type { PluginCtx } from "@executor-js/sdk";
14
15import { resolveDataDir } from "./config";
16import { makeSelfHostAppsResolver } from "./apps-resolver";
17
18const missingResolver = (): ClientResolver => ({
19 listConnections: () => Effect.succeed([]),
20 resolveConnection: () => Effect.succeed(null),
21 call: ({ integration, connection }) =>
22 Effect.fail(
23 new BindingError({
24 role: integration,
25 integration,
26 requestedConnection: connection,
27 message: "apps resolver is unavailable outside a request-scoped executor",
28 }),
29 ),
30});
31
32const createSubsystem = (dataDir: string) => {
33 const appsDir = join(dataDir, "apps");
34 const store = makeSqliteAppsStore({ path: join(appsDir, "store.sqlite") });
35 const host = makeSelfHostAppsRuntime({
36 dataDir: appsDir,
37 store,
38 resolver: missingResolver(),
39 });
40 const plugin = appsPlugin({
41 runtime: host.runtime,
42 makeResolver: ({ ctx }) => makeSelfHostAppsResolver({ ctx: ctx as PluginCtx }),
43 });
44 return {
45 runtime: host.runtime,
46 plugin,
47 close: host.close,
48 };
49};
50
51export interface SelfHostAppsSubsystem {
52 readonly runtime: AppsRuntime;
53 readonly plugin: ReturnType<typeof appsPlugin>;
54 readonly close: () => Promise<void>;
55}
56
57let current: { readonly dataDir: string; readonly subsystem: SelfHostAppsSubsystem } | undefined;
58
59export const getSelfHostAppsSubsystem = (): SelfHostAppsSubsystem => {
60 const dataDir = resolveDataDir();
61 if (current && current.dataDir === dataDir) return current.subsystem;
62 const subsystem = createSubsystem(dataDir);
63 current = { dataDir, subsystem };
64 return subsystem;
65};
66
67export const closeSelfHostAppsSubsystem = async (): Promise<void> => {
68 const subsystem = current?.subsystem;
69 current = undefined;
70 await subsystem?.close();
71};
apps/host-selfhost/src/testing/test-app.ts+4−0
@@ -30,6 +30,8 @@ import { · 29 unmodified lines
30} from "../mcp/session-store";30} from "../mcp/session-store";
31import { selfHostPlugins } from "../plugins";31import { selfHostPlugins } from "../plugins";
32import { ErrorCaptureLive } from "../observability";32import { ErrorCaptureLive } from "../observability";
33import { closeSelfHostAppsSubsystem } from "../apps";
34import { makeSelfHostAppsSyncRoute } from "../apps-route";
33 35
34// ===========================================================================36// ===========================================================================
35// Self-host TEST harness — the throwaway composition tests use to exercise the37// Self-host TEST harness — the throwaway composition tests use to exercise the
@@ -202,6 +204,7 @@ export const makeSelfHostTestApp = async ( · 166 unmodified lines
202 },204 },
203 extensions: {205 extensions: {
204 routes: [206 routes: [
207 makeSelfHostAppsSyncRoute({ identity: options.identity, db: dbHandle }),
205 HttpApiSwagger.layer(composePluginApi(selfHostPlugins).prefix("/api"), { path: "/docs" }),208 HttpApiSwagger.layer(composePluginApi(selfHostPlugins).prefix("/api"), { path: "/docs" }),
206 ],209 ],
207 },210 },
@@ -217,6 +220,7 @@ export const makeSelfHostTestApp = async ( · 9 unmodified lines
217 handler: web.handler,220 handler: web.handler,
218 dispose: async () => {221 dispose: async () => {
219 await web.dispose();222 await web.dispose();
223 await closeSelfHostAppsSubsystem();
220 await sessionStore.close();224 await sessionStore.close();
221 await dbHandle.close();225 await dbHandle.close();
222 },226 },