History

Add GitHub sync and self-host custom tools wiring

Rhys Sullivan committed 4 months ago

Commit 275446e1 parent 41eb3ae19 files · +1665 −40Browse files

Showing the changes to apps (8 of 19 files). Show all

Unified · Split

8 files
  1. apps/host-selfhost/executor.config.ts +2 −0
  2. apps/host-selfhost/package.json +2 −0
  3. apps/host-selfhost/src/app.ts +4 −0
  4. apps/host-selfhost/src/apps-resolver.ts +130 −0
  5. apps/host-selfhost/src/apps-route.ts +91 −0
  6. apps/host-selfhost/src/apps-wire.node.test.ts +549 −0
  7. apps/host-selfhost/src/apps.ts +71 −0
  8. apps/host-selfhost/src/testing/test-app.ts +4 −0
apps/host-selfhost/executor.config.ts+2−0
@@ -8,6 +8,7 @@ import { encryptedSecretsPlugin } from "@executor-js/plugin-encrypted-secrets"; · 7 unmodified lines
88 import { toolkitsPlugin } from "@executor-js/plugin-toolkits/server";
99
1010 import { resolveSecretKey } from "./src/config";
11+import { getSelfHostAppsSubsystem } from "./src/apps";
1112
1213 // ---------------------------------------------------------------------------
1314 // Single source of truth for the self-hosted app's plugin list.
@@ -36,6 +37,7 @@ export default defineExecutorConfig({ · 22 unmodified lines
3637 mcpHttpPlugin({ dangerouslyAllowStdioMCP: false }),
3738 graphqlHttpPlugin(),
3839 toolkitsPlugin({ activeToolkitSlug }),
40+ getSelfHostAppsSubsystem().plugin,
3941 // First writable secret provider -> the default for `secrets.set`.
4042 encryptedSecretsPlugin({ key: resolveSecretKey() }),
4143 ] as const,
apps/host-selfhost/package.json+2−0
@@ -26,6 +26,7 @@ · 25 unmodified lines
2626 "@executor-js/execution": "workspace:*",
2727 "@executor-js/fumadb": "workspace:*",
2828 "@executor-js/host-mcp": "workspace:*",
29+ "@executor-js/plugin-apps": "workspace:*",
2930 "@executor-js/plugin-encrypted-secrets": "workspace:*",
3031 "@executor-js/plugin-google": "workspace:*",
3132 "@executor-js/plugin-graphql": "workspace:*",
@@ -48,6 +49,7 @@ · 16 unmodified lines
4849 },
4950 "devDependencies": {
5051 "@effect/vitest": "catalog:",
52+ "@executor-js/emulate": "^0.13.2",
5153 "@executor-js/vite-plugin": "workspace:*",
5254 "@tailwindcss/vite": "catalog:",
5355 "@tanstack/router-plugin": "^1.167.12",
apps/host-selfhost/src/app.ts+4−0
@@ -22,6 +22,8 @@ import { makeSelfHostMcpSeams } from "./mcp"; · 21 unmodified lines
2222 import { selfHostPlugins } from "./plugins";
2323 import { ErrorCaptureLive } from "./observability";
2424 import { oauthCallbackSignInRedirectLocation } from "./auth/oauth-callback-login";
25+import { closeSelfHostAppsSubsystem } from "./apps";
26+import { makeSelfHostAppsSyncRoute } from "./apps-route";
2527
2628 // ===========================================================================
2729 // The self-hosted Executor app, as ONE `ExecutorApp.make` call.
@@ -119,6 +121,7 @@ export const makeSelfHostApp = async (options: MakeSelfHostAppOptions = {}) => { · 91 unmodified lines
119121 makeSelfHostAdminApiLayer({ betterAuth, db: dbHandle, mountPrefix: "/api" }),
120122 // Public system API: /api/health + /api/setup-status (unauthenticated).
121123 makeSelfHostSystemApiLayer({ betterAuth, db: dbHandle, mountPrefix: "/api" }),
124+ makeSelfHostAppsSyncRoute({ identity: identityLayer, db: dbHandle }),
122125 // Swagger UI at /docs, over the /api-prefixed spec (matches the served paths).
123126 HttpApiSwagger.layer(composePluginApi(selfHostPlugins).prefix("/api"), { path: "/docs" }),
124127 ],
@@ -139,6 +142,7 @@ export const makeSelfHostApp = async (options: MakeSelfHostAppOptions = {}) => { · 14 unmodified lines
139142 toWebHandler,
140143 betterAuth,
141144 closeDb: async () => {
145+ await closeSelfHostAppsSubsystem();
142146 await mcp.close();
143147 await dbHandle.close();
144148 },
apps/host-selfhost/src/apps-resolver.ts+130−0
@@ -0,0 +1,130 @@
1+import { Effect } from "effect";
2+
3+import {
4+ ConnectionName,
5+ IntegrationSlug,
6+ ToolAddress,
7+ isToolResult,
8+ type ConnectionRef,
9+ type ExecuteError,
10+ type Owner,
11+ type PluginCtx,
12+} from "@executor-js/sdk";
13+import {
14+ BindingError,
15+ type ClientResolver,
16+ type ConnectionCandidate,
17+} from "@executor-js/plugin-apps/api";
18+
19+const parseConnectionAddress = (address: string): ConnectionRef | null => {
20+ const parts = address.split(".");
21+ if (parts.length !== 4 || parts[0] !== "tools") return null;
22+ const [, integration, owner, name] = parts;
23+ if (!integration || !name) return null;
24+ if (owner !== "org" && owner !== "user") return null;
25+ return {
26+ owner: owner as Owner,
27+ integration: IntegrationSlug.make(integration),
28+ name: ConnectionName.make(name),
29+ };
30+};
31+
32+const bindingError = (input: {
33+ readonly integration: string;
34+ readonly connection?: string;
35+ readonly message: string;
36+ readonly cause?: unknown;
37+}): BindingError =>
38+ new BindingError({
39+ role: input.integration,
40+ integration: input.integration,
41+ requestedConnection: input.connection,
42+ message: input.message,
43+ });
44+
45+const executeErrorMessage = (cause: ExecuteError): string =>
46+ typeof cause.message === "string" && cause.message.length > 0 ? cause.message : cause._tag;
47+
48+const toCandidate = (connection: {
49+ readonly address: unknown;
50+ readonly integration: unknown;
51+ readonly name?: unknown;
52+ readonly owner?: unknown;
53+}): ConnectionCandidate => ({
54+ address: String(connection.address),
55+ integration: String(connection.integration),
56+ ...(connection.name !== undefined ? { name: String(connection.name) } : {}),
57+ ...(connection.owner !== undefined ? { owner: String(connection.owner) } : {}),
58+});
59+
60+export const makeSelfHostAppsResolver = (input: { readonly ctx: PluginCtx }): ClientResolver => ({
61+ listConnections: ({ integration }) =>
62+ input.ctx.connections.list({ integration: IntegrationSlug.make(integration) }).pipe(
63+ Effect.map((connections) => connections.map(toCandidate)),
64+ Effect.mapError((cause) =>
65+ bindingError({
66+ integration,
67+ message: `failed to list ${integration} connections`,
68+ cause,
69+ }),
70+ ),
71+ ),
72+
73+ resolveConnection: ({ connection }) =>
74+ Effect.gen(function* () {
75+ const ref = parseConnectionAddress(connection);
76+ if (!ref) return null;
77+ const row = yield* input.ctx.connections.get(ref).pipe(
78+ Effect.mapError((cause) =>
79+ bindingError({
80+ integration: String(ref.integration),
81+ connection,
82+ message: `failed to resolve connection ${connection}`,
83+ cause,
84+ }),
85+ ),
86+ );
87+ return row ? toCandidate(row) : null;
88+ }),
89+
90+ call: ({ integration, connection, path, args }) =>
91+ Effect.gen(function* () {
92+ const ref = parseConnectionAddress(connection);
93+ if (!ref) {
94+ return yield* bindingError({
95+ integration,
96+ connection,
97+ message: `invalid connection address ${connection}`,
98+ });
99+ }
100+ if (String(ref.integration) !== integration) {
101+ return yield* bindingError({
102+ integration,
103+ connection,
104+ message: `connection "${connection}" belongs to integration "${ref.integration}", not "${integration}"`,
105+ });
106+ }
107+ const tool = path.join(".");
108+ const address = ToolAddress.make(`${connection}.${tool}`);
109+ const payload = args[0] ?? {};
110+ const result = yield* input.ctx.execute(address, payload).pipe(
111+ Effect.mapError((cause) =>
112+ bindingError({
113+ integration,
114+ connection,
115+ message: executeErrorMessage(cause),
116+ cause,
117+ }),
118+ ),
119+ );
120+ if (isToolResult(result)) {
121+ if (result.ok) return result.data;
122+ return yield* bindingError({
123+ integration,
124+ connection,
125+ message: result.error.message,
126+ });
127+ }
128+ return result;
129+ }),
130+});
apps/host-selfhost/src/apps-route.ts+91−0
@@ -0,0 +1,91 @@
1+import { HttpEffect, HttpRouter } from "effect/unstable/http";
2+import { Data, Effect, Layer } from "effect";
3+
4+import {
5+ IdentityProvider,
6+ RequestOrgSlug,
7+ RequestWebOrigin,
8+ makeScopedExecutor,
9+ type Principal,
10+} from "@executor-js/api/server";
11+import { ToolAddress } from "@executor-js/sdk";
12+
13+import { SelfHostScopedExecutorSeams } from "./execution";
14+import { SelfHostDb, type SelfHostDbHandle } from "./db/self-host-db";
15+import type { SelfHostPlugins } from "./plugins";
16+
17+class AppsSyncRouteError extends Data.TaggedError("AppsSyncRouteError")<{
18+ readonly status: number;
19+ readonly message: string;
20+}> {}
21+
22+const jsonResponse = (body: unknown, status = 200): Response =>
23+ new Response(JSON.stringify(body), {
24+ status,
25+ headers: { "content-type": "application/json" },
26+ });
27+
28+const parseJson = (request: Request): Effect.Effect<unknown, AppsSyncRouteError> =>
29+ Effect.tryPromise({
30+ try: () => request.json() as Promise<unknown>,
31+ catch: () =>
32+ new AppsSyncRouteError({
33+ status: 400,
34+ message: "Invalid JSON body",
35+ }),
36+ });
37+
38+const buildExecutor = (principal: Principal, request: Request) => {
39+ const withOrigin = makeScopedExecutor<SelfHostPlugins>(
40+ principal.accountId,
41+ principal.organizationId,
42+ principal.organizationName,
43+ ).pipe(Effect.provideService(RequestWebOrigin, { origin: new URL(request.url).origin }));
44+ return principal.organizationSlug
45+ ? withOrigin.pipe(Effect.provideService(RequestOrgSlug, { slug: principal.organizationSlug }))
46+ : withOrigin;
47+};
48+
49+export interface SelfHostAppsSyncRouteDeps {
50+ readonly identity: Layer.Layer<IdentityProvider>;
51+ readonly db: SelfHostDbHandle;
52+}
53+
54+export const makeSelfHostAppsSyncRoute = ({ identity, db }: SelfHostAppsSyncRouteDeps) => {
55+ const services = Layer.merge(
56+ SelfHostScopedExecutorSeams.pipe(Layer.provide(Layer.succeed(SelfHostDb)(db))),
57+ identity,
58+ );
59+ return HttpRouter.add(
60+ "POST",
61+ "/api/apps/sources/github/sync",
62+ HttpEffect.fromWebHandler(
63+ (request): Promise<Response> =>
64+ Effect.runPromise(
65+ Effect.gen(function* () {
66+ const identityProvider = yield* IdentityProvider;
67+ const principal = yield* identityProvider.authenticate(request);
68+ const payload = yield* parseJson(request);
69+ const executor = yield* buildExecutor(principal, request);
70+ const result = yield* executor.execute(
71+ ToolAddress.make("executor.apps.sync_github_source"),
72+ payload,
73+ );
74+ return jsonResponse(result);
75+ }).pipe(
76+ Effect.catchTags({
77+ Unauthorized: () => Effect.succeed(new Response("Unauthorized", { status: 401 })),
78+ NoOrganization: () => Effect.succeed(new Response("Forbidden", { status: 403 })),
79+ Unavailable: () => Effect.succeed(new Response("Unavailable", { status: 503 })),
80+ AppsSyncRouteError: (error) =>
81+ Effect.succeed(jsonResponse({ error: error.message }, error.status)),
82+ }),
83+ Effect.catchCause(() =>
84+ Effect.succeed(new Response("Internal Server Error", { status: 500 })),
85+ ),
86+ Effect.provide(services),
87+ ),
88+ ),
89+ ),
90+ );
91+};
apps/host-selfhost/src/apps-wire.node.test.ts+549−0
@@ -0,0 +1,549 @@
1+import { randomBytes } from "node:crypto";
2+import { mkdtempSync } from "node:fs";
3+import { createServer, type IncomingMessage, type ServerResponse } from "node:http";
4+import { createServer as createNetServer } from "node:net";
5+import { tmpdir } from "node:os";
6+import { join } from "node:path";
7+
8+import { afterAll, beforeAll, expect, test } from "@effect/vitest";
9+import { createEmulator, type Emulator } from "@executor-js/emulate";
10+
11+const dataDir = mkdtempSync(join(tmpdir(), "eh-apps-wire-"));
12+const dbPath = join(dataDir, "data.db");
13+process.env.EXECUTOR_DATA_DIR = dataDir;
14+process.env.EXECUTOR_SECRET_KEY = "apps-wire-secret-key";
15+process.env.EXECUTOR_ALLOW_LOCAL_NETWORK = "true";
16+
17+const TEST_USER = "apps-user";
18+const TEST_ORG = "apps-org";
19+const OWNER = "syncer";
20+const REPO = "custom-tools";
21+const REPO_FULL_NAME = `${OWNER}/${REPO}`;
22+const GITHUB_CONNECTION = "tools.github.user.main";
23+
24+interface TestHttpServer {
25+ readonly baseUrl: string;
26+ readonly close: () => Promise<void>;
27+}
28+
29+interface SyncResult {
30+ readonly status: "published" | "up-to-date" | "failed";
31+ readonly snapshotId?: string;
32+ readonly upstreamSha?: string;
33+ readonly tools: readonly string[];
34+ readonly skipped: readonly { readonly path: string; readonly reason: string }[];
35+ readonly errors?: readonly unknown[];
36+}
37+
38+interface ToolRow {
39+ readonly address: string;
40+ readonly name: string;
41+ readonly integration: string;
42+}
43+
44+interface ExecuteResponse {
45+ readonly status: "completed" | "paused";
46+ readonly text: string;
47+ readonly structured: {
48+ readonly status?: string;
49+ readonly result?: unknown;
50+ readonly error?: unknown;
51+ };
52+ readonly isError?: boolean;
53+}
54+
55+const bearerTemplate = {
56+ slug: "bearer",
57+ type: "apiKey",
58+ label: "Bearer token",
59+ headers: { Authorization: ["Bearer ", { type: "variable", name: "token" }] },
60+};
61+
62+const jsonHeaders = {
63+ "content-type": "application/json",
64+ "x-test-user": TEST_USER,
65+ "x-test-org": TEST_ORG,
66+};
67+
68+const findFreePort = (): Promise<number> =>
69+ new Promise((resolve, reject) => {
70+ const probe = createNetServer();
71+ probe.once("error", reject);
72+ probe.listen(0, "127.0.0.1", () => {
73+ const address = probe.address();
74+ const port = typeof address === "object" && address ? address.port : 0;
75+ probe.close((error) => (error ? reject(error) : resolve(port)));
76+ });
77+ });
78+
79+const readRequestBody = async (request: IncomingMessage): Promise<Buffer | undefined> => {
80+ if (request.method === "GET" || request.method === "HEAD") return undefined;
81+ const chunks: Buffer[] = [];
82+ for await (const chunk of request) {
83+ chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk));
84+ }
85+ const body = Buffer.concat(chunks);
86+ return body.length > 0 ? body : undefined;
87+};
88+
89+const webRequestFromNode = async (request: IncomingMessage, port: number): Promise<Request> => {
90+ const headers = new Headers();
91+ for (const [name, raw] of Object.entries(request.headers)) {
92+ if (Array.isArray(raw)) {
93+ for (const value of raw) headers.append(name, value);
94+ } else if (raw !== undefined) {
95+ headers.set(name, raw);
96+ }
97+ }
98+ const body = await readRequestBody(request);
99+ return new Request(`http://127.0.0.1:${port}${request.url ?? "/"}`, {
100+ method: request.method,
101+ headers,
102+ body: body ? new Uint8Array(body) : undefined,
103+ });
104+};
105+
106+const sendWebResponse = async (webResponse: Response, response: ServerResponse): Promise<void> => {
107+ response.statusCode = webResponse.status;
108+ webResponse.headers.forEach((value, name) => response.setHeader(name, value));
109+ response.end(Buffer.from(await webResponse.arrayBuffer()));
110+};
111+
112+const startHttpServer = async (
113+ handler: (request: Request) => Promise<Response>,
114+): Promise<TestHttpServer> => {
115+ let port = 0;
116+ const server = createServer((request, response) => {
117+ void (async () => {
118+ try {
119+ await sendWebResponse(await handler(await webRequestFromNode(request, port)), response);
120+ } catch (error) {
121+ response.statusCode = 500;
122+ response.end(error instanceof Error ? error.message : String(error));
123+ }
124+ })();
125+ });
126+ await new Promise<void>((resolve, reject) => {
127+ server.once("error", reject);
128+ server.listen(0, "127.0.0.1", () => {
129+ const address = server.address();
130+ port = typeof address === "object" && address ? address.port : 0;
131+ resolve();
132+ });
133+ });
134+ return {
135+ baseUrl: `http://127.0.0.1:${port}`,
136+ close: () =>
137+ new Promise((resolve, reject) => {
138+ server.close((error) => (error ? reject(error) : resolve()));
139+ }),
140+ };
141+};
142+
143+const requestJson = async <T>(
144+ path: string,
145+ init: RequestInit = {},
146+ expectedStatus = 200,
147+): Promise<T> => {
148+ const response = await fetch(`${server.baseUrl}${path}`, init);
149+ const text = await response.text();
150+ expect(response.status).toBe(expectedStatus);
151+ return (text.length > 0 ? JSON.parse(text) : null) as T;
152+};
153+
154+const postJson = <T>(path: string, body: unknown, expectedStatus = 200): Promise<T> =>
155+ requestJson<T>(
156+ path,
157+ {
158+ method: "POST",
159+ headers: jsonHeaders,
160+ body: JSON.stringify(body),
161+ },
162+ expectedStatus,
163+ );
164+
165+const githubFetch = async <T>(
166+ emulator: Emulator,
167+ token: string,
168+ path: string,
169+ init: RequestInit = {},
170+): Promise<T> => {
171+ const headers = new Headers(init.headers);
172+ headers.set("authorization", `Bearer ${token}`);
173+ headers.set("accept", "application/vnd.github+json");
174+ if (init.body !== undefined) headers.set("content-type", "application/json");
175+ const response = await fetch(`${emulator.url}${path}`, { ...init, headers });
176+ const text = await response.text();
177+ expect(response.ok).toBe(true);
178+ return (text.length > 0 ? JSON.parse(text) : null) as T;
179+};
180+
181+const createIssue = (emulator: Emulator, token: string, title: string): Promise<unknown> =>
182+ githubFetch(emulator, token, `/repos/${OWNER}/${REPO}/issues`, {
183+ method: "POST",
184+ body: JSON.stringify({ title }),
185+ });
186+
187+const putRepoFiles = async (
188+ emulator: Emulator,
189+ token: string,
190+ files: Readonly<Record<string, string>>,
191+): Promise<string> => {
192+ const ref = await githubFetch<{ object: { sha: string } }>(
193+ emulator,
194+ token,
195+ `/repos/${OWNER}/${REPO}/git/ref/heads/main`,
196+ );
197+ const parentSha = ref.object.sha;
198+ const tree = await githubFetch<{ sha: string }>(
199+ emulator,
200+ token,
201+ `/repos/${OWNER}/${REPO}/git/trees`,
202+ {
203+ method: "POST",
204+ body: JSON.stringify({
205+ tree: Object.entries(files).map(([path, content]) => ({
206+ path,
207+ mode: "100644",
208+ type: "blob",
209+ content,
210+ })),
211+ }),
212+ },
213+ );
214+ const commit = await githubFetch<{ sha: string }>(
215+ emulator,
216+ token,
217+ `/repos/${OWNER}/${REPO}/git/commits`,
218+ {
219+ method: "POST",
220+ body: JSON.stringify({
221+ message: `Update custom tools ${randomBytes(3).toString("hex")}`,
222+ tree: tree.sha,
223+ parents: [parentSha],
224+ }),
225+ },
226+ );
227+ await githubFetch(emulator, token, `/repos/${OWNER}/${REPO}/git/refs/heads/main`, {
228+ method: "PATCH",
229+ body: JSON.stringify({ sha: commit.sha }),
230+ });
231+ return commit.sha;
232+};
233+
234+const executorJson = JSON.stringify(
235+ {
236+ $schema: "https://executor.sh/schemas/executor.json",
237+ description: "Curated tools wrapping our raw integration surface.",
238+ },
239+ null,
240+ 2,
241+);
242+
243+const dealPipelineSyncSource = `import { z } from "zod";
244+import { defineTool, integration } from "executor:app";
245+
246+/**
247+ * The deterministic-program case: pull GitHub issues into the scope database
248+ * so agents query a table instead of paging the raw API.
249+ */
250+export default defineTool({
251+ description:
252+ "Refresh the deals table from GitHub issues. Syncs issue numbers and titles " +
253+ "for pipeline questions.",
254+
255+ integrations: {
256+ github: integration("github"),
257+ },
258+
259+ input: z.object({
260+ owner: z.string(),
261+ repo: z.string(),
262+ }),
263+
264+ output: z.object({ synced: z.number() }),
265+
266+ annotations: { readOnly: false, destructive: false },
267+
268+ async handler({ owner, repo }, { github, db }) {
269+ await db.sql\`
270+ CREATE TABLE IF NOT EXISTS deals (
271+ id TEXT PRIMARY KEY,
272+ name TEXT NOT NULL,
273+ stage TEXT NOT NULL
274+ )\`;
275+
276+ const issues = await github.repos.listIssues({ owner, repo });
277+
278+ let synced = 0;
279+ for (const issue of issues) {
280+ await db.sql\`
281+ INSERT INTO deals (id, name, stage)
282+ VALUES (\${String(issue.number)}, \${issue.title}, 'open')
283+ ON CONFLICT (id) DO UPDATE SET
284+ name = excluded.name,
285+ stage = excluded.stage\`;
286+ synced++;
287+ }
288+
289+ return { synced };
290+ },
291+});
292+`;
293+
294+const findDealDocsSource = `import { z } from "zod";
295+import { defineTool, integration } from "executor:app";
296+
297+/**
298+ * The curation case: a read-only query over the scope database populated by
299+ * the sync tool.
300+ */
301+export default defineTool({
302+ description:
303+ "Find documents for a deal across synced pipeline records. Searches " +
304+ "stored issue titles.",
305+
306+ integrations: {
307+ github: integration("github"),
308+ },
309+
310+ input: z.object({
311+ limit: z.number().int().max(50).default(20),
312+ }),
313+
314+ output: z.object({
315+ documents: z.array(
316+ z.object({
317+ name: z.string(),
318+ stage: z.string(),
319+ }),
320+ ),
321+ }),
322+
323+ annotations: { readOnly: true, destructive: false },
324+
325+ async handler({ limit }, { db }) {
326+ await db.sql\`
327+ CREATE TABLE IF NOT EXISTS deals (
328+ id TEXT PRIMARY KEY,
329+ name TEXT NOT NULL,
330+ stage TEXT NOT NULL
331+ )\`;
332+ const rows = await db.sql\`
333+ SELECT name, stage FROM deals ORDER BY id LIMIT \${limit}\`;
334+
335+ return {
336+ documents: rows.map((row) => ({
337+ name: String(row.name),
338+ stage: String(row.stage),
339+ })),
340+ };
341+ },
342+});
343+`;
344+
345+const extraToolSource = `import { z } from "zod";
346+import { defineTool } from "executor:app";
347+
348+export default defineTool({
349+ description: "Return a static custom-tools health marker.",
350+ input: z.object({}),
351+ output: z.object({ ok: z.boolean() }),
352+ annotations: { readOnly: true, destructive: false },
353+ async handler() {
354+ return { ok: true };
355+ },
356+});
357+`;
358+
359+const initialFiles = (): Record<string, string> => ({
360+ "executor.json": executorJson,
361+ "tools/deal-pipeline-sync.ts": dealPipelineSyncSource,
362+ "tools/find-deal-docs.ts": findDealDocsSource,
363+});
364+
365+const registerGitHubIntegration = async (emulator: Emulator, token: string): Promise<void> => {
366+ await postJson("/api/openapi/specs", {
367+ spec: { kind: "url", url: emulator.openapiUrl },
368+ slug: "github",
369+ baseUrl: emulator.url,
370+ authenticationTemplate: [bearerTemplate],
371+ });
372+ const created = await postJson<{ address: string }>("/api/connections", {
373+ owner: "user",
374+ name: "main",
375+ integration: "github",
376+ template: "bearer",
377+ value: token,
378+ });
379+ expect(created.address).toBe(GITHUB_CONNECTION);
380+};
381+
382+const syncSource = (repo = REPO_FULL_NAME): Promise<SyncResult> =>
383+ postJson<SyncResult>("/api/apps/sources/github/sync", {
384+ repo,
385+ connection: GITHUB_CONNECTION,
386+ });
387+
388+const listAppTools = (): Promise<readonly ToolRow[]> =>
389+ requestJson<readonly ToolRow[]>("/api/tools?integration=apps", {
390+ headers: jsonHeaders,
391+ });
392+
393+const execute = (code: string): Promise<ExecuteResponse> =>
394+ postJson<ExecuteResponse>("/api/executions", { code, autoApprove: true });
395+
396+const executeResult = async (code: string): Promise<unknown> => {
397+ const response = await execute(code);
398+ expect(response.status).toBe("completed");
399+ expect(response.isError).toBe(false);
400+ return response.structured.result;
401+};
402+
403+const callAppToolCode = (toolName: string, args: unknown): string => `
404+const found = await tools.search({ namespace: "apps", query: ${JSON.stringify(toolName)}, limit: 20 });
405+const item = found.items.find((candidate) => candidate.path.endsWith(${JSON.stringify(toolName)}));
406+if (!item) return { ok: false, missing: ${JSON.stringify(toolName)}, found };
407+let fn = tools;
408+for (const segment of item.path.split(".")) fn = fn[segment];
409+const result = await fn(${JSON.stringify(args)});
410+return { path: item.path, result };
411+`;
412+
413+let github!: Emulator;
414+let server!: TestHttpServer;
415+let disposeApp: () => Promise<void> = async () => {};
416+
417+beforeAll(async () => {
418+ const emulatorPort = await findFreePort();
419+ github = await createEmulator({
420+ service: "github",
421+ port: emulatorPort,
422+ seed: {
423+ github: {
424+ users: [{ login: OWNER, name: "Syncer" }],
425+ repos: [{ owner: OWNER, name: REPO, auto_init: true }],
426+ },
427+ },
428+ });
429+ const { makeSelfHostTestApp, headerIdentityLayer } = await import("./testing/test-app");
430+ const app = await makeSelfHostTestApp({ identity: headerIdentityLayer, dbPath });
431+ disposeApp = app.dispose;
432+ server = await startHttpServer(app.handler);
433+});
434+
435+afterAll(async () => {
436+ await server?.close();
437+ await disposeApp();
438+ await github?.close();
439+});
440+
441+test("GitHub source sync publishes and invokes custom tools through self-host HTTP", async () => {
442+ const credential = await github.credentials.mint({
443+ type: "api-key",
444+ login: OWNER,
445+ scopes: ["repo", "user"],
446+ });
447+ const token = credential.token;
448+ expect(token).toBeTruthy();
449+ await registerGitHubIntegration(github, token!);
450+ await createIssue(github, token!, "Acme renewal diligence");
451+ await createIssue(github, token!, "Beta pipeline memo");
452+ await putRepoFiles(github, token!, initialFiles());
453+
454+ const unauthorized = await fetch(`${server.baseUrl}/api/apps/sources/github/sync`, {
455+ method: "POST",
456+ headers: { "content-type": "application/json" },
457+ body: JSON.stringify({ repo: REPO_FULL_NAME, connection: GITHUB_CONNECTION }),
458+ });
459+ expect(unauthorized.status).toBe(401);
460+
461+ const published = await syncSource();
462+ expect(published.status).toBe("published");
463+ expect(published.tools).toEqual(["deal-pipeline-sync", "find-deal-docs"]);
464+ expect(published.skipped).toEqual([]);
465+ const firstSnapshot = published.snapshotId;
466+ expect(firstSnapshot).toBeTruthy();
467+
468+ const listed = await listAppTools();
469+ expect(listed.map((tool) => tool.name).sort()).toEqual(["deal-pipeline-sync", "find-deal-docs"]);
470+
471+ const searchResult = await executeResult(
472+ 'const found = await tools.search({ namespace: "apps", query: "pipeline", limit: 10 }); return found.items.map((item) => item.path);',
473+ );
474+ expect(JSON.stringify(searchResult)).toContain("deal-pipeline-sync");
475+
476+ const schema = await requestJson<{ inputSchema: { properties?: Record<string, unknown> } }>(
477+ `/api/tools/schema?address=${encodeURIComponent(listed.find((tool) => tool.name === "deal-pipeline-sync")!.address)}`,
478+ { headers: jsonHeaders },
479+ );
480+ expect(schema.inputSchema.properties?.github).toMatchObject({
481+ type: "string",
482+ enum: [GITHUB_CONNECTION],
483+ });
484+
485+ const invoke = (await executeResult(
486+ callAppToolCode("deal-pipeline-sync", {
487+ github: GITHUB_CONNECTION,
488+ owner: OWNER,
489+ repo: REPO,
490+ }),
491+ )) as { result?: { ok?: boolean; data?: { synced?: number }; error?: unknown } };
492+ expect(invoke.result?.ok).toBe(true);
493+ expect(invoke.result?.data?.synced).toBe(2);
494+
495+ const ledger = await github.ledger.list();
496+ const issueList = ledger.find(
497+ (entry) =>
498+ entry.operationId === "issues/listForRepo" && entry.path === `/repos/${OWNER}/${REPO}/issues`,
499+ );
500+ expect(issueList?.identity.user?.login).toBe(OWNER);
501+
502+ const readDb = (await executeResult(callAppToolCode("find-deal-docs", { limit: 10 }))) as {
503+ result?: { ok?: boolean; data?: { documents?: readonly { name: string }[] } };
504+ };
505+ expect(readDb.result?.ok).toBe(true);
506+ expect(readDb.result?.data?.documents?.map((row) => row.name).sort()).toEqual([
507+ "Acme renewal diligence",
508+ "Beta pipeline memo",
509+ ]);
510+
511+ const upToDate = await syncSource();
512+ expect(upToDate.status).toBe("up-to-date");
513+ expect("snapshotId" in upToDate).toBe(false);
514+
515+ await putRepoFiles(github, token!, {
516+ ...initialFiles(),
517+ "tools/extra-tool.ts": extraToolSource,
518+ });
519+ const withExtra = await syncSource();
520+ expect(withExtra.status).toBe("published");
521+ expect(withExtra.snapshotId).not.toBe(firstSnapshot);
522+ expect((await listAppTools()).map((tool) => tool.name).sort()).toEqual([
523+ "deal-pipeline-sync",
524+ "extra-tool",
525+ "find-deal-docs",
526+ ]);
527+
528+ await putRepoFiles(github, token!, initialFiles());
529+ const removed = await syncSource();
530+ expect(removed.status).toBe("published");
531+ expect((await listAppTools()).map((tool) => tool.name).sort()).toEqual([
532+ "deal-pipeline-sync",
533+ "find-deal-docs",
534+ ]);
535+
536+ await putRepoFiles(github, token!, {
537+ ...initialFiles(),
538+ "workflows/x.ts": "export default {};",
539+ });
540+ const skipped = await syncSource();
541+ expect(skipped.status).toBe("published");
542+ expect(skipped.skipped).toEqual([{ path: "workflows/x.ts", reason: "not supported yet" }]);
543+ const afterSkipped = await listAppTools();
544+ expect(afterSkipped.map((tool) => tool.name).sort()).toEqual([
545+ "deal-pipeline-sync",
546+ "find-deal-docs",
547+ ]);
548+ expect(afterSkipped.some((tool) => tool.address.includes("workflow"))).toBe(false);
549+});
apps/host-selfhost/src/apps.ts+71−0
@@ -0,0 +1,71 @@
1+import { join } from "node:path";
2+
3+import { Effect } from "effect";
4+
5+import {
6+ appsPlugin,
7+ BindingError,
8+ makeSelfHostAppsRuntime,
9+ makeSqliteAppsStore,
10+ type AppsRuntime,
11+ type ClientResolver,
12+} from "@executor-js/plugin-apps/api";
13+import type { PluginCtx } from "@executor-js/sdk";
14+
15+import { resolveDataDir } from "./config";
16+import { makeSelfHostAppsResolver } from "./apps-resolver";
17+
18+const missingResolver = (): ClientResolver => ({
19+ listConnections: () => Effect.succeed([]),
20+ resolveConnection: () => Effect.succeed(null),
21+ call: ({ integration, connection }) =>
22+ Effect.fail(
23+ new BindingError({
24+ role: integration,
25+ integration,
26+ requestedConnection: connection,
27+ message: "apps resolver is unavailable outside a request-scoped executor",
28+ }),
29+ ),
30+});
31+
32+const createSubsystem = (dataDir: string) => {
33+ const appsDir = join(dataDir, "apps");
34+ const store = makeSqliteAppsStore({ path: join(appsDir, "store.sqlite") });
35+ const host = makeSelfHostAppsRuntime({
36+ dataDir: appsDir,
37+ store,
38+ resolver: missingResolver(),
39+ });
40+ const plugin = appsPlugin({
41+ runtime: host.runtime,
42+ makeResolver: ({ ctx }) => makeSelfHostAppsResolver({ ctx: ctx as PluginCtx }),
43+ });
44+ return {
45+ runtime: host.runtime,
46+ plugin,
47+ close: host.close,
48+ };
49+};
50+
51+export interface SelfHostAppsSubsystem {
52+ readonly runtime: AppsRuntime;
53+ readonly plugin: ReturnType<typeof appsPlugin>;
54+ readonly close: () => Promise<void>;
55+}
56+
57+let current: { readonly dataDir: string; readonly subsystem: SelfHostAppsSubsystem } | undefined;
58+
59+export const getSelfHostAppsSubsystem = (): SelfHostAppsSubsystem => {
60+ const dataDir = resolveDataDir();
61+ if (current && current.dataDir === dataDir) return current.subsystem;
62+ const subsystem = createSubsystem(dataDir);
63+ current = { dataDir, subsystem };
64+ return subsystem;
65+};
66+
67+export const closeSelfHostAppsSubsystem = async (): Promise<void> => {
68+ const subsystem = current?.subsystem;
69+ current = undefined;
70+ await subsystem?.close();
71+};
apps/host-selfhost/src/testing/test-app.ts+4−0
@@ -30,6 +30,8 @@ import { · 29 unmodified lines
3030 } from "../mcp/session-store";
3131 import { selfHostPlugins } from "../plugins";
3232 import { ErrorCaptureLive } from "../observability";
33+import { closeSelfHostAppsSubsystem } from "../apps";
34+import { makeSelfHostAppsSyncRoute } from "../apps-route";
3335
3436 // ===========================================================================
3537 // Self-host TEST harness — the throwaway composition tests use to exercise the
@@ -202,6 +204,7 @@ export const makeSelfHostTestApp = async ( · 166 unmodified lines
202204 },
203205 extensions: {
204206 routes: [
207+ makeSelfHostAppsSyncRoute({ identity: options.identity, db: dbHandle }),
205208 HttpApiSwagger.layer(composePluginApi(selfHostPlugins).prefix("/api"), { path: "/docs" }),
206209 ],
207210 },
@@ -217,6 +220,7 @@ export const makeSelfHostTestApp = async ( · 9 unmodified lines
217220 handler: web.handler,
218221 dispose: async () => {
219222 await web.dispose();
223+ await closeSelfHostAppsSubsystem();
220224 await sessionStore.close();
221225 await dbHandle.close();
222226 },