History

Add GitHub sync and self-host custom tools wiring

Rhys Sullivan committed 4 months ago

Commit 275446e1 parent 41eb3ae19 files · +1665 −40Browse files

Showing the changes to apps/host-selfhost/src/apps-wire.node.test.ts (1 of 19 files). Show all

Unified · Split

apps/host-selfhost/src/apps-wire.node.test.ts+549−0
@@ -0,0 +1,549 @@
1import { randomBytes } from "node:crypto";
2import { mkdtempSync } from "node:fs";
3import { createServer, type IncomingMessage, type ServerResponse } from "node:http";
4import { createServer as createNetServer } from "node:net";
5import { tmpdir } from "node:os";
6import { join } from "node:path";
7
8import { afterAll, beforeAll, expect, test } from "@effect/vitest";
9import { createEmulator, type Emulator } from "@executor-js/emulate";
10
11const dataDir = mkdtempSync(join(tmpdir(), "eh-apps-wire-"));
12const dbPath = join(dataDir, "data.db");
13process.env.EXECUTOR_DATA_DIR = dataDir;
14process.env.EXECUTOR_SECRET_KEY = "apps-wire-secret-key";
15process.env.EXECUTOR_ALLOW_LOCAL_NETWORK = "true";
16
17const TEST_USER = "apps-user";
18const TEST_ORG = "apps-org";
19const OWNER = "syncer";
20const REPO = "custom-tools";
21const REPO_FULL_NAME = `${OWNER}/${REPO}`;
22const GITHUB_CONNECTION = "tools.github.user.main";
23
24interface TestHttpServer {
25 readonly baseUrl: string;
26 readonly close: () => Promise<void>;
27}
28
29interface SyncResult {
30 readonly status: "published" | "up-to-date" | "failed";
31 readonly snapshotId?: string;
32 readonly upstreamSha?: string;
33 readonly tools: readonly string[];
34 readonly skipped: readonly { readonly path: string; readonly reason: string }[];
35 readonly errors?: readonly unknown[];
36}
37
38interface ToolRow {
39 readonly address: string;
40 readonly name: string;
41 readonly integration: string;
42}
43
44interface ExecuteResponse {
45 readonly status: "completed" | "paused";
46 readonly text: string;
47 readonly structured: {
48 readonly status?: string;
49 readonly result?: unknown;
50 readonly error?: unknown;
51 };
52 readonly isError?: boolean;
53}
54
55const bearerTemplate = {
56 slug: "bearer",
57 type: "apiKey",
58 label: "Bearer token",
59 headers: { Authorization: ["Bearer ", { type: "variable", name: "token" }] },
60};
61
62const jsonHeaders = {
63 "content-type": "application/json",
64 "x-test-user": TEST_USER,
65 "x-test-org": TEST_ORG,
66};
67
68const findFreePort = (): Promise<number> =>
69 new Promise((resolve, reject) => {
70 const probe = createNetServer();
71 probe.once("error", reject);
72 probe.listen(0, "127.0.0.1", () => {
73 const address = probe.address();
74 const port = typeof address === "object" && address ? address.port : 0;
75 probe.close((error) => (error ? reject(error) : resolve(port)));
76 });
77 });
78
79const readRequestBody = async (request: IncomingMessage): Promise<Buffer | undefined> => {
80 if (request.method === "GET" || request.method === "HEAD") return undefined;
81 const chunks: Buffer[] = [];
82 for await (const chunk of request) {
83 chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk));
84 }
85 const body = Buffer.concat(chunks);
86 return body.length > 0 ? body : undefined;
87};
88
89const webRequestFromNode = async (request: IncomingMessage, port: number): Promise<Request> => {
90 const headers = new Headers();
91 for (const [name, raw] of Object.entries(request.headers)) {
92 if (Array.isArray(raw)) {
93 for (const value of raw) headers.append(name, value);
94 } else if (raw !== undefined) {
95 headers.set(name, raw);
96 }
97 }
98 const body = await readRequestBody(request);
99 return new Request(`http://127.0.0.1:${port}${request.url ?? "/"}`, {
100 method: request.method,
101 headers,
102 body: body ? new Uint8Array(body) : undefined,
103 });
104};
105
106const sendWebResponse = async (webResponse: Response, response: ServerResponse): Promise<void> => {
107 response.statusCode = webResponse.status;
108 webResponse.headers.forEach((value, name) => response.setHeader(name, value));
109 response.end(Buffer.from(await webResponse.arrayBuffer()));
110};
111
112const startHttpServer = async (
113 handler: (request: Request) => Promise<Response>,
114): Promise<TestHttpServer> => {
115 let port = 0;
116 const server = createServer((request, response) => {
117 void (async () => {
118 try {
119 await sendWebResponse(await handler(await webRequestFromNode(request, port)), response);
120 } catch (error) {
121 response.statusCode = 500;
122 response.end(error instanceof Error ? error.message : String(error));
123 }
124 })();
125 });
126 await new Promise<void>((resolve, reject) => {
127 server.once("error", reject);
128 server.listen(0, "127.0.0.1", () => {
129 const address = server.address();
130 port = typeof address === "object" && address ? address.port : 0;
131 resolve();
132 });
133 });
134 return {
135 baseUrl: `http://127.0.0.1:${port}`,
136 close: () =>
137 new Promise((resolve, reject) => {
138 server.close((error) => (error ? reject(error) : resolve()));
139 }),
140 };
141};
142
143const requestJson = async <T>(
144 path: string,
145 init: RequestInit = {},
146 expectedStatus = 200,
147): Promise<T> => {
148 const response = await fetch(`${server.baseUrl}${path}`, init);
149 const text = await response.text();
150 expect(response.status).toBe(expectedStatus);
151 return (text.length > 0 ? JSON.parse(text) : null) as T;
152};
153
154const postJson = <T>(path: string, body: unknown, expectedStatus = 200): Promise<T> =>
155 requestJson<T>(
156 path,
157 {
158 method: "POST",
159 headers: jsonHeaders,
160 body: JSON.stringify(body),
161 },
162 expectedStatus,
163 );
164
165const githubFetch = async <T>(
166 emulator: Emulator,
167 token: string,
168 path: string,
169 init: RequestInit = {},
170): Promise<T> => {
171 const headers = new Headers(init.headers);
172 headers.set("authorization", `Bearer ${token}`);
173 headers.set("accept", "application/vnd.github+json");
174 if (init.body !== undefined) headers.set("content-type", "application/json");
175 const response = await fetch(`${emulator.url}${path}`, { ...init, headers });
176 const text = await response.text();
177 expect(response.ok).toBe(true);
178 return (text.length > 0 ? JSON.parse(text) : null) as T;
179};
180
181const createIssue = (emulator: Emulator, token: string, title: string): Promise<unknown> =>
182 githubFetch(emulator, token, `/repos/${OWNER}/${REPO}/issues`, {
183 method: "POST",
184 body: JSON.stringify({ title }),
185 });
186
187const putRepoFiles = async (
188 emulator: Emulator,
189 token: string,
190 files: Readonly<Record<string, string>>,
191): Promise<string> => {
192 const ref = await githubFetch<{ object: { sha: string } }>(
193 emulator,
194 token,
195 `/repos/${OWNER}/${REPO}/git/ref/heads/main`,
196 );
197 const parentSha = ref.object.sha;
198 const tree = await githubFetch<{ sha: string }>(
199 emulator,
200 token,
201 `/repos/${OWNER}/${REPO}/git/trees`,
202 {
203 method: "POST",
204 body: JSON.stringify({
205 tree: Object.entries(files).map(([path, content]) => ({
206 path,
207 mode: "100644",
208 type: "blob",
209 content,
210 })),
211 }),
212 },
213 );
214 const commit = await githubFetch<{ sha: string }>(
215 emulator,
216 token,
217 `/repos/${OWNER}/${REPO}/git/commits`,
218 {
219 method: "POST",
220 body: JSON.stringify({
221 message: `Update custom tools ${randomBytes(3).toString("hex")}`,
222 tree: tree.sha,
223 parents: [parentSha],
224 }),
225 },
226 );
227 await githubFetch(emulator, token, `/repos/${OWNER}/${REPO}/git/refs/heads/main`, {
228 method: "PATCH",
229 body: JSON.stringify({ sha: commit.sha }),
230 });
231 return commit.sha;
232};
233
234const executorJson = JSON.stringify(
235 {
236 $schema: "https://executor.sh/schemas/executor.json",
237 description: "Curated tools wrapping our raw integration surface.",
238 },
239 null,
240 2,
241);
242
243const dealPipelineSyncSource = `import { z } from "zod";
244import { defineTool, integration } from "executor:app";
245
246/**
247 * The deterministic-program case: pull GitHub issues into the scope database
248 * so agents query a table instead of paging the raw API.
249 */
250export default defineTool({
251 description:
252 "Refresh the deals table from GitHub issues. Syncs issue numbers and titles " +
253 "for pipeline questions.",
254
255 integrations: {
256 github: integration("github"),
257 },
258
259 input: z.object({
260 owner: z.string(),
261 repo: z.string(),
262 }),
263
264 output: z.object({ synced: z.number() }),
265
266 annotations: { readOnly: false, destructive: false },
267
268 async handler({ owner, repo }, { github, db }) {
269 await db.sql\`
270 CREATE TABLE IF NOT EXISTS deals (
271 id TEXT PRIMARY KEY,
272 name TEXT NOT NULL,
273 stage TEXT NOT NULL
274 )\`;
275
276 const issues = await github.repos.listIssues({ owner, repo });
277
278 let synced = 0;
279 for (const issue of issues) {
280 await db.sql\`
281 INSERT INTO deals (id, name, stage)
282 VALUES (\${String(issue.number)}, \${issue.title}, 'open')
283 ON CONFLICT (id) DO UPDATE SET
284 name = excluded.name,
285 stage = excluded.stage\`;
286 synced++;
287 }
288
289 return { synced };
290 },
291});
292`;
293
294const findDealDocsSource = `import { z } from "zod";
295import { defineTool, integration } from "executor:app";
296
297/**
298 * The curation case: a read-only query over the scope database populated by
299 * the sync tool.
300 */
301export default defineTool({
302 description:
303 "Find documents for a deal across synced pipeline records. Searches " +
304 "stored issue titles.",
305
306 integrations: {
307 github: integration("github"),
308 },
309
310 input: z.object({
311 limit: z.number().int().max(50).default(20),
312 }),
313
314 output: z.object({
315 documents: z.array(
316 z.object({
317 name: z.string(),
318 stage: z.string(),
319 }),
320 ),
321 }),
322
323 annotations: { readOnly: true, destructive: false },
324
325 async handler({ limit }, { db }) {
326 await db.sql\`
327 CREATE TABLE IF NOT EXISTS deals (
328 id TEXT PRIMARY KEY,
329 name TEXT NOT NULL,
330 stage TEXT NOT NULL
331 )\`;
332 const rows = await db.sql\`
333 SELECT name, stage FROM deals ORDER BY id LIMIT \${limit}\`;
334
335 return {
336 documents: rows.map((row) => ({
337 name: String(row.name),
338 stage: String(row.stage),
339 })),
340 };
341 },
342});
343`;
344
345const extraToolSource = `import { z } from "zod";
346import { defineTool } from "executor:app";
347
348export default defineTool({
349 description: "Return a static custom-tools health marker.",
350 input: z.object({}),
351 output: z.object({ ok: z.boolean() }),
352 annotations: { readOnly: true, destructive: false },
353 async handler() {
354 return { ok: true };
355 },
356});
357`;
358
359const initialFiles = (): Record<string, string> => ({
360 "executor.json": executorJson,
361 "tools/deal-pipeline-sync.ts": dealPipelineSyncSource,
362 "tools/find-deal-docs.ts": findDealDocsSource,
363});
364
365const registerGitHubIntegration = async (emulator: Emulator, token: string): Promise<void> => {
366 await postJson("/api/openapi/specs", {
367 spec: { kind: "url", url: emulator.openapiUrl },
368 slug: "github",
369 baseUrl: emulator.url,
370 authenticationTemplate: [bearerTemplate],
371 });
372 const created = await postJson<{ address: string }>("/api/connections", {
373 owner: "user",
374 name: "main",
375 integration: "github",
376 template: "bearer",
377 value: token,
378 });
379 expect(created.address).toBe(GITHUB_CONNECTION);
380};
381
382const syncSource = (repo = REPO_FULL_NAME): Promise<SyncResult> =>
383 postJson<SyncResult>("/api/apps/sources/github/sync", {
384 repo,
385 connection: GITHUB_CONNECTION,
386 });
387
388const listAppTools = (): Promise<readonly ToolRow[]> =>
389 requestJson<readonly ToolRow[]>("/api/tools?integration=apps", {
390 headers: jsonHeaders,
391 });
392
393const execute = (code: string): Promise<ExecuteResponse> =>
394 postJson<ExecuteResponse>("/api/executions", { code, autoApprove: true });
395
396const executeResult = async (code: string): Promise<unknown> => {
397 const response = await execute(code);
398 expect(response.status).toBe("completed");
399 expect(response.isError).toBe(false);
400 return response.structured.result;
401};
402
403const callAppToolCode = (toolName: string, args: unknown): string => `
404const found = await tools.search({ namespace: "apps", query: ${JSON.stringify(toolName)}, limit: 20 });
405const item = found.items.find((candidate) => candidate.path.endsWith(${JSON.stringify(toolName)}));
406if (!item) return { ok: false, missing: ${JSON.stringify(toolName)}, found };
407let fn = tools;
408for (const segment of item.path.split(".")) fn = fn[segment];
409const result = await fn(${JSON.stringify(args)});
410return { path: item.path, result };
411`;
412
413let github!: Emulator;
414let server!: TestHttpServer;
415let disposeApp: () => Promise<void> = async () => {};
416
417beforeAll(async () => {
418 const emulatorPort = await findFreePort();
419 github = await createEmulator({
420 service: "github",
421 port: emulatorPort,
422 seed: {
423 github: {
424 users: [{ login: OWNER, name: "Syncer" }],
425 repos: [{ owner: OWNER, name: REPO, auto_init: true }],
426 },
427 },
428 });
429 const { makeSelfHostTestApp, headerIdentityLayer } = await import("./testing/test-app");
430 const app = await makeSelfHostTestApp({ identity: headerIdentityLayer, dbPath });
431 disposeApp = app.dispose;
432 server = await startHttpServer(app.handler);
433});
434
435afterAll(async () => {
436 await server?.close();
437 await disposeApp();
438 await github?.close();
439});
440
441test("GitHub source sync publishes and invokes custom tools through self-host HTTP", async () => {
442 const credential = await github.credentials.mint({
443 type: "api-key",
444 login: OWNER,
445 scopes: ["repo", "user"],
446 });
447 const token = credential.token;
448 expect(token).toBeTruthy();
449 await registerGitHubIntegration(github, token!);
450 await createIssue(github, token!, "Acme renewal diligence");
451 await createIssue(github, token!, "Beta pipeline memo");
452 await putRepoFiles(github, token!, initialFiles());
453
454 const unauthorized = await fetch(`${server.baseUrl}/api/apps/sources/github/sync`, {
455 method: "POST",
456 headers: { "content-type": "application/json" },
457 body: JSON.stringify({ repo: REPO_FULL_NAME, connection: GITHUB_CONNECTION }),
458 });
459 expect(unauthorized.status).toBe(401);
460
461 const published = await syncSource();
462 expect(published.status).toBe("published");
463 expect(published.tools).toEqual(["deal-pipeline-sync", "find-deal-docs"]);
464 expect(published.skipped).toEqual([]);
465 const firstSnapshot = published.snapshotId;
466 expect(firstSnapshot).toBeTruthy();
467
468 const listed = await listAppTools();
469 expect(listed.map((tool) => tool.name).sort()).toEqual(["deal-pipeline-sync", "find-deal-docs"]);
470
471 const searchResult = await executeResult(
472 'const found = await tools.search({ namespace: "apps", query: "pipeline", limit: 10 }); return found.items.map((item) => item.path);',
473 );
474 expect(JSON.stringify(searchResult)).toContain("deal-pipeline-sync");
475
476 const schema = await requestJson<{ inputSchema: { properties?: Record<string, unknown> } }>(
477 `/api/tools/schema?address=${encodeURIComponent(listed.find((tool) => tool.name === "deal-pipeline-sync")!.address)}`,
478 { headers: jsonHeaders },
479 );
480 expect(schema.inputSchema.properties?.github).toMatchObject({
481 type: "string",
482 enum: [GITHUB_CONNECTION],
483 });
484
485 const invoke = (await executeResult(
486 callAppToolCode("deal-pipeline-sync", {
487 github: GITHUB_CONNECTION,
488 owner: OWNER,
489 repo: REPO,
490 }),
491 )) as { result?: { ok?: boolean; data?: { synced?: number }; error?: unknown } };
492 expect(invoke.result?.ok).toBe(true);
493 expect(invoke.result?.data?.synced).toBe(2);
494
495 const ledger = await github.ledger.list();
496 const issueList = ledger.find(
497 (entry) =>
498 entry.operationId === "issues/listForRepo" && entry.path === `/repos/${OWNER}/${REPO}/issues`,
499 );
500 expect(issueList?.identity.user?.login).toBe(OWNER);
501
502 const readDb = (await executeResult(callAppToolCode("find-deal-docs", { limit: 10 }))) as {
503 result?: { ok?: boolean; data?: { documents?: readonly { name: string }[] } };
504 };
505 expect(readDb.result?.ok).toBe(true);
506 expect(readDb.result?.data?.documents?.map((row) => row.name).sort()).toEqual([
507 "Acme renewal diligence",
508 "Beta pipeline memo",
509 ]);
510
511 const upToDate = await syncSource();
512 expect(upToDate.status).toBe("up-to-date");
513 expect("snapshotId" in upToDate).toBe(false);
514
515 await putRepoFiles(github, token!, {
516 ...initialFiles(),
517 "tools/extra-tool.ts": extraToolSource,
518 });
519 const withExtra = await syncSource();
520 expect(withExtra.status).toBe("published");
521 expect(withExtra.snapshotId).not.toBe(firstSnapshot);
522 expect((await listAppTools()).map((tool) => tool.name).sort()).toEqual([
523 "deal-pipeline-sync",
524 "extra-tool",
525 "find-deal-docs",
526 ]);
527
528 await putRepoFiles(github, token!, initialFiles());
529 const removed = await syncSource();
530 expect(removed.status).toBe("published");
531 expect((await listAppTools()).map((tool) => tool.name).sort()).toEqual([
532 "deal-pipeline-sync",
533 "find-deal-docs",
534 ]);
535
536 await putRepoFiles(github, token!, {
537 ...initialFiles(),
538 "workflows/x.ts": "export default {};",
539 });
540 const skipped = await syncSource();
541 expect(skipped.status).toBe("published");
542 expect(skipped.skipped).toEqual([{ path: "workflows/x.ts", reason: "not supported yet" }]);
543 const afterSkipped = await listAppTools();
544 expect(afterSkipped.map((tool) => tool.name).sort()).toEqual([
545 "deal-pipeline-sync",
546 "find-deal-docs",
547 ]);
548 expect(afterSkipped.some((tool) => tool.address.includes("workflow"))).toBe(false);
549});

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t