History

Add GitHub sync and self-host custom tools wiring

Rhys Sullivan committed 4 months ago

Commit 275446e1 parent 41eb3ae19 files · +1665 −40Browse files

Showing the changes to packages/plugins/apps/src/source (2 of 19 files). Show all

Unified · Split

2 files
  1. packages/plugins/apps/src/source/github-source.test.ts +211 −0
  2. packages/plugins/apps/src/source/github-source.ts +402 −0
packages/plugins/apps/src/source/github-source.test.ts+211−0
@@ -0,0 +1,211 @@
1+import { mkdtempSync } from "node:fs";
2+import { tmpdir } from "node:os";
3+import { join } from "node:path";
4+
5+import { describe, expect, it } from "vitest";
6+import { Effect } from "effect";
7+
8+import { makeSelfHostAppsRuntime } from "../plugin/self-host-runtime";
9+import { makeInMemoryAppsStore, makeTestResolver } from "../testing";
10+import { PUBLISH_LIMITS } from "../pipeline/publish";
11+import { fetchGitHubSource, syncGitHubSource } from "./github-source";
12+
13+const run = <A, E>(effect: Effect.Effect<A, E>): Promise<A> => Effect.runPromise(effect);
14+
15+const toolSource = (name = "ok"): string => `import { defineTool } from "executor:app";
16+import { z } from "zod";
17+export default defineTool({
18+ description: "${name}",
19+ input: z.object({ value: z.string().default("${name}") }),
20+ async handler(input) { return { value: input.value }; },
21+});`;
22+
23+const makeRuntime = () =>
24+ makeSelfHostAppsRuntime({
25+ dataDir: mkdtempSync(join(tmpdir(), "apps-gh-src-")),
26+ store: makeInMemoryAppsStore(),
27+ resolver: makeTestResolver({}),
28+ inMemory: true,
29+ });
30+
31+const json = (value: unknown, status = 200): Response =>
32+ new Response(JSON.stringify(value), {
33+ status,
34+ headers: { "content-type": "application/json" },
35+ });
36+
37+const makeGitHubFetch = (input: {
38+ readonly files: ReadonlyMap<string, string>;
39+ readonly upstreamSha?: string;
40+ readonly treeSha?: string;
41+}) => {
42+ const repoPath = "/repos/acme/tools";
43+ const upstreamSha = input.upstreamSha ?? "commit-1";
44+ const treeSha = input.treeSha ?? "tree-1";
45+ const blobBySha = new Map<string, { path: string; content: string }>();
46+ let index = 0;
47+ for (const [path, content] of input.files) {
48+ blobBySha.set(`blob-${++index}`, { path, content });
49+ }
50+ let blobCalls = 0;
51+ const fetch = (async (rawUrl: string) => {
52+ const url = new URL(rawUrl);
53+ if (url.pathname === repoPath) return json({ default_branch: "main" });
54+ if (url.pathname === `${repoPath}/git/ref/heads%2Fmain`) {
55+ return json({ object: { sha: upstreamSha } });
56+ }
57+ if (url.pathname === `${repoPath}/git/commits/${upstreamSha}`) {
58+ return json({ sha: upstreamSha, tree: { sha: treeSha } });
59+ }
60+ if (url.pathname === `${repoPath}/git/trees/${treeSha}`) {
61+ return json({
62+ tree: [...blobBySha.entries()].map(([sha, blob]) => ({
63+ path: blob.path,
64+ type: "blob",
65+ sha,
66+ size: Buffer.byteLength(blob.content, "utf8"),
67+ })),
68+ });
69+ }
70+ const blobPrefix = `${repoPath}/git/blobs/`;
71+ if (url.pathname.startsWith(blobPrefix)) {
72+ blobCalls++;
73+ const sha = decodeURIComponent(url.pathname.slice(blobPrefix.length));
74+ const blob = blobBySha.get(sha);
75+ if (!blob) return json({ message: "not found" }, 404);
76+ return json({
77+ encoding: "base64",
78+ content: Buffer.from(blob.content, "utf8").toString("base64"),
79+ });
80+ }
81+ return json({ message: "not found" }, 404);
82+ }) as typeof globalThis.fetch;
83+ return {
84+ fetch,
85+ blobCalls: () => blobCalls,
86+ };
87+};
88+
89+describe("GitHub custom-tools source", () => {
90+ it("fetches a repo fileset and publishes provenance, description, and skipped entries", async () => {
91+ const files = new Map<string, string>([
92+ [
93+ "executor.json",
94+ JSON.stringify({
95+ $schema: "https://example.test/schema",
96+ description: "Acme tools",
97+ ignored: true,
98+ }),
99+ ],
100+ ["tools/hello.ts", toolSource("hello")],
101+ ["workflows/deferred.ts", "export default {};"],
102+ ["docs/readme.md", "ignored"],
103+ ]);
104+ const github = makeGitHubFetch({ files, upstreamSha: "commit-a" });
105+ const snapshot = await run(fetchGitHubSource({ repo: "acme/tools", fetch: github.fetch }));
106+ expect([...snapshot.files.keys()].sort()).toEqual([
107+ "executor.json",
108+ "tools/hello.ts",
109+ "workflows/deferred.ts",
110+ ]);
111+ expect(snapshot.description).toBe("Acme tools");
112+
113+ const host = makeRuntime();
114+ const result = await run(
115+ syncGitHubSource({
116+ runtime: host.runtime,
117+ scope: "githubTools",
118+ repo: "acme/tools",
119+ fetch: github.fetch,
120+ }),
121+ );
122+ expect(result.status).toBe("published");
123+ expect(result.tools).toEqual(["hello"]);
124+ expect(result.skipped).toEqual([
125+ { path: "workflows/deferred.ts", reason: "not supported yet" },
126+ ]);
127+ const descriptor = await run(host.runtime.getDescriptor("githubTools"));
128+ expect(descriptor?.description).toBe("Acme tools");
129+ expect(descriptor?.source).toEqual({
130+ kind: "github",
131+ repo: "acme/tools",
132+ ref: "main",
133+ upstreamSha: "commit-a",
134+ });
135+ await host.close();
136+ });
137+
138+ it("reports up-to-date when the upstream commit SHA is unchanged", async () => {
139+ const github = makeGitHubFetch({
140+ files: new Map([["tools/hello.ts", toolSource("hello")]]),
141+ upstreamSha: "same-sha",
142+ });
143+ const host = makeRuntime();
144+ const first = await run(
145+ syncGitHubSource({
146+ runtime: host.runtime,
147+ scope: "githubTools",
148+ repo: "acme/tools",
149+ fetch: github.fetch,
150+ }),
151+ );
152+ const second = await run(
153+ syncGitHubSource({
154+ runtime: host.runtime,
155+ scope: "githubTools",
156+ repo: "acme/tools",
157+ fetch: github.fetch,
158+ }),
159+ );
160+ expect(first.status).toBe("published");
161+ expect(second).toEqual({
162+ status: "up-to-date",
163+ upstreamSha: "same-sha",
164+ tools: ["hello"],
165+ skipped: [],
166+ });
167+ await host.close();
168+ });
169+
170+ it("rejects oversized trees before fetching blobs", async () => {
171+ const files = new Map<string, string>();
172+ for (let i = 0; i < PUBLISH_LIMITS.maxFiles + 1; i++) {
173+ files.set(`tools/t${i}.ts`, toolSource(`t${i}`));
174+ }
175+ const github = makeGitHubFetch({ files });
176+ const exit = await Effect.runPromiseExit(
177+ fetchGitHubSource({ repo: "acme/tools", fetch: github.fetch }),
178+ );
179+ expect(exit._tag).toBe("Failure");
180+ expect(JSON.stringify(exit)).toContain("exceeding the limit");
181+ expect(github.blobCalls()).toBe(0);
182+ });
183+
184+ it("returns typed failure data for publish errors", async () => {
185+ const github = makeGitHubFetch({
186+ files: new Map([
187+ [
188+ "tools/bad.ts",
189+ `import { defineTool } from "executor:app";
190+import { chunk } from "lodash";
191+export default defineTool({ description: "bad", input: { type: "object" }, async handler(){ return chunk([1], 1); } });`,
192+ ],
193+ ]),
194+ upstreamSha: "bad-sha",
195+ });
196+ const host = makeRuntime();
197+ const result = await run(
198+ syncGitHubSource({
199+ runtime: host.runtime,
200+ scope: "githubTools",
201+ repo: "acme/tools",
202+ fetch: github.fetch,
203+ }),
204+ );
205+ expect(result.status).toBe("failed");
206+ expect(result.upstreamSha).toBe("bad-sha");
207+ expect(result.errors?.[0]?.stage).toBe("bundle");
208+ expect(result.errors?.[0]?.message).toContain('bare import "lodash" is not allowed');
209+ await host.close();
210+ });
211+});
packages/plugins/apps/src/source/github-source.ts+402−0
@@ -0,0 +1,402 @@
1+import { Data, Effect, Result } from "effect";
2+
3+import { PublishError, type FileDiagnostic, type SkippedArtifact } from "../pipeline/discover";
4+import { PUBLISH_LIMITS, enforcePublishLimits } from "../pipeline/publish";
5+import type { AppSourceRef } from "../pipeline/descriptor";
6+import type { AppsRuntime } from "../plugin/runtime";
7+import type { FileSet, SnapshotId } from "../seams/artifact-store";
8+
9+export interface GitHubSourceInput {
10+ readonly repo: string;
11+ readonly ref?: string;
12+ readonly token?: string | null;
13+ readonly baseUrl?: string;
14+ readonly fetch?: typeof globalThis.fetch;
15+}
16+
17+export interface GitHubSourceSnapshot {
18+ readonly files: FileSet;
19+ readonly repo: string;
20+ readonly ref: string;
21+ readonly upstreamSha: string;
22+ readonly description?: string;
23+}
24+
25+export class GitHubSourceError extends Data.TaggedError("GitHubSourceError")<{
26+ readonly message: string;
27+ readonly status?: number;
28+ readonly path?: string;
29+ readonly cause?: unknown;
30+}> {}
31+
32+export interface SyncErrorData {
33+ readonly stage: "source" | "discover" | "bundle" | "collect" | "project";
34+ readonly message: string;
35+ readonly diagnostics?: readonly FileDiagnostic[];
36+}
37+
38+export type GitHubSyncResult =
39+ | {
40+ readonly status: "published";
41+ readonly snapshotId: SnapshotId;
42+ readonly upstreamSha: string;
43+ readonly tools: readonly string[];
44+ readonly skipped: readonly SkippedArtifact[];
45+ readonly errors?: undefined;
46+ }
47+ | {
48+ readonly status: "up-to-date";
49+ readonly upstreamSha: string;
50+ readonly tools: readonly string[];
51+ readonly skipped: readonly SkippedArtifact[];
52+ readonly errors?: undefined;
53+ }
54+ | {
55+ readonly status: "failed";
56+ readonly upstreamSha?: string;
57+ readonly tools: readonly string[];
58+ readonly skipped: readonly SkippedArtifact[];
59+ readonly errors: readonly SyncErrorData[];
60+ };
61+
62+export interface SyncGitHubSourceInput extends GitHubSourceInput {
63+ readonly runtime: AppsRuntime;
64+ readonly scope: string;
65+}
66+
67+const MATERIALIZED_RE = /^(tools|workflows|ui|skills)\//;
68+
69+const trimBaseUrl = (baseUrl: string): string => baseUrl.replace(/\/+$/, "");
70+
71+const repoParts = (repo: string): { owner: string; name: string } | null => {
72+ const [owner, name, ...rest] = repo.split("/");
73+ if (!owner || !name || rest.length > 0) return null;
74+ return { owner, name };
75+};
76+
77+const encodedRepoPath = (repo: string): Effect.Effect<string, GitHubSourceError> => {
78+ const parsed = repoParts(repo);
79+ if (!parsed) {
80+ return Effect.fail(
81+ new GitHubSourceError({
82+ message: `GitHub repo must be "owner/name"; got "${repo}"`,
83+ path: repo,
84+ }),
85+ );
86+ }
87+ return Effect.succeed(`${encodeURIComponent(parsed.owner)}/${encodeURIComponent(parsed.name)}`);
88+};
89+
90+const materializes = (path: string): boolean =>
91+ path === "executor.json" || MATERIALIZED_RE.test(path);
92+
93+const isRecord = (value: unknown): value is Record<string, unknown> =>
94+ value !== null && typeof value === "object" && !Array.isArray(value);
95+
96+const asString = (value: unknown): string | undefined =>
97+ typeof value === "string" && value.length > 0 ? value : undefined;
98+
99+const requestJson = <A>(
100+ input: GitHubSourceInput,
101+ path: string,
102+): Effect.Effect<A, GitHubSourceError> =>
103+ Effect.tryPromise({
104+ try: async () => {
105+ const fetchImpl = input.fetch ?? globalThis.fetch;
106+ const headers: Record<string, string> = {
107+ accept: "application/vnd.github+json",
108+ "user-agent": "executor-apps-github-source",
109+ };
110+ if (input.token) headers.authorization = `Bearer ${input.token}`;
111+ const response = await fetchImpl(
112+ `${trimBaseUrl(input.baseUrl ?? "https://api.github.com")}${path}`,
113+ {
114+ headers,
115+ },
116+ );
117+ if (!response.ok) {
118+ throw new GitHubSourceError({
119+ message: `GitHub request failed: GET ${path} -> ${response.status}`,
120+ status: response.status,
121+ path,
122+ });
123+ }
124+ return (await response.json()) as A;
125+ },
126+ catch: (cause) =>
127+ cause instanceof GitHubSourceError
128+ ? cause
129+ : new GitHubSourceError({
130+ message: `GitHub request failed: GET ${path}`,
131+ path,
132+ cause,
133+ }),
134+ });
135+
136+interface RepoResponse {
137+ readonly default_branch?: unknown;
138+}
139+
140+interface CommitResponse {
141+ readonly sha?: unknown;
142+ readonly commit?: {
143+ readonly tree?: {
144+ readonly sha?: unknown;
145+ };
146+ };
147+}
148+
149+interface GitCommitResponse {
150+ readonly sha?: unknown;
151+ readonly tree?: {
152+ readonly sha?: unknown;
153+ };
154+}
155+
156+interface RefResponse {
157+ readonly object?: {
158+ readonly sha?: unknown;
159+ };
160+}
161+
162+interface TreeEntry {
163+ readonly path?: unknown;
164+ readonly type?: unknown;
165+ readonly sha?: unknown;
166+ readonly size?: unknown;
167+}
168+
169+interface TreeResponse {
170+ readonly tree?: readonly TreeEntry[];
171+ readonly truncated?: unknown;
172+}
173+
174+interface BlobResponse {
175+ readonly content?: unknown;
176+ readonly encoding?: unknown;
177+}
178+
179+const commitTreeSha = (commit: CommitResponse | GitCommitResponse): string | undefined => {
180+ if ("commit" in commit) return asString(commit.commit?.tree?.sha);
181+ if ("tree" in commit) return asString(commit.tree?.sha);
182+ return undefined;
183+};
184+
185+const limitError = (diagnostics: readonly FileDiagnostic[]): PublishError =>
186+ new PublishError({
187+ message: `publish payload exceeds limits (${diagnostics.length} problem(s))`,
188+ stage: "discover",
189+ diagnostics,
190+ });
191+
192+const checkTreeLimits = (entries: readonly TreeEntry[]): PublishError | null => {
193+ const diagnostics: FileDiagnostic[] = [];
194+ if (entries.length > PUBLISH_LIMITS.maxFiles) {
195+ diagnostics.push({
196+ path: "",
197+ message: `publish has ${entries.length} files, exceeding the limit of ${PUBLISH_LIMITS.maxFiles}`,
198+ });
199+ }
200+ let total = 0;
201+ for (const entry of entries) {
202+ const path = String(entry.path ?? "");
203+ const size = typeof entry.size === "number" ? entry.size : 0;
204+ total += size;
205+ if (size > PUBLISH_LIMITS.maxFileBytes) {
206+ diagnostics.push({
207+ path,
208+ message: `file is ${size} bytes, exceeding the per-file limit of ${PUBLISH_LIMITS.maxFileBytes} bytes`,
209+ });
210+ }
211+ }
212+ if (total > PUBLISH_LIMITS.maxTotalBytes) {
213+ diagnostics.push({
214+ path: "",
215+ message: `publish total is ${total} bytes, exceeding the total limit of ${PUBLISH_LIMITS.maxTotalBytes} bytes`,
216+ });
217+ }
218+ return diagnostics.length === 0 ? null : limitError(diagnostics);
219+};
220+
221+const decodeBlob = (path: string, blob: BlobResponse): Effect.Effect<string, GitHubSourceError> =>
222+ Effect.try({
223+ try: () => {
224+ const encoding = asString(blob.encoding);
225+ const content = asString(blob.content);
226+ if (!content || encoding !== "base64") {
227+ throw new GitHubSourceError({
228+ message: `GitHub blob ${path} did not return base64 content`,
229+ path,
230+ });
231+ }
232+ return Buffer.from(content.replace(/\s/g, ""), "base64").toString("utf8");
233+ },
234+ catch: (cause) =>
235+ cause instanceof GitHubSourceError
236+ ? cause
237+ : new GitHubSourceError({ message: `Failed to decode GitHub blob ${path}`, path, cause }),
238+ });
239+
240+const executorDescription = (
241+ files: FileSet,
242+): Effect.Effect<string | undefined, GitHubSourceError> =>
243+ Effect.try({
244+ try: () => {
245+ const raw = files.get("executor.json");
246+ if (!raw) return undefined;
247+ const parsed = JSON.parse(raw) as unknown;
248+ if (!isRecord(parsed)) return undefined;
249+ const description = parsed.description;
250+ return typeof description === "string" ? description : undefined;
251+ },
252+ catch: (cause) =>
253+ new GitHubSourceError({
254+ message: "executor.json is not valid JSON",
255+ path: "executor.json",
256+ cause,
257+ }),
258+ });
259+
260+export const fetchGitHubSource = (
261+ input: GitHubSourceInput,
262+): Effect.Effect<GitHubSourceSnapshot, GitHubSourceError | PublishError> =>
263+ Effect.gen(function* () {
264+ const repoPath = yield* encodedRepoPath(input.repo);
265+ const repo = yield* requestJson<RepoResponse>(input, `/repos/${repoPath}`);
266+ const ref = input.ref ?? asString(repo.default_branch) ?? "main";
267+ const branchRef = yield* requestJson<RefResponse>(
268+ input,
269+ `/repos/${repoPath}/git/ref/${encodeURIComponent(`heads/${ref}`)}`,
270+ ).pipe(Effect.result);
271+ const commitPath =
272+ Result.isSuccess(branchRef) && asString(branchRef.success.object?.sha)
273+ ? `/repos/${repoPath}/git/commits/${encodeURIComponent(String(branchRef.success.object?.sha))}`
274+ : `/repos/${repoPath}/commits/${encodeURIComponent(ref)}`;
275+ const commit = yield* requestJson<CommitResponse | GitCommitResponse>(input, commitPath);
276+ const upstreamSha = asString(commit.sha);
277+ const treeSha = commitTreeSha(commit);
278+ if (!upstreamSha || !treeSha) {
279+ return yield* new GitHubSourceError({
280+ message: `GitHub commit ${ref} did not include a commit SHA and tree SHA`,
281+ path: commitPath,
282+ });
283+ }
284+
285+ const tree = yield* requestJson<TreeResponse>(
286+ input,
287+ `/repos/${repoPath}/git/trees/${encodeURIComponent(treeSha)}?recursive=1`,
288+ );
289+ if (tree.truncated === true) {
290+ return yield* new GitHubSourceError({
291+ message: "GitHub returned a truncated repository tree; custom tools source is too large",
292+ path: `/repos/${repoPath}/git/trees/${treeSha}`,
293+ });
294+ }
295+ const entries = (tree.tree ?? []).filter(
296+ (entry) =>
297+ entry.type === "blob" &&
298+ typeof entry.path === "string" &&
299+ typeof entry.sha === "string" &&
300+ materializes(entry.path),
301+ );
302+ const treeLimitError = checkTreeLimits(entries);
303+ if (treeLimitError) return yield* Effect.fail(treeLimitError);
304+
305+ const files = new Map<string, string>();
306+ for (const entry of entries) {
307+ const path = String(entry.path);
308+ const blob = yield* requestJson<BlobResponse>(
309+ input,
310+ `/repos/${repoPath}/git/blobs/${encodeURIComponent(String(entry.sha))}`,
311+ );
312+ files.set(path, yield* decodeBlob(path, blob));
313+ }
314+ const payloadLimitError = enforcePublishLimits(files);
315+ if (payloadLimitError) return yield* Effect.fail(payloadLimitError);
316+
317+ return {
318+ files,
319+ repo: input.repo,
320+ ref,
321+ upstreamSha,
322+ description: yield* executorDescription(files),
323+ };
324+ });
325+
326+const publishErrorToSyncError = (error: PublishError): SyncErrorData => ({
327+ stage: error.stage,
328+ message: error.message,
329+ diagnostics: error.diagnostics,
330+});
331+
332+const sourceErrorToSyncError = (error: GitHubSourceError): SyncErrorData => ({
333+ stage: "source",
334+ message: error.message,
335+ diagnostics: error.path ? [{ path: error.path, message: error.message }] : [],
336+});
337+
338+const sourceRef = (snapshot: GitHubSourceSnapshot): AppSourceRef => ({
339+ kind: "github",
340+ repo: snapshot.repo,
341+ ref: snapshot.ref,
342+ upstreamSha: snapshot.upstreamSha,
343+});
344+
345+export const syncGitHubSource = (input: SyncGitHubSourceInput): Effect.Effect<GitHubSyncResult> =>
346+ Effect.gen(function* () {
347+ const fetched = yield* fetchGitHubSource(input).pipe(Effect.result);
348+ if (Result.isFailure(fetched)) {
349+ const error = fetched.failure;
350+ return {
351+ status: "failed",
352+ tools: [],
353+ skipped: [],
354+ errors: [
355+ error instanceof PublishError
356+ ? publishErrorToSyncError(error)
357+ : sourceErrorToSyncError(error),
358+ ],
359+ } satisfies GitHubSyncResult;
360+ }
361+ const snapshot = fetched.success;
362+ const current = yield* input.runtime.getDescriptor(input.scope);
363+ if (
364+ current?.source?.kind === "github" &&
365+ current.source.repo === snapshot.repo &&
366+ current.source.upstreamSha === snapshot.upstreamSha
367+ ) {
368+ return {
369+ status: "up-to-date",
370+ upstreamSha: snapshot.upstreamSha,
371+ tools: current.tools.map((tool) => tool.name),
372+ skipped: current.skipped ?? [],
373+ } satisfies GitHubSyncResult;
374+ }
375+
376+ const published = yield* input.runtime
377+ .publish({
378+ scope: input.scope,
379+ files: snapshot.files,
380+ description: snapshot.description,
381+ source: sourceRef(snapshot),
382+ message: `sync ${snapshot.repo}@${snapshot.upstreamSha}`,
383+ })
384+ .pipe(Effect.result);
385+ if (Result.isFailure(published)) {
386+ return {
387+ status: "failed",
388+ upstreamSha: snapshot.upstreamSha,
389+ tools: [],
390+ skipped: [],
391+ errors: [publishErrorToSyncError(published.failure)],
392+ } satisfies GitHubSyncResult;
393+ }
394+
395+ return {
396+ status: "published",
397+ snapshotId: published.success.snapshotId,
398+ upstreamSha: snapshot.upstreamSha,
399+ tools: published.success.descriptor.tools.map((tool) => tool.name),
400+ skipped: published.success.skipped,
401+ } satisfies GitHubSyncResult;
402+ });