History

Add GitHub sync and self-host custom tools wiring

Rhys Sullivan committed 4 months ago

Commit 275446e1 parent 41eb3ae19 files · +1665 −40Browse files

Showing the changes to packages/plugins/apps/src/source (2 of 19 files). Show all

Unified · Split

2 files
  1. packages/plugins/apps/src/source/github-source.test.ts +211 −0
  2. packages/plugins/apps/src/source/github-source.ts +402 −0
packages/plugins/apps/src/source/github-source.test.ts+211−0
@@ -0,0 +1,211 @@
1import { mkdtempSync } from "node:fs";
2import { tmpdir } from "node:os";
3import { join } from "node:path";
4
5import { describe, expect, it } from "vitest";
6import { Effect } from "effect";
7
8import { makeSelfHostAppsRuntime } from "../plugin/self-host-runtime";
9import { makeInMemoryAppsStore, makeTestResolver } from "../testing";
10import { PUBLISH_LIMITS } from "../pipeline/publish";
11import { fetchGitHubSource, syncGitHubSource } from "./github-source";
12
13const run = <A, E>(effect: Effect.Effect<A, E>): Promise<A> => Effect.runPromise(effect);
14
15const toolSource = (name = "ok"): string => `import { defineTool } from "executor:app";
16import { z } from "zod";
17export default defineTool({
18 description: "${name}",
19 input: z.object({ value: z.string().default("${name}") }),
20 async handler(input) { return { value: input.value }; },
21});`;
22
23const makeRuntime = () =>
24 makeSelfHostAppsRuntime({
25 dataDir: mkdtempSync(join(tmpdir(), "apps-gh-src-")),
26 store: makeInMemoryAppsStore(),
27 resolver: makeTestResolver({}),
28 inMemory: true,
29 });
30
31const json = (value: unknown, status = 200): Response =>
32 new Response(JSON.stringify(value), {
33 status,
34 headers: { "content-type": "application/json" },
35 });
36
37const makeGitHubFetch = (input: {
38 readonly files: ReadonlyMap<string, string>;
39 readonly upstreamSha?: string;
40 readonly treeSha?: string;
41}) => {
42 const repoPath = "/repos/acme/tools";
43 const upstreamSha = input.upstreamSha ?? "commit-1";
44 const treeSha = input.treeSha ?? "tree-1";
45 const blobBySha = new Map<string, { path: string; content: string }>();
46 let index = 0;
47 for (const [path, content] of input.files) {
48 blobBySha.set(`blob-${++index}`, { path, content });
49 }
50 let blobCalls = 0;
51 const fetch = (async (rawUrl: string) => {
52 const url = new URL(rawUrl);
53 if (url.pathname === repoPath) return json({ default_branch: "main" });
54 if (url.pathname === `${repoPath}/git/ref/heads%2Fmain`) {
55 return json({ object: { sha: upstreamSha } });
56 }
57 if (url.pathname === `${repoPath}/git/commits/${upstreamSha}`) {
58 return json({ sha: upstreamSha, tree: { sha: treeSha } });
59 }
60 if (url.pathname === `${repoPath}/git/trees/${treeSha}`) {
61 return json({
62 tree: [...blobBySha.entries()].map(([sha, blob]) => ({
63 path: blob.path,
64 type: "blob",
65 sha,
66 size: Buffer.byteLength(blob.content, "utf8"),
67 })),
68 });
69 }
70 const blobPrefix = `${repoPath}/git/blobs/`;
71 if (url.pathname.startsWith(blobPrefix)) {
72 blobCalls++;
73 const sha = decodeURIComponent(url.pathname.slice(blobPrefix.length));
74 const blob = blobBySha.get(sha);
75 if (!blob) return json({ message: "not found" }, 404);
76 return json({
77 encoding: "base64",
78 content: Buffer.from(blob.content, "utf8").toString("base64"),
79 });
80 }
81 return json({ message: "not found" }, 404);
82 }) as typeof globalThis.fetch;
83 return {
84 fetch,
85 blobCalls: () => blobCalls,
86 };
87};
88
89describe("GitHub custom-tools source", () => {
90 it("fetches a repo fileset and publishes provenance, description, and skipped entries", async () => {
91 const files = new Map<string, string>([
92 [
93 "executor.json",
94 JSON.stringify({
95 $schema: "https://example.test/schema",
96 description: "Acme tools",
97 ignored: true,
98 }),
99 ],
100 ["tools/hello.ts", toolSource("hello")],
101 ["workflows/deferred.ts", "export default {};"],
102 ["docs/readme.md", "ignored"],
103 ]);
104 const github = makeGitHubFetch({ files, upstreamSha: "commit-a" });
105 const snapshot = await run(fetchGitHubSource({ repo: "acme/tools", fetch: github.fetch }));
106 expect([...snapshot.files.keys()].sort()).toEqual([
107 "executor.json",
108 "tools/hello.ts",
109 "workflows/deferred.ts",
110 ]);
111 expect(snapshot.description).toBe("Acme tools");
112
113 const host = makeRuntime();
114 const result = await run(
115 syncGitHubSource({
116 runtime: host.runtime,
117 scope: "githubTools",
118 repo: "acme/tools",
119 fetch: github.fetch,
120 }),
121 );
122 expect(result.status).toBe("published");
123 expect(result.tools).toEqual(["hello"]);
124 expect(result.skipped).toEqual([
125 { path: "workflows/deferred.ts", reason: "not supported yet" },
126 ]);
127 const descriptor = await run(host.runtime.getDescriptor("githubTools"));
128 expect(descriptor?.description).toBe("Acme tools");
129 expect(descriptor?.source).toEqual({
130 kind: "github",
131 repo: "acme/tools",
132 ref: "main",
133 upstreamSha: "commit-a",
134 });
135 await host.close();
136 });
137
138 it("reports up-to-date when the upstream commit SHA is unchanged", async () => {
139 const github = makeGitHubFetch({
140 files: new Map([["tools/hello.ts", toolSource("hello")]]),
141 upstreamSha: "same-sha",
142 });
143 const host = makeRuntime();
144 const first = await run(
145 syncGitHubSource({
146 runtime: host.runtime,
147 scope: "githubTools",
148 repo: "acme/tools",
149 fetch: github.fetch,
150 }),
151 );
152 const second = await run(
153 syncGitHubSource({
154 runtime: host.runtime,
155 scope: "githubTools",
156 repo: "acme/tools",
157 fetch: github.fetch,
158 }),
159 );
160 expect(first.status).toBe("published");
161 expect(second).toEqual({
162 status: "up-to-date",
163 upstreamSha: "same-sha",
164 tools: ["hello"],
165 skipped: [],
166 });
167 await host.close();
168 });
169
170 it("rejects oversized trees before fetching blobs", async () => {
171 const files = new Map<string, string>();
172 for (let i = 0; i < PUBLISH_LIMITS.maxFiles + 1; i++) {
173 files.set(`tools/t${i}.ts`, toolSource(`t${i}`));
174 }
175 const github = makeGitHubFetch({ files });
176 const exit = await Effect.runPromiseExit(
177 fetchGitHubSource({ repo: "acme/tools", fetch: github.fetch }),
178 );
179 expect(exit._tag).toBe("Failure");
180 expect(JSON.stringify(exit)).toContain("exceeding the limit");
181 expect(github.blobCalls()).toBe(0);
182 });
183
184 it("returns typed failure data for publish errors", async () => {
185 const github = makeGitHubFetch({
186 files: new Map([
187 [
188 "tools/bad.ts",
189 `import { defineTool } from "executor:app";
190import { chunk } from "lodash";
191export default defineTool({ description: "bad", input: { type: "object" }, async handler(){ return chunk([1], 1); } });`,
192 ],
193 ]),
194 upstreamSha: "bad-sha",
195 });
196 const host = makeRuntime();
197 const result = await run(
198 syncGitHubSource({
199 runtime: host.runtime,
200 scope: "githubTools",
201 repo: "acme/tools",
202 fetch: github.fetch,
203 }),
204 );
205 expect(result.status).toBe("failed");
206 expect(result.upstreamSha).toBe("bad-sha");
207 expect(result.errors?.[0]?.stage).toBe("bundle");
208 expect(result.errors?.[0]?.message).toContain('bare import "lodash" is not allowed');
209 await host.close();
210 });
211});
packages/plugins/apps/src/source/github-source.ts+402−0
@@ -0,0 +1,402 @@
1import { Data, Effect, Result } from "effect";
2
3import { PublishError, type FileDiagnostic, type SkippedArtifact } from "../pipeline/discover";
4import { PUBLISH_LIMITS, enforcePublishLimits } from "../pipeline/publish";
5import type { AppSourceRef } from "../pipeline/descriptor";
6import type { AppsRuntime } from "../plugin/runtime";
7import type { FileSet, SnapshotId } from "../seams/artifact-store";
8
9export interface GitHubSourceInput {
10 readonly repo: string;
11 readonly ref?: string;
12 readonly token?: string | null;
13 readonly baseUrl?: string;
14 readonly fetch?: typeof globalThis.fetch;
15}
16
17export interface GitHubSourceSnapshot {
18 readonly files: FileSet;
19 readonly repo: string;
20 readonly ref: string;
21 readonly upstreamSha: string;
22 readonly description?: string;
23}
24
25export class GitHubSourceError extends Data.TaggedError("GitHubSourceError")<{
26 readonly message: string;
27 readonly status?: number;
28 readonly path?: string;
29 readonly cause?: unknown;
30}> {}
31
32export interface SyncErrorData {
33 readonly stage: "source" | "discover" | "bundle" | "collect" | "project";
34 readonly message: string;
35 readonly diagnostics?: readonly FileDiagnostic[];
36}
37
38export type GitHubSyncResult =
39 | {
40 readonly status: "published";
41 readonly snapshotId: SnapshotId;
42 readonly upstreamSha: string;
43 readonly tools: readonly string[];
44 readonly skipped: readonly SkippedArtifact[];
45 readonly errors?: undefined;
46 }
47 | {
48 readonly status: "up-to-date";
49 readonly upstreamSha: string;
50 readonly tools: readonly string[];
51 readonly skipped: readonly SkippedArtifact[];
52 readonly errors?: undefined;
53 }
54 | {
55 readonly status: "failed";
56 readonly upstreamSha?: string;
57 readonly tools: readonly string[];
58 readonly skipped: readonly SkippedArtifact[];
59 readonly errors: readonly SyncErrorData[];
60 };
61
62export interface SyncGitHubSourceInput extends GitHubSourceInput {
63 readonly runtime: AppsRuntime;
64 readonly scope: string;
65}
66
67const MATERIALIZED_RE = /^(tools|workflows|ui|skills)\//;
68
69const trimBaseUrl = (baseUrl: string): string => baseUrl.replace(/\/+$/, "");
70
71const repoParts = (repo: string): { owner: string; name: string } | null => {
72 const [owner, name, ...rest] = repo.split("/");
73 if (!owner || !name || rest.length > 0) return null;
74 return { owner, name };
75};
76
77const encodedRepoPath = (repo: string): Effect.Effect<string, GitHubSourceError> => {
78 const parsed = repoParts(repo);
79 if (!parsed) {
80 return Effect.fail(
81 new GitHubSourceError({
82 message: `GitHub repo must be "owner/name"; got "${repo}"`,
83 path: repo,
84 }),
85 );
86 }
87 return Effect.succeed(`${encodeURIComponent(parsed.owner)}/${encodeURIComponent(parsed.name)}`);
88};
89
90const materializes = (path: string): boolean =>
91 path === "executor.json" || MATERIALIZED_RE.test(path);
92
93const isRecord = (value: unknown): value is Record<string, unknown> =>
94 value !== null && typeof value === "object" && !Array.isArray(value);
95
96const asString = (value: unknown): string | undefined =>
97 typeof value === "string" && value.length > 0 ? value : undefined;
98
99const requestJson = <A>(
100 input: GitHubSourceInput,
101 path: string,
102): Effect.Effect<A, GitHubSourceError> =>
103 Effect.tryPromise({
104 try: async () => {
105 const fetchImpl = input.fetch ?? globalThis.fetch;
106 const headers: Record<string, string> = {
107 accept: "application/vnd.github+json",
108 "user-agent": "executor-apps-github-source",
109 };
110 if (input.token) headers.authorization = `Bearer ${input.token}`;
111 const response = await fetchImpl(
112 `${trimBaseUrl(input.baseUrl ?? "https://api.github.com")}${path}`,
113 {
114 headers,
115 },
116 );
117 if (!response.ok) {
118 throw new GitHubSourceError({
119 message: `GitHub request failed: GET ${path} -> ${response.status}`,
120 status: response.status,
121 path,
122 });
123 }
124 return (await response.json()) as A;
125 },
126 catch: (cause) =>
127 cause instanceof GitHubSourceError
128 ? cause
129 : new GitHubSourceError({
130 message: `GitHub request failed: GET ${path}`,
131 path,
132 cause,
133 }),
134 });
135
136interface RepoResponse {
137 readonly default_branch?: unknown;
138}
139
140interface CommitResponse {
141 readonly sha?: unknown;
142 readonly commit?: {
143 readonly tree?: {
144 readonly sha?: unknown;
145 };
146 };
147}
148
149interface GitCommitResponse {
150 readonly sha?: unknown;
151 readonly tree?: {
152 readonly sha?: unknown;
153 };
154}
155
156interface RefResponse {
157 readonly object?: {
158 readonly sha?: unknown;
159 };
160}
161
162interface TreeEntry {
163 readonly path?: unknown;
164 readonly type?: unknown;
165 readonly sha?: unknown;
166 readonly size?: unknown;
167}
168
169interface TreeResponse {
170 readonly tree?: readonly TreeEntry[];
171 readonly truncated?: unknown;
172}
173
174interface BlobResponse {
175 readonly content?: unknown;
176 readonly encoding?: unknown;
177}
178
179const commitTreeSha = (commit: CommitResponse | GitCommitResponse): string | undefined => {
180 if ("commit" in commit) return asString(commit.commit?.tree?.sha);
181 if ("tree" in commit) return asString(commit.tree?.sha);
182 return undefined;
183};
184
185const limitError = (diagnostics: readonly FileDiagnostic[]): PublishError =>
186 new PublishError({
187 message: `publish payload exceeds limits (${diagnostics.length} problem(s))`,
188 stage: "discover",
189 diagnostics,
190 });
191
192const checkTreeLimits = (entries: readonly TreeEntry[]): PublishError | null => {
193 const diagnostics: FileDiagnostic[] = [];
194 if (entries.length > PUBLISH_LIMITS.maxFiles) {
195 diagnostics.push({
196 path: "",
197 message: `publish has ${entries.length} files, exceeding the limit of ${PUBLISH_LIMITS.maxFiles}`,
198 });
199 }
200 let total = 0;
201 for (const entry of entries) {
202 const path = String(entry.path ?? "");
203 const size = typeof entry.size === "number" ? entry.size : 0;
204 total += size;
205 if (size > PUBLISH_LIMITS.maxFileBytes) {
206 diagnostics.push({
207 path,
208 message: `file is ${size} bytes, exceeding the per-file limit of ${PUBLISH_LIMITS.maxFileBytes} bytes`,
209 });
210 }
211 }
212 if (total > PUBLISH_LIMITS.maxTotalBytes) {
213 diagnostics.push({
214 path: "",
215 message: `publish total is ${total} bytes, exceeding the total limit of ${PUBLISH_LIMITS.maxTotalBytes} bytes`,
216 });
217 }
218 return diagnostics.length === 0 ? null : limitError(diagnostics);
219};
220
221const decodeBlob = (path: string, blob: BlobResponse): Effect.Effect<string, GitHubSourceError> =>
222 Effect.try({
223 try: () => {
224 const encoding = asString(blob.encoding);
225 const content = asString(blob.content);
226 if (!content || encoding !== "base64") {
227 throw new GitHubSourceError({
228 message: `GitHub blob ${path} did not return base64 content`,
229 path,
230 });
231 }
232 return Buffer.from(content.replace(/\s/g, ""), "base64").toString("utf8");
233 },
234 catch: (cause) =>
235 cause instanceof GitHubSourceError
236 ? cause
237 : new GitHubSourceError({ message: `Failed to decode GitHub blob ${path}`, path, cause }),
238 });
239
240const executorDescription = (
241 files: FileSet,
242): Effect.Effect<string | undefined, GitHubSourceError> =>
243 Effect.try({
244 try: () => {
245 const raw = files.get("executor.json");
246 if (!raw) return undefined;
247 const parsed = JSON.parse(raw) as unknown;
248 if (!isRecord(parsed)) return undefined;
249 const description = parsed.description;
250 return typeof description === "string" ? description : undefined;
251 },
252 catch: (cause) =>
253 new GitHubSourceError({
254 message: "executor.json is not valid JSON",
255 path: "executor.json",
256 cause,
257 }),
258 });
259
260export const fetchGitHubSource = (
261 input: GitHubSourceInput,
262): Effect.Effect<GitHubSourceSnapshot, GitHubSourceError | PublishError> =>
263 Effect.gen(function* () {
264 const repoPath = yield* encodedRepoPath(input.repo);
265 const repo = yield* requestJson<RepoResponse>(input, `/repos/${repoPath}`);
266 const ref = input.ref ?? asString(repo.default_branch) ?? "main";
267 const branchRef = yield* requestJson<RefResponse>(
268 input,
269 `/repos/${repoPath}/git/ref/${encodeURIComponent(`heads/${ref}`)}`,
270 ).pipe(Effect.result);
271 const commitPath =
272 Result.isSuccess(branchRef) && asString(branchRef.success.object?.sha)
273 ? `/repos/${repoPath}/git/commits/${encodeURIComponent(String(branchRef.success.object?.sha))}`
274 : `/repos/${repoPath}/commits/${encodeURIComponent(ref)}`;
275 const commit = yield* requestJson<CommitResponse | GitCommitResponse>(input, commitPath);
276 const upstreamSha = asString(commit.sha);
277 const treeSha = commitTreeSha(commit);
278 if (!upstreamSha || !treeSha) {
279 return yield* new GitHubSourceError({
280 message: `GitHub commit ${ref} did not include a commit SHA and tree SHA`,
281 path: commitPath,
282 });
283 }
284
285 const tree = yield* requestJson<TreeResponse>(
286 input,
287 `/repos/${repoPath}/git/trees/${encodeURIComponent(treeSha)}?recursive=1`,
288 );
289 if (tree.truncated === true) {
290 return yield* new GitHubSourceError({
291 message: "GitHub returned a truncated repository tree; custom tools source is too large",
292 path: `/repos/${repoPath}/git/trees/${treeSha}`,
293 });
294 }
295 const entries = (tree.tree ?? []).filter(
296 (entry) =>
297 entry.type === "blob" &&
298 typeof entry.path === "string" &&
299 typeof entry.sha === "string" &&
300 materializes(entry.path),
301 );
302 const treeLimitError = checkTreeLimits(entries);
303 if (treeLimitError) return yield* Effect.fail(treeLimitError);
304
305 const files = new Map<string, string>();
306 for (const entry of entries) {
307 const path = String(entry.path);
308 const blob = yield* requestJson<BlobResponse>(
309 input,
310 `/repos/${repoPath}/git/blobs/${encodeURIComponent(String(entry.sha))}`,
311 );
312 files.set(path, yield* decodeBlob(path, blob));
313 }
314 const payloadLimitError = enforcePublishLimits(files);
315 if (payloadLimitError) return yield* Effect.fail(payloadLimitError);
316
317 return {
318 files,
319 repo: input.repo,
320 ref,
321 upstreamSha,
322 description: yield* executorDescription(files),
323 };
324 });
325
326const publishErrorToSyncError = (error: PublishError): SyncErrorData => ({
327 stage: error.stage,
328 message: error.message,
329 diagnostics: error.diagnostics,
330});
331
332const sourceErrorToSyncError = (error: GitHubSourceError): SyncErrorData => ({
333 stage: "source",
334 message: error.message,
335 diagnostics: error.path ? [{ path: error.path, message: error.message }] : [],
336});
337
338const sourceRef = (snapshot: GitHubSourceSnapshot): AppSourceRef => ({
339 kind: "github",
340 repo: snapshot.repo,
341 ref: snapshot.ref,
342 upstreamSha: snapshot.upstreamSha,
343});
344
345export const syncGitHubSource = (input: SyncGitHubSourceInput): Effect.Effect<GitHubSyncResult> =>
346 Effect.gen(function* () {
347 const fetched = yield* fetchGitHubSource(input).pipe(Effect.result);
348 if (Result.isFailure(fetched)) {
349 const error = fetched.failure;
350 return {
351 status: "failed",
352 tools: [],
353 skipped: [],
354 errors: [
355 error instanceof PublishError
356 ? publishErrorToSyncError(error)
357 : sourceErrorToSyncError(error),
358 ],
359 } satisfies GitHubSyncResult;
360 }
361 const snapshot = fetched.success;
362 const current = yield* input.runtime.getDescriptor(input.scope);
363 if (
364 current?.source?.kind === "github" &&
365 current.source.repo === snapshot.repo &&
366 current.source.upstreamSha === snapshot.upstreamSha
367 ) {
368 return {
369 status: "up-to-date",
370 upstreamSha: snapshot.upstreamSha,
371 tools: current.tools.map((tool) => tool.name),
372 skipped: current.skipped ?? [],
373 } satisfies GitHubSyncResult;
374 }
375
376 const published = yield* input.runtime
377 .publish({
378 scope: input.scope,
379 files: snapshot.files,
380 description: snapshot.description,
381 source: sourceRef(snapshot),
382 message: `sync ${snapshot.repo}@${snapshot.upstreamSha}`,
383 })
384 .pipe(Effect.result);
385 if (Result.isFailure(published)) {
386 return {
387 status: "failed",
388 upstreamSha: snapshot.upstreamSha,
389 tools: [],
390 skipped: [],
391 errors: [publishErrorToSyncError(published.failure)],
392 } satisfies GitHubSyncResult;
393 }
394
395 return {
396 status: "published",
397 snapshotId: published.success.snapshotId,
398 upstreamSha: snapshot.upstreamSha,
399 tools: published.success.descriptor.tools.map((tool) => tool.name),
400 skipped: published.success.skipped,
401 } satisfies GitHubSyncResult;
402 });