import { mkdtempSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { describe, expect, it } from "@effect/vitest"; import { Effect, Exit } from "effect"; import { makeSelfHostAppsRuntime } from "../plugin/self-host-runtime"; import { makeInMemoryAppsStore, makeTestResolver } from "../testing"; import { PUBLISH_LIMITS } from "../pipeline/publish"; import { scopeAddress } from "../seams/scope-address"; import { fetchGitHubSource, parseGitHubSourceUrl, syncGitHubSource } from "./github-source"; const run = (effect: Effect.Effect): Promise => Effect.runPromise(effect); const toolSource = (name = "ok"): string => `import { defineTool } from "executor:app"; import { z } from "zod"; export default defineTool({ description: "${name}", input: z.object({ value: z.string().default("${name}") }), async handler(input) { return { value: input.value }; }, });`; const makeRuntime = () => makeSelfHostAppsRuntime({ dataDir: mkdtempSync(join(tmpdir(), "apps-gh-src-")), store: makeInMemoryAppsStore(), resolver: makeTestResolver({}), inMemory: true, }); const json = (value: unknown, status = 200): Response => new Response(JSON.stringify(value), { status, headers: { "content-type": "application/json" }, }); interface GitHubTreeFixtureEntry { readonly path: string; readonly type: string; readonly mode?: string; readonly content?: string; readonly size?: number; } const makeGitHubFetch = (input: { readonly files: ReadonlyMap; readonly entries?: readonly GitHubTreeFixtureEntry[]; readonly upstreamSha?: string; readonly treeSha?: string; }) => { const repoPath = "/repos/acme/tools"; const upstreamSha = input.upstreamSha ?? "commit-1"; const treeSha = input.treeSha ?? "tree-1"; const blobBySha = new Map(); const treeEntries: readonly GitHubTreeFixtureEntry[] = input.entries ?? [...input.files].map(([path, content]) => ({ path, type: "blob", mode: "100644", content, })); let index = 0; const shaByPath = new Map(); for (const entry of treeEntries) { if (entry.content !== undefined) { const sha = `blob-${++index}`; shaByPath.set(entry.path, sha); blobBySha.set(sha, { path: entry.path, content: entry.content }); } } let blobCalls = 0; const authHeaders: (string | null)[] = []; const fetch = (async (rawUrl: string, init?: RequestInit) => { authHeaders.push(new Headers(init?.headers).get("authorization")); const url = new URL(rawUrl); if (url.pathname === repoPath) return json({ default_branch: "main" }); if (url.pathname === `${repoPath}/git/ref/heads%2Fmain`) { return json({ object: { sha: upstreamSha } }); } if (url.pathname === `${repoPath}/git/commits/${upstreamSha}`) { return json({ sha: upstreamSha, tree: { sha: treeSha } }); } if (url.pathname === `${repoPath}/git/trees/${treeSha}`) { return json({ tree: treeEntries.map((entry, entryIndex) => { const sha = shaByPath.get(entry.path) ?? `tree-${entryIndex}`; return { path: entry.path, type: entry.type, mode: entry.mode, sha, size: entry.size ?? Buffer.byteLength(entry.content ?? "", "utf8"), }; }), }); } const blobPrefix = `${repoPath}/git/blobs/`; if (url.pathname.startsWith(blobPrefix)) { blobCalls++; const sha = decodeURIComponent(url.pathname.slice(blobPrefix.length)); const blob = blobBySha.get(sha); if (!blob) return json({ message: "not found" }, 404); return json({ encoding: "base64", content: Buffer.from(blob.content, "utf8").toString("base64"), }); } return json({ message: "not found" }, 404); }) as typeof globalThis.fetch; return { fetch, blobCalls: () => blobCalls, authHeaders: () => authHeaders, }; }; describe("GitHub custom-tools source", () => { it("parses strict GitHub source URLs", () => { expect(parseGitHubSourceUrl("https://github.com/UsefulSoftwareCo/executor")).toEqual({ ok: true, value: { owner: "UsefulSoftwareCo", name: "executor", repo: "UsefulSoftwareCo/executor", url: "https://github.com/UsefulSoftwareCo/executor", }, }); expect(parseGitHubSourceUrl("https://github.com/UsefulSoftwareCo/executor.git/")).toEqual({ ok: true, value: { owner: "UsefulSoftwareCo", name: "executor", repo: "UsefulSoftwareCo/executor", url: "https://github.com/UsefulSoftwareCo/executor", }, }); expect(parseGitHubSourceUrl("https://github.com/UsefulSoftwareCo/executor/tree/main")).toEqual({ ok: true, value: { owner: "UsefulSoftwareCo", name: "executor", repo: "UsefulSoftwareCo/executor", ref: "main", url: "https://github.com/UsefulSoftwareCo/executor/tree/main", }, }); expect( parseGitHubSourceUrl("https://github.com/UsefulSoftwareCo/executor/tree/feature/tools"), ).toEqual({ ok: true, value: { owner: "UsefulSoftwareCo", name: "executor", repo: "UsefulSoftwareCo/executor", ref: "feature/tools", url: "https://github.com/UsefulSoftwareCo/executor/tree/feature/tools", }, }); expect( parseGitHubSourceUrl( "https://github.com/UsefulSoftwareCo/executor/commit/0123456789abcdef0123456789abcdef01234567", ), ).toEqual({ ok: true, value: { owner: "UsefulSoftwareCo", name: "executor", repo: "UsefulSoftwareCo/executor", ref: "0123456789abcdef0123456789abcdef01234567", url: "https://github.com/UsefulSoftwareCo/executor/commit/0123456789abcdef0123456789abcdef01234567", }, }); expect(parseGitHubSourceUrl("UsefulSoftwareCo/executor")).toEqual({ ok: true, value: { owner: "UsefulSoftwareCo", name: "executor", repo: "UsefulSoftwareCo/executor", url: "https://github.com/UsefulSoftwareCo/executor", }, }); expect(parseGitHubSourceUrl("https://gitlab.com/UsefulSoftwareCo/executor")).toEqual({ ok: false, message: "GitHub source URLs must use github.com.", }); expect(parseGitHubSourceUrl("https://github.com/UsefulSoftwareCo")).toEqual({ ok: false, message: "Use a GitHub repo URL like https://github.com/owner/repo, optionally with /tree/ or /commit/.", }); expect(parseGitHubSourceUrl("https://github.com/UsefulSoftwareCo/executor/issues")).toEqual({ ok: false, message: "Use a GitHub repo URL like https://github.com/owner/repo, optionally with /tree/ or /commit/.", }); expect( parseGitHubSourceUrl("https://github.com/UsefulSoftwareCo/executor/commit/main"), ).toEqual({ ok: false, message: "GitHub commit URLs must include a commit SHA.", }); }); it("fetches a repo fileset and publishes provenance, description, and skipped entries", async () => { const files = new Map([ [ "executor.json", JSON.stringify({ $schema: "https://example.test/schema", description: "Acme tools", ignored: true, }), ], ["tools/hello.ts", toolSource("hello")], ["workflows/deferred.ts", "export default {};"], ["docs/readme.md", "ignored"], ]); const github = makeGitHubFetch({ files, upstreamSha: "commit-a" }); const snapshot = await run( fetchGitHubSource({ url: "https://github.com/acme/tools", fetch: github.fetch }), ); expect([...snapshot.files.keys()].sort()).toEqual(["executor.json", "tools/hello.ts"]); expect(github.authHeaders().every((header) => header === null)).toBe(true); expect(snapshot.skipped).toEqual([ { path: "workflows/deferred.ts", reason: "not supported yet" }, { path: "docs/readme.md", reason: "ignored" }, ]); expect(snapshot.description).toBe("Acme tools"); const host = makeRuntime(); const result = await run( syncGitHubSource({ runtime: host.runtime, scope: "githubTools", url: "https://github.com/acme/tools", fetch: github.fetch, }), ); expect(result.status).toBe("published"); expect(result.tools).toEqual(["hello"]); expect(result.skipped).toEqual([ { path: "workflows/deferred.ts", reason: "not supported yet" }, { path: "docs/readme.md", reason: "ignored" }, ]); const descriptor = await run(host.runtime.getDescriptor("githubTools")); expect(descriptor?.description).toBe("Acme tools"); expect(descriptor?.source).toEqual({ kind: "github", url: "https://github.com/acme/tools", repo: "acme/tools", ref: "main", upstreamSha: "commit-a", skipped: [ { path: "workflows/deferred.ts", reason: "not supported yet" }, { path: "docs/readme.md", reason: "ignored" }, ], }); await host.close(); }); it("reports up-to-date when the upstream commit SHA is unchanged", async () => { const github = makeGitHubFetch({ files: new Map([["tools/hello.ts", toolSource("hello")]]), upstreamSha: "same-sha", }); const host = makeRuntime(); const first = await run( syncGitHubSource({ runtime: host.runtime, scope: "githubTools", url: "https://github.com/acme/tools", fetch: github.fetch, }), ); const second = await run( syncGitHubSource({ runtime: host.runtime, scope: "githubTools", url: "https://github.com/acme/tools", fetch: github.fetch, }), ); expect(first.status).toBe("published"); expect(second).toEqual({ status: "up-to-date", upstreamSha: "same-sha", tools: ["hello"], skipped: [], }); await host.close(); }); it("rejects oversized trees before fetching blobs", async () => { const files = new Map(); for (let i = 0; i < PUBLISH_LIMITS.maxFiles + 1; i++) { files.set(`tools/t${i}.ts`, toolSource(`t${i}`)); } const github = makeGitHubFetch({ files }); const exit = await Effect.runPromiseExit( fetchGitHubSource({ url: "https://github.com/acme/tools", fetch: github.fetch }), ); expect(Exit.isFailure(exit)).toBe(true); expect(JSON.stringify(exit)).toContain("exceeding the limit"); expect(github.blobCalls()).toBe(0); }); it("returns typed failure data for publish errors", async () => { const github = makeGitHubFetch({ files: new Map([ [ "tools/bad.ts", `import { defineTool } from "executor:app"; import { chunk } from "lodash"; export default defineTool({ description: "bad", input: { type: "object" }, async handler(){ return chunk([1], 1); } });`, ], ]), upstreamSha: "bad-sha", }); const host = makeRuntime(); const result = await run( syncGitHubSource({ runtime: host.runtime, scope: "githubTools", url: "https://github.com/acme/tools", fetch: github.fetch, }), ); expect(result.status).toBe("failed"); expect(result.upstreamSha).toBe("bad-sha"); expect(result.errors?.[0]?.stage).toBe("bundle"); expect(result.errors?.[0]?.message).toContain('bare import "lodash" is not allowed'); await host.close(); }); it("skips unsupported tree entries and invalid paths without committing them", async () => { const github = makeGitHubFetch({ files: new Map(), entries: [ { path: "tools/ok.ts", type: "blob", mode: "100644", content: toolSource("ok"), }, { path: "tools/link.ts", type: "blob", mode: "120000", content: "../outside.ts", }, { path: "tools/submodule.ts", type: "commit", mode: "160000", }, { path: "tools/Bad.Name.ts", type: "blob", mode: "100644", content: toolSource("bad"), }, ], upstreamSha: "mixed-sha", }); const host = makeRuntime(); const result = await run( syncGitHubSource({ runtime: host.runtime, scope: "githubTools", url: "https://github.com/acme/tools", token: "source-token", fetch: github.fetch, }), ); expect(result.status).toBe("published"); expect(github.authHeaders().every((header) => header === "Bearer source-token")).toBe(true); expect(result.tools).toEqual(["ok"]); expect(result.skipped).toEqual([ { path: "tools/link.ts", reason: "unsupported file type" }, { path: "tools/submodule.ts", reason: "unsupported file type" }, { path: "tools/Bad.Name.ts", reason: "ignored" }, ]); const descriptor = await run(host.runtime.getDescriptor("githubTools")); expect(descriptor?.tools.map((tool) => tool.name)).toEqual(["ok"]); const scopeStore = await run( host.runtime.deps.artifactStore.forScope(scopeAddress("org", "githubTools")), ); const snapshotPaths = await run(scopeStore.list(descriptor!.snapshotId as never)); expect(snapshotPaths).toContain("tools/ok.ts"); expect(snapshotPaths).not.toContain("tools/link.ts"); expect(snapshotPaths).not.toContain("tools/submodule.ts"); expect(snapshotPaths).not.toContain("tools/Bad.Name.ts"); await host.close(); }); });