import { mkdtempSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { describe, expect, it } from "@effect/vitest";
import { Effect, Exit } from "effect";
import { makeSelfHostAppsRuntime } from "../plugin/self-host-runtime";
import { makeInMemoryAppsStore, makeTestResolver } from "../testing";
import { PUBLISH_LIMITS } from "../pipeline/publish";
import { scopeAddress } from "../seams/scope-address";
import { fetchGitHubSource, parseGitHubSourceUrl, syncGitHubSource } from "./github-source";
const run = (effect: Effect.Effect): Promise => Effect.runPromise(effect);
const toolSource = (name = "ok"): string => `import { defineTool } from "executor:app";
import { z } from "zod";
export default defineTool({
description: "${name}",
input: z.object({ value: z.string().default("${name}") }),
async handler(input) { return { value: input.value }; },
});`;
const makeRuntime = () =>
makeSelfHostAppsRuntime({
dataDir: mkdtempSync(join(tmpdir(), "apps-gh-src-")),
store: makeInMemoryAppsStore(),
resolver: makeTestResolver({}),
inMemory: true,
});
const json = (value: unknown, status = 200): Response =>
new Response(JSON.stringify(value), {
status,
headers: { "content-type": "application/json" },
});
interface GitHubTreeFixtureEntry {
readonly path: string;
readonly type: string;
readonly mode?: string;
readonly content?: string;
readonly size?: number;
}
const makeGitHubFetch = (input: {
readonly files: ReadonlyMap;
readonly entries?: readonly GitHubTreeFixtureEntry[];
readonly upstreamSha?: string;
readonly treeSha?: string;
}) => {
const repoPath = "/repos/acme/tools";
const upstreamSha = input.upstreamSha ?? "commit-1";
const treeSha = input.treeSha ?? "tree-1";
const blobBySha = new Map();
const treeEntries: readonly GitHubTreeFixtureEntry[] =
input.entries ??
[...input.files].map(([path, content]) => ({
path,
type: "blob",
mode: "100644",
content,
}));
let index = 0;
const shaByPath = new Map();
for (const entry of treeEntries) {
if (entry.content !== undefined) {
const sha = `blob-${++index}`;
shaByPath.set(entry.path, sha);
blobBySha.set(sha, { path: entry.path, content: entry.content });
}
}
let blobCalls = 0;
const authHeaders: (string | null)[] = [];
const fetch = (async (rawUrl: string, init?: RequestInit) => {
authHeaders.push(new Headers(init?.headers).get("authorization"));
const url = new URL(rawUrl);
if (url.pathname === repoPath) return json({ default_branch: "main" });
if (url.pathname === `${repoPath}/git/ref/heads%2Fmain`) {
return json({ object: { sha: upstreamSha } });
}
if (url.pathname === `${repoPath}/git/commits/${upstreamSha}`) {
return json({ sha: upstreamSha, tree: { sha: treeSha } });
}
if (url.pathname === `${repoPath}/git/trees/${treeSha}`) {
return json({
tree: treeEntries.map((entry, entryIndex) => {
const sha = shaByPath.get(entry.path) ?? `tree-${entryIndex}`;
return {
path: entry.path,
type: entry.type,
mode: entry.mode,
sha,
size: entry.size ?? Buffer.byteLength(entry.content ?? "", "utf8"),
};
}),
});
}
const blobPrefix = `${repoPath}/git/blobs/`;
if (url.pathname.startsWith(blobPrefix)) {
blobCalls++;
const sha = decodeURIComponent(url.pathname.slice(blobPrefix.length));
const blob = blobBySha.get(sha);
if (!blob) return json({ message: "not found" }, 404);
return json({
encoding: "base64",
content: Buffer.from(blob.content, "utf8").toString("base64"),
});
}
return json({ message: "not found" }, 404);
}) as typeof globalThis.fetch;
return {
fetch,
blobCalls: () => blobCalls,
authHeaders: () => authHeaders,
};
};
describe("GitHub custom-tools source", () => {
it("parses strict GitHub source URLs", () => {
expect(parseGitHubSourceUrl("https://github.com/UsefulSoftwareCo/executor")).toEqual({
ok: true,
value: {
owner: "UsefulSoftwareCo",
name: "executor",
repo: "UsefulSoftwareCo/executor",
url: "https://github.com/UsefulSoftwareCo/executor",
},
});
expect(parseGitHubSourceUrl("https://github.com/UsefulSoftwareCo/executor.git/")).toEqual({
ok: true,
value: {
owner: "UsefulSoftwareCo",
name: "executor",
repo: "UsefulSoftwareCo/executor",
url: "https://github.com/UsefulSoftwareCo/executor",
},
});
expect(parseGitHubSourceUrl("https://github.com/UsefulSoftwareCo/executor/tree/main")).toEqual({
ok: true,
value: {
owner: "UsefulSoftwareCo",
name: "executor",
repo: "UsefulSoftwareCo/executor",
ref: "main",
url: "https://github.com/UsefulSoftwareCo/executor/tree/main",
},
});
expect(
parseGitHubSourceUrl("https://github.com/UsefulSoftwareCo/executor/tree/feature/tools"),
).toEqual({
ok: true,
value: {
owner: "UsefulSoftwareCo",
name: "executor",
repo: "UsefulSoftwareCo/executor",
ref: "feature/tools",
url: "https://github.com/UsefulSoftwareCo/executor/tree/feature/tools",
},
});
expect(
parseGitHubSourceUrl(
"https://github.com/UsefulSoftwareCo/executor/commit/0123456789abcdef0123456789abcdef01234567",
),
).toEqual({
ok: true,
value: {
owner: "UsefulSoftwareCo",
name: "executor",
repo: "UsefulSoftwareCo/executor",
ref: "0123456789abcdef0123456789abcdef01234567",
url: "https://github.com/UsefulSoftwareCo/executor/commit/0123456789abcdef0123456789abcdef01234567",
},
});
expect(parseGitHubSourceUrl("UsefulSoftwareCo/executor")).toEqual({
ok: true,
value: {
owner: "UsefulSoftwareCo",
name: "executor",
repo: "UsefulSoftwareCo/executor",
url: "https://github.com/UsefulSoftwareCo/executor",
},
});
expect(parseGitHubSourceUrl("https://gitlab.com/UsefulSoftwareCo/executor")).toEqual({
ok: false,
message: "GitHub source URLs must use github.com.",
});
expect(parseGitHubSourceUrl("https://github.com/UsefulSoftwareCo")).toEqual({
ok: false,
message:
"Use a GitHub repo URL like https://github.com/owner/repo, optionally with /tree/[ or /commit/.",
});
expect(parseGitHubSourceUrl("https://github.com/UsefulSoftwareCo/executor/issues")).toEqual({
ok: false,
message:
"Use a GitHub repo URL like https://github.com/owner/repo, optionally with /tree/][ or /commit/.",
});
expect(
parseGitHubSourceUrl("https://github.com/UsefulSoftwareCo/executor/commit/main"),
).toEqual({
ok: false,
message: "GitHub commit URLs must include a commit SHA.",
});
});
it("fetches a repo fileset and publishes provenance, description, and skipped entries", async () => {
const files = new Map([
[
"executor.json",
JSON.stringify({
$schema: "https://example.test/schema",
description: "Acme tools",
ignored: true,
}),
],
["tools/hello.ts", toolSource("hello")],
["workflows/deferred.ts", "export default {};"],
["docs/readme.md", "ignored"],
]);
const github = makeGitHubFetch({ files, upstreamSha: "commit-a" });
const snapshot = await run(
fetchGitHubSource({ url: "https://github.com/acme/tools", fetch: github.fetch }),
);
expect([...snapshot.files.keys()].sort()).toEqual(["executor.json", "tools/hello.ts"]);
expect(github.authHeaders().every((header) => header === null)).toBe(true);
expect(snapshot.skipped).toEqual([
{ path: "workflows/deferred.ts", reason: "not supported yet" },
{ path: "docs/readme.md", reason: "ignored" },
]);
expect(snapshot.description).toBe("Acme tools");
const host = makeRuntime();
const result = await run(
syncGitHubSource({
runtime: host.runtime,
scope: "githubTools",
url: "https://github.com/acme/tools",
fetch: github.fetch,
}),
);
expect(result.status).toBe("published");
expect(result.tools).toEqual(["hello"]);
expect(result.skipped).toEqual([
{ path: "workflows/deferred.ts", reason: "not supported yet" },
{ path: "docs/readme.md", reason: "ignored" },
]);
const descriptor = await run(host.runtime.getDescriptor("githubTools"));
expect(descriptor?.description).toBe("Acme tools");
expect(descriptor?.source).toEqual({
kind: "github",
url: "https://github.com/acme/tools",
repo: "acme/tools",
ref: "main",
upstreamSha: "commit-a",
skipped: [
{ path: "workflows/deferred.ts", reason: "not supported yet" },
{ path: "docs/readme.md", reason: "ignored" },
],
});
await host.close();
});
it("reports up-to-date when the upstream commit SHA is unchanged", async () => {
const github = makeGitHubFetch({
files: new Map([["tools/hello.ts", toolSource("hello")]]),
upstreamSha: "same-sha",
});
const host = makeRuntime();
const first = await run(
syncGitHubSource({
runtime: host.runtime,
scope: "githubTools",
url: "https://github.com/acme/tools",
fetch: github.fetch,
}),
);
const second = await run(
syncGitHubSource({
runtime: host.runtime,
scope: "githubTools",
url: "https://github.com/acme/tools",
fetch: github.fetch,
}),
);
expect(first.status).toBe("published");
expect(second).toEqual({
status: "up-to-date",
upstreamSha: "same-sha",
tools: ["hello"],
skipped: [],
});
await host.close();
});
it("rejects oversized trees before fetching blobs", async () => {
const files = new Map();
for (let i = 0; i < PUBLISH_LIMITS.maxFiles + 1; i++) {
files.set(`tools/t${i}.ts`, toolSource(`t${i}`));
}
const github = makeGitHubFetch({ files });
const exit = await Effect.runPromiseExit(
fetchGitHubSource({ url: "https://github.com/acme/tools", fetch: github.fetch }),
);
expect(Exit.isFailure(exit)).toBe(true);
expect(JSON.stringify(exit)).toContain("exceeding the limit");
expect(github.blobCalls()).toBe(0);
});
it("returns typed failure data for publish errors", async () => {
const github = makeGitHubFetch({
files: new Map([
[
"tools/bad.ts",
`import { defineTool } from "executor:app";
import { chunk } from "lodash";
export default defineTool({ description: "bad", input: { type: "object" }, async handler(){ return chunk([1], 1); } });`,
],
]),
upstreamSha: "bad-sha",
});
const host = makeRuntime();
const result = await run(
syncGitHubSource({
runtime: host.runtime,
scope: "githubTools",
url: "https://github.com/acme/tools",
fetch: github.fetch,
}),
);
expect(result.status).toBe("failed");
expect(result.upstreamSha).toBe("bad-sha");
expect(result.errors?.[0]?.stage).toBe("bundle");
expect(result.errors?.[0]?.message).toContain('bare import "lodash" is not allowed');
await host.close();
});
it("skips unsupported tree entries and invalid paths without committing them", async () => {
const github = makeGitHubFetch({
files: new Map(),
entries: [
{
path: "tools/ok.ts",
type: "blob",
mode: "100644",
content: toolSource("ok"),
},
{
path: "tools/link.ts",
type: "blob",
mode: "120000",
content: "../outside.ts",
},
{
path: "tools/submodule.ts",
type: "commit",
mode: "160000",
},
{
path: "tools/Bad.Name.ts",
type: "blob",
mode: "100644",
content: toolSource("bad"),
},
],
upstreamSha: "mixed-sha",
});
const host = makeRuntime();
const result = await run(
syncGitHubSource({
runtime: host.runtime,
scope: "githubTools",
url: "https://github.com/acme/tools",
token: "source-token",
fetch: github.fetch,
}),
);
expect(result.status).toBe("published");
expect(github.authHeaders().every((header) => header === "Bearer source-token")).toBe(true);
expect(result.tools).toEqual(["ok"]);
expect(result.skipped).toEqual([
{ path: "tools/link.ts", reason: "unsupported file type" },
{ path: "tools/submodule.ts", reason: "unsupported file type" },
{ path: "tools/Bad.Name.ts", reason: "ignored" },
]);
const descriptor = await run(host.runtime.getDescriptor("githubTools"));
expect(descriptor?.tools.map((tool) => tool.name)).toEqual(["ok"]);
const scopeStore = await run(
host.runtime.deps.artifactStore.forScope(scopeAddress("org", "githubTools")),
);
const snapshotPaths = await run(scopeStore.list(descriptor!.snapshotId as never));
expect(snapshotPaths).toContain("tools/ok.ts");
expect(snapshotPaths).not.toContain("tools/link.ts");
expect(snapshotPaths).not.toContain("tools/submodule.ts");
expect(snapshotPaths).not.toContain("tools/Bad.Name.ts");
await host.close();
});
});
]