import { Data, Effect, Result } from "effect"; import { PublishError, type FileDiagnostic, type SkippedArtifact } from "../pipeline/discover"; import { PUBLISH_LIMITS, enforcePublishLimits } from "../pipeline/publish"; import type { AppSourceRef } from "../pipeline/descriptor"; import type { AppsRuntime } from "../plugin/runtime"; import type { FileSet, SnapshotId } from "../seams/artifact-store"; export interface GitHubSourceInput { readonly repo: string; readonly ref?: string; readonly token?: string | null; readonly baseUrl?: string; readonly fetch?: typeof globalThis.fetch; } export interface GitHubSourceSnapshot { readonly files: FileSet; readonly repo: string; readonly ref: string; readonly upstreamSha: string; readonly description?: string; } export class GitHubSourceError extends Data.TaggedError("GitHubSourceError")<{ readonly message: string; readonly status?: number; readonly path?: string; readonly cause?: unknown; }> {} export interface SyncErrorData { readonly stage: "source" | "discover" | "bundle" | "collect" | "project"; readonly message: string; readonly diagnostics?: readonly FileDiagnostic[]; } export type GitHubSyncResult = | { readonly status: "published"; readonly snapshotId: SnapshotId; readonly upstreamSha: string; readonly tools: readonly string[]; readonly skipped: readonly SkippedArtifact[]; readonly errors?: undefined; } | { readonly status: "up-to-date"; readonly upstreamSha: string; readonly tools: readonly string[]; readonly skipped: readonly SkippedArtifact[]; readonly errors?: undefined; } | { readonly status: "failed"; readonly upstreamSha?: string; readonly tools: readonly string[]; readonly skipped: readonly SkippedArtifact[]; readonly errors: readonly SyncErrorData[]; }; export interface SyncGitHubSourceInput extends GitHubSourceInput { readonly runtime: AppsRuntime; readonly scope: string; } const MATERIALIZED_RE = /^(tools|workflows|ui|skills)\//; const trimBaseUrl = (baseUrl: string): string => baseUrl.replace(/\/+$/, ""); const repoParts = (repo: string): { owner: string; name: string } | null => { const [owner, name, ...rest] = repo.split("/"); if (!owner || !name || rest.length > 0) return null; return { owner, name }; }; const encodedRepoPath = (repo: string): Effect.Effect => { const parsed = repoParts(repo); if (!parsed) { return Effect.fail( new GitHubSourceError({ message: `GitHub repo must be "owner/name"; got "${repo}"`, path: repo, }), ); } return Effect.succeed(`${encodeURIComponent(parsed.owner)}/${encodeURIComponent(parsed.name)}`); }; const materializes = (path: string): boolean => path === "executor.json" || MATERIALIZED_RE.test(path); const isRecord = (value: unknown): value is Record => value !== null && typeof value === "object" && !Array.isArray(value); const asString = (value: unknown): string | undefined => typeof value === "string" && value.length > 0 ? value : undefined; const requestJson = ( input: GitHubSourceInput, path: string, ): Effect.Effect => Effect.tryPromise({ try: async () => { const fetchImpl = input.fetch ?? globalThis.fetch; const headers: Record = { accept: "application/vnd.github+json", "user-agent": "executor-apps-github-source", }; if (input.token) headers.authorization = `Bearer ${input.token}`; const response = await fetchImpl( `${trimBaseUrl(input.baseUrl ?? "https://api.github.com")}${path}`, { headers, }, ); if (!response.ok) { throw new GitHubSourceError({ message: `GitHub request failed: GET ${path} -> ${response.status}`, status: response.status, path, }); } return (await response.json()) as A; }, catch: (cause) => cause instanceof GitHubSourceError ? cause : new GitHubSourceError({ message: `GitHub request failed: GET ${path}`, path, cause, }), }); interface RepoResponse { readonly default_branch?: unknown; } interface CommitResponse { readonly sha?: unknown; readonly commit?: { readonly tree?: { readonly sha?: unknown; }; }; } interface GitCommitResponse { readonly sha?: unknown; readonly tree?: { readonly sha?: unknown; }; } interface RefResponse { readonly object?: { readonly sha?: unknown; }; } interface TreeEntry { readonly path?: unknown; readonly type?: unknown; readonly sha?: unknown; readonly size?: unknown; } interface TreeResponse { readonly tree?: readonly TreeEntry[]; readonly truncated?: unknown; } interface BlobResponse { readonly content?: unknown; readonly encoding?: unknown; } const commitTreeSha = (commit: CommitResponse | GitCommitResponse): string | undefined => { if ("commit" in commit) return asString(commit.commit?.tree?.sha); if ("tree" in commit) return asString(commit.tree?.sha); return undefined; }; const limitError = (diagnostics: readonly FileDiagnostic[]): PublishError => new PublishError({ message: `publish payload exceeds limits (${diagnostics.length} problem(s))`, stage: "discover", diagnostics, }); const checkTreeLimits = (entries: readonly TreeEntry[]): PublishError | null => { const diagnostics: FileDiagnostic[] = []; if (entries.length > PUBLISH_LIMITS.maxFiles) { diagnostics.push({ path: "", message: `publish has ${entries.length} files, exceeding the limit of ${PUBLISH_LIMITS.maxFiles}`, }); } let total = 0; for (const entry of entries) { const path = String(entry.path ?? ""); const size = typeof entry.size === "number" ? entry.size : 0; total += size; if (size > PUBLISH_LIMITS.maxFileBytes) { diagnostics.push({ path, message: `file is ${size} bytes, exceeding the per-file limit of ${PUBLISH_LIMITS.maxFileBytes} bytes`, }); } } if (total > PUBLISH_LIMITS.maxTotalBytes) { diagnostics.push({ path: "", message: `publish total is ${total} bytes, exceeding the total limit of ${PUBLISH_LIMITS.maxTotalBytes} bytes`, }); } return diagnostics.length === 0 ? null : limitError(diagnostics); }; const decodeBlob = (path: string, blob: BlobResponse): Effect.Effect => Effect.try({ try: () => { const encoding = asString(blob.encoding); const content = asString(blob.content); if (!content || encoding !== "base64") { throw new GitHubSourceError({ message: `GitHub blob ${path} did not return base64 content`, path, }); } return Buffer.from(content.replace(/\s/g, ""), "base64").toString("utf8"); }, catch: (cause) => cause instanceof GitHubSourceError ? cause : new GitHubSourceError({ message: `Failed to decode GitHub blob ${path}`, path, cause }), }); const executorDescription = ( files: FileSet, ): Effect.Effect => Effect.try({ try: () => { const raw = files.get("executor.json"); if (!raw) return undefined; const parsed = JSON.parse(raw) as unknown; if (!isRecord(parsed)) return undefined; const description = parsed.description; return typeof description === "string" ? description : undefined; }, catch: (cause) => new GitHubSourceError({ message: "executor.json is not valid JSON", path: "executor.json", cause, }), }); export const fetchGitHubSource = ( input: GitHubSourceInput, ): Effect.Effect => Effect.gen(function* () { const repoPath = yield* encodedRepoPath(input.repo); const repo = yield* requestJson(input, `/repos/${repoPath}`); const ref = input.ref ?? asString(repo.default_branch) ?? "main"; const branchRef = yield* requestJson( input, `/repos/${repoPath}/git/ref/${encodeURIComponent(`heads/${ref}`)}`, ).pipe(Effect.result); const commitPath = Result.isSuccess(branchRef) && asString(branchRef.success.object?.sha) ? `/repos/${repoPath}/git/commits/${encodeURIComponent(String(branchRef.success.object?.sha))}` : `/repos/${repoPath}/commits/${encodeURIComponent(ref)}`; const commit = yield* requestJson(input, commitPath); const upstreamSha = asString(commit.sha); const treeSha = commitTreeSha(commit); if (!upstreamSha || !treeSha) { return yield* new GitHubSourceError({ message: `GitHub commit ${ref} did not include a commit SHA and tree SHA`, path: commitPath, }); } const tree = yield* requestJson( input, `/repos/${repoPath}/git/trees/${encodeURIComponent(treeSha)}?recursive=1`, ); if (tree.truncated === true) { return yield* new GitHubSourceError({ message: "GitHub returned a truncated repository tree; custom tools source is too large", path: `/repos/${repoPath}/git/trees/${treeSha}`, }); } const entries = (tree.tree ?? []).filter( (entry) => entry.type === "blob" && typeof entry.path === "string" && typeof entry.sha === "string" && materializes(entry.path), ); const treeLimitError = checkTreeLimits(entries); if (treeLimitError) return yield* Effect.fail(treeLimitError); const files = new Map(); for (const entry of entries) { const path = String(entry.path); const blob = yield* requestJson( input, `/repos/${repoPath}/git/blobs/${encodeURIComponent(String(entry.sha))}`, ); files.set(path, yield* decodeBlob(path, blob)); } const payloadLimitError = enforcePublishLimits(files); if (payloadLimitError) return yield* Effect.fail(payloadLimitError); return { files, repo: input.repo, ref, upstreamSha, description: yield* executorDescription(files), }; }); const publishErrorToSyncError = (error: PublishError): SyncErrorData => ({ stage: error.stage, message: error.message, diagnostics: error.diagnostics, }); const sourceErrorToSyncError = (error: GitHubSourceError): SyncErrorData => ({ stage: "source", message: error.message, diagnostics: error.path ? [{ path: error.path, message: error.message }] : [], }); const sourceRef = (snapshot: GitHubSourceSnapshot): AppSourceRef => ({ kind: "github", repo: snapshot.repo, ref: snapshot.ref, upstreamSha: snapshot.upstreamSha, }); export const syncGitHubSource = (input: SyncGitHubSourceInput): Effect.Effect => Effect.gen(function* () { const fetched = yield* fetchGitHubSource(input).pipe(Effect.result); if (Result.isFailure(fetched)) { const error = fetched.failure; return { status: "failed", tools: [], skipped: [], errors: [ error instanceof PublishError ? publishErrorToSyncError(error) : sourceErrorToSyncError(error), ], } satisfies GitHubSyncResult; } const snapshot = fetched.success; const current = yield* input.runtime.getDescriptor(input.scope); if ( current?.source?.kind === "github" && current.source.repo === snapshot.repo && current.source.upstreamSha === snapshot.upstreamSha ) { return { status: "up-to-date", upstreamSha: snapshot.upstreamSha, tools: current.tools.map((tool) => tool.name), skipped: current.skipped ?? [], } satisfies GitHubSyncResult; } const published = yield* input.runtime .publish({ scope: input.scope, files: snapshot.files, description: snapshot.description, source: sourceRef(snapshot), message: `sync ${snapshot.repo}@${snapshot.upstreamSha}`, }) .pipe(Effect.result); if (Result.isFailure(published)) { return { status: "failed", upstreamSha: snapshot.upstreamSha, tools: [], skipped: [], errors: [publishErrorToSyncError(published.failure)], } satisfies GitHubSyncResult; } return { status: "published", snapshotId: published.success.snapshotId, upstreamSha: snapshot.upstreamSha, tools: published.success.descriptor.tools.map((tool) => tool.name), skipped: published.success.skipped, } satisfies GitHubSyncResult; });