import { Data, Effect, Predicate, Result, Schema } from "effect"; import { PublishError, type FileDiagnostic } from "../pipeline/discover"; import { PUBLISH_LIMITS, enforcePublishLimits } from "../pipeline/publish"; import type { AppSourceRef, SourceSkippedArtifact } from "../pipeline/descriptor"; import type { AppsRuntime } from "../plugin/runtime"; import type { FileSet, SnapshotId } from "../seams/artifact-store"; import { parseGitHubSourceUrl, type ParsedGitHubSourceUrl } from "./github-url"; export interface GitHubSourceInput { readonly url: string; readonly ref?: string; readonly token?: string | null; readonly baseUrl?: string; readonly fetch?: typeof globalThis.fetch; } export interface GitHubSourceSnapshot { readonly files: FileSet; readonly url: string; readonly repo: string; readonly ref: string; readonly upstreamSha: string; readonly description?: string; readonly skipped: readonly GitHubSkippedArtifact[]; } export class GitHubSourceError extends Data.TaggedError("GitHubSourceError")<{ readonly message: string; readonly status?: number; readonly path?: string; readonly cause?: unknown; }> {} export interface SyncErrorData { readonly stage: "source" | "discover" | "bundle" | "collect" | "project"; readonly message: string; readonly diagnostics?: readonly FileDiagnostic[]; } export type GitHubSkippedArtifact = SourceSkippedArtifact; export type GitHubSyncResult = | { readonly status: "published"; readonly snapshotId: SnapshotId; readonly upstreamSha: string; readonly tools: readonly string[]; readonly skipped: readonly GitHubSkippedArtifact[]; readonly errors?: undefined; } | { readonly status: "up-to-date"; readonly upstreamSha: string; readonly tools: readonly string[]; readonly skipped: readonly GitHubSkippedArtifact[]; readonly errors?: undefined; } | { readonly status: "failed"; readonly upstreamSha?: string; readonly tools: readonly string[]; readonly skipped: readonly GitHubSkippedArtifact[]; readonly errors: readonly SyncErrorData[]; }; export interface SyncGitHubSourceInput extends GitHubSourceInput { readonly runtime: AppsRuntime; readonly tenant?: string; readonly scope: string; } const TOOL_RE = /^tools\/([a-z0-9][a-z0-9-]*)\.(ts|tsx|js|jsx)$/; const DEFERRED_RE = /^(workflows|ui|skills)\//; const REGULAR_FILE_MODES = new Set(["100644", "100755"]); const trimBaseUrl = (baseUrl: string): string => baseUrl.replace(/\/+$/, ""); const encodedRepoPath = (source: ParsedGitHubSourceUrl): string => `${encodeURIComponent(source.owner)}/${encodeURIComponent(source.name)}`; const parseSourceInput = ( input: GitHubSourceInput, ): Effect.Effect => { const parsed = parseGitHubSourceUrl(input.url, { ref: input.ref }); return parsed.ok ? Effect.succeed(parsed.value) : Effect.fail( new GitHubSourceError({ message: parsed.message, path: input.url, }), ); }; const acceptedPath = (path: string): boolean => path === "executor.json" || TOOL_RE.test(path); const isRecord = (value: unknown): value is Record => value !== null && typeof value === "object" && !Array.isArray(value); const asString = (value: unknown): string | undefined => typeof value === "string" && value.length > 0 ? value : undefined; const requestJson = ( input: GitHubSourceInput, path: string, ): Effect.Effect => Effect.gen(function* () { const response = yield* Effect.tryPromise({ try: async () => { const fetchImpl = input.fetch ?? globalThis.fetch; const headers: Record = { accept: "application/vnd.github+json", "user-agent": "executor-apps-github-source", }; if (input.token) headers.authorization = `Bearer ${input.token}`; return fetchImpl(`${trimBaseUrl(input.baseUrl ?? "https://api.github.com")}${path}`, { headers, }); }, catch: (cause) => new GitHubSourceError({ message: `GitHub request failed: GET ${path}`, path, cause, }), }); if (!response.ok) { return yield* new GitHubSourceError({ message: `GitHub request failed: GET ${path} -> ${response.status}`, status: response.status, path, }); } return yield* Effect.tryPromise({ try: () => response.json() as Promise, catch: (cause) => new GitHubSourceError({ message: `GitHub response was not valid JSON: GET ${path}`, path, cause, }), }); }); interface RepoResponse { readonly default_branch?: unknown; } interface CommitResponse { readonly sha?: unknown; readonly commit?: { readonly tree?: { readonly sha?: unknown; }; }; } interface GitCommitResponse { readonly sha?: unknown; readonly tree?: { readonly sha?: unknown; }; } interface RefResponse { readonly object?: { readonly sha?: unknown; }; } interface TreeEntry { readonly path?: unknown; readonly type?: unknown; readonly mode?: unknown; readonly sha?: unknown; readonly size?: unknown; } interface TreeResponse { readonly tree?: readonly TreeEntry[]; readonly truncated?: unknown; } interface BlobResponse { readonly content?: unknown; readonly encoding?: unknown; } const commitTreeSha = (commit: CommitResponse | GitCommitResponse): string | undefined => { if ("commit" in commit) return asString(commit.commit?.tree?.sha); if ("tree" in commit) return asString(commit.tree?.sha); return undefined; }; const limitError = (diagnostics: readonly FileDiagnostic[]): PublishError => new PublishError({ message: `publish payload exceeds limits (${diagnostics.length} problem(s))`, stage: "discover", diagnostics, }); const checkTreeLimits = (entries: readonly TreeEntry[]): PublishError | null => { const diagnostics: FileDiagnostic[] = []; if (entries.length > PUBLISH_LIMITS.maxFiles) { diagnostics.push({ path: "", message: `publish has ${entries.length} files, exceeding the limit of ${PUBLISH_LIMITS.maxFiles}`, }); } let total = 0; for (const entry of entries) { const path = String(entry.path ?? ""); const size = typeof entry.size === "number" ? entry.size : 0; total += size; if (size > PUBLISH_LIMITS.maxFileBytes) { diagnostics.push({ path, message: `file is ${size} bytes, exceeding the per-file limit of ${PUBLISH_LIMITS.maxFileBytes} bytes`, }); } } if (total > PUBLISH_LIMITS.maxTotalBytes) { diagnostics.push({ path: "", message: `publish total is ${total} bytes, exceeding the total limit of ${PUBLISH_LIMITS.maxTotalBytes} bytes`, }); } return diagnostics.length === 0 ? null : limitError(diagnostics); }; const classifyTreeEntry = ( entry: TreeEntry, ): | { readonly kind: "fetch"; readonly entry: TreeEntry & { readonly path: string; readonly sha: string }; } | { readonly kind: "skip"; readonly skipped: GitHubSkippedArtifact } | null => { const path = asString(entry.path); if (!path) return null; const type = asString(entry.type); const mode = asString(entry.mode); if (type === "tree") return null; if (type !== "blob" || !mode || !REGULAR_FILE_MODES.has(mode)) { return { kind: "skip", skipped: { path, reason: "unsupported file type" } }; } const sha = asString(entry.sha); if (!sha) return { kind: "skip", skipped: { path, reason: "unsupported file type" } }; if (acceptedPath(path)) return { kind: "fetch", entry: { ...entry, path, sha } }; if (DEFERRED_RE.test(path)) { return { kind: "skip", skipped: { path, reason: "not supported yet" } }; } return { kind: "skip", skipped: { path, reason: "ignored" } }; }; const decodeBlob = (path: string, blob: BlobResponse): Effect.Effect => Effect.gen(function* () { const encoding = asString(blob.encoding); const content = asString(blob.content); if (!content || encoding !== "base64") { return yield* new GitHubSourceError({ message: `GitHub blob ${path} did not return base64 content`, path, }); } return Buffer.from(content.replace(/\s/g, ""), "base64").toString("utf8"); }); const decodeExecutorJson = Schema.decodeUnknownEffect(Schema.UnknownFromJsonString); const executorDescription = ( files: FileSet, ): Effect.Effect => Effect.gen(function* () { const raw = files.get("executor.json"); if (!raw) return undefined; const parsed = yield* decodeExecutorJson(raw).pipe( Effect.mapError( (cause) => new GitHubSourceError({ message: "executor.json is not valid JSON", path: "executor.json", cause, }), ), ); if (!isRecord(parsed)) return undefined; const description = parsed.description; return typeof description === "string" ? description : undefined; }); export const fetchGitHubSource = ( input: GitHubSourceInput, ): Effect.Effect => Effect.gen(function* () { const source = yield* parseSourceInput(input); const repoPath = encodedRepoPath(source); const repo = yield* requestJson(input, `/repos/${repoPath}`); const ref = source.ref ?? asString(repo.default_branch) ?? "main"; const branchRef = yield* requestJson( input, `/repos/${repoPath}/git/ref/${encodeURIComponent(`heads/${ref}`)}`, ).pipe(Effect.result); const commitPath = Result.isSuccess(branchRef) && asString(branchRef.success.object?.sha) ? `/repos/${repoPath}/git/commits/${encodeURIComponent(String(branchRef.success.object?.sha))}` : `/repos/${repoPath}/commits/${encodeURIComponent(ref)}`; const commit = yield* requestJson(input, commitPath); const upstreamSha = asString(commit.sha); const treeSha = commitTreeSha(commit); if (!upstreamSha || !treeSha) { return yield* new GitHubSourceError({ message: `GitHub commit ${ref} did not include a commit SHA and tree SHA`, path: commitPath, }); } const tree = yield* requestJson( input, `/repos/${repoPath}/git/trees/${encodeURIComponent(treeSha)}?recursive=1`, ); if (tree.truncated === true) { return yield* new GitHubSourceError({ message: "GitHub returned a truncated repository tree; custom tools source is too large", path: `/repos/${repoPath}/git/trees/${treeSha}`, }); } const entries: (TreeEntry & { readonly path: string; readonly sha: string })[] = []; const skipped: GitHubSkippedArtifact[] = []; for (const entry of tree.tree ?? []) { const classified = classifyTreeEntry(entry); if (!classified) continue; if (classified.kind === "fetch") entries.push(classified.entry); else skipped.push(classified.skipped); } const treeLimitError = checkTreeLimits(entries); if (treeLimitError) return yield* treeLimitError; const files = new Map(); for (const entry of entries) { const path = String(entry.path); const blob = yield* requestJson( input, `/repos/${repoPath}/git/blobs/${encodeURIComponent(String(entry.sha))}`, ); files.set(path, yield* decodeBlob(path, blob)); } const payloadLimitError = enforcePublishLimits(files); if (payloadLimitError) return yield* payloadLimitError; return { files, url: source.ref ? source.url : `https://github.com/${source.repo}`, repo: source.repo, ref, upstreamSha, description: yield* executorDescription(files), skipped, }; }); const publishErrorToSyncError = (publishFailure: PublishError): SyncErrorData => { const stage = publishFailure.stage; const message = publishFailure.message; const diagnostics = publishFailure.diagnostics; return { stage, message, diagnostics }; }; const sourceErrorToSyncError = (sourceFailure: GitHubSourceError): SyncErrorData => { const message = sourceFailure.message; const path = sourceFailure.path; return { stage: "source", message, diagnostics: path ? [{ path, message }] : [], }; }; const sourceRef = (snapshot: GitHubSourceSnapshot): AppSourceRef => ({ kind: "github", url: snapshot.url, repo: snapshot.repo, ref: snapshot.ref, upstreamSha: snapshot.upstreamSha, skipped: snapshot.skipped, }); export const syncGitHubSource = (input: SyncGitHubSourceInput): Effect.Effect => Effect.gen(function* () { const fetched = yield* fetchGitHubSource(input).pipe(Effect.result); if (Result.isFailure(fetched)) { const error = fetched.failure; return { status: "failed", tools: [], skipped: [], errors: [ Predicate.isTagged("PublishError")(error) ? publishErrorToSyncError(error) : sourceErrorToSyncError(error), ], } satisfies GitHubSyncResult; } const snapshot = fetched.success; const current = input.tenant ? yield* input.runtime.getDescriptor(input.tenant, input.scope) : yield* input.runtime.getDescriptor(input.scope); if ( current?.source?.kind === "github" && current.source.repo === snapshot.repo && current.source.upstreamSha === snapshot.upstreamSha ) { return { status: "up-to-date", upstreamSha: snapshot.upstreamSha, tools: current.tools.map((tool) => tool.name), skipped: [...snapshot.skipped, ...(current.skipped ?? [])], } satisfies GitHubSyncResult; } const published = yield* input.runtime .publish({ tenant: input.tenant, scope: input.scope, files: snapshot.files, description: snapshot.description, source: sourceRef(snapshot), message: `sync ${snapshot.repo}@${snapshot.upstreamSha}`, }) .pipe(Effect.result); if (Result.isFailure(published)) { return { status: "failed", upstreamSha: snapshot.upstreamSha, tools: [], skipped: snapshot.skipped, errors: [publishErrorToSyncError(published.failure)], } satisfies GitHubSyncResult; } return { status: "published", snapshotId: published.success.snapshotId, upstreamSha: snapshot.upstreamSha, tools: published.success.descriptor.tools.map((tool) => tool.name), skipped: [...snapshot.skipped, ...published.success.skipped], } satisfies GitHubSyncResult; }); export { parseGitHubSourceUrl } from "./github-url"; export type { ParsedGitHubSourceUrl } from "./github-url";