Showing the changes to packages/plugins/apps/src/source/github-source.ts (1 of 19 files). Show all
Unified · Split
| @@ -0,0 +1,402 @@ | ||
| 1 | +import { Data, Effect, Result } from "effect"; | |
| 2 | + | |
| 3 | +import { PublishError, type FileDiagnostic, type SkippedArtifact } from "../pipeline/discover"; | |
| 4 | +import { PUBLISH_LIMITS, enforcePublishLimits } from "../pipeline/publish"; | |
| 5 | +import type { AppSourceRef } from "../pipeline/descriptor"; | |
| 6 | +import type { AppsRuntime } from "../plugin/runtime"; | |
| 7 | +import type { FileSet, SnapshotId } from "../seams/artifact-store"; | |
| 8 | + | |
| 9 | +export interface GitHubSourceInput { | |
| 10 | + readonly repo: string; | |
| 11 | + readonly ref?: string; | |
| 12 | + readonly token?: string | null; | |
| 13 | + readonly baseUrl?: string; | |
| 14 | + readonly fetch?: typeof globalThis.fetch; | |
| 15 | +} | |
| 16 | + | |
| 17 | +export interface GitHubSourceSnapshot { | |
| 18 | + readonly files: FileSet; | |
| 19 | + readonly repo: string; | |
| 20 | + readonly ref: string; | |
| 21 | + readonly upstreamSha: string; | |
| 22 | + readonly description?: string; | |
| 23 | +} | |
| 24 | + | |
| 25 | +export class GitHubSourceError extends Data.TaggedError("GitHubSourceError")<{ | |
| 26 | + readonly message: string; | |
| 27 | + readonly status?: number; | |
| 28 | + readonly path?: string; | |
| 29 | + readonly cause?: unknown; | |
| 30 | +}> {} | |
| 31 | + | |
| 32 | +export interface SyncErrorData { | |
| 33 | + readonly stage: "source" | "discover" | "bundle" | "collect" | "project"; | |
| 34 | + readonly message: string; | |
| 35 | + readonly diagnostics?: readonly FileDiagnostic[]; | |
| 36 | +} | |
| 37 | + | |
| 38 | +export type GitHubSyncResult = | |
| 39 | + | { | |
| 40 | + readonly status: "published"; | |
| 41 | + readonly snapshotId: SnapshotId; | |
| 42 | + readonly upstreamSha: string; | |
| 43 | + readonly tools: readonly string[]; | |
| 44 | + readonly skipped: readonly SkippedArtifact[]; | |
| 45 | + readonly errors?: undefined; | |
| 46 | + } | |
| 47 | + | { | |
| 48 | + readonly status: "up-to-date"; | |
| 49 | + readonly upstreamSha: string; | |
| 50 | + readonly tools: readonly string[]; | |
| 51 | + readonly skipped: readonly SkippedArtifact[]; | |
| 52 | + readonly errors?: undefined; | |
| 53 | + } | |
| 54 | + | { | |
| 55 | + readonly status: "failed"; | |
| 56 | + readonly upstreamSha?: string; | |
| 57 | + readonly tools: readonly string[]; | |
| 58 | + readonly skipped: readonly SkippedArtifact[]; | |
| 59 | + readonly errors: readonly SyncErrorData[]; | |
| 60 | + }; | |
| 61 | + | |
| 62 | +export interface SyncGitHubSourceInput extends GitHubSourceInput { | |
| 63 | + readonly runtime: AppsRuntime; | |
| 64 | + readonly scope: string; | |
| 65 | +} | |
| 66 | + | |
| 67 | +const MATERIALIZED_RE = /^(tools|workflows|ui|skills)\//; | |
| 68 | + | |
| 69 | +const trimBaseUrl = (baseUrl: string): string => baseUrl.replace(/\/+$/, ""); | |
| 70 | + | |
| 71 | +const repoParts = (repo: string): { owner: string; name: string } | null => { | |
| 72 | + const [owner, name, ...rest] = repo.split("/"); | |
| 73 | + if (!owner || !name || rest.length > 0) return null; | |
| 74 | + return { owner, name }; | |
| 75 | +}; | |
| 76 | + | |
| 77 | +const encodedRepoPath = (repo: string): Effect.Effect<string, GitHubSourceError> => { | |
| 78 | + const parsed = repoParts(repo); | |
| 79 | + if (!parsed) { | |
| 80 | + return Effect.fail( | |
| 81 | + new GitHubSourceError({ | |
| 82 | + message: `GitHub repo must be "owner/name"; got "${repo}"`, | |
| 83 | + path: repo, | |
| 84 | + }), | |
| 85 | + ); | |
| 86 | + } | |
| 87 | + return Effect.succeed(`${encodeURIComponent(parsed.owner)}/${encodeURIComponent(parsed.name)}`); | |
| 88 | +}; | |
| 89 | + | |
| 90 | +const materializes = (path: string): boolean => | |
| 91 | + path === "executor.json" || MATERIALIZED_RE.test(path); | |
| 92 | + | |
| 93 | +const isRecord = (value: unknown): value is Record<string, unknown> => | |
| 94 | + value !== null && typeof value === "object" && !Array.isArray(value); | |
| 95 | + | |
| 96 | +const asString = (value: unknown): string | undefined => | |
| 97 | + typeof value === "string" && value.length > 0 ? value : undefined; | |
| 98 | + | |
| 99 | +const requestJson = <A>( | |
| 100 | + input: GitHubSourceInput, | |
| 101 | + path: string, | |
| 102 | +): Effect.Effect<A, GitHubSourceError> => | |
| 103 | + Effect.tryPromise({ | |
| 104 | + try: async () => { | |
| 105 | + const fetchImpl = input.fetch ?? globalThis.fetch; | |
| 106 | + const headers: Record<string, string> = { | |
| 107 | + accept: "application/vnd.github+json", | |
| 108 | + "user-agent": "executor-apps-github-source", | |
| 109 | + }; | |
| 110 | + if (input.token) headers.authorization = `Bearer ${input.token}`; | |
| 111 | + const response = await fetchImpl( | |
| 112 | + `${trimBaseUrl(input.baseUrl ?? "https://api.github.com")}${path}`, | |
| 113 | + { | |
| 114 | + headers, | |
| 115 | + }, | |
| 116 | + ); | |
| 117 | + if (!response.ok) { | |
| 118 | + throw new GitHubSourceError({ | |
| 119 | + message: `GitHub request failed: GET ${path} -> ${response.status}`, | |
| 120 | + status: response.status, | |
| 121 | + path, | |
| 122 | + }); | |
| 123 | + } | |
| 124 | + return (await response.json()) as A; | |
| 125 | + }, | |
| 126 | + catch: (cause) => | |
| 127 | + cause instanceof GitHubSourceError | |
| 128 | + ? cause | |
| 129 | + : new GitHubSourceError({ | |
| 130 | + message: `GitHub request failed: GET ${path}`, | |
| 131 | + path, | |
| 132 | + cause, | |
| 133 | + }), | |
| 134 | + }); | |
| 135 | + | |
| 136 | +interface RepoResponse { | |
| 137 | + readonly default_branch?: unknown; | |
| 138 | +} | |
| 139 | + | |
| 140 | +interface CommitResponse { | |
| 141 | + readonly sha?: unknown; | |
| 142 | + readonly commit?: { | |
| 143 | + readonly tree?: { | |
| 144 | + readonly sha?: unknown; | |
| 145 | + }; | |
| 146 | + }; | |
| 147 | +} | |
| 148 | + | |
| 149 | +interface GitCommitResponse { | |
| 150 | + readonly sha?: unknown; | |
| 151 | + readonly tree?: { | |
| 152 | + readonly sha?: unknown; | |
| 153 | + }; | |
| 154 | +} | |
| 155 | + | |
| 156 | +interface RefResponse { | |
| 157 | + readonly object?: { | |
| 158 | + readonly sha?: unknown; | |
| 159 | + }; | |
| 160 | +} | |
| 161 | + | |
| 162 | +interface TreeEntry { | |
| 163 | + readonly path?: unknown; | |
| 164 | + readonly type?: unknown; | |
| 165 | + readonly sha?: unknown; | |
| 166 | + readonly size?: unknown; | |
| 167 | +} | |
| 168 | + | |
| 169 | +interface TreeResponse { | |
| 170 | + readonly tree?: readonly TreeEntry[]; | |
| 171 | + readonly truncated?: unknown; | |
| 172 | +} | |
| 173 | + | |
| 174 | +interface BlobResponse { | |
| 175 | + readonly content?: unknown; | |
| 176 | + readonly encoding?: unknown; | |
| 177 | +} | |
| 178 | + | |
| 179 | +const commitTreeSha = (commit: CommitResponse | GitCommitResponse): string | undefined => { | |
| 180 | + if ("commit" in commit) return asString(commit.commit?.tree?.sha); | |
| 181 | + if ("tree" in commit) return asString(commit.tree?.sha); | |
| 182 | + return undefined; | |
| 183 | +}; | |
| 184 | + | |
| 185 | +const limitError = (diagnostics: readonly FileDiagnostic[]): PublishError => | |
| 186 | + new PublishError({ | |
| 187 | + message: `publish payload exceeds limits (${diagnostics.length} problem(s))`, | |
| 188 | + stage: "discover", | |
| 189 | + diagnostics, | |
| 190 | + }); | |
| 191 | + | |
| 192 | +const checkTreeLimits = (entries: readonly TreeEntry[]): PublishError | null => { | |
| 193 | + const diagnostics: FileDiagnostic[] = []; | |
| 194 | + if (entries.length > PUBLISH_LIMITS.maxFiles) { | |
| 195 | + diagnostics.push({ | |
| 196 | + path: "", | |
| 197 | + message: `publish has ${entries.length} files, exceeding the limit of ${PUBLISH_LIMITS.maxFiles}`, | |
| 198 | + }); | |
| 199 | + } | |
| 200 | + let total = 0; | |
| 201 | + for (const entry of entries) { | |
| 202 | + const path = String(entry.path ?? ""); | |
| 203 | + const size = typeof entry.size === "number" ? entry.size : 0; | |
| 204 | + total += size; | |
| 205 | + if (size > PUBLISH_LIMITS.maxFileBytes) { | |
| 206 | + diagnostics.push({ | |
| 207 | + path, | |
| 208 | + message: `file is ${size} bytes, exceeding the per-file limit of ${PUBLISH_LIMITS.maxFileBytes} bytes`, | |
| 209 | + }); | |
| 210 | + } | |
| 211 | + } | |
| 212 | + if (total > PUBLISH_LIMITS.maxTotalBytes) { | |
| 213 | + diagnostics.push({ | |
| 214 | + path: "", | |
| 215 | + message: `publish total is ${total} bytes, exceeding the total limit of ${PUBLISH_LIMITS.maxTotalBytes} bytes`, | |
| 216 | + }); | |
| 217 | + } | |
| 218 | + return diagnostics.length === 0 ? null : limitError(diagnostics); | |
| 219 | +}; | |
| 220 | + | |
| 221 | +const decodeBlob = (path: string, blob: BlobResponse): Effect.Effect<string, GitHubSourceError> => | |
| 222 | + Effect.try({ | |
| 223 | + try: () => { | |
| 224 | + const encoding = asString(blob.encoding); | |
| 225 | + const content = asString(blob.content); | |
| 226 | + if (!content || encoding !== "base64") { | |
| 227 | + throw new GitHubSourceError({ | |
| 228 | + message: `GitHub blob ${path} did not return base64 content`, | |
| 229 | + path, | |
| 230 | + }); | |
| 231 | + } | |
| 232 | + return Buffer.from(content.replace(/\s/g, ""), "base64").toString("utf8"); | |
| 233 | + }, | |
| 234 | + catch: (cause) => | |
| 235 | + cause instanceof GitHubSourceError | |
| 236 | + ? cause | |
| 237 | + : new GitHubSourceError({ message: `Failed to decode GitHub blob ${path}`, path, cause }), | |
| 238 | + }); | |
| 239 | + | |
| 240 | +const executorDescription = ( | |
| 241 | + files: FileSet, | |
| 242 | +): Effect.Effect<string | undefined, GitHubSourceError> => | |
| 243 | + Effect.try({ | |
| 244 | + try: () => { | |
| 245 | + const raw = files.get("executor.json"); | |
| 246 | + if (!raw) return undefined; | |
| 247 | + const parsed = JSON.parse(raw) as unknown; | |
| 248 | + if (!isRecord(parsed)) return undefined; | |
| 249 | + const description = parsed.description; | |
| 250 | + return typeof description === "string" ? description : undefined; | |
| 251 | + }, | |
| 252 | + catch: (cause) => | |
| 253 | + new GitHubSourceError({ | |
| 254 | + message: "executor.json is not valid JSON", | |
| 255 | + path: "executor.json", | |
| 256 | + cause, | |
| 257 | + }), | |
| 258 | + }); | |
| 259 | + | |
| 260 | +export const fetchGitHubSource = ( | |
| 261 | + input: GitHubSourceInput, | |
| 262 | +): Effect.Effect<GitHubSourceSnapshot, GitHubSourceError | PublishError> => | |
| 263 | + Effect.gen(function* () { | |
| 264 | + const repoPath = yield* encodedRepoPath(input.repo); | |
| 265 | + const repo = yield* requestJson<RepoResponse>(input, `/repos/${repoPath}`); | |
| 266 | + const ref = input.ref ?? asString(repo.default_branch) ?? "main"; | |
| 267 | + const branchRef = yield* requestJson<RefResponse>( | |
| 268 | + input, | |
| 269 | + `/repos/${repoPath}/git/ref/${encodeURIComponent(`heads/${ref}`)}`, | |
| 270 | + ).pipe(Effect.result); | |
| 271 | + const commitPath = | |
| 272 | + Result.isSuccess(branchRef) && asString(branchRef.success.object?.sha) | |
| 273 | + ? `/repos/${repoPath}/git/commits/${encodeURIComponent(String(branchRef.success.object?.sha))}` | |
| 274 | + : `/repos/${repoPath}/commits/${encodeURIComponent(ref)}`; | |
| 275 | + const commit = yield* requestJson<CommitResponse | GitCommitResponse>(input, commitPath); | |
| 276 | + const upstreamSha = asString(commit.sha); | |
| 277 | + const treeSha = commitTreeSha(commit); | |
| 278 | + if (!upstreamSha || !treeSha) { | |
| 279 | + return yield* new GitHubSourceError({ | |
| 280 | + message: `GitHub commit ${ref} did not include a commit SHA and tree SHA`, | |
| 281 | + path: commitPath, | |
| 282 | + }); | |
| 283 | + } | |
| 284 | + | |
| 285 | + const tree = yield* requestJson<TreeResponse>( | |
| 286 | + input, | |
| 287 | + `/repos/${repoPath}/git/trees/${encodeURIComponent(treeSha)}?recursive=1`, | |
| 288 | + ); | |
| 289 | + if (tree.truncated === true) { | |
| 290 | + return yield* new GitHubSourceError({ | |
| 291 | + message: "GitHub returned a truncated repository tree; custom tools source is too large", | |
| 292 | + path: `/repos/${repoPath}/git/trees/${treeSha}`, | |
| 293 | + }); | |
| 294 | + } | |
| 295 | + const entries = (tree.tree ?? []).filter( | |
| 296 | + (entry) => | |
| 297 | + entry.type === "blob" && | |
| 298 | + typeof entry.path === "string" && | |
| 299 | + typeof entry.sha === "string" && | |
| 300 | + materializes(entry.path), | |
| 301 | + ); | |
| 302 | + const treeLimitError = checkTreeLimits(entries); | |
| 303 | + if (treeLimitError) return yield* Effect.fail(treeLimitError); | |
| 304 | + | |
| 305 | + const files = new Map<string, string>(); | |
| 306 | + for (const entry of entries) { | |
| 307 | + const path = String(entry.path); | |
| 308 | + const blob = yield* requestJson<BlobResponse>( | |
| 309 | + input, | |
| 310 | + `/repos/${repoPath}/git/blobs/${encodeURIComponent(String(entry.sha))}`, | |
| 311 | + ); | |
| 312 | + files.set(path, yield* decodeBlob(path, blob)); | |
| 313 | + } | |
| 314 | + const payloadLimitError = enforcePublishLimits(files); | |
| 315 | + if (payloadLimitError) return yield* Effect.fail(payloadLimitError); | |
| 316 | + | |
| 317 | + return { | |
| 318 | + files, | |
| 319 | + repo: input.repo, | |
| 320 | + ref, | |
| 321 | + upstreamSha, | |
| 322 | + description: yield* executorDescription(files), | |
| 323 | + }; | |
| 324 | + }); | |
| 325 | + | |
| 326 | +const publishErrorToSyncError = (error: PublishError): SyncErrorData => ({ | |
| 327 | + stage: error.stage, | |
| 328 | + message: error.message, | |
| 329 | + diagnostics: error.diagnostics, | |
| 330 | +}); | |
| 331 | + | |
| 332 | +const sourceErrorToSyncError = (error: GitHubSourceError): SyncErrorData => ({ | |
| 333 | + stage: "source", | |
| 334 | + message: error.message, | |
| 335 | + diagnostics: error.path ? [{ path: error.path, message: error.message }] : [], | |
| 336 | +}); | |
| 337 | + | |
| 338 | +const sourceRef = (snapshot: GitHubSourceSnapshot): AppSourceRef => ({ | |
| 339 | + kind: "github", | |
| 340 | + repo: snapshot.repo, | |
| 341 | + ref: snapshot.ref, | |
| 342 | + upstreamSha: snapshot.upstreamSha, | |
| 343 | +}); | |
| 344 | + | |
| 345 | +export const syncGitHubSource = (input: SyncGitHubSourceInput): Effect.Effect<GitHubSyncResult> => | |
| 346 | + Effect.gen(function* () { | |
| 347 | + const fetched = yield* fetchGitHubSource(input).pipe(Effect.result); | |
| 348 | + if (Result.isFailure(fetched)) { | |
| 349 | + const error = fetched.failure; | |
| 350 | + return { | |
| 351 | + status: "failed", | |
| 352 | + tools: [], | |
| 353 | + skipped: [], | |
| 354 | + errors: [ | |
| 355 | + error instanceof PublishError | |
| 356 | + ? publishErrorToSyncError(error) | |
| 357 | + : sourceErrorToSyncError(error), | |
| 358 | + ], | |
| 359 | + } satisfies GitHubSyncResult; | |
| 360 | + } | |
| 361 | + const snapshot = fetched.success; | |
| 362 | + const current = yield* input.runtime.getDescriptor(input.scope); | |
| 363 | + if ( | |
| 364 | + current?.source?.kind === "github" && | |
| 365 | + current.source.repo === snapshot.repo && | |
| 366 | + current.source.upstreamSha === snapshot.upstreamSha | |
| 367 | + ) { | |
| 368 | + return { | |
| 369 | + status: "up-to-date", | |
| 370 | + upstreamSha: snapshot.upstreamSha, | |
| 371 | + tools: current.tools.map((tool) => tool.name), | |
| 372 | + skipped: current.skipped ?? [], | |
| 373 | + } satisfies GitHubSyncResult; | |
| 374 | + } | |
| 375 | + | |
| 376 | + const published = yield* input.runtime | |
| 377 | + .publish({ | |
| 378 | + scope: input.scope, | |
| 379 | + files: snapshot.files, | |
| 380 | + description: snapshot.description, | |
| 381 | + source: sourceRef(snapshot), | |
| 382 | + message: `sync ${snapshot.repo}@${snapshot.upstreamSha}`, | |
| 383 | + }) | |
| 384 | + .pipe(Effect.result); | |
| 385 | + if (Result.isFailure(published)) { | |
| 386 | + return { | |
| 387 | + status: "failed", | |
| 388 | + upstreamSha: snapshot.upstreamSha, | |
| 389 | + tools: [], | |
| 390 | + skipped: [], | |
| 391 | + errors: [publishErrorToSyncError(published.failure)], | |
| 392 | + } satisfies GitHubSyncResult; | |
| 393 | + } | |
| 394 | + | |
| 395 | + return { | |
| 396 | + status: "published", | |
| 397 | + snapshotId: published.success.snapshotId, | |
| 398 | + upstreamSha: snapshot.upstreamSha, | |
| 399 | + tools: published.success.descriptor.tools.map((tool) => tool.name), | |
| 400 | + skipped: published.success.skipped, | |
| 401 | + } satisfies GitHubSyncResult; | |
| 402 | + }); | |