Showing the changes to packages/plugins/apps/src/source/github-source.ts (1 of 19 files). Show all
Unified · Split
| @@ -0,0 +1,402 @@ | |||
| 1 | import { Data, Effect, Result } from "effect"; | ||
| 2 | |||
| 3 | import { PublishError, type FileDiagnostic, type SkippedArtifact } from "../pipeline/discover"; | ||
| 4 | import { PUBLISH_LIMITS, enforcePublishLimits } from "../pipeline/publish"; | ||
| 5 | import type { AppSourceRef } from "../pipeline/descriptor"; | ||
| 6 | import type { AppsRuntime } from "../plugin/runtime"; | ||
| 7 | import type { FileSet, SnapshotId } from "../seams/artifact-store"; | ||
| 8 | |||
| 9 | export interface GitHubSourceInput { | ||
| 10 | readonly repo: string; | ||
| 11 | readonly ref?: string; | ||
| 12 | readonly token?: string | null; | ||
| 13 | readonly baseUrl?: string; | ||
| 14 | readonly fetch?: typeof globalThis.fetch; | ||
| 15 | } | ||
| 16 | |||
| 17 | export interface GitHubSourceSnapshot { | ||
| 18 | readonly files: FileSet; | ||
| 19 | readonly repo: string; | ||
| 20 | readonly ref: string; | ||
| 21 | readonly upstreamSha: string; | ||
| 22 | readonly description?: string; | ||
| 23 | } | ||
| 24 | |||
| 25 | export class GitHubSourceError extends Data.TaggedError("GitHubSourceError")<{ | ||
| 26 | readonly message: string; | ||
| 27 | readonly status?: number; | ||
| 28 | readonly path?: string; | ||
| 29 | readonly cause?: unknown; | ||
| 30 | }> {} | ||
| 31 | |||
| 32 | export interface SyncErrorData { | ||
| 33 | readonly stage: "source" | "discover" | "bundle" | "collect" | "project"; | ||
| 34 | readonly message: string; | ||
| 35 | readonly diagnostics?: readonly FileDiagnostic[]; | ||
| 36 | } | ||
| 37 | |||
| 38 | export type GitHubSyncResult = | ||
| 39 | | { | ||
| 40 | readonly status: "published"; | ||
| 41 | readonly snapshotId: SnapshotId; | ||
| 42 | readonly upstreamSha: string; | ||
| 43 | readonly tools: readonly string[]; | ||
| 44 | readonly skipped: readonly SkippedArtifact[]; | ||
| 45 | readonly errors?: undefined; | ||
| 46 | } | ||
| 47 | | { | ||
| 48 | readonly status: "up-to-date"; | ||
| 49 | readonly upstreamSha: string; | ||
| 50 | readonly tools: readonly string[]; | ||
| 51 | readonly skipped: readonly SkippedArtifact[]; | ||
| 52 | readonly errors?: undefined; | ||
| 53 | } | ||
| 54 | | { | ||
| 55 | readonly status: "failed"; | ||
| 56 | readonly upstreamSha?: string; | ||
| 57 | readonly tools: readonly string[]; | ||
| 58 | readonly skipped: readonly SkippedArtifact[]; | ||
| 59 | readonly errors: readonly SyncErrorData[]; | ||
| 60 | }; | ||
| 61 | |||
| 62 | export interface SyncGitHubSourceInput extends GitHubSourceInput { | ||
| 63 | readonly runtime: AppsRuntime; | ||
| 64 | readonly scope: string; | ||
| 65 | } | ||
| 66 | |||
| 67 | const MATERIALIZED_RE = /^(tools|workflows|ui|skills)\//; | ||
| 68 | |||
| 69 | const trimBaseUrl = (baseUrl: string): string => baseUrl.replace(/\/+$/, ""); | ||
| 70 | |||
| 71 | const repoParts = (repo: string): { owner: string; name: string } | null => { | ||
| 72 | const [owner, name, ...rest] = repo.split("/"); | ||
| 73 | if (!owner || !name || rest.length > 0) return null; | ||
| 74 | return { owner, name }; | ||
| 75 | }; | ||
| 76 | |||
| 77 | const encodedRepoPath = (repo: string): Effect.Effect<string, GitHubSourceError> => { | ||
| 78 | const parsed = repoParts(repo); | ||
| 79 | if (!parsed) { | ||
| 80 | return Effect.fail( | ||
| 81 | new GitHubSourceError({ | ||
| 82 | message: `GitHub repo must be "owner/name"; got "${repo}"`, | ||
| 83 | path: repo, | ||
| 84 | }), | ||
| 85 | ); | ||
| 86 | } | ||
| 87 | return Effect.succeed(`${encodeURIComponent(parsed.owner)}/${encodeURIComponent(parsed.name)}`); | ||
| 88 | }; | ||
| 89 | |||
| 90 | const materializes = (path: string): boolean => | ||
| 91 | path === "executor.json" || MATERIALIZED_RE.test(path); | ||
| 92 | |||
| 93 | const isRecord = (value: unknown): value is Record<string, unknown> => | ||
| 94 | value !== null && typeof value === "object" && !Array.isArray(value); | ||
| 95 | |||
| 96 | const asString = (value: unknown): string | undefined => | ||
| 97 | typeof value === "string" && value.length > 0 ? value : undefined; | ||
| 98 | |||
| 99 | const requestJson = <A>( | ||
| 100 | input: GitHubSourceInput, | ||
| 101 | path: string, | ||
| 102 | ): Effect.Effect<A, GitHubSourceError> => | ||
| 103 | Effect.tryPromise({ | ||
| 104 | try: async () => { | ||
| 105 | const fetchImpl = input.fetch ?? globalThis.fetch; | ||
| 106 | const headers: Record<string, string> = { | ||
| 107 | accept: "application/vnd.github+json", | ||
| 108 | "user-agent": "executor-apps-github-source", | ||
| 109 | }; | ||
| 110 | if (input.token) headers.authorization = `Bearer ${input.token}`; | ||
| 111 | const response = await fetchImpl( | ||
| 112 | `${trimBaseUrl(input.baseUrl ?? "https://api.github.com")}${path}`, | ||
| 113 | { | ||
| 114 | headers, | ||
| 115 | }, | ||
| 116 | ); | ||
| 117 | if (!response.ok) { | ||
| 118 | throw new GitHubSourceError({ | ||
| 119 | message: `GitHub request failed: GET ${path} -> ${response.status}`, | ||
| 120 | status: response.status, | ||
| 121 | path, | ||
| 122 | }); | ||
| 123 | } | ||
| 124 | return (await response.json()) as A; | ||
| 125 | }, | ||
| 126 | catch: (cause) => | ||
| 127 | cause instanceof GitHubSourceError | ||
| 128 | ? cause | ||
| 129 | : new GitHubSourceError({ | ||
| 130 | message: `GitHub request failed: GET ${path}`, | ||
| 131 | path, | ||
| 132 | cause, | ||
| 133 | }), | ||
| 134 | }); | ||
| 135 | |||
| 136 | interface RepoResponse { | ||
| 137 | readonly default_branch?: unknown; | ||
| 138 | } | ||
| 139 | |||
| 140 | interface CommitResponse { | ||
| 141 | readonly sha?: unknown; | ||
| 142 | readonly commit?: { | ||
| 143 | readonly tree?: { | ||
| 144 | readonly sha?: unknown; | ||
| 145 | }; | ||
| 146 | }; | ||
| 147 | } | ||
| 148 | |||
| 149 | interface GitCommitResponse { | ||
| 150 | readonly sha?: unknown; | ||
| 151 | readonly tree?: { | ||
| 152 | readonly sha?: unknown; | ||
| 153 | }; | ||
| 154 | } | ||
| 155 | |||
| 156 | interface RefResponse { | ||
| 157 | readonly object?: { | ||
| 158 | readonly sha?: unknown; | ||
| 159 | }; | ||
| 160 | } | ||
| 161 | |||
| 162 | interface TreeEntry { | ||
| 163 | readonly path?: unknown; | ||
| 164 | readonly type?: unknown; | ||
| 165 | readonly sha?: unknown; | ||
| 166 | readonly size?: unknown; | ||
| 167 | } | ||
| 168 | |||
| 169 | interface TreeResponse { | ||
| 170 | readonly tree?: readonly TreeEntry[]; | ||
| 171 | readonly truncated?: unknown; | ||
| 172 | } | ||
| 173 | |||
| 174 | interface BlobResponse { | ||
| 175 | readonly content?: unknown; | ||
| 176 | readonly encoding?: unknown; | ||
| 177 | } | ||
| 178 | |||
| 179 | const commitTreeSha = (commit: CommitResponse | GitCommitResponse): string | undefined => { | ||
| 180 | if ("commit" in commit) return asString(commit.commit?.tree?.sha); | ||
| 181 | if ("tree" in commit) return asString(commit.tree?.sha); | ||
| 182 | return undefined; | ||
| 183 | }; | ||
| 184 | |||
| 185 | const limitError = (diagnostics: readonly FileDiagnostic[]): PublishError => | ||
| 186 | new PublishError({ | ||
| 187 | message: `publish payload exceeds limits (${diagnostics.length} problem(s))`, | ||
| 188 | stage: "discover", | ||
| 189 | diagnostics, | ||
| 190 | }); | ||
| 191 | |||
| 192 | const checkTreeLimits = (entries: readonly TreeEntry[]): PublishError | null => { | ||
| 193 | const diagnostics: FileDiagnostic[] = []; | ||
| 194 | if (entries.length > PUBLISH_LIMITS.maxFiles) { | ||
| 195 | diagnostics.push({ | ||
| 196 | path: "", | ||
| 197 | message: `publish has ${entries.length} files, exceeding the limit of ${PUBLISH_LIMITS.maxFiles}`, | ||
| 198 | }); | ||
| 199 | } | ||
| 200 | let total = 0; | ||
| 201 | for (const entry of entries) { | ||
| 202 | const path = String(entry.path ?? ""); | ||
| 203 | const size = typeof entry.size === "number" ? entry.size : 0; | ||
| 204 | total += size; | ||
| 205 | if (size > PUBLISH_LIMITS.maxFileBytes) { | ||
| 206 | diagnostics.push({ | ||
| 207 | path, | ||
| 208 | message: `file is ${size} bytes, exceeding the per-file limit of ${PUBLISH_LIMITS.maxFileBytes} bytes`, | ||
| 209 | }); | ||
| 210 | } | ||
| 211 | } | ||
| 212 | if (total > PUBLISH_LIMITS.maxTotalBytes) { | ||
| 213 | diagnostics.push({ | ||
| 214 | path: "", | ||
| 215 | message: `publish total is ${total} bytes, exceeding the total limit of ${PUBLISH_LIMITS.maxTotalBytes} bytes`, | ||
| 216 | }); | ||
| 217 | } | ||
| 218 | return diagnostics.length === 0 ? null : limitError(diagnostics); | ||
| 219 | }; | ||
| 220 | |||
| 221 | const decodeBlob = (path: string, blob: BlobResponse): Effect.Effect<string, GitHubSourceError> => | ||
| 222 | Effect.try({ | ||
| 223 | try: () => { | ||
| 224 | const encoding = asString(blob.encoding); | ||
| 225 | const content = asString(blob.content); | ||
| 226 | if (!content || encoding !== "base64") { | ||
| 227 | throw new GitHubSourceError({ | ||
| 228 | message: `GitHub blob ${path} did not return base64 content`, | ||
| 229 | path, | ||
| 230 | }); | ||
| 231 | } | ||
| 232 | return Buffer.from(content.replace(/\s/g, ""), "base64").toString("utf8"); | ||
| 233 | }, | ||
| 234 | catch: (cause) => | ||
| 235 | cause instanceof GitHubSourceError | ||
| 236 | ? cause | ||
| 237 | : new GitHubSourceError({ message: `Failed to decode GitHub blob ${path}`, path, cause }), | ||
| 238 | }); | ||
| 239 | |||
| 240 | const executorDescription = ( | ||
| 241 | files: FileSet, | ||
| 242 | ): Effect.Effect<string | undefined, GitHubSourceError> => | ||
| 243 | Effect.try({ | ||
| 244 | try: () => { | ||
| 245 | const raw = files.get("executor.json"); | ||
| 246 | if (!raw) return undefined; | ||
| 247 | const parsed = JSON.parse(raw) as unknown; | ||
| 248 | if (!isRecord(parsed)) return undefined; | ||
| 249 | const description = parsed.description; | ||
| 250 | return typeof description === "string" ? description : undefined; | ||
| 251 | }, | ||
| 252 | catch: (cause) => | ||
| 253 | new GitHubSourceError({ | ||
| 254 | message: "executor.json is not valid JSON", | ||
| 255 | path: "executor.json", | ||
| 256 | cause, | ||
| 257 | }), | ||
| 258 | }); | ||
| 259 | |||
| 260 | export const fetchGitHubSource = ( | ||
| 261 | input: GitHubSourceInput, | ||
| 262 | ): Effect.Effect<GitHubSourceSnapshot, GitHubSourceError | PublishError> => | ||
| 263 | Effect.gen(function* () { | ||
| 264 | const repoPath = yield* encodedRepoPath(input.repo); | ||
| 265 | const repo = yield* requestJson<RepoResponse>(input, `/repos/${repoPath}`); | ||
| 266 | const ref = input.ref ?? asString(repo.default_branch) ?? "main"; | ||
| 267 | const branchRef = yield* requestJson<RefResponse>( | ||
| 268 | input, | ||
| 269 | `/repos/${repoPath}/git/ref/${encodeURIComponent(`heads/${ref}`)}`, | ||
| 270 | ).pipe(Effect.result); | ||
| 271 | const commitPath = | ||
| 272 | Result.isSuccess(branchRef) && asString(branchRef.success.object?.sha) | ||
| 273 | ? `/repos/${repoPath}/git/commits/${encodeURIComponent(String(branchRef.success.object?.sha))}` | ||
| 274 | : `/repos/${repoPath}/commits/${encodeURIComponent(ref)}`; | ||
| 275 | const commit = yield* requestJson<CommitResponse | GitCommitResponse>(input, commitPath); | ||
| 276 | const upstreamSha = asString(commit.sha); | ||
| 277 | const treeSha = commitTreeSha(commit); | ||
| 278 | if (!upstreamSha || !treeSha) { | ||
| 279 | return yield* new GitHubSourceError({ | ||
| 280 | message: `GitHub commit ${ref} did not include a commit SHA and tree SHA`, | ||
| 281 | path: commitPath, | ||
| 282 | }); | ||
| 283 | } | ||
| 284 | |||
| 285 | const tree = yield* requestJson<TreeResponse>( | ||
| 286 | input, | ||
| 287 | `/repos/${repoPath}/git/trees/${encodeURIComponent(treeSha)}?recursive=1`, | ||
| 288 | ); | ||
| 289 | if (tree.truncated === true) { | ||
| 290 | return yield* new GitHubSourceError({ | ||
| 291 | message: "GitHub returned a truncated repository tree; custom tools source is too large", | ||
| 292 | path: `/repos/${repoPath}/git/trees/${treeSha}`, | ||
| 293 | }); | ||
| 294 | } | ||
| 295 | const entries = (tree.tree ?? []).filter( | ||
| 296 | (entry) => | ||
| 297 | entry.type === "blob" && | ||
| 298 | typeof entry.path === "string" && | ||
| 299 | typeof entry.sha === "string" && | ||
| 300 | materializes(entry.path), | ||
| 301 | ); | ||
| 302 | const treeLimitError = checkTreeLimits(entries); | ||
| 303 | if (treeLimitError) return yield* Effect.fail(treeLimitError); | ||
| 304 | |||
| 305 | const files = new Map<string, string>(); | ||
| 306 | for (const entry of entries) { | ||
| 307 | const path = String(entry.path); | ||
| 308 | const blob = yield* requestJson<BlobResponse>( | ||
| 309 | input, | ||
| 310 | `/repos/${repoPath}/git/blobs/${encodeURIComponent(String(entry.sha))}`, | ||
| 311 | ); | ||
| 312 | files.set(path, yield* decodeBlob(path, blob)); | ||
| 313 | } | ||
| 314 | const payloadLimitError = enforcePublishLimits(files); | ||
| 315 | if (payloadLimitError) return yield* Effect.fail(payloadLimitError); | ||
| 316 | |||
| 317 | return { | ||
| 318 | files, | ||
| 319 | repo: input.repo, | ||
| 320 | ref, | ||
| 321 | upstreamSha, | ||
| 322 | description: yield* executorDescription(files), | ||
| 323 | }; | ||
| 324 | }); | ||
| 325 | |||
| 326 | const publishErrorToSyncError = (error: PublishError): SyncErrorData => ({ | ||
| 327 | stage: error.stage, | ||
| 328 | message: error.message, | ||
| 329 | diagnostics: error.diagnostics, | ||
| 330 | }); | ||
| 331 | |||
| 332 | const sourceErrorToSyncError = (error: GitHubSourceError): SyncErrorData => ({ | ||
| 333 | stage: "source", | ||
| 334 | message: error.message, | ||
| 335 | diagnostics: error.path ? [{ path: error.path, message: error.message }] : [], | ||
| 336 | }); | ||
| 337 | |||
| 338 | const sourceRef = (snapshot: GitHubSourceSnapshot): AppSourceRef => ({ | ||
| 339 | kind: "github", | ||
| 340 | repo: snapshot.repo, | ||
| 341 | ref: snapshot.ref, | ||
| 342 | upstreamSha: snapshot.upstreamSha, | ||
| 343 | }); | ||
| 344 | |||
| 345 | export const syncGitHubSource = (input: SyncGitHubSourceInput): Effect.Effect<GitHubSyncResult> => | ||
| 346 | Effect.gen(function* () { | ||
| 347 | const fetched = yield* fetchGitHubSource(input).pipe(Effect.result); | ||
| 348 | if (Result.isFailure(fetched)) { | ||
| 349 | const error = fetched.failure; | ||
| 350 | return { | ||
| 351 | status: "failed", | ||
| 352 | tools: [], | ||
| 353 | skipped: [], | ||
| 354 | errors: [ | ||
| 355 | error instanceof PublishError | ||
| 356 | ? publishErrorToSyncError(error) | ||
| 357 | : sourceErrorToSyncError(error), | ||
| 358 | ], | ||
| 359 | } satisfies GitHubSyncResult; | ||
| 360 | } | ||
| 361 | const snapshot = fetched.success; | ||
| 362 | const current = yield* input.runtime.getDescriptor(input.scope); | ||
| 363 | if ( | ||
| 364 | current?.source?.kind === "github" && | ||
| 365 | current.source.repo === snapshot.repo && | ||
| 366 | current.source.upstreamSha === snapshot.upstreamSha | ||
| 367 | ) { | ||
| 368 | return { | ||
| 369 | status: "up-to-date", | ||
| 370 | upstreamSha: snapshot.upstreamSha, | ||
| 371 | tools: current.tools.map((tool) => tool.name), | ||
| 372 | skipped: current.skipped ?? [], | ||
| 373 | } satisfies GitHubSyncResult; | ||
| 374 | } | ||
| 375 | |||
| 376 | const published = yield* input.runtime | ||
| 377 | .publish({ | ||
| 378 | scope: input.scope, | ||
| 379 | files: snapshot.files, | ||
| 380 | description: snapshot.description, | ||
| 381 | source: sourceRef(snapshot), | ||
| 382 | message: `sync ${snapshot.repo}@${snapshot.upstreamSha}`, | ||
| 383 | }) | ||
| 384 | .pipe(Effect.result); | ||
| 385 | if (Result.isFailure(published)) { | ||
| 386 | return { | ||
| 387 | status: "failed", | ||
| 388 | upstreamSha: snapshot.upstreamSha, | ||
| 389 | tools: [], | ||
| 390 | skipped: [], | ||
| 391 | errors: [publishErrorToSyncError(published.failure)], | ||
| 392 | } satisfies GitHubSyncResult; | ||
| 393 | } | ||
| 394 | |||
| 395 | return { | ||
| 396 | status: "published", | ||
| 397 | snapshotId: published.success.snapshotId, | ||
| 398 | upstreamSha: snapshot.upstreamSha, | ||
| 399 | tools: published.success.descriptor.tools.map((tool) => tool.name), | ||
| 400 | skipped: published.success.skipped, | ||
| 401 | } satisfies GitHubSyncResult; | ||
| 402 | }); | ||