History

Filter GitHub source entries safely

Rhys Sullivan committed 4 months ago

Commit 9e77e341 parent bb155804 files · +159 −29Browse files

Showing the changes to packages/plugins/apps/src/source/github-source.ts (1 of 4 files). Show all

Unified · Split

packages/plugins/apps/src/source/github-source.ts+55−17
@@ -1,6 +1,6 @@
11 import { Data, Effect, Result } from "effect";
22
3−import { PublishError, type FileDiagnostic, type SkippedArtifact } from "../pipeline/discover";
3+import { PublishError, type FileDiagnostic } from "../pipeline/discover";
44 import { PUBLISH_LIMITS, enforcePublishLimits } from "../pipeline/publish";
55 import type { AppSourceRef } from "../pipeline/descriptor";
66 import type { AppsRuntime } from "../plugin/runtime";
@@ -20,6 +20,7 @@ export interface GitHubSourceSnapshot { · 13 unmodified lines
2020 readonly ref: string;
2121 readonly upstreamSha: string;
2222 readonly description?: string;
23+ readonly skipped: readonly GitHubSkippedArtifact[];
2324 }
2425
2526 export class GitHubSourceError extends Data.TaggedError("GitHubSourceError")<{
@@ -35,27 +36,32 @@ export interface SyncErrorData { · 9 unmodified lines
3536 readonly diagnostics?: readonly FileDiagnostic[];
3637 }
3738
39+export interface GitHubSkippedArtifact {
40+ readonly path: string;
41+ readonly reason: "not supported yet" | "unsupported file type" | "ignored";
42+}
43+
3844 export type GitHubSyncResult =
3945 | {
4046 readonly status: "published";
4147 readonly snapshotId: SnapshotId;
4248 readonly upstreamSha: string;
4349 readonly tools: readonly string[];
44− readonly skipped: readonly SkippedArtifact[];
50+ readonly skipped: readonly GitHubSkippedArtifact[];
4551 readonly errors?: undefined;
4652 }
4753 | {
4854 readonly status: "up-to-date";
4955 readonly upstreamSha: string;
5056 readonly tools: readonly string[];
51− readonly skipped: readonly SkippedArtifact[];
57+ readonly skipped: readonly GitHubSkippedArtifact[];
5258 readonly errors?: undefined;
5359 }
5460 | {
5561 readonly status: "failed";
5662 readonly upstreamSha?: string;
5763 readonly tools: readonly string[];
58− readonly skipped: readonly SkippedArtifact[];
64+ readonly skipped: readonly GitHubSkippedArtifact[];
5965 readonly errors: readonly SyncErrorData[];
6066 };
6167
@@ -65,7 +71,9 @@ export interface SyncGitHubSourceInput extends GitHubSourceInput { · 3 unmodified lines
6571 readonly scope: string;
6672 }
6773
68−const MATERIALIZED_RE = /^(tools|workflows|ui|skills)\//;
74+const TOOL_RE = /^tools\/([a-z0-9][a-z0-9-]*)\.(ts|tsx|js|jsx)$/;
75+const DEFERRED_RE = /^(workflows|ui|skills)\//;
76+const REGULAR_FILE_MODES = new Set(["100644", "100755"]);
6977
7078 const trimBaseUrl = (baseUrl: string): string => baseUrl.replace(/\/+$/, "");
7179
@@ -88,8 +96,7 @@ const encodedRepoPath = (repo: string): Effect.Effect<string, GitHubSourceError> · 16 unmodified lines
8896 return Effect.succeed(`${encodeURIComponent(parsed.owner)}/${encodeURIComponent(parsed.name)}`);
8997 };
9098
91−const materializes = (path: string): boolean =>
92− path === "executor.json" || MATERIALIZED_RE.test(path);
99+const acceptedPath = (path: string): boolean => path === "executor.json" || TOOL_RE.test(path);
93100
94101 const isRecord = (value: unknown): value is Record<string, unknown> =>
95102 value !== null && typeof value === "object" && !Array.isArray(value);
@@ -163,6 +170,7 @@ interface RefResponse { · 67 unmodified lines
163170 interface TreeEntry {
164171 readonly path?: unknown;
165172 readonly type?: unknown;
173+ readonly mode?: unknown;
166174 readonly sha?: unknown;
167175 readonly size?: unknown;
168176 }
@@ -219,6 +227,34 @@ const checkTreeLimits = (entries: readonly TreeEntry[]): PublishError | null => · 50 unmodified lines
219227 return diagnostics.length === 0 ? null : limitError(diagnostics);
220228 };
221229
230+const classifyTreeEntry = (
231+ entry: TreeEntry,
232+):
233+ | {
234+ readonly kind: "fetch";
235+ readonly entry: TreeEntry & { readonly path: string; readonly sha: string };
236+ }
237+ | { readonly kind: "skip"; readonly skipped: GitHubSkippedArtifact }
238+ | null => {
239+ const path = asString(entry.path);
240+ if (!path) return null;
241+
242+ const type = asString(entry.type);
243+ const mode = asString(entry.mode);
244+ if (type === "tree") return null;
245+ if (type !== "blob" || !mode || !REGULAR_FILE_MODES.has(mode)) {
246+ return { kind: "skip", skipped: { path, reason: "unsupported file type" } };
247+ }
248+
249+ const sha = asString(entry.sha);
250+ if (!sha) return { kind: "skip", skipped: { path, reason: "unsupported file type" } };
251+ if (acceptedPath(path)) return { kind: "fetch", entry: { ...entry, path, sha } };
252+ if (DEFERRED_RE.test(path)) {
253+ return { kind: "skip", skipped: { path, reason: "not supported yet" } };
254+ }
255+ return { kind: "skip", skipped: { path, reason: "ignored" } };
256+};
257+
222258 const decodeBlob = (path: string, blob: BlobResponse): Effect.Effect<string, GitHubSourceError> =>
223259 Effect.try({
224260 try: () => {
@@ -293,13 +329,14 @@ export const fetchGitHubSource = ( · 68 unmodified lines
293329 path: `/repos/${repoPath}/git/trees/${treeSha}`,
294330 });
295331 }
296− const entries = (tree.tree ?? []).filter(
297− (entry) =>
298− entry.type === "blob" &&
299− typeof entry.path === "string" &&
300− typeof entry.sha === "string" &&
301− materializes(entry.path),
302− );
332+ const entries: (TreeEntry & { readonly path: string; readonly sha: string })[] = [];
333+ const skipped: GitHubSkippedArtifact[] = [];
334+ for (const entry of tree.tree ?? []) {
335+ const classified = classifyTreeEntry(entry);
336+ if (!classified) continue;
337+ if (classified.kind === "fetch") entries.push(classified.entry);
338+ else skipped.push(classified.skipped);
339+ }
303340 const treeLimitError = checkTreeLimits(entries);
304341 if (treeLimitError) return yield* Effect.fail(treeLimitError);
305342
@@ -321,6 +358,7 @@ export const fetchGitHubSource = ( · 15 unmodified lines
321358 ref,
322359 upstreamSha,
323360 description: yield* executorDescription(files),
361+ skipped,
324362 };
325363 });
326364
@@ -372,7 +410,7 @@ export const syncGitHubSource = (input: SyncGitHubSourceInput): Effect.Effect<Gi · 45 unmodified lines
372410 status: "up-to-date",
373411 upstreamSha: snapshot.upstreamSha,
374412 tools: current.tools.map((tool) => tool.name),
375− skipped: current.skipped ?? [],
413+ skipped: [...snapshot.skipped, ...(current.skipped ?? [])],
376414 } satisfies GitHubSyncResult;
377415 }
378416
@@ -391,7 +429,7 @@ export const syncGitHubSource = (input: SyncGitHubSourceInput): Effect.Effect<Gi · 12 unmodified lines
391429 status: "failed",
392430 upstreamSha: snapshot.upstreamSha,
393431 tools: [],
394− skipped: [],
432+ skipped: snapshot.skipped,
395433 errors: [publishErrorToSyncError(published.failure)],
396434 } satisfies GitHubSyncResult;
397435 }
@@ -401,6 +439,6 @@ export const syncGitHubSource = (input: SyncGitHubSourceInput): Effect.Effect<Gi · 3 unmodified lines
401439 snapshotId: published.success.snapshotId,
402440 upstreamSha: snapshot.upstreamSha,
403441 tools: published.success.descriptor.tools.map((tool) => tool.name),
404− skipped: published.success.skipped,
442+ skipped: [...snapshot.skipped, ...published.success.skipped],
405443 } satisfies GitHubSyncResult;
406444 });