Filter GitHub source entries safely

Rhys Sullivan committed 4 months ago

Commit 9e77e341 parent bb155804 files · +159 −29Browse files

Showing the changes to packages/plugins/apps/src/source/github-source.ts (1 of 4 files). Show all

Unified · Split

packages/plugins/apps/src/source/github-source.ts+55−17

 import { Data, Effect, Result } from "effect";
-import { PublishError, type FileDiagnostic, type SkippedArtifact } from "../pipeline/discover";
+import { PublishError, type FileDiagnostic } from "../pipeline/discover";
 import { PUBLISH_LIMITS, enforcePublishLimits } from "../pipeline/publish";
 import type { AppSourceRef } from "../pipeline/descriptor";
 import type { AppsRuntime } from "../plugin/runtime";
   readonly ref: string;
   readonly upstreamSha: string;
   readonly description?: string;
+  readonly skipped: readonly GitHubSkippedArtifact[];
 }
 export class GitHubSourceError extends Data.TaggedError("GitHubSourceError")<{
   readonly diagnostics?: readonly FileDiagnostic[];
 }
+export interface GitHubSkippedArtifact {
+  readonly path: string;
+  readonly reason: "not supported yet" | "unsupported file type" | "ignored";
+}
 export type GitHubSyncResult =
   | {
       readonly status: "published";
       readonly snapshotId: SnapshotId;
       readonly upstreamSha: string;
       readonly tools: readonly string[];
-      readonly skipped: readonly SkippedArtifact[];
+      readonly skipped: readonly GitHubSkippedArtifact[];
       readonly errors?: undefined;
     }
   | {
       readonly status: "up-to-date";
       readonly upstreamSha: string;
       readonly tools: readonly string[];
-      readonly skipped: readonly SkippedArtifact[];
+      readonly skipped: readonly GitHubSkippedArtifact[];
       readonly errors?: undefined;
     }
   | {
       readonly status: "failed";
       readonly upstreamSha?: string;
       readonly tools: readonly string[];
-      readonly skipped: readonly SkippedArtifact[];
+      readonly skipped: readonly GitHubSkippedArtifact[];
       readonly errors: readonly SyncErrorData[];
     };
   readonly scope: string;
 }
-const MATERIALIZED_RE = /^(tools|workflows|ui|skills)\//;
+const TOOL_RE = /^tools\/([a-z0-9][a-z0-9-]*)\.(ts|tsx|js|jsx)$/;
+const DEFERRED_RE = /^(workflows|ui|skills)\//;
+const REGULAR_FILE_MODES = new Set(["100644", "100755"]);
 const trimBaseUrl = (baseUrl: string): string => baseUrl.replace(/\/+$/, "");
   return Effect.succeed(`${encodeURIComponent(parsed.owner)}/${encodeURIComponent(parsed.name)}`);
 };
-const materializes = (path: string): boolean =>
+const acceptedPath = (path: string): boolean => path === "executor.json" || TOOL_RE.test(path);
-  path === "executor.json" || MATERIALIZED_RE.test(path);
 const isRecord = (value: unknown): value is Record<string, unknown> =>
   value !== null && typeof value === "object" && !Array.isArray(value);
 interface TreeEntry {
   readonly path?: unknown;
   readonly type?: unknown;
+  readonly mode?: unknown;
   readonly sha?: unknown;
   readonly size?: unknown;
 }
   return diagnostics.length === 0 ? null : limitError(diagnostics);
 };
+const classifyTreeEntry = (
+  entry: TreeEntry,
+):
+  | {
+      readonly kind: "fetch";
+      readonly entry: TreeEntry & { readonly path: string; readonly sha: string };
+    }
+  | { readonly kind: "skip"; readonly skipped: GitHubSkippedArtifact }
+  | null => {
+  const path = asString(entry.path);
+  if (!path) return null;
+  const type = asString(entry.type);
+  const mode = asString(entry.mode);
+  if (type === "tree") return null;
+  if (type !== "blob" || !mode || !REGULAR_FILE_MODES.has(mode)) {
+    return { kind: "skip", skipped: { path, reason: "unsupported file type" } };
+  }
+  const sha = asString(entry.sha);
+  if (!sha) return { kind: "skip", skipped: { path, reason: "unsupported file type" } };
+  if (acceptedPath(path)) return { kind: "fetch", entry: { ...entry, path, sha } };
+  if (DEFERRED_RE.test(path)) {
+    return { kind: "skip", skipped: { path, reason: "not supported yet" } };
+  }
+  return { kind: "skip", skipped: { path, reason: "ignored" } };
+};
 const decodeBlob = (path: string, blob: BlobResponse): Effect.Effect<string, GitHubSourceError> =>
   Effect.try({
     try: () => {
         path: `/repos/${repoPath}/git/trees/${treeSha}`,
       });
     }
-    const entries = (tree.tree ?? []).filter(
+    const entries: (TreeEntry & { readonly path: string; readonly sha: string })[] = [];
-      (entry) =>
+    const skipped: GitHubSkippedArtifact[] = [];
-        entry.type === "blob" &&
+    for (const entry of tree.tree ?? []) {
-        typeof entry.path === "string" &&
+      const classified = classifyTreeEntry(entry);
-        typeof entry.sha === "string" &&
+      if (!classified) continue;
-        materializes(entry.path),
+      if (classified.kind === "fetch") entries.push(classified.entry);
-    );
+      else skipped.push(classified.skipped);
+    }
     const treeLimitError = checkTreeLimits(entries);
     if (treeLimitError) return yield* Effect.fail(treeLimitError);
       ref,
       upstreamSha,
       description: yield* executorDescription(files),
+      skipped,
     };
   });
         status: "up-to-date",
         upstreamSha: snapshot.upstreamSha,
         tools: current.tools.map((tool) => tool.name),
-        skipped: current.skipped ?? [],
+        skipped: [...snapshot.skipped, ...(current.skipped ?? [])],
       } satisfies GitHubSyncResult;
     }
         status: "failed",
         upstreamSha: snapshot.upstreamSha,
         tools: [],
-        skipped: [],
+        skipped: snapshot.skipped,
         errors: [publishErrorToSyncError(published.failure)],
       } satisfies GitHubSyncResult;
     }
       snapshotId: published.success.snapshotId,
       upstreamSha: snapshot.upstreamSha,
       tools: published.success.descriptor.tools.map((tool) => tool.name),
-      skipped: published.success.skipped,
+      skipped: [...snapshot.skipped, ...published.success.skipped],
     } satisfies GitHubSyncResult;
   });