History

Filter GitHub source entries safely

Rhys Sullivan committed 4 months ago

Commit 9e77e341 parent bb155804 files · +159 −29Browse files

Unified · Split

4 files
  1. packages/plugins/apps/src/plugin/bindings.ts +2 −0
  2. packages/plugins/apps/src/plugin/tenant-isolation.test.ts +1 −0
  3. packages/plugins/apps/src/source/github-source.test.ts +101 −12
  4. packages/plugins/apps/src/source/github-source.ts +55 −17
packages/plugins/apps/src/plugin/bindings.ts+2−0
@@ -259,6 +259,8 @@ export const buildBridge = (context: BindingContext): HandleBridge => ({ · 258 unmodified lines
259259 return Effect.fail(invokeErr(`no binding for role ${role}`));
260260 }
261261 if (path.length === 1 && path[0] === "account") {
262+ // Connection-derived placeholders only; this is not verified upstream
263+ // account identity.
262264 return Effect.succeed({ email: connectionName, login: connectionName, name: connectionName });
263265 }
264266
packages/plugins/apps/src/plugin/tenant-isolation.test.ts+1−0
@@ -51,6 +51,7 @@ const makeGitHubFetch = (input: { readonly toolName: string; readonly upstreamSh · 50 unmodified lines
5151 {
5252 path: `tools/${input.toolName}.ts`,
5353 type: "blob",
54+ mode: "100644",
5455 sha: `${input.upstreamSha}-blob`,
5556 size: Buffer.byteLength(content, "utf8"),
5657 },
packages/plugins/apps/src/source/github-source.test.ts+101−12
@@ -8,6 +8,7 @@ import { Effect } from "effect"; · 7 unmodified lines
88 import { makeSelfHostAppsRuntime } from "../plugin/self-host-runtime";
99 import { makeInMemoryAppsStore, makeTestResolver } from "../testing";
1010 import { PUBLISH_LIMITS } from "../pipeline/publish";
11+import { scopeAddress } from "../seams/scope-address";
1112 import { fetchGitHubSource, syncGitHubSource } from "./github-source";
1213
1314 const run = <A, E>(effect: Effect.Effect<A, E>): Promise<A> => Effect.runPromise(effect);
@@ -34,8 +35,17 @@ const json = (value: unknown, status = 200): Response => · 20 unmodified lines
3435 headers: { "content-type": "application/json" },
3536 });
3637
38+interface GitHubTreeFixtureEntry {
39+ readonly path: string;
40+ readonly type: string;
41+ readonly mode?: string;
42+ readonly content?: string;
43+ readonly size?: number;
44+}
45+
3746 const makeGitHubFetch = (input: {
3847 readonly files: ReadonlyMap<string, string>;
48+ readonly entries?: readonly GitHubTreeFixtureEntry[];
3949 readonly upstreamSha?: string;
4050 readonly treeSha?: string;
4151 }) => {
@@ -43,9 +53,22 @@ const makeGitHubFetch = (input: { · 1 unmodified line
4353 const upstreamSha = input.upstreamSha ?? "commit-1";
4454 const treeSha = input.treeSha ?? "tree-1";
4555 const blobBySha = new Map<string, { path: string; content: string }>();
56+ const treeEntries: readonly GitHubTreeFixtureEntry[] =
57+ input.entries ??
58+ [...input.files].map(([path, content]) => ({
59+ path,
60+ type: "blob",
61+ mode: "100644",
62+ content,
63+ }));
4664 let index = 0;
47− for (const [path, content] of input.files) {
48− blobBySha.set(`blob-${++index}`, { path, content });
65+ const shaByPath = new Map<string, string>();
66+ for (const entry of treeEntries) {
67+ if (entry.content !== undefined) {
68+ const sha = `blob-${++index}`;
69+ shaByPath.set(entry.path, sha);
70+ blobBySha.set(sha, { path: entry.path, content: entry.content });
71+ }
4972 }
5073 let blobCalls = 0;
5174 const fetch = (async (rawUrl: string) => {
@@ -59,12 +82,16 @@ const makeGitHubFetch = (input: { · 7 unmodified lines
5982 }
6083 if (url.pathname === `${repoPath}/git/trees/${treeSha}`) {
6184 return json({
62− tree: [...blobBySha.entries()].map(([sha, blob]) => ({
63− path: blob.path,
64− type: "blob",
65− sha,
66− size: Buffer.byteLength(blob.content, "utf8"),
67− })),
85+ tree: treeEntries.map((entry, entryIndex) => {
86+ const sha = shaByPath.get(entry.path) ?? `tree-${entryIndex}`;
87+ return {
88+ path: entry.path,
89+ type: entry.type,
90+ mode: entry.mode,
91+ sha,
92+ size: entry.size ?? Buffer.byteLength(entry.content ?? "", "utf8"),
93+ };
94+ }),
6895 });
6996 }
7097 const blobPrefix = `${repoPath}/git/blobs/`;
@@ -103,10 +130,10 @@ describe("GitHub custom-tools source", () => { · 32 unmodified lines
103130 ]);
104131 const github = makeGitHubFetch({ files, upstreamSha: "commit-a" });
105132 const snapshot = await run(fetchGitHubSource({ repo: "acme/tools", fetch: github.fetch }));
106− expect([...snapshot.files.keys()].sort()).toEqual([
107− "executor.json",
108− "tools/hello.ts",
109− "workflows/deferred.ts",
133+ expect([...snapshot.files.keys()].sort()).toEqual(["executor.json", "tools/hello.ts"]);
134+ expect(snapshot.skipped).toEqual([
135+ { path: "workflows/deferred.ts", reason: "not supported yet" },
136+ { path: "docs/readme.md", reason: "ignored" },
110137 ]);
111138 expect(snapshot.description).toBe("Acme tools");
112139
@@ -123,6 +150,7 @@ describe("GitHub custom-tools source", () => { · 10 unmodified lines
123150 expect(result.tools).toEqual(["hello"]);
124151 expect(result.skipped).toEqual([
125152 { path: "workflows/deferred.ts", reason: "not supported yet" },
153+ { path: "docs/readme.md", reason: "ignored" },
126154 ]);
127155 const descriptor = await run(host.runtime.getDescriptor("githubTools"));
128156 expect(descriptor?.description).toBe("Acme tools");
@@ -208,4 +236,65 @@ export default defineTool({ description: "bad", input: { type: "object" }, async · 79 unmodified lines
208236 expect(result.errors?.[0]?.message).toContain('bare import "lodash" is not allowed');
209237 await host.close();
210238 });
239+
240+ it("skips unsupported tree entries and invalid paths without committing them", async () => {
241+ const github = makeGitHubFetch({
242+ files: new Map(),
243+ entries: [
244+ {
245+ path: "tools/ok.ts",
246+ type: "blob",
247+ mode: "100644",
248+ content: toolSource("ok"),
249+ },
250+ {
251+ path: "tools/link.ts",
252+ type: "blob",
253+ mode: "120000",
254+ content: "../outside.ts",
255+ },
256+ {
257+ path: "tools/submodule.ts",
258+ type: "commit",
259+ mode: "160000",
260+ },
261+ {
262+ path: "tools/Bad.Name.ts",
263+ type: "blob",
264+ mode: "100644",
265+ content: toolSource("bad"),
266+ },
267+ ],
268+ upstreamSha: "mixed-sha",
269+ });
270+ const host = makeRuntime();
271+ const result = await run(
272+ syncGitHubSource({
273+ runtime: host.runtime,
274+ scope: "githubTools",
275+ repo: "acme/tools",
276+ fetch: github.fetch,
277+ }),
278+ );
279+
280+ expect(result.status).toBe("published");
281+ expect(result.tools).toEqual(["ok"]);
282+ expect(result.skipped).toEqual([
283+ { path: "tools/link.ts", reason: "unsupported file type" },
284+ { path: "tools/submodule.ts", reason: "unsupported file type" },
285+ { path: "tools/Bad.Name.ts", reason: "ignored" },
286+ ]);
287+
288+ const descriptor = await run(host.runtime.getDescriptor("githubTools"));
289+ expect(descriptor?.tools.map((tool) => tool.name)).toEqual(["ok"]);
290+ const scopeStore = await run(
291+ host.runtime.deps.artifactStore.forScope(scopeAddress("org", "githubTools")),
292+ );
293+ const snapshotPaths = await run(scopeStore.list(descriptor!.snapshotId as never));
294+ expect(snapshotPaths).toContain("tools/ok.ts");
295+ expect(snapshotPaths).not.toContain("tools/link.ts");
296+ expect(snapshotPaths).not.toContain("tools/submodule.ts");
297+ expect(snapshotPaths).not.toContain("tools/Bad.Name.ts");
298+ await host.close();
299+ });
211300 });
packages/plugins/apps/src/source/github-source.ts+55−17
@@ -1,6 +1,6 @@
11 import { Data, Effect, Result } from "effect";
22
3−import { PublishError, type FileDiagnostic, type SkippedArtifact } from "../pipeline/discover";
3+import { PublishError, type FileDiagnostic } from "../pipeline/discover";
44 import { PUBLISH_LIMITS, enforcePublishLimits } from "../pipeline/publish";
55 import type { AppSourceRef } from "../pipeline/descriptor";
66 import type { AppsRuntime } from "../plugin/runtime";
@@ -20,6 +20,7 @@ export interface GitHubSourceSnapshot { · 13 unmodified lines
2020 readonly ref: string;
2121 readonly upstreamSha: string;
2222 readonly description?: string;
23+ readonly skipped: readonly GitHubSkippedArtifact[];
2324 }
2425
2526 export class GitHubSourceError extends Data.TaggedError("GitHubSourceError")<{
@@ -35,27 +36,32 @@ export interface SyncErrorData { · 9 unmodified lines
3536 readonly diagnostics?: readonly FileDiagnostic[];
3637 }
3738
39+export interface GitHubSkippedArtifact {
40+ readonly path: string;
41+ readonly reason: "not supported yet" | "unsupported file type" | "ignored";
42+}
43+
3844 export type GitHubSyncResult =
3945 | {
4046 readonly status: "published";
4147 readonly snapshotId: SnapshotId;
4248 readonly upstreamSha: string;
4349 readonly tools: readonly string[];
44− readonly skipped: readonly SkippedArtifact[];
50+ readonly skipped: readonly GitHubSkippedArtifact[];
4551 readonly errors?: undefined;
4652 }
4753 | {
4854 readonly status: "up-to-date";
4955 readonly upstreamSha: string;
5056 readonly tools: readonly string[];
51− readonly skipped: readonly SkippedArtifact[];
57+ readonly skipped: readonly GitHubSkippedArtifact[];
5258 readonly errors?: undefined;
5359 }
5460 | {
5561 readonly status: "failed";
5662 readonly upstreamSha?: string;
5763 readonly tools: readonly string[];
58− readonly skipped: readonly SkippedArtifact[];
64+ readonly skipped: readonly GitHubSkippedArtifact[];
5965 readonly errors: readonly SyncErrorData[];
6066 };
6167
@@ -65,7 +71,9 @@ export interface SyncGitHubSourceInput extends GitHubSourceInput { · 3 unmodified lines
6571 readonly scope: string;
6672 }
6773
68−const MATERIALIZED_RE = /^(tools|workflows|ui|skills)\//;
74+const TOOL_RE = /^tools\/([a-z0-9][a-z0-9-]*)\.(ts|tsx|js|jsx)$/;
75+const DEFERRED_RE = /^(workflows|ui|skills)\//;
76+const REGULAR_FILE_MODES = new Set(["100644", "100755"]);
6977
7078 const trimBaseUrl = (baseUrl: string): string => baseUrl.replace(/\/+$/, "");
7179
@@ -88,8 +96,7 @@ const encodedRepoPath = (repo: string): Effect.Effect<string, GitHubSourceError> · 16 unmodified lines
8896 return Effect.succeed(`${encodeURIComponent(parsed.owner)}/${encodeURIComponent(parsed.name)}`);
8997 };
9098
91−const materializes = (path: string): boolean =>
92− path === "executor.json" || MATERIALIZED_RE.test(path);
99+const acceptedPath = (path: string): boolean => path === "executor.json" || TOOL_RE.test(path);
93100
94101 const isRecord = (value: unknown): value is Record<string, unknown> =>
95102 value !== null && typeof value === "object" && !Array.isArray(value);
@@ -163,6 +170,7 @@ interface RefResponse { · 67 unmodified lines
163170 interface TreeEntry {
164171 readonly path?: unknown;
165172 readonly type?: unknown;
173+ readonly mode?: unknown;
166174 readonly sha?: unknown;
167175 readonly size?: unknown;
168176 }
@@ -219,6 +227,34 @@ const checkTreeLimits = (entries: readonly TreeEntry[]): PublishError | null => · 50 unmodified lines
219227 return diagnostics.length === 0 ? null : limitError(diagnostics);
220228 };
221229
230+const classifyTreeEntry = (
231+ entry: TreeEntry,
232+):
233+ | {
234+ readonly kind: "fetch";
235+ readonly entry: TreeEntry & { readonly path: string; readonly sha: string };
236+ }
237+ | { readonly kind: "skip"; readonly skipped: GitHubSkippedArtifact }
238+ | null => {
239+ const path = asString(entry.path);
240+ if (!path) return null;
241+
242+ const type = asString(entry.type);
243+ const mode = asString(entry.mode);
244+ if (type === "tree") return null;
245+ if (type !== "blob" || !mode || !REGULAR_FILE_MODES.has(mode)) {
246+ return { kind: "skip", skipped: { path, reason: "unsupported file type" } };
247+ }
248+
249+ const sha = asString(entry.sha);
250+ if (!sha) return { kind: "skip", skipped: { path, reason: "unsupported file type" } };
251+ if (acceptedPath(path)) return { kind: "fetch", entry: { ...entry, path, sha } };
252+ if (DEFERRED_RE.test(path)) {
253+ return { kind: "skip", skipped: { path, reason: "not supported yet" } };
254+ }
255+ return { kind: "skip", skipped: { path, reason: "ignored" } };
256+};
257+
222258 const decodeBlob = (path: string, blob: BlobResponse): Effect.Effect<string, GitHubSourceError> =>
223259 Effect.try({
224260 try: () => {
@@ -293,13 +329,14 @@ export const fetchGitHubSource = ( · 68 unmodified lines
293329 path: `/repos/${repoPath}/git/trees/${treeSha}`,
294330 });
295331 }
296− const entries = (tree.tree ?? []).filter(
297− (entry) =>
298− entry.type === "blob" &&
299− typeof entry.path === "string" &&
300− typeof entry.sha === "string" &&
301− materializes(entry.path),
302− );
332+ const entries: (TreeEntry & { readonly path: string; readonly sha: string })[] = [];
333+ const skipped: GitHubSkippedArtifact[] = [];
334+ for (const entry of tree.tree ?? []) {
335+ const classified = classifyTreeEntry(entry);
336+ if (!classified) continue;
337+ if (classified.kind === "fetch") entries.push(classified.entry);
338+ else skipped.push(classified.skipped);
339+ }
303340 const treeLimitError = checkTreeLimits(entries);
304341 if (treeLimitError) return yield* Effect.fail(treeLimitError);
305342
@@ -321,6 +358,7 @@ export const fetchGitHubSource = ( · 15 unmodified lines
321358 ref,
322359 upstreamSha,
323360 description: yield* executorDescription(files),
361+ skipped,
324362 };
325363 });
326364
@@ -372,7 +410,7 @@ export const syncGitHubSource = (input: SyncGitHubSourceInput): Effect.Effect<Gi · 45 unmodified lines
372410 status: "up-to-date",
373411 upstreamSha: snapshot.upstreamSha,
374412 tools: current.tools.map((tool) => tool.name),
375− skipped: current.skipped ?? [],
413+ skipped: [...snapshot.skipped, ...(current.skipped ?? [])],
376414 } satisfies GitHubSyncResult;
377415 }
378416
@@ -391,7 +429,7 @@ export const syncGitHubSource = (input: SyncGitHubSourceInput): Effect.Effect<Gi · 12 unmodified lines
391429 status: "failed",
392430 upstreamSha: snapshot.upstreamSha,
393431 tools: [],
394− skipped: [],
432+ skipped: snapshot.skipped,
395433 errors: [publishErrorToSyncError(published.failure)],
396434 } satisfies GitHubSyncResult;
397435 }
@@ -401,6 +439,6 @@ export const syncGitHubSource = (input: SyncGitHubSourceInput): Effect.Effect<Gi · 3 unmodified lines
401439 snapshotId: published.success.snapshotId,
402440 upstreamSha: snapshot.upstreamSha,
403441 tools: published.success.descriptor.tools.map((tool) => tool.name),
404− skipped: published.success.skipped,
442+ skipped: [...snapshot.skipped, ...published.success.skipped],
405443 } satisfies GitHubSyncResult;
406444 });