Add GitHub sync and self-host custom tools wiring

Rhys Sullivan committed 4 months ago

Commit 275446e1 parent 41eb3ae19 files · +1665 −40Browse files

Showing the changes to packages/plugins (8 of 19 files). Show all

Unified · Split

8 files

packages/plugins/apps/src/api.ts +11 −0
packages/plugins/apps/src/index.ts +10 −0
packages/plugins/apps/src/pipeline/descriptor.ts +16 −1
packages/plugins/apps/src/pipeline/publish.ts +8 −1
packages/plugins/apps/src/plugin/apps-plugin.ts +136 −37
packages/plugins/apps/src/plugin/runtime.ts +5 −1
packages/plugins/apps/src/source/github-source.test.ts +211 −0
packages/plugins/apps/src/source/github-source.ts +402 −0

@@ -13,6 +13,7 @@ export { · 12 unmodified lines
13	13	type SelfHostAppsRuntime,
14	14	type SelfHostAppsRuntimeOptions,
15	15	} from "./plugin/self-host-runtime";
	16	+export { type AppsRuntime } from "./plugin/runtime";
16	17	export { makeSqliteAppsStore } from "./backing/sqlite-apps-store";
17	18	export {
18	19	BindingError,
@@ -20,3 +21,13 @@ export { · 1 unmodified line
20	21	type ConnectionCandidate,
21	22	type RoleBindings,
22	23	} from "./plugin/bindings";
	24	+export {
	25	+ fetchGitHubSource,
	26	+ syncGitHubSource,
	27	+ GitHubSourceError,
	28	+ type GitHubSourceInput,
	29	+ type GitHubSourceSnapshot,
	30	+ type GitHubSyncResult,
	31	+ type SyncErrorData,
	32	+ type SyncGitHubSourceInput,
	33	+} from "./source/github-source";

packages/plugins/apps/src/index.ts+10−0

@@ -16,6 +16,16 @@ export { · 15 unmodified lines
16	16	type PublishOutput,
17	17	type PublishDeps,
18	18	} from "./pipeline/publish";
	19	+export {
	20	+ fetchGitHubSource,
	21	+ syncGitHubSource,
	22	+ GitHubSourceError,
	23	+ type GitHubSourceInput,
	24	+ type GitHubSourceSnapshot,
	25	+ type GitHubSyncResult,
	26	+ type SyncErrorData,
	27	+ type SyncGitHubSourceInput,
	28	+} from "./source/github-source";
19	29
20	30	export { makeAppsRuntime, type AppsRuntime, type AppsRuntimeDeps } from "./plugin/runtime";
21	31	export {

packages/plugins/apps/src/pipeline/descriptor.ts+16−1

@@ -7,7 +7,7 @@ · 6 unmodified lines
7	7
8	8	/** Descriptor schema version. Bumped on any breaking shape change. A reader
9	9	* refuses a descriptor from a version it does not understand. */
10		−export const DESCRIPTOR_VERSION = 2 as const;
	10	+export const DESCRIPTOR_VERSION = 3 as const;
11	11
12	12	/** Where an entry came from: path + content hash. Lets a projection point back
13	13	* at the exact source bytes without re-reading the snapshot, and makes the
@@ -31,6 +31,15 @@ export interface IntegrationDecl { · 17 unmodified lines
31	31	readonly integration: string;
32	32	}
33	33
	34	+export interface GitHubSourceRef {
	35	+ readonly kind: "github";
	36	+ readonly repo: string;
	37	+ readonly ref: string;
	38	+ readonly upstreamSha: string;
	39	+}
	40	+
	41	+export type AppSourceRef = GitHubSourceRef;
	42	+
34	43	export interface ToolDescriptor {
35	44	/** Path identity, e.g. `issues-sync` (from `tools/issues-sync.ts`). */
36	45	readonly name: string;
@@ -62,6 +71,8 @@ export const GUIDE_ENTRIES_KEY = "skills"; · 25 unmodified lines
62	71	export interface AppDescriptor {
63	72	readonly version: typeof DESCRIPTOR_VERSION;
64	73	readonly scope: string;
	74	+ readonly description?: string;
	75	+ readonly source?: AppSourceRef;
65	76	/** The snapshot (commit hash) this descriptor was extracted from. */
66	77	readonly snapshotId: string;
67	78	/** The toolchain that produced the compiled bundles. */
@@ -70,6 +81,10 @@ export interface AppDescriptor { · 2 unmodified lines
70	81	readonly [FLOW_ENTRIES_KEY]: readonly DeferredDescriptor[];
71	82	readonly ui: readonly DeferredDescriptor[];
72	83	readonly [GUIDE_ENTRIES_KEY]: readonly DeferredDescriptor[];
	84	+ readonly skipped: readonly {
	85	+ readonly path: string;
	86	+ readonly reason: "not supported yet";
	87	+ }[];
73	88	/** Shared JSON-schema `$defs` reachable from tool schemas. */
74	89	readonly definitions?: Record<string, unknown>;
75	90	}

packages/plugins/apps/src/pipeline/publish.ts+8−1

@@ -10,6 +10,7 @@ import { · 9 unmodified lines
10	10	GUIDE_ENTRIES_KEY,
11	11	stableStringify,
12	12	type AppDescriptor,
	13	+ type AppSourceRef,
13	14	type IntegrationDecl,
14	15	type ModuleSourceRef,
15	16	type ToolDescriptor,
@@ -73,6 +74,8 @@ export interface PublishInput { · 57 unmodified lines
73	74	readonly scope: string;
74	75	readonly files: FileSet;
75	76	readonly commitMessage?: string;
	77	+ readonly description?: string;
	78	+ readonly source?: AppSourceRef;
76	79	}
77	80
78	81	export interface PublishOutput {
@@ -219,14 +222,18 @@ const assemble = (deps: PublishDeps, files: FileSet): Effect.Effect<AssembledApp · 140 unmodified lines
219	222	const descriptorBody = (
220	223	scope: string,
221	224	assembled: AssembledApp,
	225	+ input: Pick<PublishInput, "description" \| "source">,
222	226	): Omit<AppDescriptor, "snapshotId"> => ({
223	227	version: DESCRIPTOR_VERSION,
224	228	scope,
	229	+ ...(input.description !== undefined ? { description: input.description } : {}),
	230	+ ...(input.source !== undefined ? { source: input.source } : {}),
225	231	toolchain: toolchainRef(),
226	232	tools: assembled.tools,
227	233	[FLOW_ENTRIES_KEY]: [],
228	234	ui: [],
229	235	[GUIDE_ENTRIES_KEY]: [],
	236	+ skipped: assembled.skipped,
230	237	});
231	238
232	239	export const publish = (
@@ -238,7 +245,7 @@ export const publish = ( · 5 unmodified lines
238	245	if (overLimit) return yield* Effect.fail(overLimit);
239	246
240	247	const assembled = yield* assemble(deps, input.files);
241		− const body = descriptorBody(input.scope, assembled);
	248	+ const body = descriptorBody(input.scope, assembled, input);
242	249
243	250	const scopeStore = yield* deps.artifactStore
244	251	.forScope(input.scope)

packages/plugins/apps/src/plugin/apps-plugin.ts+136−37

@@ -4,10 +4,12 @@ import { · 3 unmodified lines
4	4	definePlugin,
5	5	tool,
6	6	ToolName,
7		− ConnectionName,
8	7	IntegrationSlug,
9	8	AuthTemplateSlug,
10	9	connectionIdentifier,
	10	+ type ConnectionRef,
	11	+ type Owner,
	12	+ type PluginCtx,
11	13	type ResolveToolsInput,
12	14	type ResolveToolsResult,
13	15	type InvokeToolInput,
@@ -18,6 +20,7 @@ import type { AppsRuntime } from "./runtime"; · 4 unmodified lines
18	20	import { makeAppsStore } from "./store";
19	21	import type { ClientResolver, ConnectionCandidate } from "./bindings";
20	22	import type { IntegrationDecl, ToolDescriptor } from "../pipeline/descriptor";
	23	+import { syncGitHubSource, type GitHubSyncResult } from "../source/github-source";
21	24
22	25	export const APPS_INTEGRATION_SLUG = "apps";
23	26	export const APPS_PLUGIN_ID = "apps";
@@ -42,6 +45,40 @@ const isRecord = (value: unknown): value is Record<string, unknown> => · 18 unmodified lines
42	45
43	46	const unique = (values: readonly string[]): readonly string[] => [...new Set(values)];
44	47
	48	+const syncFailure = (message: string, path?: string): GitHubSyncResult => ({
	49	+ status: "failed",
	50	+ tools: [],
	51	+ skipped: [],
	52	+ errors: [
	53	+ {
	54	+ stage: "source",
	55	+ message,
	56	+ ...(path ? { diagnostics: [{ path, message }] } : {}),
	57	+ },
	58	+ ],
	59	+});
	60	+
	61	+const sourceScopeFor = (input: { repo: string; connection: string }): string =>
	62	+ String(connectionIdentifier(`github ${input.connection} ${input.repo}`, "githubSource"));
	63	+
	64	+const parseConnectionAddress = (address: string): ConnectionRef \| null => {
	65	+ const parts = address.split(".");
	66	+ if (parts.length !== 4 \|\| parts[0] !== "tools") return null;
	67	+ const [, integration, owner, name] = parts;
	68	+ if (!integration \|\| !name) return null;
	69	+ if (owner !== "org" && owner !== "user") return null;
	70	+ return {
	71	+ owner: owner as Owner,
	72	+ integration: IntegrationSlug.make(integration),
	73	+ name: connectionIdentifier(name),
	74	+ };
	75	+};
	76	+
	77	+const configBaseUrl = (config: unknown): string \| undefined =>
	78	+ isRecord(config) && typeof config.baseUrl === "string" && config.baseUrl.length > 0
	79	+ ? config.baseUrl
	80	+ : undefined;
	81	+
45	82	const projectInputSchema = (
46	83	schema: unknown,
47	84	integrations: Readonly<Record<string, IntegrationDecl>>,
@@ -94,6 +131,49 @@ export const appsPlugin = definePlugin((options?: AppsPluginOptions) => { · 46 unmodified lines
94	131	Effect.map((mapped) => mapped ?? scopeFromConnection(connectionName)),
95	132	);
96	133
	134	+ const ensureCatalogConnection = (scope: string, ctx: PluginCtx<AppsStoreShape>) =>
	135	+ Effect.gen(function* () {
	136	+ const slug = IntegrationSlug.make(APPS_INTEGRATION_SLUG);
	137	+ const existing = yield* ctx.core.integrations
	138	+ .get(slug)
	139	+ .pipe(Effect.orElseSucceed(() => null));
	140	+ if (!existing) {
	141	+ yield* ctx.core.integrations.register({
	142	+ slug,
	143	+ name: "Apps",
	144	+ description: "User-authored, published custom tools.",
	145	+ config: {},
	146	+ canRemove: false,
	147	+ canRefresh: true,
	148	+ });
	149	+ }
	150	+ const rawName = connectionNameForScope(scope);
	151	+ const connName = connectionIdentifier(rawName);
	152	+ const conns = yield* ctx.connections
	153	+ .list({ integration: slug })
	154	+ .pipe(Effect.orElseSucceed(() => []));
	155	+ if (!conns.some((c) => String(c.name) === String(connName))) {
	156	+ yield* ctx.connections.create({
	157	+ owner: "user",
	158	+ name: connName,
	159	+ integration: slug,
	160	+ template: AuthTemplateSlug.make("none"),
	161	+ value: "",
	162	+ });
	163	+ }
	164	+ yield* runtime.deps.store
	165	+ .putScopeForConnection(String(rawName), scope)
	166	+ .pipe(Effect.orElseSucceed(() => undefined));
	167	+ yield* runtime.deps.store
	168	+ .putScopeForConnection(String(connName), scope)
	169	+ .pipe(Effect.orElseSucceed(() => undefined));
	170	+ return {
	171	+ scope,
	172	+ connection: String(connName),
	173	+ ref: { owner: "user" as const, integration: slug, name: connName },
	174	+ };
	175	+ });
	176	+
97	177	return {
98	178	id: APPS_PLUGIN_ID as "apps",
99	179	packageName: "@executor-js/plugin-apps",
@@ -135,59 +215,78 @@ export const appsPlugin = definePlugin((options?: AppsPluginOptions) => { · 35 unmodified lines
135	215	Effect.gen(function* () {
136	216	const scope =
137	217	(args as { scope?: string } \| undefined)?.scope ?? DEFAULT_CATALOG_SCOPE;
138		− const slug = IntegrationSlug.make(APPS_INTEGRATION_SLUG);
139		− const existing = yield* ctx.core.integrations
140		− .get(slug)
141		− .pipe(Effect.orElseSucceed(() => null));
142		− if (!existing) {
143		− yield* ctx.core.integrations.register({
144		− slug,
145		− name: "Apps",
146		− description: "User-authored, published custom tools.",
147		− config: {},
148		− canRemove: false,
149		− canRefresh: true,
150		− });
	218	+ const connected = yield* ensureCatalogConnection(scope, ctx);
	219	+ return { scope, connection: connected.connection };
	220	+ }),
	221	+ }),
	222	+ tool<AppsStoreShape>({
	223	+ name: "sync_github_source",
	224	+ description: "Sync a GitHub repository containing custom tool source files.",
	225	+ execute: (args, { ctx }) =>
	226	+ Effect.gen(function* () {
	227	+ const payload = isRecord(args) ? args : {};
	228	+ const repo = typeof payload.repo === "string" ? payload.repo : "";
	229	+ const connection = typeof payload.connection === "string" ? payload.connection : "";
	230	+ const ref = typeof payload.ref === "string" ? payload.ref : undefined;
	231	+ const explicitScope = typeof payload.scope === "string" ? payload.scope : undefined;
	232	+ if (!repo) return syncFailure('sync_github_source requires "repo"');
	233	+ if (!connection) return syncFailure('sync_github_source requires "connection"');
	234	+
	235	+ const sourceConnection = parseConnectionAddress(connection);
	236	+ if (!sourceConnection) {
	237	+ return syncFailure(
	238	+ `GitHub connection must be a connection address like tools.github.user.main; got "${connection}"`,
	239	+ connection,
	240	+ );
151	241	}
152		− const connName = ConnectionName.make(connectionNameForScope(scope));
153		− const conns = yield* ctx.connections
154		− .list({ integration: slug })
155		− .pipe(Effect.orElseSucceed(() => []));
156		− if (!conns.some((c) => String(c.name) === String(connName))) {
157		− yield* ctx.connections.create({
158		− owner: "user",
159		− name: connName,
160		− integration: slug,
161		− template: AuthTemplateSlug.make("none"),
162		− value: "",
163		− });
	242	+ if (String(sourceConnection.integration) !== "github") {
	243	+ return syncFailure(
	244	+ `GitHub source sync requires a github connection; got "${sourceConnection.integration}"`,
	245	+ connection,
	246	+ );
164	247	}
165		− yield* runtime.deps.store
166		− .putScopeForConnection(String(connName), scope)
167		− .pipe(Effect.orElseSucceed(() => undefined));
168		− const normalized = connectionIdentifier(String(connName));
169		− if (normalized !== String(connName)) {
170		− yield* runtime.deps.store
171		− .putScopeForConnection(normalized, scope)
172		− .pipe(Effect.orElseSucceed(() => undefined));
	248	+ const existing = yield* ctx.connections
	249	+ .get(sourceConnection)
	250	+ .pipe(Effect.orElseSucceed(() => null));
	251	+ if (!existing) return syncFailure(`GitHub connection not found: ${connection}`);
	252	+ const token = yield* ctx.connections
	253	+ .resolveValue(sourceConnection)
	254	+ .pipe(Effect.orElseSucceed(() => null));
	255	+ const integration = yield* ctx.core.integrations
	256	+ .get(sourceConnection.integration)
	257	+ .pipe(Effect.orElseSucceed(() => null));
	258	+ const scope = explicitScope ?? sourceScopeFor({ repo, connection });
	259	+ const result = yield* syncGitHubSource({
	260	+ runtime,
	261	+ scope,
	262	+ repo,
	263	+ ref,
	264	+ token,
	265	+ baseUrl: configBaseUrl(integration?.config),
	266	+ });
	267	+ if (result.status === "published") {
	268	+ const catalog = yield* ensureCatalogConnection(scope, ctx);
	269	+ yield* ctx.connections.refresh(catalog.ref).pipe(Effect.orElseSucceed(() => []));
173	270	}
174		− return { scope, connection: String(connName) };
	271	+ return result;
175	272	}),
176	273	}),
177	274	],
178	275	},
179	276	],
180	277
181		− resolveTools: ({ connection }: ResolveToolsInput<AppsStoreShape>) =>
	278	+ resolveTools: ({ ctx, connection }: ResolveToolsInput<AppsStoreShape>) =>
182	279	Effect.gen(function* () {
183	280	const scope = yield* scopeFor(String(connection.name));
184	281	const descriptor = yield* runtime.getDescriptor(scope);
185	282	if (!descriptor) return { tools: [] } satisfies ResolveToolsResult;
	283	+ const resolver =
	284	+ makeResolver && ctx ? makeResolver({ ctx, scope, tool: "*" }) : runtime.deps.resolver;
186	285	const tools: ToolDef[] = [];
187	286	for (const t of descriptor.tools) {
188	287	const byRole: Record<string, readonly ConnectionCandidate[]> = {};
189	288	for (const [role, decl] of Object.entries(t.integrations)) {
190		− byRole[role] = yield* runtime.deps.resolver
	289	+ byRole[role] = yield* resolver
191	290	.listConnections({ integration: decl.integration })
192	291	.pipe(Effect.orElseSucceed(() => []));
193	292	}

packages/plugins/apps/src/plugin/runtime.ts+5−1

@@ -13,7 +13,7 @@ import { · 12 unmodified lines
13	13	type PublishOutput,
14	14	} from "../pipeline/publish";
15	15	import { PublishError } from "../pipeline/discover";
16		−import type { AppDescriptor, ToolDescriptor } from "../pipeline/descriptor";
	16	+import type { AppDescriptor, AppSourceRef, ToolDescriptor } from "../pipeline/descriptor";
17	17	import { bundleEntry } from "../pipeline/bundle";
18	18	import {
19	19	buildBridge,
@@ -42,6 +42,8 @@ export interface AppsRuntime { · 22 unmodified lines
42	42	readonly scope: string;
43	43	readonly files: ReadonlyMap<string, string>;
44	44	readonly message?: string;
	45	+ readonly description?: string;
	46	+ readonly source?: AppSourceRef;
45	47	}) => Effect.Effect<PublishOutput, PublishError>;
46	48	readonly getDescriptor: (scope: string) => Effect.Effect<AppDescriptor \| null>;
47	49	/** Re-derive the published-descriptor pointer from the latest committed
@@ -264,6 +266,8 @@ export const makeAppsRuntime = (deps: AppsRuntimeDeps): AppsRuntime => { · 216 unmodified lines
264	266	scope: input.scope,
265	267	files: input.files,
266	268	commitMessage: input.message,
	269	+ description: input.description,
	270	+ source: input.source,
267	271	},
268	272	);
269	273	yield* putDescriptorPointer(out.descriptor);

packages/plugins/apps/src/source/github-source.test.ts+211−0

@@ -0,0 +1,211 @@
	1	+import { mkdtempSync } from "node:fs";
	2	+import { tmpdir } from "node:os";
	3	+import { join } from "node:path";
	4	+
	5	+import { describe, expect, it } from "vitest";
	6	+import { Effect } from "effect";
	7	+
	8	+import { makeSelfHostAppsRuntime } from "../plugin/self-host-runtime";
	9	+import { makeInMemoryAppsStore, makeTestResolver } from "../testing";
	10	+import { PUBLISH_LIMITS } from "../pipeline/publish";
	11	+import { fetchGitHubSource, syncGitHubSource } from "./github-source";
	12	+
	13	+const run = <A, E>(effect: Effect.Effect<A, E>): Promise<A> => Effect.runPromise(effect);
	14	+
	15	+const toolSource = (name = "ok"): string => `import { defineTool } from "executor:app";
	16	+import { z } from "zod";
	17	+export default defineTool({
	18	+ description: "${name}",
	19	+ input: z.object({ value: z.string().default("${name}") }),
	20	+ async handler(input) { return { value: input.value }; },
	21	+});`;
	22	+
	23	+const makeRuntime = () =>
	24	+ makeSelfHostAppsRuntime({
	25	+ dataDir: mkdtempSync(join(tmpdir(), "apps-gh-src-")),
	26	+ store: makeInMemoryAppsStore(),
	27	+ resolver: makeTestResolver({}),
	28	+ inMemory: true,
	29	+ });
	30	+
	31	+const json = (value: unknown, status = 200): Response =>
	32	+ new Response(JSON.stringify(value), {
	33	+ status,
	34	+ headers: { "content-type": "application/json" },
	35	+ });
	36	+
	37	+const makeGitHubFetch = (input: {
	38	+ readonly files: ReadonlyMap<string, string>;
	39	+ readonly upstreamSha?: string;
	40	+ readonly treeSha?: string;
	41	+}) => {
	42	+ const repoPath = "/repos/acme/tools";
	43	+ const upstreamSha = input.upstreamSha ?? "commit-1";
	44	+ const treeSha = input.treeSha ?? "tree-1";
	45	+ const blobBySha = new Map<string, { path: string; content: string }>();
	46	+ let index = 0;
	47	+ for (const [path, content] of input.files) {
	48	+ blobBySha.set(`blob-${++index}`, { path, content });
	49	+ }
	50	+ let blobCalls = 0;
	51	+ const fetch = (async (rawUrl: string) => {
	52	+ const url = new URL(rawUrl);
	53	+ if (url.pathname === repoPath) return json({ default_branch: "main" });
	54	+ if (url.pathname === `${repoPath}/git/ref/heads%2Fmain`) {
	55	+ return json({ object: { sha: upstreamSha } });
	56	+ }
	57	+ if (url.pathname === `${repoPath}/git/commits/${upstreamSha}`) {
	58	+ return json({ sha: upstreamSha, tree: { sha: treeSha } });
	59	+ }
	60	+ if (url.pathname === `${repoPath}/git/trees/${treeSha}`) {
	61	+ return json({
	62	+ tree: [...blobBySha.entries()].map(([sha, blob]) => ({
	63	+ path: blob.path,
	64	+ type: "blob",
	65	+ sha,
	66	+ size: Buffer.byteLength(blob.content, "utf8"),
	67	+ })),
	68	+ });
	69	+ }
	70	+ const blobPrefix = `${repoPath}/git/blobs/`;
	71	+ if (url.pathname.startsWith(blobPrefix)) {
	72	+ blobCalls++;
	73	+ const sha = decodeURIComponent(url.pathname.slice(blobPrefix.length));
	74	+ const blob = blobBySha.get(sha);
	75	+ if (!blob) return json({ message: "not found" }, 404);
	76	+ return json({
	77	+ encoding: "base64",
	78	+ content: Buffer.from(blob.content, "utf8").toString("base64"),
	79	+ });
	80	+ }
	81	+ return json({ message: "not found" }, 404);
	82	+ }) as typeof globalThis.fetch;
	83	+ return {
	84	+ fetch,
	85	+ blobCalls: () => blobCalls,
	86	+ };
	87	+};
	88	+
	89	+describe("GitHub custom-tools source", () => {
	90	+ it("fetches a repo fileset and publishes provenance, description, and skipped entries", async () => {
	91	+ const files = new Map<string, string>([
	92	+ [
	93	+ "executor.json",
	94	+ JSON.stringify({
	95	+ $schema: "https://example.test/schema",
	96	+ description: "Acme tools",
	97	+ ignored: true,
	98	+ }),
	99	+ ],
	100	+ ["tools/hello.ts", toolSource("hello")],
	101	+ ["workflows/deferred.ts", "export default {};"],
	102	+ ["docs/readme.md", "ignored"],
	103	+ ]);
	104	+ const github = makeGitHubFetch({ files, upstreamSha: "commit-a" });
	105	+ const snapshot = await run(fetchGitHubSource({ repo: "acme/tools", fetch: github.fetch }));
	106	+ expect([...snapshot.files.keys()].sort()).toEqual([
	107	+ "executor.json",
	108	+ "tools/hello.ts",
	109	+ "workflows/deferred.ts",
	110	+ ]);
	111	+ expect(snapshot.description).toBe("Acme tools");
	112	+
	113	+ const host = makeRuntime();
	114	+ const result = await run(
	115	+ syncGitHubSource({
	116	+ runtime: host.runtime,
	117	+ scope: "githubTools",
	118	+ repo: "acme/tools",
	119	+ fetch: github.fetch,
	120	+ }),
	121	+ );
	122	+ expect(result.status).toBe("published");
	123	+ expect(result.tools).toEqual(["hello"]);
	124	+ expect(result.skipped).toEqual([
	125	+ { path: "workflows/deferred.ts", reason: "not supported yet" },
	126	+ ]);
	127	+ const descriptor = await run(host.runtime.getDescriptor("githubTools"));
	128	+ expect(descriptor?.description).toBe("Acme tools");
	129	+ expect(descriptor?.source).toEqual({
	130	+ kind: "github",
	131	+ repo: "acme/tools",
	132	+ ref: "main",
	133	+ upstreamSha: "commit-a",
	134	+ });
	135	+ await host.close();
	136	+ });
	137	+
	138	+ it("reports up-to-date when the upstream commit SHA is unchanged", async () => {
	139	+ const github = makeGitHubFetch({
	140	+ files: new Map([["tools/hello.ts", toolSource("hello")]]),
	141	+ upstreamSha: "same-sha",
	142	+ });
	143	+ const host = makeRuntime();
	144	+ const first = await run(
	145	+ syncGitHubSource({
	146	+ runtime: host.runtime,
	147	+ scope: "githubTools",
	148	+ repo: "acme/tools",
	149	+ fetch: github.fetch,
	150	+ }),
	151	+ );
	152	+ const second = await run(
	153	+ syncGitHubSource({
	154	+ runtime: host.runtime,
	155	+ scope: "githubTools",
	156	+ repo: "acme/tools",
	157	+ fetch: github.fetch,
	158	+ }),
	159	+ );
	160	+ expect(first.status).toBe("published");
	161	+ expect(second).toEqual({
	162	+ status: "up-to-date",
	163	+ upstreamSha: "same-sha",
	164	+ tools: ["hello"],
	165	+ skipped: [],
	166	+ });
	167	+ await host.close();
	168	+ });
	169	+
	170	+ it("rejects oversized trees before fetching blobs", async () => {
	171	+ const files = new Map<string, string>();
	172	+ for (let i = 0; i < PUBLISH_LIMITS.maxFiles + 1; i++) {
	173	+ files.set(`tools/t${i}.ts`, toolSource(`t${i}`));
	174	+ }
	175	+ const github = makeGitHubFetch({ files });
	176	+ const exit = await Effect.runPromiseExit(
	177	+ fetchGitHubSource({ repo: "acme/tools", fetch: github.fetch }),
	178	+ );
	179	+ expect(exit._tag).toBe("Failure");
	180	+ expect(JSON.stringify(exit)).toContain("exceeding the limit");
	181	+ expect(github.blobCalls()).toBe(0);
	182	+ });
	183	+
	184	+ it("returns typed failure data for publish errors", async () => {
	185	+ const github = makeGitHubFetch({
	186	+ files: new Map([
	187	+ [
	188	+ "tools/bad.ts",
	189	+ `import { defineTool } from "executor:app";
	190	+import { chunk } from "lodash";
	191	+export default defineTool({ description: "bad", input: { type: "object" }, async handler(){ return chunk([1], 1); } });`,
	192	+ ],
	193	+ ]),
	194	+ upstreamSha: "bad-sha",
	195	+ });
	196	+ const host = makeRuntime();
	197	+ const result = await run(
	198	+ syncGitHubSource({
	199	+ runtime: host.runtime,
	200	+ scope: "githubTools",
	201	+ repo: "acme/tools",
	202	+ fetch: github.fetch,
	203	+ }),
	204	+ );
	205	+ expect(result.status).toBe("failed");
	206	+ expect(result.upstreamSha).toBe("bad-sha");
	207	+ expect(result.errors?.[0]?.stage).toBe("bundle");
	208	+ expect(result.errors?.[0]?.message).toContain('bare import "lodash" is not allowed');
	209	+ await host.close();
	210	+ });
	211	+});

packages/plugins/apps/src/source/github-source.ts+402−0

@@ -0,0 +1,402 @@
	1	+import { Data, Effect, Result } from "effect";
	2	+
	3	+import { PublishError, type FileDiagnostic, type SkippedArtifact } from "../pipeline/discover";
	4	+import { PUBLISH_LIMITS, enforcePublishLimits } from "../pipeline/publish";
	5	+import type { AppSourceRef } from "../pipeline/descriptor";
	6	+import type { AppsRuntime } from "../plugin/runtime";
	7	+import type { FileSet, SnapshotId } from "../seams/artifact-store";
	8	+
	9	+export interface GitHubSourceInput {
	10	+ readonly repo: string;
	11	+ readonly ref?: string;
	12	+ readonly token?: string \| null;
	13	+ readonly baseUrl?: string;
	14	+ readonly fetch?: typeof globalThis.fetch;
	15	+}
	16	+
	17	+export interface GitHubSourceSnapshot {
	18	+ readonly files: FileSet;
	19	+ readonly repo: string;
	20	+ readonly ref: string;
	21	+ readonly upstreamSha: string;
	22	+ readonly description?: string;
	23	+}
	24	+
	25	+export class GitHubSourceError extends Data.TaggedError("GitHubSourceError")<{
	26	+ readonly message: string;
	27	+ readonly status?: number;
	28	+ readonly path?: string;
	29	+ readonly cause?: unknown;
	30	+}> {}
	31	+
	32	+export interface SyncErrorData {
	33	+ readonly stage: "source" \| "discover" \| "bundle" \| "collect" \| "project";
	34	+ readonly message: string;
	35	+ readonly diagnostics?: readonly FileDiagnostic[];
	36	+}
	37	+
	38	+export type GitHubSyncResult =
	39	+ \| {
	40	+ readonly status: "published";
	41	+ readonly snapshotId: SnapshotId;
	42	+ readonly upstreamSha: string;
	43	+ readonly tools: readonly string[];
	44	+ readonly skipped: readonly SkippedArtifact[];
	45	+ readonly errors?: undefined;
	46	+ }
	47	+ \| {
	48	+ readonly status: "up-to-date";
	49	+ readonly upstreamSha: string;
	50	+ readonly tools: readonly string[];
	51	+ readonly skipped: readonly SkippedArtifact[];
	52	+ readonly errors?: undefined;
	53	+ }
	54	+ \| {
	55	+ readonly status: "failed";
	56	+ readonly upstreamSha?: string;
	57	+ readonly tools: readonly string[];
	58	+ readonly skipped: readonly SkippedArtifact[];
	59	+ readonly errors: readonly SyncErrorData[];
	60	+ };
	61	+
	62	+export interface SyncGitHubSourceInput extends GitHubSourceInput {
	63	+ readonly runtime: AppsRuntime;
	64	+ readonly scope: string;
	65	+}
	66	+
	67	+const MATERIALIZED_RE = /^(tools\|workflows\|ui\|skills)\//;
	68	+
	69	+const trimBaseUrl = (baseUrl: string): string => baseUrl.replace(/\/+$/, "");
	70	+
	71	+const repoParts = (repo: string): { owner: string; name: string } \| null => {
	72	+ const [owner, name, ...rest] = repo.split("/");
	73	+ if (!owner \|\| !name \|\| rest.length > 0) return null;
	74	+ return { owner, name };
	75	+};
	76	+
	77	+const encodedRepoPath = (repo: string): Effect.Effect<string, GitHubSourceError> => {
	78	+ const parsed = repoParts(repo);
	79	+ if (!parsed) {
	80	+ return Effect.fail(
	81	+ new GitHubSourceError({
	82	+ message: `GitHub repo must be "owner/name"; got "${repo}"`,
	83	+ path: repo,
	84	+ }),
	85	+ );
	86	+ }
	87	+ return Effect.succeed(`${encodeURIComponent(parsed.owner)}/${encodeURIComponent(parsed.name)}`);
	88	+};
	89	+
	90	+const materializes = (path: string): boolean =>
	91	+ path === "executor.json" \|\| MATERIALIZED_RE.test(path);
	92	+
	93	+const isRecord = (value: unknown): value is Record<string, unknown> =>
	94	+ value !== null && typeof value === "object" && !Array.isArray(value);
	95	+
	96	+const asString = (value: unknown): string \| undefined =>
	97	+ typeof value === "string" && value.length > 0 ? value : undefined;
	98	+
	99	+const requestJson = <A>(
	100	+ input: GitHubSourceInput,
	101	+ path: string,
	102	+): Effect.Effect<A, GitHubSourceError> =>
	103	+ Effect.tryPromise({
	104	+ try: async () => {
	105	+ const fetchImpl = input.fetch ?? globalThis.fetch;
	106	+ const headers: Record<string, string> = {
	107	+ accept: "application/vnd.github+json",
	108	+ "user-agent": "executor-apps-github-source",
	109	+ };
	110	+ if (input.token) headers.authorization = `Bearer ${input.token}`;
	111	+ const response = await fetchImpl(
	112	+ `${trimBaseUrl(input.baseUrl ?? "https://api.github.com")}${path}`,
	113	+ {
	114	+ headers,
	115	+ },
	116	+ );
	117	+ if (!response.ok) {
	118	+ throw new GitHubSourceError({
	119	+ message: `GitHub request failed: GET ${path} -> ${response.status}`,
	120	+ status: response.status,
	121	+ path,
	122	+ });
	123	+ }
	124	+ return (await response.json()) as A;
	125	+ },
	126	+ catch: (cause) =>
	127	+ cause instanceof GitHubSourceError
	128	+ ? cause
	129	+ : new GitHubSourceError({
	130	+ message: `GitHub request failed: GET ${path}`,
	131	+ path,
	132	+ cause,
	133	+ }),
	134	+ });
	135	+
	136	+interface RepoResponse {
	137	+ readonly default_branch?: unknown;
	138	+}
	139	+
	140	+interface CommitResponse {
	141	+ readonly sha?: unknown;
	142	+ readonly commit?: {
	143	+ readonly tree?: {
	144	+ readonly sha?: unknown;
	145	+ };
	146	+ };
	147	+}
	148	+
	149	+interface GitCommitResponse {
	150	+ readonly sha?: unknown;
	151	+ readonly tree?: {
	152	+ readonly sha?: unknown;
	153	+ };
	154	+}
	155	+
	156	+interface RefResponse {
	157	+ readonly object?: {
	158	+ readonly sha?: unknown;
	159	+ };
	160	+}
	161	+
	162	+interface TreeEntry {
	163	+ readonly path?: unknown;
	164	+ readonly type?: unknown;
	165	+ readonly sha?: unknown;
	166	+ readonly size?: unknown;
	167	+}
	168	+
	169	+interface TreeResponse {
	170	+ readonly tree?: readonly TreeEntry[];
	171	+ readonly truncated?: unknown;
	172	+}
	173	+
	174	+interface BlobResponse {
	175	+ readonly content?: unknown;
	176	+ readonly encoding?: unknown;
	177	+}
	178	+
	179	+const commitTreeSha = (commit: CommitResponse \| GitCommitResponse): string \| undefined => {
	180	+ if ("commit" in commit) return asString(commit.commit?.tree?.sha);
	181	+ if ("tree" in commit) return asString(commit.tree?.sha);
	182	+ return undefined;
	183	+};
	184	+
	185	+const limitError = (diagnostics: readonly FileDiagnostic[]): PublishError =>
	186	+ new PublishError({
	187	+ message: `publish payload exceeds limits (${diagnostics.length} problem(s))`,
	188	+ stage: "discover",
	189	+ diagnostics,
	190	+ });
	191	+
	192	+const checkTreeLimits = (entries: readonly TreeEntry[]): PublishError \| null => {
	193	+ const diagnostics: FileDiagnostic[] = [];
	194	+ if (entries.length > PUBLISH_LIMITS.maxFiles) {
	195	+ diagnostics.push({
	196	+ path: "",
	197	+ message: `publish has ${entries.length} files, exceeding the limit of ${PUBLISH_LIMITS.maxFiles}`,
	198	+ });
	199	+ }
	200	+ let total = 0;
	201	+ for (const entry of entries) {
	202	+ const path = String(entry.path ?? "");
	203	+ const size = typeof entry.size === "number" ? entry.size : 0;
	204	+ total += size;
	205	+ if (size > PUBLISH_LIMITS.maxFileBytes) {
	206	+ diagnostics.push({
	207	+ path,
	208	+ message: `file is ${size} bytes, exceeding the per-file limit of ${PUBLISH_LIMITS.maxFileBytes} bytes`,
	209	+ });
	210	+ }
	211	+ }
	212	+ if (total > PUBLISH_LIMITS.maxTotalBytes) {
	213	+ diagnostics.push({
	214	+ path: "",
	215	+ message: `publish total is ${total} bytes, exceeding the total limit of ${PUBLISH_LIMITS.maxTotalBytes} bytes`,
	216	+ });
	217	+ }
	218	+ return diagnostics.length === 0 ? null : limitError(diagnostics);
	219	+};
	220	+
	221	+const decodeBlob = (path: string, blob: BlobResponse): Effect.Effect<string, GitHubSourceError> =>
	222	+ Effect.try({
	223	+ try: () => {
	224	+ const encoding = asString(blob.encoding);
	225	+ const content = asString(blob.content);
	226	+ if (!content \|\| encoding !== "base64") {
	227	+ throw new GitHubSourceError({
	228	+ message: `GitHub blob ${path} did not return base64 content`,
	229	+ path,
	230	+ });
	231	+ }
	232	+ return Buffer.from(content.replace(/\s/g, ""), "base64").toString("utf8");
	233	+ },
	234	+ catch: (cause) =>
	235	+ cause instanceof GitHubSourceError
	236	+ ? cause
	237	+ : new GitHubSourceError({ message: `Failed to decode GitHub blob ${path}`, path, cause }),
	238	+ });
	239	+
	240	+const executorDescription = (
	241	+ files: FileSet,
	242	+): Effect.Effect<string \| undefined, GitHubSourceError> =>
	243	+ Effect.try({
	244	+ try: () => {
	245	+ const raw = files.get("executor.json");
	246	+ if (!raw) return undefined;
	247	+ const parsed = JSON.parse(raw) as unknown;
	248	+ if (!isRecord(parsed)) return undefined;
	249	+ const description = parsed.description;
	250	+ return typeof description === "string" ? description : undefined;
	251	+ },
	252	+ catch: (cause) =>
	253	+ new GitHubSourceError({
	254	+ message: "executor.json is not valid JSON",
	255	+ path: "executor.json",
	256	+ cause,
	257	+ }),
	258	+ });
	259	+
	260	+export const fetchGitHubSource = (
	261	+ input: GitHubSourceInput,
	262	+): Effect.Effect<GitHubSourceSnapshot, GitHubSourceError \| PublishError> =>
	263	+ Effect.gen(function* () {
	264	+ const repoPath = yield* encodedRepoPath(input.repo);
	265	+ const repo = yield* requestJson<RepoResponse>(input, `/repos/${repoPath}`);
	266	+ const ref = input.ref ?? asString(repo.default_branch) ?? "main";
	267	+ const branchRef = yield* requestJson<RefResponse>(
	268	+ input,
	269	+ `/repos/${repoPath}/git/ref/${encodeURIComponent(`heads/${ref}`)}`,
	270	+ ).pipe(Effect.result);
	271	+ const commitPath =
	272	+ Result.isSuccess(branchRef) && asString(branchRef.success.object?.sha)
	273	+ ? `/repos/${repoPath}/git/commits/${encodeURIComponent(String(branchRef.success.object?.sha))}`
	274	+ : `/repos/${repoPath}/commits/${encodeURIComponent(ref)}`;
	275	+ const commit = yield* requestJson<CommitResponse \| GitCommitResponse>(input, commitPath);
	276	+ const upstreamSha = asString(commit.sha);
	277	+ const treeSha = commitTreeSha(commit);
	278	+ if (!upstreamSha \|\| !treeSha) {
	279	+ return yield* new GitHubSourceError({
	280	+ message: `GitHub commit ${ref} did not include a commit SHA and tree SHA`,
	281	+ path: commitPath,
	282	+ });
	283	+ }
	284	+
	285	+ const tree = yield* requestJson<TreeResponse>(
	286	+ input,
	287	+ `/repos/${repoPath}/git/trees/${encodeURIComponent(treeSha)}?recursive=1`,
	288	+ );
	289	+ if (tree.truncated === true) {
	290	+ return yield* new GitHubSourceError({
	291	+ message: "GitHub returned a truncated repository tree; custom tools source is too large",
	292	+ path: `/repos/${repoPath}/git/trees/${treeSha}`,
	293	+ });
	294	+ }
	295	+ const entries = (tree.tree ?? []).filter(
	296	+ (entry) =>
	297	+ entry.type === "blob" &&
	298	+ typeof entry.path === "string" &&
	299	+ typeof entry.sha === "string" &&
	300	+ materializes(entry.path),
	301	+ );
	302	+ const treeLimitError = checkTreeLimits(entries);
	303	+ if (treeLimitError) return yield* Effect.fail(treeLimitError);
	304	+
	305	+ const files = new Map<string, string>();
	306	+ for (const entry of entries) {
	307	+ const path = String(entry.path);
	308	+ const blob = yield* requestJson<BlobResponse>(
	309	+ input,
	310	+ `/repos/${repoPath}/git/blobs/${encodeURIComponent(String(entry.sha))}`,
	311	+ );
	312	+ files.set(path, yield* decodeBlob(path, blob));
	313	+ }
	314	+ const payloadLimitError = enforcePublishLimits(files);
	315	+ if (payloadLimitError) return yield* Effect.fail(payloadLimitError);
	316	+
	317	+ return {
	318	+ files,
	319	+ repo: input.repo,
	320	+ ref,
	321	+ upstreamSha,
	322	+ description: yield* executorDescription(files),
	323	+ };
	324	+ });
	325	+
	326	+const publishErrorToSyncError = (error: PublishError): SyncErrorData => ({
	327	+ stage: error.stage,
	328	+ message: error.message,
	329	+ diagnostics: error.diagnostics,
	330	+});
	331	+
	332	+const sourceErrorToSyncError = (error: GitHubSourceError): SyncErrorData => ({
	333	+ stage: "source",
	334	+ message: error.message,
	335	+ diagnostics: error.path ? [{ path: error.path, message: error.message }] : [],
	336	+});
	337	+
	338	+const sourceRef = (snapshot: GitHubSourceSnapshot): AppSourceRef => ({
	339	+ kind: "github",
	340	+ repo: snapshot.repo,
	341	+ ref: snapshot.ref,
	342	+ upstreamSha: snapshot.upstreamSha,
	343	+});
	344	+
	345	+export const syncGitHubSource = (input: SyncGitHubSourceInput): Effect.Effect<GitHubSyncResult> =>
	346	+ Effect.gen(function* () {
	347	+ const fetched = yield* fetchGitHubSource(input).pipe(Effect.result);
	348	+ if (Result.isFailure(fetched)) {
	349	+ const error = fetched.failure;
	350	+ return {
	351	+ status: "failed",
	352	+ tools: [],
	353	+ skipped: [],
	354	+ errors: [
	355	+ error instanceof PublishError
	356	+ ? publishErrorToSyncError(error)
	357	+ : sourceErrorToSyncError(error),
	358	+ ],
	359	+ } satisfies GitHubSyncResult;
	360	+ }
	361	+ const snapshot = fetched.success;
	362	+ const current = yield* input.runtime.getDescriptor(input.scope);
	363	+ if (
	364	+ current?.source?.kind === "github" &&
	365	+ current.source.repo === snapshot.repo &&
	366	+ current.source.upstreamSha === snapshot.upstreamSha
	367	+ ) {
	368	+ return {
	369	+ status: "up-to-date",
	370	+ upstreamSha: snapshot.upstreamSha,
	371	+ tools: current.tools.map((tool) => tool.name),
	372	+ skipped: current.skipped ?? [],
	373	+ } satisfies GitHubSyncResult;
	374	+ }
	375	+
	376	+ const published = yield* input.runtime
	377	+ .publish({
	378	+ scope: input.scope,
	379	+ files: snapshot.files,
	380	+ description: snapshot.description,
	381	+ source: sourceRef(snapshot),
	382	+ message: `sync ${snapshot.repo}@${snapshot.upstreamSha}`,
	383	+ })
	384	+ .pipe(Effect.result);
	385	+ if (Result.isFailure(published)) {
	386	+ return {
	387	+ status: "failed",
	388	+ upstreamSha: snapshot.upstreamSha,
	389	+ tools: [],
	390	+ skipped: [],
	391	+ errors: [publishErrorToSyncError(published.failure)],
	392	+ } satisfies GitHubSyncResult;
	393	+ }
	394	+
	395	+ return {
	396	+ status: "published",
	397	+ snapshotId: published.success.snapshotId,
	398	+ upstreamSha: snapshot.upstreamSha,
	399	+ tools: published.success.descriptor.tools.map((tool) => tool.name),
	400	+ skipped: published.success.skipped,
	401	+ } satisfies GitHubSyncResult;
	402	+ });