lael-labs/executor

packages/plugins/apps/src/api.ts+11−0

@@ -13,6 +13,7 @@ export { · 12 unmodified lines
13	type SelfHostAppsRuntime,	13	type SelfHostAppsRuntime,
14	type SelfHostAppsRuntimeOptions,	14	type SelfHostAppsRuntimeOptions,
15	} from "./plugin/self-host-runtime";	15	} from "./plugin/self-host-runtime";
		16	export { type AppsRuntime } from "./plugin/runtime";
16	export { makeSqliteAppsStore } from "./backing/sqlite-apps-store";	17	export { makeSqliteAppsStore } from "./backing/sqlite-apps-store";
17	export {	18	export {
18	BindingError,	19	BindingError,
@@ -20,3 +21,13 @@ export { · 1 unmodified line
20	type ConnectionCandidate,	21	type ConnectionCandidate,
21	type RoleBindings,	22	type RoleBindings,
22	} from "./plugin/bindings";	23	} from "./plugin/bindings";
		24	export {
		25	fetchGitHubSource,
		26	syncGitHubSource,
		27	GitHubSourceError,
		28	type GitHubSourceInput,
		29	type GitHubSourceSnapshot,
		30	type GitHubSyncResult,
		31	type SyncErrorData,
		32	type SyncGitHubSourceInput,
		33	} from "./source/github-source";

packages/plugins/apps/src/index.ts+10−0

@@ -16,6 +16,16 @@ export { · 15 unmodified lines
16	type PublishOutput,	16	type PublishOutput,
17	type PublishDeps,	17	type PublishDeps,
18	} from "./pipeline/publish";	18	} from "./pipeline/publish";
		19	export {
		20	fetchGitHubSource,
		21	syncGitHubSource,
		22	GitHubSourceError,
		23	type GitHubSourceInput,
		24	type GitHubSourceSnapshot,
		25	type GitHubSyncResult,
		26	type SyncErrorData,
		27	type SyncGitHubSourceInput,
		28	} from "./source/github-source";
19		29
20	export { makeAppsRuntime, type AppsRuntime, type AppsRuntimeDeps } from "./plugin/runtime";	30	export { makeAppsRuntime, type AppsRuntime, type AppsRuntimeDeps } from "./plugin/runtime";
21	export {	31	export {

packages/plugins/apps/src/pipeline/descriptor.ts+16−1

 /** Descriptor schema version. Bumped on any breaking shape change. A reader
  *  refuses a descriptor from a version it does not understand. */
-export const DESCRIPTOR_VERSION = 2 as const;
+export const DESCRIPTOR_VERSION = 3 as const;
 /** Where an entry came from: path + content hash. Lets a projection point back
  *  at the exact source bytes without re-reading the snapshot, and makes the
   readonly integration: string;
 }
+export interface GitHubSourceRef {
+  readonly kind: "github";
+  readonly repo: string;
+  readonly ref: string;
+  readonly upstreamSha: string;
+}
+export type AppSourceRef = GitHubSourceRef;
 export interface ToolDescriptor {
   /** Path identity, e.g. `issues-sync` (from `tools/issues-sync.ts`). */
   readonly name: string;
 export interface AppDescriptor {
   readonly version: typeof DESCRIPTOR_VERSION;
   readonly scope: string;
+  readonly description?: string;
+  readonly source?: AppSourceRef;
   /** The snapshot (commit hash) this descriptor was extracted from. */
   readonly snapshotId: string;
   /** The toolchain that produced the compiled bundles. */
   readonly [FLOW_ENTRIES_KEY]: readonly DeferredDescriptor[];
   readonly ui: readonly DeferredDescriptor[];
   readonly [GUIDE_ENTRIES_KEY]: readonly DeferredDescriptor[];
+  readonly skipped: readonly {
+    readonly path: string;
+    readonly reason: "not supported yet";
+  }[];
   /** Shared JSON-schema `$defs` reachable from tool schemas. */
   readonly definitions?: Record<string, unknown>;
 }

packages/plugins/apps/src/pipeline/publish.ts+8−1

   GUIDE_ENTRIES_KEY,
   stableStringify,
   type AppDescriptor,
+  type AppSourceRef,
   type IntegrationDecl,
   type ModuleSourceRef,
   type ToolDescriptor,
   readonly scope: string;
   readonly files: FileSet;
   readonly commitMessage?: string;
+  readonly description?: string;
+  readonly source?: AppSourceRef;
 }
 export interface PublishOutput {
 const descriptorBody = (
   scope: string,
   assembled: AssembledApp,
+  input: Pick<PublishInput, "description" | "source">,
 ): Omit<AppDescriptor, "snapshotId"> => ({
   version: DESCRIPTOR_VERSION,
   scope,
+  ...(input.description !== undefined ? { description: input.description } : {}),
+  ...(input.source !== undefined ? { source: input.source } : {}),
   toolchain: toolchainRef(),
   tools: assembled.tools,
   [FLOW_ENTRIES_KEY]: [],
   ui: [],
   [GUIDE_ENTRIES_KEY]: [],
+  skipped: assembled.skipped,
 });
 export const publish = (
     if (overLimit) return yield* Effect.fail(overLimit);
     const assembled = yield* assemble(deps, input.files);
-    const body = descriptorBody(input.scope, assembled);
+    const body = descriptorBody(input.scope, assembled, input);
     const scopeStore = yield* deps.artifactStore
       .forScope(input.scope)

packages/plugins/apps/src/plugin/apps-plugin.ts+136−37

@@ -4,10 +4,12 @@ import { · 3 unmodified lines
4	definePlugin,	4	definePlugin,
5	tool,	5	tool,
6	ToolName,	6	ToolName,
7	ConnectionName,
8	IntegrationSlug,	7	IntegrationSlug,
9	AuthTemplateSlug,	8	AuthTemplateSlug,
10	connectionIdentifier,	9	connectionIdentifier,
		10	type ConnectionRef,
		11	type Owner,
		12	type PluginCtx,
11	type ResolveToolsInput,	13	type ResolveToolsInput,
12	type ResolveToolsResult,	14	type ResolveToolsResult,
13	type InvokeToolInput,	15	type InvokeToolInput,
@@ -18,6 +20,7 @@ import type { AppsRuntime } from "./runtime"; · 4 unmodified lines
18	import { makeAppsStore } from "./store";	20	import { makeAppsStore } from "./store";
19	import type { ClientResolver, ConnectionCandidate } from "./bindings";	21	import type { ClientResolver, ConnectionCandidate } from "./bindings";
20	import type { IntegrationDecl, ToolDescriptor } from "../pipeline/descriptor";	22	import type { IntegrationDecl, ToolDescriptor } from "../pipeline/descriptor";
		23	import { syncGitHubSource, type GitHubSyncResult } from "../source/github-source";
21		24
22	export const APPS_INTEGRATION_SLUG = "apps";	25	export const APPS_INTEGRATION_SLUG = "apps";
23	export const APPS_PLUGIN_ID = "apps";	26	export const APPS_PLUGIN_ID = "apps";
@@ -42,6 +45,40 @@ const isRecord = (value: unknown): value is Record<string, unknown> => · 18 unmodified lines
42		45
43	const unique = (values: readonly string[]): readonly string[] => [...new Set(values)];	46	const unique = (values: readonly string[]): readonly string[] => [...new Set(values)];
44		47
		48	const syncFailure = (message: string, path?: string): GitHubSyncResult => ({
		49	status: "failed",
		50	tools: [],
		51	skipped: [],
		52	errors: [
		53	{
		54	stage: "source",
		55	message,
		56	...(path ? { diagnostics: [{ path, message }] } : {}),
		57	},
		58	],
		59	});
		60
		61	const sourceScopeFor = (input: { repo: string; connection: string }): string =>
		62	String(connectionIdentifier(`github ${input.connection} ${input.repo}`, "githubSource"));
		63
		64	const parseConnectionAddress = (address: string): ConnectionRef \| null => {
		65	const parts = address.split(".");
		66	if (parts.length !== 4 \|\| parts[0] !== "tools") return null;
		67	const [, integration, owner, name] = parts;
		68	if (!integration \|\| !name) return null;
		69	if (owner !== "org" && owner !== "user") return null;
		70	return {
		71	owner: owner as Owner,
		72	integration: IntegrationSlug.make(integration),
		73	name: connectionIdentifier(name),
		74	};
		75	};
		76
		77	const configBaseUrl = (config: unknown): string \| undefined =>
		78	isRecord(config) && typeof config.baseUrl === "string" && config.baseUrl.length > 0
		79	? config.baseUrl
		80	: undefined;
		81
45	const projectInputSchema = (	82	const projectInputSchema = (
46	schema: unknown,	83	schema: unknown,
47	integrations: Readonly<Record<string, IntegrationDecl>>,	84	integrations: Readonly<Record<string, IntegrationDecl>>,
@@ -94,6 +131,49 @@ export const appsPlugin = definePlugin((options?: AppsPluginOptions) => { · 46 unmodified lines
94	Effect.map((mapped) => mapped ?? scopeFromConnection(connectionName)),	131	Effect.map((mapped) => mapped ?? scopeFromConnection(connectionName)),
95	);	132	);
96		133
		134	const ensureCatalogConnection = (scope: string, ctx: PluginCtx<AppsStoreShape>) =>
		135	Effect.gen(function* () {
		136	const slug = IntegrationSlug.make(APPS_INTEGRATION_SLUG);
		137	const existing = yield* ctx.core.integrations
		138	.get(slug)
		139	.pipe(Effect.orElseSucceed(() => null));
		140	if (!existing) {
		141	yield* ctx.core.integrations.register({
		142	slug,
		143	name: "Apps",
		144	description: "User-authored, published custom tools.",
		145	config: {},
		146	canRemove: false,
		147	canRefresh: true,
		148	});
		149	}
		150	const rawName = connectionNameForScope(scope);
		151	const connName = connectionIdentifier(rawName);
		152	const conns = yield* ctx.connections
		153	.list({ integration: slug })
		154	.pipe(Effect.orElseSucceed(() => []));
		155	if (!conns.some((c) => String(c.name) === String(connName))) {
		156	yield* ctx.connections.create({
		157	owner: "user",
		158	name: connName,
		159	integration: slug,
		160	template: AuthTemplateSlug.make("none"),
		161	value: "",
		162	});
		163	}
		164	yield* runtime.deps.store
		165	.putScopeForConnection(String(rawName), scope)
		166	.pipe(Effect.orElseSucceed(() => undefined));
		167	yield* runtime.deps.store
		168	.putScopeForConnection(String(connName), scope)
		169	.pipe(Effect.orElseSucceed(() => undefined));
		170	return {
		171	scope,
		172	connection: String(connName),
		173	ref: { owner: "user" as const, integration: slug, name: connName },
		174	};
		175	});
		176
97	return {	177	return {
98	id: APPS_PLUGIN_ID as "apps",	178	id: APPS_PLUGIN_ID as "apps",
99	packageName: "@executor-js/plugin-apps",	179	packageName: "@executor-js/plugin-apps",
@@ -135,59 +215,78 @@ export const appsPlugin = definePlugin((options?: AppsPluginOptions) => { · 35 unmodified lines
135	Effect.gen(function* () {	215	Effect.gen(function* () {
136	const scope =	216	const scope =
137	(args as { scope?: string } \| undefined)?.scope ?? DEFAULT_CATALOG_SCOPE;	217	(args as { scope?: string } \| undefined)?.scope ?? DEFAULT_CATALOG_SCOPE;
138	const slug = IntegrationSlug.make(APPS_INTEGRATION_SLUG);	218	const connected = yield* ensureCatalogConnection(scope, ctx);
139	const existing = yield* ctx.core.integrations	219	return { scope, connection: connected.connection };
140	.get(slug)	220	}),
141	.pipe(Effect.orElseSucceed(() => null));	221	}),
142	if (!existing) {	222	tool<AppsStoreShape>({
143	yield* ctx.core.integrations.register({	223	name: "sync_github_source",
144	slug,	224	description: "Sync a GitHub repository containing custom tool source files.",
145	name: "Apps",	225	execute: (args, { ctx }) =>
146	description: "User-authored, published custom tools.",	226	Effect.gen(function* () {
147	config: {},	227	const payload = isRecord(args) ? args : {};
148	canRemove: false,	228	const repo = typeof payload.repo === "string" ? payload.repo : "";
149	canRefresh: true,	229	const connection = typeof payload.connection === "string" ? payload.connection : "";
150	});	230	const ref = typeof payload.ref === "string" ? payload.ref : undefined;
		231	const explicitScope = typeof payload.scope === "string" ? payload.scope : undefined;
		232	if (!repo) return syncFailure('sync_github_source requires "repo"');
		233	if (!connection) return syncFailure('sync_github_source requires "connection"');
		234
		235	const sourceConnection = parseConnectionAddress(connection);
		236	if (!sourceConnection) {
		237	return syncFailure(
		238	`GitHub connection must be a connection address like tools.github.user.main; got "${connection}"`,
		239	connection,
		240	);
151	}	241	}
152	const connName = ConnectionName.make(connectionNameForScope(scope));	242	if (String(sourceConnection.integration) !== "github") {
153	const conns = yield* ctx.connections	243	return syncFailure(
154	.list({ integration: slug })	244	`GitHub source sync requires a github connection; got "${sourceConnection.integration}"`,
155	.pipe(Effect.orElseSucceed(() => []));	245	connection,
156	if (!conns.some((c) => String(c.name) === String(connName))) {	246	);
157	yield* ctx.connections.create({
158	owner: "user",
159	name: connName,
160	integration: slug,
161	template: AuthTemplateSlug.make("none"),
162	value: "",
163	});
164	}	247	}
165	yield* runtime.deps.store	248	const existing = yield* ctx.connections
166	.putScopeForConnection(String(connName), scope)	249	.get(sourceConnection)
167	.pipe(Effect.orElseSucceed(() => undefined));	250	.pipe(Effect.orElseSucceed(() => null));
168	const normalized = connectionIdentifier(String(connName));	251	if (!existing) return syncFailure(`GitHub connection not found: ${connection}`);
169	if (normalized !== String(connName)) {	252	const token = yield* ctx.connections
170	yield* runtime.deps.store	253	.resolveValue(sourceConnection)
171	.putScopeForConnection(normalized, scope)	254	.pipe(Effect.orElseSucceed(() => null));
172	.pipe(Effect.orElseSucceed(() => undefined));	255	const integration = yield* ctx.core.integrations
		256	.get(sourceConnection.integration)
		257	.pipe(Effect.orElseSucceed(() => null));
		258	const scope = explicitScope ?? sourceScopeFor({ repo, connection });
		259	const result = yield* syncGitHubSource({
		260	runtime,
		261	scope,
		262	repo,
		263	ref,
		264	token,
		265	baseUrl: configBaseUrl(integration?.config),
		266	});
		267	if (result.status === "published") {
		268	const catalog = yield* ensureCatalogConnection(scope, ctx);
		269	yield* ctx.connections.refresh(catalog.ref).pipe(Effect.orElseSucceed(() => []));
173	}	270	}
174	return { scope, connection: String(connName) };	271	return result;
175	}),	272	}),
176	}),	273	}),
177	],	274	],
178	},	275	},
179	],	276	],
180		277
181	resolveTools: ({ connection }: ResolveToolsInput<AppsStoreShape>) =>	278	resolveTools: ({ ctx, connection }: ResolveToolsInput<AppsStoreShape>) =>
182	Effect.gen(function* () {	279	Effect.gen(function* () {
183	const scope = yield* scopeFor(String(connection.name));	280	const scope = yield* scopeFor(String(connection.name));
184	const descriptor = yield* runtime.getDescriptor(scope);	281	const descriptor = yield* runtime.getDescriptor(scope);
185	if (!descriptor) return { tools: [] } satisfies ResolveToolsResult;	282	if (!descriptor) return { tools: [] } satisfies ResolveToolsResult;
		283	const resolver =
		284	makeResolver && ctx ? makeResolver({ ctx, scope, tool: "*" }) : runtime.deps.resolver;
186	const tools: ToolDef[] = [];	285	const tools: ToolDef[] = [];
187	for (const t of descriptor.tools) {	286	for (const t of descriptor.tools) {
188	const byRole: Record<string, readonly ConnectionCandidate[]> = {};	287	const byRole: Record<string, readonly ConnectionCandidate[]> = {};
189	for (const [role, decl] of Object.entries(t.integrations)) {	288	for (const [role, decl] of Object.entries(t.integrations)) {
190	byRole[role] = yield* runtime.deps.resolver	289	byRole[role] = yield* resolver
191	.listConnections({ integration: decl.integration })	290	.listConnections({ integration: decl.integration })
192	.pipe(Effect.orElseSucceed(() => []));	291	.pipe(Effect.orElseSucceed(() => []));
193	}	292	}

packages/plugins/apps/src/plugin/runtime.ts+5−1

   type PublishOutput,
 } from "../pipeline/publish";
 import { PublishError } from "../pipeline/discover";
-import type { AppDescriptor, ToolDescriptor } from "../pipeline/descriptor";
+import type { AppDescriptor, AppSourceRef, ToolDescriptor } from "../pipeline/descriptor";
 import { bundleEntry } from "../pipeline/bundle";
 import {
   buildBridge,
     readonly scope: string;
     readonly files: ReadonlyMap<string, string>;
     readonly message?: string;
+    readonly description?: string;
+    readonly source?: AppSourceRef;
   }) => Effect.Effect<PublishOutput, PublishError>;
   readonly getDescriptor: (scope: string) => Effect.Effect<AppDescriptor | null>;
   /** Re-derive the published-descriptor pointer from the latest committed
                       scope: input.scope,
                       files: input.files,
                       commitMessage: input.message,
+                      description: input.description,
+                      source: input.source,
                     },
                   );
                   yield* putDescriptorPointer(out.descriptor);

packages/plugins/apps/src/source/github-source.test.ts+211−0

@@ -0,0 +1,211 @@
	1	import { mkdtempSync } from "node:fs";
	2	import { tmpdir } from "node:os";
	3	import { join } from "node:path";
	4
	5	import { describe, expect, it } from "vitest";
	6	import { Effect } from "effect";
	7
	8	import { makeSelfHostAppsRuntime } from "../plugin/self-host-runtime";
	9	import { makeInMemoryAppsStore, makeTestResolver } from "../testing";
	10	import { PUBLISH_LIMITS } from "../pipeline/publish";
	11	import { fetchGitHubSource, syncGitHubSource } from "./github-source";
	12
	13	const run = <A, E>(effect: Effect.Effect<A, E>): Promise<A> => Effect.runPromise(effect);
	14
	15	const toolSource = (name = "ok"): string => `import { defineTool } from "executor:app";
	16	import { z } from "zod";
	17	export default defineTool({
	18	description: "${name}",
	19	input: z.object({ value: z.string().default("${name}") }),
	20	async handler(input) { return { value: input.value }; },
	21	});`;
	22
	23	const makeRuntime = () =>
	24	makeSelfHostAppsRuntime({
	25	dataDir: mkdtempSync(join(tmpdir(), "apps-gh-src-")),
	26	store: makeInMemoryAppsStore(),
	27	resolver: makeTestResolver({}),
	28	inMemory: true,
	29	});
	30
	31	const json = (value: unknown, status = 200): Response =>
	32	new Response(JSON.stringify(value), {
	33	status,
	34	headers: { "content-type": "application/json" },
	35	});
	36
	37	const makeGitHubFetch = (input: {
	38	readonly files: ReadonlyMap<string, string>;
	39	readonly upstreamSha?: string;
	40	readonly treeSha?: string;
	41	}) => {
	42	const repoPath = "/repos/acme/tools";
	43	const upstreamSha = input.upstreamSha ?? "commit-1";
	44	const treeSha = input.treeSha ?? "tree-1";
	45	const blobBySha = new Map<string, { path: string; content: string }>();
	46	let index = 0;
	47	for (const [path, content] of input.files) {
	48	blobBySha.set(`blob-${++index}`, { path, content });
	49	}
	50	let blobCalls = 0;
	51	const fetch = (async (rawUrl: string) => {
	52	const url = new URL(rawUrl);
	53	if (url.pathname === repoPath) return json({ default_branch: "main" });
	54	if (url.pathname === `${repoPath}/git/ref/heads%2Fmain`) {
	55	return json({ object: { sha: upstreamSha } });
	56	}
	57	if (url.pathname === `${repoPath}/git/commits/${upstreamSha}`) {
	58	return json({ sha: upstreamSha, tree: { sha: treeSha } });
	59	}
	60	if (url.pathname === `${repoPath}/git/trees/${treeSha}`) {
	61	return json({
	62	tree: [...blobBySha.entries()].map(([sha, blob]) => ({
	63	path: blob.path,
	64	type: "blob",
	65	sha,
	66	size: Buffer.byteLength(blob.content, "utf8"),
	67	})),
	68	});
	69	}
	70	const blobPrefix = `${repoPath}/git/blobs/`;
	71	if (url.pathname.startsWith(blobPrefix)) {
	72	blobCalls++;
	73	const sha = decodeURIComponent(url.pathname.slice(blobPrefix.length));
	74	const blob = blobBySha.get(sha);
	75	if (!blob) return json({ message: "not found" }, 404);
	76	return json({
	77	encoding: "base64",
	78	content: Buffer.from(blob.content, "utf8").toString("base64"),
	79	});
	80	}
	81	return json({ message: "not found" }, 404);
	82	}) as typeof globalThis.fetch;
	83	return {
	84	fetch,
	85	blobCalls: () => blobCalls,
	86	};
	87	};
	88
	89	describe("GitHub custom-tools source", () => {
	90	it("fetches a repo fileset and publishes provenance, description, and skipped entries", async () => {
	91	const files = new Map<string, string>([
	92	[
	93	"executor.json",
	94	JSON.stringify({
	95	$schema: "https://example.test/schema",
	96	description: "Acme tools",
	97	ignored: true,
	98	}),
	99	],
	100	["tools/hello.ts", toolSource("hello")],
	101	["workflows/deferred.ts", "export default {};"],
	102	["docs/readme.md", "ignored"],
	103	]);
	104	const github = makeGitHubFetch({ files, upstreamSha: "commit-a" });
	105	const snapshot = await run(fetchGitHubSource({ repo: "acme/tools", fetch: github.fetch }));
	106	expect([...snapshot.files.keys()].sort()).toEqual([
	107	"executor.json",
	108	"tools/hello.ts",
	109	"workflows/deferred.ts",
	110	]);
	111	expect(snapshot.description).toBe("Acme tools");
	112
	113	const host = makeRuntime();
	114	const result = await run(
	115	syncGitHubSource({
	116	runtime: host.runtime,
	117	scope: "githubTools",
	118	repo: "acme/tools",
	119	fetch: github.fetch,
	120	}),
	121	);
	122	expect(result.status).toBe("published");
	123	expect(result.tools).toEqual(["hello"]);
	124	expect(result.skipped).toEqual([
	125	{ path: "workflows/deferred.ts", reason: "not supported yet" },
	126	]);
	127	const descriptor = await run(host.runtime.getDescriptor("githubTools"));
	128	expect(descriptor?.description).toBe("Acme tools");
	129	expect(descriptor?.source).toEqual({
	130	kind: "github",
	131	repo: "acme/tools",
	132	ref: "main",
	133	upstreamSha: "commit-a",
	134	});
	135	await host.close();
	136	});
	137
	138	it("reports up-to-date when the upstream commit SHA is unchanged", async () => {
	139	const github = makeGitHubFetch({
	140	files: new Map([["tools/hello.ts", toolSource("hello")]]),
	141	upstreamSha: "same-sha",
	142	});
	143	const host = makeRuntime();
	144	const first = await run(
	145	syncGitHubSource({
	146	runtime: host.runtime,
	147	scope: "githubTools",
	148	repo: "acme/tools",
	149	fetch: github.fetch,
	150	}),
	151	);
	152	const second = await run(
	153	syncGitHubSource({
	154	runtime: host.runtime,
	155	scope: "githubTools",
	156	repo: "acme/tools",
	157	fetch: github.fetch,
	158	}),
	159	);
	160	expect(first.status).toBe("published");
	161	expect(second).toEqual({
	162	status: "up-to-date",
	163	upstreamSha: "same-sha",
	164	tools: ["hello"],
	165	skipped: [],
	166	});
	167	await host.close();
	168	});
	169
	170	it("rejects oversized trees before fetching blobs", async () => {
	171	const files = new Map<string, string>();
	172	for (let i = 0; i < PUBLISH_LIMITS.maxFiles + 1; i++) {
	173	files.set(`tools/t${i}.ts`, toolSource(`t${i}`));
	174	}
	175	const github = makeGitHubFetch({ files });
	176	const exit = await Effect.runPromiseExit(
	177	fetchGitHubSource({ repo: "acme/tools", fetch: github.fetch }),
	178	);
	179	expect(exit._tag).toBe("Failure");
	180	expect(JSON.stringify(exit)).toContain("exceeding the limit");
	181	expect(github.blobCalls()).toBe(0);
	182	});
	183
	184	it("returns typed failure data for publish errors", async () => {
	185	const github = makeGitHubFetch({
	186	files: new Map([
	187	[
	188	"tools/bad.ts",
	189	`import { defineTool } from "executor:app";
	190	import { chunk } from "lodash";
	191	export default defineTool({ description: "bad", input: { type: "object" }, async handler(){ return chunk([1], 1); } });`,
	192	],
	193	]),
	194	upstreamSha: "bad-sha",
	195	});
	196	const host = makeRuntime();
	197	const result = await run(
	198	syncGitHubSource({
	199	runtime: host.runtime,
	200	scope: "githubTools",
	201	repo: "acme/tools",
	202	fetch: github.fetch,
	203	}),
	204	);
	205	expect(result.status).toBe("failed");
	206	expect(result.upstreamSha).toBe("bad-sha");
	207	expect(result.errors?.[0]?.stage).toBe("bundle");
	208	expect(result.errors?.[0]?.message).toContain('bare import "lodash" is not allowed');
	209	await host.close();
	210	});
	211	});

packages/plugins/apps/src/source/github-source.ts+402−0

@@ -0,0 +1,402 @@
	1	import { Data, Effect, Result } from "effect";
	2
	3	import { PublishError, type FileDiagnostic, type SkippedArtifact } from "../pipeline/discover";
	4	import { PUBLISH_LIMITS, enforcePublishLimits } from "../pipeline/publish";
	5	import type { AppSourceRef } from "../pipeline/descriptor";
	6	import type { AppsRuntime } from "../plugin/runtime";
	7	import type { FileSet, SnapshotId } from "../seams/artifact-store";
	8
	9	export interface GitHubSourceInput {
	10	readonly repo: string;
	11	readonly ref?: string;
	12	readonly token?: string \| null;
	13	readonly baseUrl?: string;
	14	readonly fetch?: typeof globalThis.fetch;
	15	}
	16
	17	export interface GitHubSourceSnapshot {
	18	readonly files: FileSet;
	19	readonly repo: string;
	20	readonly ref: string;
	21	readonly upstreamSha: string;
	22	readonly description?: string;
	23	}
	24
	25	export class GitHubSourceError extends Data.TaggedError("GitHubSourceError")<{
	26	readonly message: string;
	27	readonly status?: number;
	28	readonly path?: string;
	29	readonly cause?: unknown;
	30	}> {}
	31
	32	export interface SyncErrorData {
	33	readonly stage: "source" \| "discover" \| "bundle" \| "collect" \| "project";
	34	readonly message: string;
	35	readonly diagnostics?: readonly FileDiagnostic[];
	36	}
	37
	38	export type GitHubSyncResult =
	39	\| {
	40	readonly status: "published";
	41	readonly snapshotId: SnapshotId;
	42	readonly upstreamSha: string;
	43	readonly tools: readonly string[];
	44	readonly skipped: readonly SkippedArtifact[];
	45	readonly errors?: undefined;
	46	}
	47	\| {
	48	readonly status: "up-to-date";
	49	readonly upstreamSha: string;
	50	readonly tools: readonly string[];
	51	readonly skipped: readonly SkippedArtifact[];
	52	readonly errors?: undefined;
	53	}
	54	\| {
	55	readonly status: "failed";
	56	readonly upstreamSha?: string;
	57	readonly tools: readonly string[];
	58	readonly skipped: readonly SkippedArtifact[];
	59	readonly errors: readonly SyncErrorData[];
	60	};
	61
	62	export interface SyncGitHubSourceInput extends GitHubSourceInput {
	63	readonly runtime: AppsRuntime;
	64	readonly scope: string;
	65	}
	66
	67	const MATERIALIZED_RE = /^(tools\|workflows\|ui\|skills)\//;
	68
	69	const trimBaseUrl = (baseUrl: string): string => baseUrl.replace(/\/+$/, "");
	70
	71	const repoParts = (repo: string): { owner: string; name: string } \| null => {
	72	const [owner, name, ...rest] = repo.split("/");
	73	if (!owner \|\| !name \|\| rest.length > 0) return null;
	74	return { owner, name };
	75	};
	76
	77	const encodedRepoPath = (repo: string): Effect.Effect<string, GitHubSourceError> => {
	78	const parsed = repoParts(repo);
	79	if (!parsed) {
	80	return Effect.fail(
	81	new GitHubSourceError({
	82	message: `GitHub repo must be "owner/name"; got "${repo}"`,
	83	path: repo,
	84	}),
	85	);
	86	}
	87	return Effect.succeed(`${encodeURIComponent(parsed.owner)}/${encodeURIComponent(parsed.name)}`);
	88	};
	89
	90	const materializes = (path: string): boolean =>
	91	path === "executor.json" \|\| MATERIALIZED_RE.test(path);
	92
	93	const isRecord = (value: unknown): value is Record<string, unknown> =>
	94	value !== null && typeof value === "object" && !Array.isArray(value);
	95
	96	const asString = (value: unknown): string \| undefined =>
	97	typeof value === "string" && value.length > 0 ? value : undefined;
	98
	99	const requestJson = <A>(
	100	input: GitHubSourceInput,
	101	path: string,
	102	): Effect.Effect<A, GitHubSourceError> =>
	103	Effect.tryPromise({
	104	try: async () => {
	105	const fetchImpl = input.fetch ?? globalThis.fetch;
	106	const headers: Record<string, string> = {
	107	accept: "application/vnd.github+json",
	108	"user-agent": "executor-apps-github-source",
	109	};
	110	if (input.token) headers.authorization = `Bearer ${input.token}`;
	111	const response = await fetchImpl(
	112	`${trimBaseUrl(input.baseUrl ?? "https://api.github.com")}${path}`,
	113	{
	114	headers,
	115	},
	116	);
	117	if (!response.ok) {
	118	throw new GitHubSourceError({
	119	message: `GitHub request failed: GET ${path} -> ${response.status}`,
	120	status: response.status,
	121	path,
	122	});
	123	}
	124	return (await response.json()) as A;
	125	},
	126	catch: (cause) =>
	127	cause instanceof GitHubSourceError
	128	? cause
	129	: new GitHubSourceError({
	130	message: `GitHub request failed: GET ${path}`,
	131	path,
	132	cause,
	133	}),
	134	});
	135
	136	interface RepoResponse {
	137	readonly default_branch?: unknown;
	138	}
	139
	140	interface CommitResponse {
	141	readonly sha?: unknown;
	142	readonly commit?: {
	143	readonly tree?: {
	144	readonly sha?: unknown;
	145	};
	146	};
	147	}
	148
	149	interface GitCommitResponse {
	150	readonly sha?: unknown;
	151	readonly tree?: {
	152	readonly sha?: unknown;
	153	};
	154	}
	155
	156	interface RefResponse {
	157	readonly object?: {
	158	readonly sha?: unknown;
	159	};
	160	}
	161
	162	interface TreeEntry {
	163	readonly path?: unknown;
	164	readonly type?: unknown;
	165	readonly sha?: unknown;
	166	readonly size?: unknown;
	167	}
	168
	169	interface TreeResponse {
	170	readonly tree?: readonly TreeEntry[];
	171	readonly truncated?: unknown;
	172	}
	173
	174	interface BlobResponse {
	175	readonly content?: unknown;
	176	readonly encoding?: unknown;
	177	}
	178
	179	const commitTreeSha = (commit: CommitResponse \| GitCommitResponse): string \| undefined => {
	180	if ("commit" in commit) return asString(commit.commit?.tree?.sha);
	181	if ("tree" in commit) return asString(commit.tree?.sha);
	182	return undefined;
	183	};
	184
	185	const limitError = (diagnostics: readonly FileDiagnostic[]): PublishError =>
	186	new PublishError({
	187	message: `publish payload exceeds limits (${diagnostics.length} problem(s))`,
	188	stage: "discover",
	189	diagnostics,
	190	});
	191
	192	const checkTreeLimits = (entries: readonly TreeEntry[]): PublishError \| null => {
	193	const diagnostics: FileDiagnostic[] = [];
	194	if (entries.length > PUBLISH_LIMITS.maxFiles) {
	195	diagnostics.push({
	196	path: "",
	197	message: `publish has ${entries.length} files, exceeding the limit of ${PUBLISH_LIMITS.maxFiles}`,
	198	});
	199	}
	200	let total = 0;
	201	for (const entry of entries) {
	202	const path = String(entry.path ?? "");
	203	const size = typeof entry.size === "number" ? entry.size : 0;
	204	total += size;
	205	if (size > PUBLISH_LIMITS.maxFileBytes) {
	206	diagnostics.push({
	207	path,
	208	message: `file is ${size} bytes, exceeding the per-file limit of ${PUBLISH_LIMITS.maxFileBytes} bytes`,
	209	});
	210	}
	211	}
	212	if (total > PUBLISH_LIMITS.maxTotalBytes) {
	213	diagnostics.push({
	214	path: "",
	215	message: `publish total is ${total} bytes, exceeding the total limit of ${PUBLISH_LIMITS.maxTotalBytes} bytes`,
	216	});
	217	}
	218	return diagnostics.length === 0 ? null : limitError(diagnostics);
	219	};
	220
	221	const decodeBlob = (path: string, blob: BlobResponse): Effect.Effect<string, GitHubSourceError> =>
	222	Effect.try({
	223	try: () => {
	224	const encoding = asString(blob.encoding);
	225	const content = asString(blob.content);
	226	if (!content \|\| encoding !== "base64") {
	227	throw new GitHubSourceError({
	228	message: `GitHub blob ${path} did not return base64 content`,
	229	path,
	230	});
	231	}
	232	return Buffer.from(content.replace(/\s/g, ""), "base64").toString("utf8");
	233	},
	234	catch: (cause) =>
	235	cause instanceof GitHubSourceError
	236	? cause
	237	: new GitHubSourceError({ message: `Failed to decode GitHub blob ${path}`, path, cause }),
	238	});
	239
	240	const executorDescription = (
	241	files: FileSet,
	242	): Effect.Effect<string \| undefined, GitHubSourceError> =>
	243	Effect.try({
	244	try: () => {
	245	const raw = files.get("executor.json");
	246	if (!raw) return undefined;
	247	const parsed = JSON.parse(raw) as unknown;
	248	if (!isRecord(parsed)) return undefined;
	249	const description = parsed.description;
	250	return typeof description === "string" ? description : undefined;
	251	},
	252	catch: (cause) =>
	253	new GitHubSourceError({
	254	message: "executor.json is not valid JSON",
	255	path: "executor.json",
	256	cause,
	257	}),
	258	});
	259
	260	export const fetchGitHubSource = (
	261	input: GitHubSourceInput,
	262	): Effect.Effect<GitHubSourceSnapshot, GitHubSourceError \| PublishError> =>
	263	Effect.gen(function* () {
	264	const repoPath = yield* encodedRepoPath(input.repo);
	265	const repo = yield* requestJson<RepoResponse>(input, `/repos/${repoPath}`);
	266	const ref = input.ref ?? asString(repo.default_branch) ?? "main";
	267	const branchRef = yield* requestJson<RefResponse>(
	268	input,
	269	`/repos/${repoPath}/git/ref/${encodeURIComponent(`heads/${ref}`)}`,
	270	).pipe(Effect.result);
	271	const commitPath =
	272	Result.isSuccess(branchRef) && asString(branchRef.success.object?.sha)
	273	? `/repos/${repoPath}/git/commits/${encodeURIComponent(String(branchRef.success.object?.sha))}`
	274	: `/repos/${repoPath}/commits/${encodeURIComponent(ref)}`;
	275	const commit = yield* requestJson<CommitResponse \| GitCommitResponse>(input, commitPath);
	276	const upstreamSha = asString(commit.sha);
	277	const treeSha = commitTreeSha(commit);
	278	if (!upstreamSha \|\| !treeSha) {
	279	return yield* new GitHubSourceError({
	280	message: `GitHub commit ${ref} did not include a commit SHA and tree SHA`,
	281	path: commitPath,
	282	});
	283	}
	284
	285	const tree = yield* requestJson<TreeResponse>(
	286	input,
	287	`/repos/${repoPath}/git/trees/${encodeURIComponent(treeSha)}?recursive=1`,
	288	);
	289	if (tree.truncated === true) {
	290	return yield* new GitHubSourceError({
	291	message: "GitHub returned a truncated repository tree; custom tools source is too large",
	292	path: `/repos/${repoPath}/git/trees/${treeSha}`,
	293	});
	294	}
	295	const entries = (tree.tree ?? []).filter(
	296	(entry) =>
	297	entry.type === "blob" &&
	298	typeof entry.path === "string" &&
	299	typeof entry.sha === "string" &&
	300	materializes(entry.path),
	301	);
	302	const treeLimitError = checkTreeLimits(entries);
	303	if (treeLimitError) return yield* Effect.fail(treeLimitError);
	304
	305	const files = new Map<string, string>();
	306	for (const entry of entries) {
	307	const path = String(entry.path);
	308	const blob = yield* requestJson<BlobResponse>(
	309	input,
	310	`/repos/${repoPath}/git/blobs/${encodeURIComponent(String(entry.sha))}`,
	311	);
	312	files.set(path, yield* decodeBlob(path, blob));
	313	}
	314	const payloadLimitError = enforcePublishLimits(files);
	315	if (payloadLimitError) return yield* Effect.fail(payloadLimitError);
	316
	317	return {
	318	files,
	319	repo: input.repo,
	320	ref,
	321	upstreamSha,
	322	description: yield* executorDescription(files),
	323	};
	324	});
	325
	326	const publishErrorToSyncError = (error: PublishError): SyncErrorData => ({
	327	stage: error.stage,
	328	message: error.message,
	329	diagnostics: error.diagnostics,
	330	});
	331
	332	const sourceErrorToSyncError = (error: GitHubSourceError): SyncErrorData => ({
	333	stage: "source",
	334	message: error.message,
	335	diagnostics: error.path ? [{ path: error.path, message: error.message }] : [],
	336	});
	337
	338	const sourceRef = (snapshot: GitHubSourceSnapshot): AppSourceRef => ({
	339	kind: "github",
	340	repo: snapshot.repo,
	341	ref: snapshot.ref,
	342	upstreamSha: snapshot.upstreamSha,
	343	});
	344
	345	export const syncGitHubSource = (input: SyncGitHubSourceInput): Effect.Effect<GitHubSyncResult> =>
	346	Effect.gen(function* () {
	347	const fetched = yield* fetchGitHubSource(input).pipe(Effect.result);
	348	if (Result.isFailure(fetched)) {
	349	const error = fetched.failure;
	350	return {
	351	status: "failed",
	352	tools: [],
	353	skipped: [],
	354	errors: [
	355	error instanceof PublishError
	356	? publishErrorToSyncError(error)
	357	: sourceErrorToSyncError(error),
	358	],
	359	} satisfies GitHubSyncResult;
	360	}
	361	const snapshot = fetched.success;
	362	const current = yield* input.runtime.getDescriptor(input.scope);
	363	if (
	364	current?.source?.kind === "github" &&
	365	current.source.repo === snapshot.repo &&
	366	current.source.upstreamSha === snapshot.upstreamSha
	367	) {
	368	return {
	369	status: "up-to-date",
	370	upstreamSha: snapshot.upstreamSha,
	371	tools: current.tools.map((tool) => tool.name),
	372	skipped: current.skipped ?? [],
	373	} satisfies GitHubSyncResult;
	374	}
	375
	376	const published = yield* input.runtime
	377	.publish({
	378	scope: input.scope,
	379	files: snapshot.files,
	380	description: snapshot.description,
	381	source: sourceRef(snapshot),
	382	message: `sync ${snapshot.repo}@${snapshot.upstreamSha}`,
	383	})
	384	.pipe(Effect.result);
	385	if (Result.isFailure(published)) {
	386	return {
	387	status: "failed",
	388	upstreamSha: snapshot.upstreamSha,
	389	tools: [],
	390	skipped: [],
	391	errors: [publishErrorToSyncError(published.failure)],
	392	} satisfies GitHubSyncResult;
	393	}
	394
	395	return {
	396	status: "published",
	397	snapshotId: published.success.snapshotId,
	398	upstreamSha: snapshot.upstreamSha,
	399	tools: published.success.descriptor.tools.map((tool) => tool.name),
	400	skipped: published.success.skipped,
	401	} satisfies GitHubSyncResult;
	402	});