Package provider service split migration

Rhys Sullivan committed 4 months ago

Commit df7ca6e1 parent a75506527 files · +303 −159Browse files

Unified · Split

27 files

apps/host-cloudflare/package.json +1 −0
apps/host-cloudflare/src/db/data-migrations.ts +2 −2
apps/host-cloudflare/src/worker.e2e.node.test.ts +39 −3
apps/host-selfhost/package.json +1 −0
apps/host-selfhost/src/db/data-migrations.ts +1 −1
apps/local/package.json +1 −0
apps/local/src/db/data-migrations.ts +1 −1
bun.lock +25 −0
e2e/scenarios/google-photos-preset-ui.test.ts +22 −21
e2e/scenarios/microsoft-emulator.test.ts +10 −5
e2e/scenarios/microsoft-graph-default.test.ts +22 −24
e2e/scenarios/microsoft-graph-full.test.ts +32 −33
e2e/scenarios/oauth-client-handoff.test.ts +11 −6
e2e/scenarios/provider-plugins-ui.test.ts +29 −39
package.json +1 −0
packages/plugins/provider-service-split/CHANGELOG.md +7 −0
packages/plugins/provider-service-split/package.json +33 −0
packages/plugins/provider-service-split/src/index.ts +2 −0
packages/plugins/provider-service-split/src/planner.test.ts +3 −6
packages/plugins/provider-service-split/src/planner.ts +4 −6
packages/plugins/provider-service-split/src/sqlite.test.ts +5 −5
packages/plugins/provider-service-split/src/sqlite.ts +3 −5
packages/plugins/provider-service-split/tsconfig.json +23 −0
packages/plugins/provider-service-split/tsup.config.ts +12 −0
packages/plugins/provider-service-split/vitest.config.ts +8 −0
scripts/migration/service-split-postgres-cli.ts +1 −1
tests/migration/service-split-postgres-cli.test.ts +4 −1

@@ -29,6 +29,7 @@ · 28 unmodified lines
29	29	"@executor-js/plugin-mcp": "workspace:*",
30	30	"@executor-js/plugin-microsoft": "workspace:*",
31	31	"@executor-js/plugin-openapi": "workspace:*",
	32	+ "@executor-js/plugin-provider-service-split": "workspace:*",
32	33	"@executor-js/plugin-toolkits": "workspace:*",
33	34	"@executor-js/react": "workspace:*",
34	35	"@executor-js/runtime-quickjs": "workspace:*",

apps/host-cloudflare/src/db/data-migrations.ts+2−2

@@ -12,8 +12,8 @@ import { googleOpenApiOwnershipDataMigration } from "@executor-js/plugin-google" · 11 unmodified lines
12	12	import {
13	13	providerServiceSplitDataMigration,
14	14	runSqliteProviderServiceSplitMigration,
15		−} from "../../../../scripts/migration/service-split-sqlite";
16		−import type { BlobRow } from "../../../../scripts/migration/service-split-planner";
	15	+ type BlobRow,
	16	+} from "@executor-js/plugin-provider-service-split";
17	17
18	18	const TX_CONTROL = new Set(["BEGIN", "BEGIN TRANSACTION", "COMMIT", "ROLLBACK"]);
19	19

apps/host-cloudflare/src/worker.e2e.node.test.ts+39−3

@@ -6,6 +6,7 @@ import { fileURLToPath } from "node:url"; · 5 unmodified lines
6	6	import { afterAll, beforeAll, describe, expect, it } from "@effect/vitest";
7	7	import { Schema } from "effect";
8	8	import { unstable_dev, type Unstable_DevWorker } from "wrangler";
	9	+import { microsoftCatalog } from "@executor-js/plugin-microsoft";
9	10
10	11	// ---------------------------------------------------------------------------
11	12	// End-to-end test for the Cloudflare host: boots the REAL worker on workerd via
@@ -167,13 +168,48 @@ describe("cloudflare host e2e (workerd/miniflare)", () => { · 155 unmodified lines
167	168	expect(integration?.slug).toBe(slug);
168	169	}, 60_000);
169	170
170		− it("registers the Microsoft Graph API route in the runtime plugin set", async () => {
171		− const res = await worker.fetch("/api/microsoft/graph", {
	171	+ it("exposes Microsoft catalog presets through the OpenAPI integration catalog", async () => {
	172	+ const microsoftFilesPreset = microsoftCatalog.find(
	173	+ (preset) => preset.defaultSlug === "microsoft_files",
	174	+ );
	175	+ expect(microsoftFilesPreset).toBeTruthy();
	176	+
	177	+ const removedProviderRoute = await worker.fetch("/api/microsoft/graph", {
172	178	method: "POST",
173	179	headers: { "content-type": "application/json" },
174	180	body: JSON.stringify({}),
175	181	});
176		− expect(res.status).toBe(400);
	182	+ expect(removedProviderRoute.status).toBe(404);
	183	+
	184	+ const slug = `microsoft-files-${runId}`;
	185	+ const add = await worker.fetch("/api/openapi/specs", {
	186	+ method: "POST",
	187	+ headers: { "content-type": "application/json" },
	188	+ body: JSON.stringify({
	189	+ spec: { kind: "blob", value: SPEC },
	190	+ slug,
	191	+ name: microsoftFilesPreset?.name,
	192	+ description: microsoftFilesPreset?.summary,
	193	+ family: microsoftFilesPreset?.family,
	194	+ authenticationTemplate: microsoftFilesPreset?.authTemplate,
	195	+ }),
	196	+ });
	197	+ expect(add.status).toBe(200);
	198	+
	199	+ const res = await worker.fetch(`/api/integrations/${slug}`);
	200	+ expect(res.status).toBe(200);
	201	+ const integration = (await res.json()) as {
	202	+ readonly slug: string;
	203	+ readonly kind: string;
	204	+ readonly family?: string;
	205	+ readonly authMethods: readonly { readonly template: string; readonly kind: string }[];
	206	+ };
	207	+ expect(integration.slug).toBe(slug);
	208	+ expect(integration.kind).toBe("openapi");
	209	+ expect(integration.family).toBe("microsoft");
	210	+ expect(integration.authMethods.some((method) => method.template === "azureAdDelegated")).toBe(
	211	+ true,
	212	+ );
177	213	});
178	214
179	215	it("gates the API when dev-auth is on but treats the request as the dev admin", async () => {

apps/host-selfhost/package.json+1−0

@@ -32,6 +32,7 @@ · 31 unmodified lines
32	32	"@executor-js/plugin-mcp": "workspace:*",
33	33	"@executor-js/plugin-microsoft": "workspace:*",
34	34	"@executor-js/plugin-openapi": "workspace:*",
	35	+ "@executor-js/plugin-provider-service-split": "workspace:*",
35	36	"@executor-js/plugin-toolkits": "workspace:*",
36	37	"@executor-js/react": "workspace:*",
37	38	"@executor-js/runtime-quickjs": "workspace:*",

apps/host-selfhost/src/db/data-migrations.ts+1−1

@@ -14,7 +14,7 @@ import { · 13 unmodified lines
14	14	import { graphqlIntrospectionBlobDataMigration } from "@executor-js/plugin-graphql";
15	15	import { googleOpenApiOwnershipDataMigration } from "@executor-js/plugin-google";
16	16
17		−import { providerServiceSplitDataMigration } from "../../../../scripts/migration/service-split-sqlite";
	17	+import { providerServiceSplitDataMigration } from "@executor-js/plugin-provider-service-split";
18	18	import { authConfigTransforms } from "./auth-config-migration";
19	19
20	20	export const selfHostDataMigrations: readonly SqliteDataMigration[] = [

apps/local/package.json+1−0

@@ -39,6 +39,7 @@ · 38 unmodified lines
39	39	"@executor-js/plugin-microsoft": "workspace:*",
40	40	"@executor-js/plugin-onepassword": "workspace:*",
41	41	"@executor-js/plugin-openapi": "workspace:*",
	42	+ "@executor-js/plugin-provider-service-split": "workspace:*",
42	43	"@executor-js/plugin-toolkits": "workspace:*",
43	44	"@executor-js/react": "workspace:*",
44	45	"@executor-js/runtime-quickjs": "workspace:*",

apps/local/src/db/data-migrations.ts+1−1

@@ -19,7 +19,7 @@ import { · 18 unmodified lines
19	19	import { graphqlIntrospectionBlobDataMigration } from "@executor-js/plugin-graphql";
20	20	import { googleOpenApiOwnershipDataMigration } from "@executor-js/plugin-google";
21	21
22		−import { providerServiceSplitDataMigration } from "../../../../scripts/migration/service-split-sqlite";
	22	+import { providerServiceSplitDataMigration } from "@executor-js/plugin-provider-service-split";
23	23	import { authConfigTransforms } from "./auth-config-migration";
24	24	import { LOCAL_V1_V2_LEDGER_NAME } from "./v1-v2-migration";
25	25

bun.lock+25−0

@@ -10,6 +10,7 @@ · 9 unmodified lines
10	10	"@effect/language-service": "^0.85.1",
11	11	"@effect/tsgo": "^0.5.2",
12	12	"@effect/vitest": "catalog:",
	13	+ "@executor-js/plugin-provider-service-split": "workspace:*",
13	14	"@typescript/native-preview": "^7.0.0-dev.20260410.1",
14	15	"@vitest/expect": "catalog:",
15	16	"@vitest/mocker": "catalog:",
@@ -181,6 +182,7 @@ · 165 unmodified lines
181	182	"@executor-js/plugin-mcp": "workspace:*",
182	183	"@executor-js/plugin-microsoft": "workspace:*",
183	184	"@executor-js/plugin-openapi": "workspace:*",
	185	+ "@executor-js/plugin-provider-service-split": "workspace:*",
184	186	"@executor-js/plugin-toolkits": "workspace:*",
185	187	"@executor-js/react": "workspace:*",
186	188	"@executor-js/runtime-quickjs": "workspace:*",
@@ -230,6 +232,7 @@ · 43 unmodified lines
230	232	"@executor-js/plugin-mcp": "workspace:*",
231	233	"@executor-js/plugin-microsoft": "workspace:*",
232	234	"@executor-js/plugin-openapi": "workspace:*",
	235	+ "@executor-js/plugin-provider-service-split": "workspace:*",
233	236	"@executor-js/plugin-toolkits": "workspace:*",
234	237	"@executor-js/react": "workspace:*",
235	238	"@executor-js/runtime-quickjs": "workspace:*",
@@ -283,6 +286,7 @@ · 47 unmodified lines
283	286	"@executor-js/plugin-microsoft": "workspace:*",
284	287	"@executor-js/plugin-onepassword": "workspace:*",
285	288	"@executor-js/plugin-openapi": "workspace:*",
	289	+ "@executor-js/plugin-provider-service-split": "workspace:*",
286	290	"@executor-js/plugin-toolkits": "workspace:*",
287	291	"@executor-js/react": "workspace:*",
288	292	"@executor-js/runtime-quickjs": "workspace:*",
@@ -1095,6 +1099,25 @@ · 806 unmodified lines
1095	1099	"react",
1096	1100	],
1097	1101	},
	1102	+ "packages/plugins/provider-service-split": {
	1103	+ "name": "@executor-js/plugin-provider-service-split",
	1104	+ "version": "0.0.0",
	1105	+ "dependencies": {
	1106	+ "@executor-js/plugin-google": "workspace:*",
	1107	+ "@executor-js/plugin-microsoft": "workspace:*",
	1108	+ "@executor-js/plugin-openapi": "workspace:*",
	1109	+ "@executor-js/sdk": "workspace:*",
	1110	+ "effect": "catalog:",
	1111	+ },
	1112	+ "devDependencies": {
	1113	+ "@effect/vitest": "catalog:",
	1114	+ "@types/node": "catalog:",
	1115	+ "bun-types": "catalog:",
	1116	+ "tsup": "catalog:",
	1117	+ "typescript": "catalog:",
	1118	+ "vitest": "catalog:",
	1119	+ },
	1120	+ },
1098	1121	"packages/plugins/toolkits": {
1099	1122	"name": "@executor-js/plugin-toolkits",
1100	1123	"version": "1.5.21",
@@ -1797,6 +1820,8 @@ · 696 unmodified lines
1797	1820
1798	1821	"@executor-js/plugin-openapi": ["@executor-js/plugin-openapi@workspace:packages/plugins/openapi"],
1799	1822
	1823	+ "@executor-js/plugin-provider-service-split": ["@executor-js/plugin-provider-service-split@workspace:packages/plugins/provider-service-split"],
	1824	+
1800	1825	"@executor-js/plugin-toolkits": ["@executor-js/plugin-toolkits@workspace:packages/plugins/toolkits"],
1801	1826
1802	1827	"@executor-js/plugin-workos-vault": ["@executor-js/plugin-workos-vault@workspace:packages/plugins/workos-vault"],

e2e/scenarios/google-photos-preset-ui.test.ts+22−21

@@ -5,7 +5,7 @@ import { scenario } from "../src/scenario"; · 4 unmodified lines
5	5	import { Browser, Target } from "../src/services";
6	6
7	7	scenario(
8		− "Google Photos: the focused preset opens a Photos-scoped add flow",
	8	+ "Google Photos: separated catalog presets open a Photos service add flow",
9	9	{},
10	10	Effect.gen(function* () {
11	11	const target = yield* Target;
@@ -13,30 +13,31 @@ scenario( · 1 unmodified line
13	13	const identity = yield* target.newIdentity();
14	14
15	15	yield* browser.session(identity, async ({ page, step }) => {
16		− await step("Find the Google Photos preset from the integrations picker", async () => {
17		− await page.goto("/integrations", { waitUntil: "networkidle" });
18		− await page.getByRole("button", { name: "Connect" }).click();
19		− const dialog = page.getByRole("dialog", { name: "Connect an integration" });
20		− await dialog.waitFor();
21		− await dialog.getByPlaceholder(/Search or paste a URL/).fill("google photos");
22		− await dialog.getByRole("link", { name: /^Google Photos\b/ }).waitFor();
23		− });
	16	+ await step(
	17	+ "Find the separated Google Photos presets from the integrations picker",
	18	+ async () => {
	19	+ await page.goto("/integrations", { waitUntil: "networkidle" });
	20	+ await page.getByRole("button", { name: "Connect" }).click();
	21	+ const dialog = page.getByRole("dialog", { name: "Connect an integration" });
	22	+ await dialog.waitFor();
	23	+ await dialog.getByPlaceholder(/Search or paste a URL/).fill("google photos");
	24	+ await dialog.getByRole("link", { name: /^Google Photos Library\b/ }).waitFor();
	25	+ await dialog.getByRole("link", { name: /^Google Photos Picker\b/ }).waitFor();
	26	+ },
	27	+ );
24	28
25		− await step("Open the Google Photos scoped add flow", async () => {
	29	+ await step("Open the Google Photos Library add flow", async () => {
26	30	const dialog = page.getByRole("dialog", { name: "Connect an integration" });
27		− await dialog.getByRole("link", { name: /^Google Photos\b/ }).click();
28		− await page.waitForURL(/\/integrations\/add\/google/);
29		− await page.getByRole("heading", { name: "Add Google integration" }).waitFor();
	31	+ await dialog.getByRole("link", { name: /^Google Photos Library\b/ }).click();
	32	+ await page.waitForURL(/\/integrations\/add\/openapi/);
	33	+ await page.getByRole("heading", { name: "Add OpenAPI integration" }).waitFor();
30	34	});
31	35
32		− await step("The Photos preset defaults to the focused namespace and products", async () => {
33		− await page.locator('input[value="Google Photos"]').waitFor();
34		− await page.locator('input[value="google_photos"]').waitFor();
35		− await page.getByText("Google Photos Library").first().waitFor();
36		− await page.getByText("Google Photos Picker").first().waitFor();
37		− await page.getByText("2 Google APIs").waitFor();
38		− expect(await page.locator('input[value="Google"]').count()).toBe(0);
39		− expect(await page.locator('input[value="google"]').count()).toBe(0);
	36	+ await step("The Library link carries the focused OpenAPI preset", async () => {
	37	+ const url = new URL(page.url());
	38	+ expect(url.searchParams.get("preset")).toBe("google-photos-library");
	39	+ expect(url.searchParams.get("url")).toContain("photoslibrary");
	40	+ await expect.poll(() => page.locator("textarea").inputValue()).toContain("photoslibrary");
40	41	});
41	42	});
42	43	}),

e2e/scenarios/microsoft-emulator.test.ts+10−5

@@ -86,10 +86,7 @@ return { ok: result.ok, path: item.path, result: result.ok ? result.data : resul · 85 unmodified lines
86	86
87	87	scenario(
88	88	"Microsoft · client credentials against the emulator mint a Graph connection and call /users",
89		− {
90		− skip: "phase 3 will replace this deleted microsoft.addGraph scenario with catalog-driven Microsoft e2e coverage",
91		− timeout: 180_000,
92		− },
	89	+ { timeout: 180_000 },
93	90	Effect.scoped(
94	91	Effect.gen(function* () {
95	92	const target = yield* Target;
@@ -115,7 +112,15 @@ scenario( · 19 unmodified lines
115	112	name: "Microsoft Graph Emulator",
116	113	baseUrl: emulator.url,
117	114	family: "microsoft",
118		− specFormat: "microsoft-graph",
	115	+ authenticationTemplate: [
	116	+ {
	117	+ slug: MICROSOFT_CLIENT_CREDENTIALS_AUTH_TEMPLATE_SLUG,
	118	+ kind: "oauth2",
	119	+ authorizationUrl: oauth.authorizationUrl,
	120	+ tokenUrl: oauth.tokenUrl,
	121	+ scopes: ["https://graph.microsoft.com/.default"],
	122	+ },
	123	+ ],
119	124	},
120	125	});
121	126

e2e/scenarios/microsoft-graph-default.test.ts+22−24

@@ -6,7 +6,6 @@ import { composePluginApi } from "@executor-js/api/server"; · 5 unmodified lines
6	6	import {
7	7	MICROSOFT_AUTH_TEMPLATE_SLUG,
8	8	MICROSOFT_AUTHORIZATION_URL,
9		− MICROSOFT_GRAPH_DEFAULT_PRESET_IDS,
10	9	MICROSOFT_GRAPH_OPENAPI_URL,
11	10	MICROSOFT_TOKEN_URL,
12	11	microsoftCatalog,
@@ -32,17 +31,17 @@ type ToolView = { · 19 unmodified lines
32	31	};
33	32
34	33	const unique = (prefix: string) => `${prefix}_${randomBytes(4).toString("hex")}`;
	34	+const MICROSOFT_FILES_PRESET_ID = "files";
35	35
36		−// Compiling the ~37MB Graph spec inside dev workerd needs more headroom than
37		−// GitHub's 2-core runners have: /api/microsoft/graph 500s and the dev stack is
38		−// dead for every scenario after it in the shard. Local runs (and the
39		−// production Workers streaming path) are unaffected — CI-only quarantine.
	36	+// Compiling the Graph spec inside dev workerd needs more headroom than GitHub's
	37	+// 2-core runners have. Local runs and the production Workers streaming path are
	38	+// unaffected, so CI keeps this narrow catalog scenario quarantined.
40	39	const CI_GRAPH_SPEC_SKIP = process.env.CI
41	40	? "compiling the full Microsoft Graph spec exhausts the 2-core CI runner and kills the dev stack for the rest of the shard"
42	41	: undefined;
43	42
44	43	scenario(
45		− "Microsoft Graph: default add stores common Microsoft 365 workloads",
	44	+ "Microsoft Graph: catalog add stores the OneDrive files service",
46	45	{ timeout: 180_000, skip: CI_GRAPH_SPEC_SKIP },
47	46	Effect.gen(function* () {
48	47	const target = yield* Target;
@@ -50,7 +49,7 @@ scenario( · 1 unmodified line
50	49	const identity = yield* target.newIdentity();
51	50	const client = yield* makeApiClient(api, identity);
52	51
53		− const integration = unique("msgraph_default");
	52	+ const integration = unique("msgraph_files");
54	53	const connection = ConnectionName.make("main");
55	54	const oauthClient = OAuthClientSlug.make(unique("msgraph_app"));
56	55
@@ -60,10 +59,10 @@ scenario( · 3 unmodified lines
60	59	payload: {
61	60	spec: {
62	61	kind: "url",
63		− url: `${MICROSOFT_GRAPH_OPENAPI_URL}#preset=${MICROSOFT_GRAPH_DEFAULT_PRESET_IDS[0]}`,
	62	+ url: `${MICROSOFT_GRAPH_OPENAPI_URL}#preset=${MICROSOFT_FILES_PRESET_ID}`,
64	63	},
65	64	slug: integration,
66		− name: "Microsoft Graph Defaults",
	65	+ name: "Microsoft Graph Files",
67	66	family: "microsoft",
68	67	specFormat: "microsoft-graph",
69	68	},
@@ -71,8 +70,8 @@ scenario( · 1 unmodified line
71	70	expect(added.slug, "the Microsoft Graph source keeps the requested slug").toBe(integration);
72	71	expect(
73	72	added.toolCount,
74		− "the default Microsoft Graph add extracts common user-facing operations",
75		− ).toBeGreaterThan(100);
	73	+ "the Microsoft files catalog preset extracts a focused operation subtree",
	74	+ ).toBeGreaterThan(10);
76	75
77	76	const config = yield* client.openapi.getConfig({
78	77	params: { slug: integration },
@@ -82,10 +81,9 @@ scenario( · 3 unmodified lines
82	81	? [...template.scopes]
83	82	: [],
84	83	);
85		− expect(
86		− delegatedScopes,
87		− "default delegated OAuth asks for common workload scopes",
88		− ).toContain("User.Read");
	84	+ expect(delegatedScopes, "the files delegated OAuth asks for file scopes").toContain(
	85	+ "Files.ReadWrite.All",
	86	+ );
89	87
90	88	yield* client.oauth.createClient({
91	89	payload: {
@@ -116,11 +114,11 @@ scenario( · 24 unmodified lines
116	114	const authorizeUrl = new URL(authorizationUrl \|\| "https://invalid.example");
117	115	expect(
118	116	authorizeUrl.toString().length,
119		− "default Microsoft Graph OAuth authorize URLs stay under ordinary proxy limits",
	117	+ "Microsoft files OAuth authorize URLs stay under ordinary proxy limits",
120	118	).toBeLessThan(2_000);
121	119	expect(
122	120	authorizeUrl.searchParams.get("scope"),
123		− "default Microsoft Graph OAuth asks for common workload scopes",
	121	+ "Microsoft files OAuth asks for the focused scope set",
124	122	).toBe(delegatedScopes?.join(" "));
125	123
126	124	yield* client.connections.create({
@@ -137,15 +135,15 @@ scenario( · 10 unmodified lines
137	135	query: { integration: IntegrationSlug.make(integration), connection },
138	136	});
139	137	const names = tools.map((tool: ToolView) => tool.name);
140		− const messageTools = names.filter((name) => name.toLowerCase().includes("message"));
141		− const siteTools = names.filter((name) => name.toLowerCase().includes("site"));
142		− expect(
143		− messageTools,
144		− "the retrieved catalog includes Microsoft message operations",
145		− ).not.toEqual([]);
146		− expect(siteTools, "the retrieved catalog includes SharePoint site operations").not.toEqual(
	138	+ const driveTools = names.filter((name) => name.toLowerCase().includes("drive"));
	139	+ const shareTools = names.filter((name) => name.toLowerCase().includes("share"));
	140	+ expect(driveTools, "the retrieved catalog includes Microsoft drive operations").not.toEqual(
147	141	[],
148	142	);
	143	+ expect(
	144	+ shareTools,
	145	+ "the retrieved catalog includes Microsoft sharing operations",
	146	+ ).not.toEqual([]);
149	147	}),
150	148	Effect.gen(function* () {
151	149	yield* client.connections

e2e/scenarios/microsoft-graph-full.test.ts+32−33

@@ -5,8 +5,6 @@ import { Effect } from "effect"; · 4 unmodified lines
5	5	import { composePluginApi } from "@executor-js/api/server";
6	6	import {
7	7	MICROSOFT_AUTH_TEMPLATE_SLUG,
8		− MICROSOFT_GRAPH_ALL_PRESET_IDS,
9		− MICROSOFT_GRAPH_DELEGATED_DEFAULT_SCOPES,
10	8	MICROSOFT_GRAPH_OPENAPI_URL,
11	9	microsoftCatalog,
12	10	microsoftGraphAdapter,
@@ -26,18 +24,21 @@ type ToolView = { · 13 unmodified lines
26	24	};
27	25
28	26	const unique = (prefix: string) => `${prefix}_${randomBytes(4).toString("hex")}`;
	27	+const MICROSOFT_FILES_PRESET_ID = "files";
	28	+const MICROSOFT_FILES_DELEGATED_SCOPES = [
	29	+ "offline_access",
	30	+ "User.Read",
	31	+ "Files.ReadWrite.All",
	32	+ "Sites.ReadWrite.All",
	33	+] as const;
29	34
30		−// Adding every Graph workload pulls the full Microsoft Graph OpenAPI document
31		−// (~37MB, ~16.5k operations) and persists a binding per operation. That whole-
32		−// document path used to 503 on the Cloudflare worker: parsing the spec, and
33		−// then re-parsing it on every tools/list, each rebuilt a ~300MB JS tree that
34		−// blew the 128MB isolate. This scenario is the regression guard for both sites
35		−// at real scale: the add streams the compile + persist, and tools/list serves
36		−// the catalog back from the persisted bindings (+ the content-addressed defs
37		−// blob) without ever re-parsing the spec. It drives only the public API, so a
38		−// green run is evidence the full catalog lands and serves end to end.
	35	+// Adding a catalog service extracts only that service's Microsoft Graph subtree
	36	+// and persists a binding per operation. This is the regression guard for both
	37	+// former worker pressure sites: the add streams compile and persist, and
	38	+// tools/list serves from persisted bindings plus the content-addressed defs blob
	39	+// without re-parsing the Graph spec.
39	40	scenario(
40		− "Microsoft Graph: the full catalog adds and serves without re-parsing the spec",
	41	+ "Microsoft Graph: the files catalog service adds and serves without re-parsing the spec",
41	42	{ timeout: 300_000 },
42	43	Effect.gen(function* () {
43	44	const target = yield* Target;
@@ -45,30 +46,30 @@ scenario( · 1 unmodified line
45	46	const identity = yield* target.newIdentity();
46	47	const client = yield* makeApiClient(api, identity);
47	48
48		− const integration = unique("msgraph_full");
	49	+ const integration = unique("msgraph_files");
49	50	const connection = ConnectionName.make("main");
50	51
51	52	yield* Effect.ensuring(
52	53	Effect.gen(function* () {
53		− // Add path (1st former OOM site): the full spec is fetched and
54		− // stream-compiled into one persisted binding per operation.
	54	+ // Add path, first former OOM site: the Graph spec is fetched and
	55	+ // stream-compiled into one persisted binding per selected operation.
55	56	const added = yield* client.openapi.addSpec({
56	57	payload: {
57	58	spec: {
58	59	kind: "url",
59		− url: `${MICROSOFT_GRAPH_OPENAPI_URL}#preset=${MICROSOFT_GRAPH_ALL_PRESET_IDS[0]}`,
	60	+ url: `${MICROSOFT_GRAPH_OPENAPI_URL}#preset=${MICROSOFT_FILES_PRESET_ID}`,
60	61	},
61	62	slug: integration,
62		− name: "Microsoft Graph (full)",
	63	+ name: "Microsoft Graph Files",
63	64	family: "microsoft",
64	65	specFormat: "microsoft-graph",
65	66	},
66	67	});
67		− expect(added.slug, "the full Graph source keeps the requested slug").toBe(integration);
	68	+ expect(added.slug, "the Microsoft files source keeps the requested slug").toBe(integration);
68	69	expect(
69	70	added.toolCount,
70		− "adding every Graph workload extracts the whole catalog (thousands of operations)",
71		− ).toBeGreaterThan(5_000);
	71	+ "adding the files catalog service extracts a focused Graph operation subtree",
	72	+ ).toBeGreaterThan(10);
72	73
73	74	const config = yield* client.openapi.getConfig({ params: { slug: integration } });
74	75	const delegatedScopes = config?.authenticationTemplate?.flatMap((template) =>
@@ -78,8 +79,8 @@ scenario( · 3 unmodified lines
78	79	);
79	80	expect(
80	81	delegatedScopes,
81		− "full Graph delegates the app-registration default scope set",
82		− ).toEqual([...MICROSOFT_GRAPH_DELEGATED_DEFAULT_SCOPES]);
	82	+ "the files service delegates only the file-service scope set",
	83	+ ).toEqual([...MICROSOFT_FILES_DELEGATED_SCOPES]);
83	84
84	85	yield* client.connections.create({
85	86	payload: {
@@ -91,24 +92,22 @@ scenario( · 5 unmodified lines
91	92	},
92	93	});
93	94
94		− // Serve path (2nd former OOM site): tools/list rebuilds the catalog from
95		− // the persisted bindings. The whole catalog must come back, with real
96		− // descriptions, and without re-parsing the 37MB spec.
	95	+ // Serve path, second former OOM site: tools/list rebuilds the catalog
	96	+ // from persisted bindings, with real descriptions, and without
	97	+ // re-parsing the Graph spec.
97	98	const tools = yield* client.tools.list({
98	99	query: { integration: IntegrationSlug.make(integration), connection },
99	100	});
100	101	expect(
101	102	tools.length,
102		− "the served catalog returns the whole set of operations, not a re-parse failure",
103		− ).toBeGreaterThan(5_000);
	103	+ "the served catalog returns the files operation subtree, not a re-parse failure",
	104	+ ).toBeGreaterThan(10);
104	105
105	106	const names = tools.map((tool: ToolView) => tool.name);
106		− const messageTools = names.filter((name) => name.toLowerCase().includes("message"));
107		− const siteTools = names.filter((name) => name.toLowerCase().includes("site"));
108		− const userTools = names.filter((name) => name.toLowerCase().includes("user"));
109		− expect(messageTools, "the served catalog spans mail operations").not.toEqual([]);
110		− expect(siteTools, "the served catalog spans SharePoint site operations").not.toEqual([]);
111		− expect(userTools, "the served catalog spans directory user operations").not.toEqual([]);
	107	+ const driveTools = names.filter((name) => name.toLowerCase().includes("drive"));
	108	+ const shareTools = names.filter((name) => name.toLowerCase().includes("share"));
	109	+ expect(driveTools, "the served catalog spans drive operations").not.toEqual([]);
	110	+ expect(shareTools, "the served catalog spans sharing operations").not.toEqual([]);
112	111	}),
113	112	Effect.gen(function* () {
114	113	yield* client.connections

e2e/scenarios/oauth-client-handoff.test.ts+11−6

@@ -316,10 +316,7 @@ const requireOAuthClientCredential = (credential: IssuedCredential) => · 315 unmodified lines
316	316
317	317	scenario(
318	318	"OAuth client · agent hands off, the human enters the secret in the browser, and the app connects",
319		− {
320		− skip: "phase 3 will replace this deleted microsoft.addGraph scenario with catalog-driven Microsoft e2e coverage",
321		− timeout: 240_000,
322		− },
	319	+ { timeout: 240_000 },
323	320	Effect.gen(function* () {
324	321	const target = yield* Target;
325	322	const { client: makeApiClient } = yield* Api;
@@ -361,7 +358,15 @@ scenario( · 35 unmodified lines
361	358	name: "Microsoft Graph Emulator",
362	359	baseUrl: emulator.baseUrl,
363	360	family: "microsoft",
364		− specFormat: "microsoft-graph",
	361	+ authenticationTemplate: [
	362	+ {
	363	+ slug: template,
	364	+ kind: "oauth2",
	365	+ authorizationUrl: oauth.authorizationUrl,
	366	+ tokenUrl: oauth.tokenUrl,
	367	+ scopes: ["https://graph.microsoft.com/.default"],
	368	+ },
	369	+ ],
365	370	},
366	371	});
367	372
@@ -420,7 +425,7 @@ scenario( · 52 unmodified lines
420	425	});
421	426
422	427	await step("Register the app", async () => {
423		− await page.getByRole("button", { name: "Register app" }).click();
	428	+ await page.getByRole("button", { name: "Register app", exact: true }).click();
424	429	// onCreated returns to the Add-connection view — the register form closes.
425	430	await page
426	431	.getByRole("heading", { name: "Register OAuth app" })

e2e/scenarios/provider-plugins-ui.test.ts+29−39

@@ -5,7 +5,7 @@ import { scenario } from "../src/scenario"; · 4 unmodified lines
5	5	import { Browser, Target } from "../src/services";
6	6
7	7	scenario(
8		− "Provider plugins · Google and Microsoft own their add flows outside OpenAPI",
	8	+ "Provider catalog · Google and Microsoft services are OpenAPI presets",
9	9	{},
10	10	Effect.gen(function* () {
11	11	const target = yield* Target;
@@ -19,56 +19,46 @@ scenario( · 7 unmodified lines
19	19	await page.getByRole("dialog", { name: "Connect an integration" }).waitFor();
20	20	});
21	21
22		− await step(
23		− "The provider list exposes OpenAPI, Google, and Microsoft separately",
24		− async () => {
25		− const dialog = page.getByRole("dialog", {
26		− name: "Connect an integration",
27		− });
28		− await dialog.getByRole("link", { name: "OpenAPI", exact: true }).waitFor();
29		− await dialog.getByRole("link", { name: "Google", exact: true }).waitFor();
30		− await dialog.getByRole("link", { name: "Microsoft", exact: true }).waitFor();
31		− },
32		− );
	22	+ await step("The picker exposes OpenAPI plus provider service presets", async () => {
	23	+ const dialog = page.getByRole("dialog", {
	24	+ name: "Connect an integration",
	25	+ });
	26	+ const search = dialog.getByPlaceholder(/Search or paste a URL/);
	27	+ await dialog.getByRole("link", { name: "OpenAPI", exact: true }).waitFor();
	28	+
	29	+ await search.fill("gmail");
	30	+ await dialog.getByRole("link", { name: /^Gmail\b/ }).waitFor();
	31	+
	32	+ await search.fill("onedrive");
	33	+ await dialog.getByRole("link", { name: /^OneDrive Files\b/ }).waitFor();
	34	+ });
33	35
34	36	await step("OpenAPI add remains generic", async () => {
35	37	await page.goto("/integrations/add/openapi", {
36	38	waitUntil: "domcontentloaded",
37	39	});
38		− await page.getByRole("heading", { name: "Add OpenAPI Integration" }).waitFor();
	40	+ await page.getByRole("heading", { name: "Add OpenAPI integration" }).waitFor();
39	41	await page.getByText("OpenAPI Spec").waitFor();
40	42	expect(await page.getByText("Customize your Google connection").count()).toBe(0);
41	43	expect(await page.getByText("Customize Microsoft Graph").count()).toBe(0);
42	44	});
43	45
44		− await step("Google has its own product picker", async () => {
45		− await page.goto("/integrations/add/google", {
46		− waitUntil: "domcontentloaded",
47		− });
48		− await page.getByRole("heading", { name: "Add Google integration" }).waitFor();
49		− await page.getByText("Customize your Google connection").waitFor();
50		− await page.getByText("Gmail").first().waitFor();
51		− await page.getByText("Google Calendar").first().waitFor();
	46	+ await step("A Google service preset opens the OpenAPI add flow", async () => {
	47	+ await page.goto(
	48	+ "/integrations/add/openapi?preset=google-gmail&url=https%3A%2F%2Fwww.googleapis.com%2Fdiscovery%2Fv1%2Fapis%2Fgmail%2Fv1%2Frest",
	49	+ { waitUntil: "domcontentloaded" },
	50	+ );
	51	+ await page.getByRole("heading", { name: "Add OpenAPI integration" }).waitFor();
	52	+ await expect.poll(() => page.locator("textarea").inputValue()).toContain("gmail");
52	53	});
53	54
54		− await step("Microsoft has its own Graph scope picker", async () => {
55		− await page.goto("/integrations/add/microsoft", {
56		− waitUntil: "domcontentloaded",
57		− });
58		− await page.getByRole("heading", { name: "Add Microsoft integration" }).waitFor();
59		− await page.getByText("Customize Microsoft Graph").waitFor();
60		− expect(await page.getByText("All Microsoft Graph", { exact: true }).count()).toBe(0);
61		− await page.getByText("Productivity").waitFor();
62		− await page.getByText("Directory and identity").waitFor();
63		− await page.getByText("Outlook Mail").waitFor();
64		− await page.getByText("OneDrive Files").waitFor();
65		− await page.getByText("OneNote", { exact: true }).waitFor();
66		− await page.getByText("Teams Channels").waitFor();
67		− await page.locator('img[src*="svgl.app/library/microsoft-outlook.svg"]').first().waitFor();
68		− await page.locator('img[src*="svgl.app/library/microsoft-onedrive.svg"]').waitFor();
69		− await page.getByRole("button", { name: /View scopes/ }).click();
70		− await page.getByText("offline_access").waitFor();
71		− await page.getByText("Mail.ReadWrite").waitFor();
	55	+ await step("A Microsoft service preset opens the OpenAPI add flow", async () => {
	56	+ await page.goto(
	57	+ "/integrations/add/openapi?preset=microsoft-files&url=https%3A%2F%2Fraw.githubusercontent.com%2Fmicrosoftgraph%2Fmsgraph-metadata%2Fmaster%2Fopenapi%2Fv1.0%2Fopenapi.yaml%23preset%3Dfiles",
	58	+ { waitUntil: "domcontentloaded" },
	59	+ );
	60	+ await page.getByRole("heading", { name: "Add OpenAPI integration" }).waitFor();
	61	+ await expect.poll(() => page.locator("textarea").inputValue()).toContain("preset=files");
72	62	});
73	63	});
74	64	}),

package.json+1−0

@@ -75,6 +75,7 @@ · 74 unmodified lines
75	75	"@effect/language-service": "^0.85.1",
76	76	"@effect/tsgo": "^0.5.2",
77	77	"@effect/vitest": "catalog:",
	78	+ "@executor-js/plugin-provider-service-split": "workspace:*",
78	79	"@typescript/native-preview": "^7.0.0-dev.20260410.1",
79	80	"@vitest/expect": "catalog:",
80	81	"@vitest/mocker": "catalog:",

packages/plugins/provider-service-split/CHANGELOG.md+7−0

@@ -0,0 +1,7 @@
	1	+# @executor-js/plugin-provider-service-split
	2	+
	3	+## 0.0.0
	4	+
	5	+### Patch Changes
	6	+
	7	+- Initial internal package.

packages/plugins/provider-service-split/package.json+33−0

@@ -0,0 +1,33 @@
	1	+{
	2	+ "name": "@executor-js/plugin-provider-service-split",
	3	+ "version": "0.0.0",
	4	+ "private": true,
	5	+ "type": "module",
	6	+ "exports": {
	7	+ ".": "./src/index.ts",
	8	+ "./planner": "./src/planner.ts",
	9	+ "./sqlite": "./src/sqlite.ts"
	10	+ },
	11	+ "scripts": {
	12	+ "build": "tsup && (tsc --declaration --emitDeclarationOnly --outDir dist --rootDir src \|\| true)",
	13	+ "typecheck": "tsgo --noEmit",
	14	+ "test": "vitest run",
	15	+ "test:watch": "vitest",
	16	+ "typecheck:slow": "bunx tsc --noEmit -p tsconfig.json"
	17	+ },
	18	+ "dependencies": {
	19	+ "@executor-js/plugin-google": "workspace:*",
	20	+ "@executor-js/plugin-microsoft": "workspace:*",
	21	+ "@executor-js/plugin-openapi": "workspace:*",
	22	+ "@executor-js/sdk": "workspace:*",
	23	+ "effect": "catalog:"
	24	+ },
	25	+ "devDependencies": {
	26	+ "@effect/vitest": "catalog:",
	27	+ "@types/node": "catalog:",
	28	+ "bun-types": "catalog:",
	29	+ "tsup": "catalog:",
	30	+ "typescript": "catalog:",
	31	+ "vitest": "catalog:"
	32	+ }
	33	+}

packages/plugins/provider-service-split/src/index.ts+2−0

@@ -0,0 +1,2 @@
	1	+export * from "./planner";
	2	+export * from "./sqlite";

packages/plugins/provider-service-split/src/planner.test.tsrenamed from tests/migration/service-split-planner.test.ts+3−6

@@ -1,9 +1,6 @@
1	1	import { describe, expect, it } from "@effect/vitest";
2	2	import { Effect } from "effect";
3		−import {
4		− streamOperationBindingsFromStructure,
5		− structuralSplit,
6		−} from "../../packages/plugins/openapi/src/sdk";
	3	+import { streamOperationBindingsFromStructure, structuralSplit } from "@executor-js/plugin-openapi";
7	4
8	5	import {
9	6	googlePresetIdForTool,
@@ -19,8 +16,8 @@ import { · 9 unmodified lines
19	16	type PluginStorageRow,
20	17	type ToolPolicyRow,
21	18	type ToolRow,
22		−} from "../../scripts/migration/service-split-planner";
23		−import { googleCatalog } from "../../packages/plugins/google/src/sdk/presets";
	19	+} from "./planner";
	20	+import { googleCatalog } from "@executor-js/plugin-google";
24	21
25	22	const now = "2026-01-01T00:00:00.000Z";
26	23

packages/plugins/provider-service-split/src/planner.tsrenamed from scripts/migration/service-split-planner.ts+4−6

@@ -1,14 +1,12 @@
	1	+/* oxlint-disable executor/no-try-catch-or-throw, executor/no-error-constructor, executor/no-instanceof-error, executor/no-unknown-error-message, executor/no-json-parse -- boundary: one-shot provider service split planner preserves the throwing dry-run contract used by migration tooling */
1	2	import { createHash } from "node:crypto";
2	3
3		−import {
4		− googleCatalog,
5		− googlePresetForDiscoveryUrl,
6		−} from "../../packages/plugins/google/src/sdk/presets";
	4	+import { googleCatalog, googlePresetForDiscoveryUrl } from "@executor-js/plugin-google";
7	5	import {
8	6	MICROSOFT_GRAPH_DEFAULT_PRESET_IDS,
9	7	microsoftCatalog,
10		−} from "../../packages/plugins/microsoft/src/sdk/presets";
11		−import type { OpenApiPreset } from "../../packages/plugins/openapi/src/sdk/presets";
	8	+} from "@executor-js/plugin-microsoft";
	9	+import type { OpenApiPreset } from "@executor-js/plugin-openapi/presets";
12	10
13	11	type MonolithPluginId = "google" \| "microsoft";
14	12

packages/plugins/provider-service-split/src/sqlite.test.tsrenamed from tests/migration/service-split-sqlite.test.ts+5−5

@@ -1,18 +1,18 @@
1	1	import { describe, expect, it } from "@effect/vitest";
2	2	import { Effect, Schema } from "effect";
3		−import { collectTables } from "../../packages/core/sdk/src/executor";
4	3	import {
	4	+ collectTables,
5	5	DataMigrationError,
6	6	runSqliteDataMigrations,
7	7	type SqliteDataMigrationClient,
8		−} from "../../packages/core/sdk/src/sqlite-data-migrations";
9		−import { createSqliteTestFumaDb } from "../../packages/core/sdk/src/sqlite-test-db";
	8	+} from "@executor-js/sdk";
	9	+import { createSqliteTestFumaDb } from "@executor-js/sdk/testing";
10	10
11		−import { operationStorageKey } from "../../scripts/migration/service-split-planner";
	11	+import { operationStorageKey } from "./planner";
12	12	import {
13	13	providerServiceSplitDataMigration,
14	14	runSqliteProviderServiceSplitMigration,
15		−} from "../../scripts/migration/service-split-sqlite";
	15	+} from "./sqlite";
16	16
17	17	const now = 1_780_000_000_000;
18	18	const parseJson = Schema.decodeUnknownSync(Schema.fromJsonString(Schema.Unknown));

packages/plugins/provider-service-split/src/sqlite.tsrenamed from scripts/migration/service-split-sqlite.ts+3−5

@@ -1,8 +1,6 @@
	1	+/* oxlint-disable executor/no-try-catch-or-throw, executor/no-error-constructor, executor/no-json-parse -- boundary: one-shot provider service split data migration preserves legacy SQLite row coercion and typed DataMigrationError wrapping */
1	2	import { Effect } from "effect";
2		−import {
3		− DataMigrationError,
4		− type SqliteDataMigrationClient,
5		−} from "../../packages/core/sdk/src/sqlite-data-migrations";
	3	+import { DataMigrationError, type SqliteDataMigrationClient } from "@executor-js/sdk";
6	4
7	5	import {
8	6	operationStorageKey,
@@ -17,7 +15,7 @@ import { · 8 unmodified lines
17	15	type PluginStorageRow,
18	16	type ToolPolicyRow,
19	17	type ToolRow,
20		−} from "./service-split-planner";
	18	+} from "./planner";
21	19
22	20	const MIGRATION_NAME = "2026-07-08-provider-service-split";
23	21	const LEDGER_TABLE = "provider_service_split_org_migration";

packages/plugins/provider-service-split/tsconfig.json+23−0

@@ -0,0 +1,23 @@
	1	+{
	2	+ "compilerOptions": {
	3	+ "target": "ES2022",
	4	+ "module": "ESNext",
	5	+ "moduleResolution": "Bundler",
	6	+ "strict": true,
	7	+ "skipLibCheck": true,
	8	+ "lib": ["ES2022", "DOM"],
	9	+ "types": ["bun-types", "node"],
	10	+ "noUnusedLocals": true,
	11	+ "noImplicitOverride": true,
	12	+ "jsx": "react-jsx",
	13	+ "plugins": [
	14	+ {
	15	+ "name": "@effect/language-service",
	16	+ "ignoreEffectSuggestionsInTscExitCode": true,
	17	+ "ignoreEffectWarningsInTscExitCode": true,
	18	+ "diagnosticSeverity": {}
	19	+ }
	20	+ ]
	21	+ },
	22	+ "include": ["src/*/.ts"]
	23	+}

packages/plugins/provider-service-split/tsup.config.ts+12−0

@@ -0,0 +1,12 @@
	1	+import { defineConfig } from "tsup";
	2	+
	3	+export default defineConfig({
	4	+ entry: {
	5	+ index: "src/index.ts",
	6	+ },
	7	+ format: ["esm"],
	8	+ dts: false,
	9	+ sourcemap: true,
	10	+ clean: true,
	11	+ external: [/^@executor-js\//, /^effect/, /^@effect\//],
	12	+});

packages/plugins/provider-service-split/vitest.config.ts+8−0

@@ -0,0 +1,8 @@
	1	+import { defineConfig } from "vitest/config";
	2	+
	3	+export default defineConfig({
	4	+ test: {
	5	+ include: ["src/*/.test.ts"],
	6	+ testTimeout: 15_000,
	7	+ },
	8	+});

scripts/migration/service-split-postgres-cli.ts+1−1

@@ -22,7 +22,7 @@ import { · 21 unmodified lines
22	22	type PluginStorageRow,
23	23	type ToolPolicyRow,
24	24	type ToolRow,
25		−} from "./service-split-planner";
	25	+} from "@executor-js/plugin-provider-service-split/planner";
26	26
27	27	export interface SqlClient {
28	28	readonly unsafe: <T extends readonly object[] = readonly Record<string, unknown>[]>(

tests/migration/service-split-postgres-cli.test.ts+4−1

@@ -10,7 +10,10 @@ import { · 9 unmodified lines
10	10	readDatabaseInput,
11	11	type SqlClient,
12	12	} from "../../scripts/migration/service-split-postgres-cli";
13		−import { operationStorageKey, planMigration } from "../../scripts/migration/service-split-planner";
	13	+import {
	14	+ operationStorageKey,
	15	+ planMigration,
	16	+} from "@executor-js/plugin-provider-service-split/planner";
14	17
15	18	const now = "2026-01-01T00:00:00.000Z";
16	19	const decodeJson = Schema.decodeUnknownSync(Schema.fromJsonString(Schema.Unknown));

Package provider service split migration

Keyboard shortcuts

General

Issues and pull requests

Issues, pull requests and Inbox

Repository code