History

Package provider service split migration

Rhys Sullivan committed 4 months ago

Commit df7ca6e1 parent a75506527 files · +303 −159Browse files

Showing the changes to scripts (3 of 27 files). Show all

Unified · Split

3 files
  1. scripts/migration/service-split-planner.ts +0 −1334
  2. scripts/migration/service-split-postgres-cli.ts +1 −1
  3. scripts/migration/service-split-sqlite.ts +0 −551
scripts/migration/service-split-planner.ts+0−1334
@@ -1,1334 +0,0 @@
1−import { createHash } from "node:crypto";
2−
3−import {
4− googleCatalog,
5− googlePresetForDiscoveryUrl,
6−} from "../../packages/plugins/google/src/sdk/presets";
7−import {
8− MICROSOFT_GRAPH_DEFAULT_PRESET_IDS,
9− microsoftCatalog,
10−} from "../../packages/plugins/microsoft/src/sdk/presets";
11−import type { OpenApiPreset } from "../../packages/plugins/openapi/src/sdk/presets";
12−
13−type MonolithPluginId = "google" | "microsoft";
14−
15−const normalizeUrl = (url: string): string => {
16− const trimmed = url.trim();
17− if (!URL.canParse(trimmed)) return trimmed.replace(/\/$/, "");
18− const parsed = new URL(trimmed);
19− parsed.hash = "";
20− parsed.searchParams.sort();
21− return parsed.toString().replace(/\/$/, "");
22−};
23−
24−const googleCatalogByDiscoveryUrl = new Map(
25− googleCatalog.flatMap((preset) =>
26− preset.url ? [[normalizeUrl(preset.url), preset] as const] : [],
27− ),
28−);
29−
30−const matchPattern = (pattern: string, toolId: string): boolean => {
31− if (pattern === "*") return true;
32− const patternSegments = pattern.split(".");
33− const toolSegments = toolId.split(".");
34− for (let index = 0; index < patternSegments.length; index += 1) {
35− const segment = patternSegments[index]!;
36− if (segment === "*") {
37− if (index === patternSegments.length - 1) return toolSegments.length >= index;
38− if (index >= toolSegments.length) return false;
39− continue;
40− }
41− if (index >= toolSegments.length || toolSegments[index] !== segment) return false;
42− }
43− return patternSegments.length === toolSegments.length;
44−};
45−
46−export type PluginId = "openapi";
47−
48−export interface IntegrationRow {
49− readonly tenant: string;
50− readonly slug: string;
51− readonly plugin_id: string;
52− readonly name: string | null;
53− readonly description: string | null;
54− readonly config: unknown;
55− readonly health_check?: unknown;
56− readonly config_revised_at?: string | number | bigint | null;
57− readonly can_remove: boolean;
58− readonly can_refresh: boolean;
59− readonly created_at: string;
60− readonly updated_at: string;
61− readonly row_id: string;
62−}
63−
64−export interface ConnectionRow {
65− readonly tenant: string;
66− readonly owner: string;
67− readonly subject: string;
68− readonly integration: string;
69− readonly name: string;
70− readonly template: string;
71− readonly provider: string;
72− readonly item_ids: unknown;
73− readonly identity_label: string | null;
74− readonly description?: string | null;
75− readonly last_health?: unknown;
76− readonly tools_synced_at?: string | number | bigint | null;
77− readonly oauth_client: string | null;
78− readonly oauth_client_owner: string | null;
79− readonly refresh_item_id: string | null;
80− readonly expires_at: string | number | bigint | null;
81− readonly oauth_scope: string | null;
82− readonly oauth_token_url?: string | null;
83− readonly provider_state: unknown;
84− readonly created_at: string;
85− readonly updated_at: string;
86− readonly row_id: string;
87−}
88−
89−export interface ToolRow {
90− readonly tenant: string;
91− readonly owner: string;
92− readonly subject: string;
93− readonly integration: string;
94− readonly connection: string;
95− readonly plugin_id: string;
96− readonly name: string;
97− readonly description?: string;
98− readonly input_schema?: unknown;
99− readonly output_schema?: unknown;
100− readonly annotations?: unknown;
101− readonly created_at?: string;
102− readonly updated_at?: string;
103− readonly row_id: string;
104−}
105−
106−export interface PluginStorageRow {
107− readonly tenant: string;
108− readonly owner: string;
109− readonly subject: string;
110− readonly plugin_id: string;
111− readonly collection: string;
112− readonly key: string;
113− readonly data: unknown;
114− readonly created_at: string;
115− readonly updated_at: string;
116− readonly row_id: string;
117−}
118−
119−export interface BlobRow {
120− readonly id: string;
121− readonly namespace: string;
122− readonly key: string;
123−}
124−
125−export interface ToolPolicyRow {
126− readonly tenant: string;
127− readonly owner: string;
128− readonly subject: string;
129− readonly id: string;
130− readonly pattern: string;
131− readonly action: string;
132− readonly position: string;
133− readonly created_at: string;
134− readonly updated_at: string;
135− readonly row_id: string;
136−}
137−
138−export interface MigrationInput {
139− readonly integrations: readonly IntegrationRow[];
140− readonly connections: readonly ConnectionRow[];
141− readonly tools: readonly ToolRow[];
142− readonly pluginStorage?: readonly PluginStorageRow[];
143− readonly blobs?: readonly BlobRow[];
144− readonly policies: readonly ToolPolicyRow[];
145− readonly completedTenants?: readonly string[];
146− readonly trafficLastTenant?: string;
147− readonly collectPolicyErrors?: boolean;
148− readonly orphanPolicyMode?: "hard_error" | "retarget_all";
149− readonly blobBackend?: "database" | "external";
150− readonly assumeExternalBlobSourcePresent?: boolean;
151− readonly bootRailCreateToolImpliedServices?: boolean;
152−}
153−
154−export interface ServiceTarget {
155− readonly family: MonolithPluginId;
156− readonly pluginId: PluginId;
157− readonly presetId: string;
158− readonly slug: string;
159− readonly name: string;
160− readonly description: string;
161− readonly sourceUrl?: string;
162− readonly specFormat: "google-discovery" | "microsoft-graph";
163− readonly authenticationTemplate?: readonly unknown[];
164− readonly healthCheck?: unknown;
165−}
166−
167−export interface PlannedIntegration {
168− readonly source: Pick<IntegrationRow, "tenant" | "slug" | "plugin_id" | "name">;
169− readonly sourceContributions: readonly {
170− readonly source: IntegrationRow;
171− readonly specHash?: string;
172− readonly operationsToBuild: number;
173− readonly operationToolNames: readonly string[];
174− readonly specBlobPresent: boolean;
175− readonly defsBlobPresent: boolean;
176− }[];
177− readonly target: ServiceTarget;
178− readonly action: "create" | "skip_existing";
179− readonly config: unknown;
180− readonly healthCheck?: unknown;
181− readonly servingState: {
182− readonly specHash?: string;
183− readonly specSource: string;
184− readonly blobBackend: "database" | "external";
185− readonly specBlobPresent: boolean;
186− readonly defsBlobPresent: boolean;
187− readonly operationsToBuild: number;
188− readonly operationToolNames: readonly string[];
189− readonly expectedZeroOperations: boolean;
190− };
191−}
192−
193−export interface PlannedBlobCopy {
194− readonly source: Pick<IntegrationRow, "tenant" | "slug" | "plugin_id">;
195− readonly specHash: string;
196− readonly key: string;
197− readonly sourceNamespace: string;
198− readonly targetNamespace: string;
199− readonly backend: "database" | "external";
200− readonly sourcePresent: boolean;
201− readonly targetPresent: boolean;
202− readonly sourceObjectName: string;
203− readonly targetObjectName: string;
204−}
205−
206−export interface PlannedConnection {
207− readonly source: Pick<ConnectionRow, "tenant" | "owner" | "subject" | "integration" | "name">;
208− readonly targetIntegration: string;
209− readonly action: "clone" | "skip_existing";
210− readonly tokenReuse: "copy_item_ids_and_oauth_columns";
211−}
212−
213−export interface PlannedPolicyRewrite {
214− readonly policy: Pick<
215− ToolPolicyRow,
216− "tenant" | "owner" | "subject" | "id" | "pattern" | "action" | "position"
217− >;
218− readonly action: "rewrite";
219− readonly afterPatterns: readonly string[];
220− readonly matchedServices: readonly string[];
221−}
222−
223−export interface OrgPlan {
224− readonly tenant: string;
225− readonly tenantHash: string;
226− readonly completed: boolean;
227− readonly integrations: readonly PlannedIntegration[];
228− readonly connections: readonly PlannedConnection[];
229− readonly policies: readonly PlannedPolicyRewrite[];
230− readonly blobCopies: readonly PlannedBlobCopy[];
231− readonly deleteMonoliths: readonly Pick<
232− IntegrationRow,
233− "tenant" | "slug" | "plugin_id" | "name"
234− >[];
235− readonly clonedToolRows: number;
236− readonly operationsToBuild: number;
237− readonly hardErrors: readonly string[];
238−}
239−
240−export interface MigrationPlan {
241− readonly orgs: readonly OrgPlan[];
242− readonly summary: {
243− readonly orgs: number;
244− readonly completedOrgs: number;
245− readonly integrationsCreate: number;
246− readonly integrationsSkipExisting: number;
247− readonly connectionsClone: number;
248− readonly connectionsSkipExisting: number;
249− readonly policiesRewrite: number;
250− readonly policiesSkip: number;
251− readonly policyRowsAfter: number;
252− readonly monolithDeletes: number;
253− readonly clonedToolRows: number;
254− readonly operationsToBuild: number;
255− readonly integrationsMissingSpecBlob: number;
256− readonly integrationsMissingDefsBlob: number;
257− readonly hardErrorOrgs: number;
258− readonly policyHardErrors: number;
259− };
260−}
261−
262−const GOOGLE_IDENTITY_DISCOVERY_URL = "https://www.googleapis.com/discovery/v1/apis/oauth2/v2/rest";
263−
264−const GOOGLE_TOOL_PREFIX_TO_PRESET_ID: ReadonlyMap<string, string> = new Map([
265− ["calendar", "google-calendar"],
266− ["gmail", "google-gmail"],
267− ["sheets", "google-sheets"],
268− ["drive", "google-drive"],
269− ["docs", "google-docs"],
270− ["slides", "google-slides"],
271− ["forms", "google-forms"],
272− ["tasks", "google-tasks"],
273− ["people", "google-people"],
274− ["photoslibrary", "google-photos-library"],
275− ["photospicker", "google-photos-picker"],
276− ["chat", "google-chat"],
277− ["keep", "google-keep"],
278− ["youtube", "google-youtube-data"],
279− ["searchconsole", "google-search-console"],
280− ["webmasters", "google-search-console"],
281− ["classroom", "google-classroom"],
282− ["directory", "google-admin-directory"],
283− ["reports", "google-admin-reports"],
284− ["script", "google-apps-script"],
285− ["bigquery", "google-bigquery"],
286− ["cloudresourcemanager", "google-cloud-resource-manager"],
287−]);
288−
289−const unique = <T>(values: Iterable<T>): readonly T[] => [...new Set(values)];
290−
291−export const tenantHash = (tenant: string): string =>
292− createHash("sha256").update(tenant).digest("hex").slice(0, 12);
293−
294−const isRecord = (value: unknown): value is Record<string, unknown> =>
295− typeof value === "object" && value !== null && !Array.isArray(value);
296−
297−const recordFromJsonLike = (value: unknown): Record<string, unknown> => {
298− if (isRecord(value)) return value;
299− if (typeof value !== "string") return {};
300− try {
301− const parsed = JSON.parse(value) as unknown;
302− return isRecord(parsed) ? parsed : {};
303− } catch {
304− return {};
305− }
306−};
307−
308−const stringArray = (value: unknown): readonly string[] =>
309− Array.isArray(value) ? value.filter((item): item is string => typeof item === "string") : [];
310−
311−const configRecord = (integration: IntegrationRow): Record<string, unknown> =>
312− recordFromJsonLike(integration.config);
313−
314−const toolAddress = (
315− tool: Pick<ToolRow, "integration" | "owner" | "connection" | "name">,
316−): string => `${tool.integration}.${tool.owner}.${tool.connection}.${tool.name}`;
317−
318−const withoutToolsPrefix = (pattern: string): string =>
319− pattern.startsWith("tools.") ? pattern.slice("tools.".length) : pattern;
320−
321−const withOriginalToolsPrefix = (original: string, rewrittenTail: string): string =>
322− original.startsWith("tools.") ? `tools.${rewrittenTail}` : rewrittenTail;
323−
324−const integrationPatternSegment = (
325− pattern: string,
326−): { readonly prefix: boolean; readonly integration: string } => {
327− const prefix = pattern.startsWith("tools.");
328− const tail = prefix ? pattern.slice("tools.".length) : pattern;
329− return { prefix, integration: tail.split(".")[0] ?? "" };
330−};
331−
332−const googleCatalogById: ReadonlyMap<string, OpenApiPreset> = new Map(
333− googleCatalog.map((preset) => [preset.id, preset]),
334−);
335−
336−const microsoftCatalogByMonolithPresetId: ReadonlyMap<string, OpenApiPreset> = new Map(
337− microsoftCatalog.map((preset) => [preset.id.replace(/^microsoft-/, ""), preset]),
338−);
339−
340−const MICROSOFT_UNMIGRATABLE_UMBRELLA_PRESET_IDS = new Set(["me-surface", "users", "groups"]);
341−const MICROSOFT_TOOL_IMPLIED_PRESET_IDS = new Set(MICROSOFT_GRAPH_DEFAULT_PRESET_IDS);
342−const microsoftCatalogBackedPresetIds = new Set(
343− microsoftCatalog
344− .filter((preset) => preset.defaultSlug && preset.defaultSlug.length > 0)
345− .map((preset) => preset.id.replace(/^microsoft-/, "")),
346−);
347−
348−export const microsoftToolFirstSegmentPresetIds: ReadonlyMap<string, readonly string[]> = new Map([
349− ["chats", ["teams-chat"]],
350− ["chatsChat", ["teams-chat"]],
351− ["communications", ["meetings-calls"]],
352− ["communicationsCall", ["meetings-calls"]],
353− ["communicationsOnlineMeeting", ["meetings-calls"]],
354− ["communicationsPresence", ["meetings-calls"]],
355− ["drives", ["files"]],
356− ["drivesDrive", ["files"]],
357− ["drivesDriveItem", ["files"]],
358− ["groupsDrive", ["files"]],
359− ["groupsDrives", ["files"]],
360− ["groupsOnenote", ["onenote"]],
361− ["groupsTeam", ["teams-channels"]],
362− ["meCalendar", ["calendar"]],
363− ["meCalendarGroups", ["calendar"]],
364− ["meCalendars", ["calendar"]],
365− ["meCalendarView", ["calendar"]],
366− ["meChat", ["teams-chat"]],
367− ["meChats", ["teams-chat"]],
368− ["meContact", ["contacts"]],
369− ["meContactFolder", ["contacts"]],
370− ["meContactFolders", ["contacts"]],
371− ["meContacts", ["contacts"]],
372− ["meDrive", ["files"]],
373− ["meDrives", ["files"]],
374− ["meEvent", ["calendar"]],
375− ["meEvents", ["calendar"]],
376− ["meFindMeetingTimes", ["calendar"]],
377− ["meFollowedSites", ["files"]],
378− ["meGetMailTips", ["mail"]],
379− ["meInferenceClassification", ["mail"]],
380− ["meJoinedTeams", ["teams-channels"]],
381− ["meMailFolder", ["mail"]],
382− ["meMailFolders", ["mail"]],
383− ["meMailboxSettings", ["mail"]],
384− ["meMessage", ["mail"]],
385− ["meMessages", ["mail"]],
386− ["meOnlineMeeting", ["meetings-calls"]],
387− ["meOnlineMeetings", ["meetings-calls"]],
388− ["meOnenote", ["onenote"]],
389− ["meOutlook", ["mail"]],
390− ["meOutlookUser", ["mail"]],
391− ["mePeople", ["contacts"]],
392− ["mePerson", ["contacts"]],
393− ["mePhoto", ["profile"]],
394− ["meProfilePhoto", ["profile"]],
395− ["meReminderView", ["calendar"]],
396− ["meSendMail", ["mail"]],
397− ["meSite", ["files"]],
398− ["meTeam", ["teams-channels"]],
399− ["meTodo", ["tasks"]],
400− ["meUser", ["profile"]],
401− ["meUserActions", ["profile"]],
402− ["meUserFunctions", ["profile"]],
403− ["shares", ["files"]],
404− ["sites", ["sites"]],
405− ["sitesOnenote", ["onenote"]],
406− ["sitesSite", ["sites"]],
407− ["teams", ["teams-channels"]],
408− ["teamsChannel", ["teams-channels"]],
409− ["teamsTeam", ["teams-channels"]],
410− ["teamsTemplates", ["teams-channels"]],
411− ["teamwork", ["teams-channels"]],
412− ["usersCalendar", ["calendar"]],
413− ["usersCalendarGroups", ["calendar"]],
414− ["usersCalendars", ["calendar"]],
415− ["usersCalendarView", ["calendar"]],
416− ["usersContact", ["contacts"]],
417− ["usersContactFolder", ["contacts"]],
418− ["usersContactFolders", ["contacts"]],
419− ["usersContacts", ["contacts"]],
420− ["usersDrive", ["files"]],
421− ["usersDrives", ["files"]],
422− ["usersEvent", ["calendar"]],
423− ["usersEvents", ["calendar"]],
424− ["usersFindMeetingTimes", ["calendar"]],
425− ["usersMailFolder", ["mail"]],
426− ["usersMailFolders", ["mail"]],
427− ["usersMessage", ["mail"]],
428− ["usersMessages", ["mail"]],
429− ["usersOnlineMeeting", ["meetings-calls"]],
430− ["usersOnlineMeetings", ["meetings-calls"]],
431− ["usersOnenote", ["onenote"]],
432− ["usersOutlook", ["mail"]],
433− ["usersPeople", ["contacts"]],
434− ["usersPerson", ["contacts"]],
435− ["usersReminderView", ["calendar"]],
436− ["usersSendMail", ["mail"]],
437− ["usersTodo", ["tasks"]],
438− ["usersUser", ["profile"]],
439− ["usersUserActions", ["profile"]],
440− ["usersUserFunctions", ["profile"]],
441−]);
442−
443−const MICROSOFT_WORKBOOK_FIRST_SEGMENTS = new Set([
444− "drivesDriveItem",
445− "groupsDrive",
446− "meDrive",
447− "usersDrive",
448−]);
449−
450−const microsoftFirstSegmentOwnershipKey = (firstSegment: string): string | undefined =>
451− [...microsoftToolFirstSegmentPresetIds.keys()]
452− .filter(
453− (key) =>
454− firstSegment === key ||
455− (firstSegment.startsWith(key) &&
456− firstSegment[key.length]?.toUpperCase() === firstSegment[key.length]),
457− )
458− .sort((left, right) => right.length - left.length)[0];
459−
460−const serviceSlugForPreset = (family: MonolithPluginId, presetId: string): string | undefined => {
461− const preset =
462− family === "google"
463− ? googleCatalogById.get(presetId)
464− : microsoftCatalogByMonolithPresetId.get(presetId);
465− return preset?.defaultSlug;
466−};
467−
468−const serviceTargetForPreset = (
469− family: MonolithPluginId,
470− presetId: string,
471−): ServiceTarget | undefined => {
472− if (family === "google") {
473− const preset = googleCatalogById.get(presetId);
474− if (!preset) return undefined;
475− return {
476− family,
477− pluginId: "openapi",
478− presetId,
479− slug: preset.defaultSlug ?? preset.id,
480− name: preset.name,
481− description: preset.summary,
482− ...(preset.url ? { sourceUrl: preset.url } : {}),
483− specFormat: "google-discovery",
484− ...(preset.authTemplate ? { authenticationTemplate: preset.authTemplate } : {}),
485− ...(preset.healthCheck ? { healthCheck: preset.healthCheck } : {}),
486− };
487− }
488− const preset = microsoftCatalogByMonolithPresetId.get(presetId);
489− if (!preset) return undefined;
490− return {
491− family,
492− pluginId: "openapi",
493− presetId,
494− slug: preset.defaultSlug ?? preset.id,
495− name: preset.name,
496− description: preset.summary,
497− ...(preset.url ? { sourceUrl: preset.url } : {}),
498− specFormat: "microsoft-graph",
499− ...(preset.authTemplate ? { authenticationTemplate: preset.authTemplate } : {}),
500− ...(preset.healthCheck ? { healthCheck: preset.healthCheck } : {}),
501− };
502−};
503−
504−const googlePresetIdsFromConfig = (integration: IntegrationRow): readonly string[] => {
505− const urls = stringArray(configRecord(integration).googleDiscoveryUrls);
506− return unique(
507− urls.flatMap((url) => {
508− if (url === GOOGLE_IDENTITY_DISCOVERY_URL || url.includes("/oauth2/")) return [];
509− const preset =
510− googleCatalogByDiscoveryUrl.get(normalizeUrl(url)) ?? googlePresetForDiscoveryUrl(url);
511− return preset ? [preset.id] : [];
512− }),
513− );
514−};
515−
516−const microsoftPresetIdsFromConfig = (integration: IntegrationRow): readonly string[] => {
517− const config = configRecord(integration);
518− const configured = [
519− ...stringArray(config.microsoftGraphPresetIds),
520− ...stringArray(config.microsoftGraphScopePresetIds),
521− ];
522− if (configured.length > 0) return unique(configured);
523− throw new Error(
524− `Microsoft monolith ${tenantHash(integration.tenant)}/${integration.slug} has no stored microsoftGraphPresetIds; refusing to fabricate ${MICROSOFT_GRAPH_DEFAULT_PRESET_IDS.length} default workloads`,
525− );
526−};
527−
528−const microsoftUnmigratablePresetIds = (presetIds: readonly string[]): readonly string[] =>
529− presetIds.filter(
530− (presetId) =>
531− MICROSOFT_UNMIGRATABLE_UMBRELLA_PRESET_IDS.has(presetId) ||
532− !microsoftCatalogBackedPresetIds.has(presetId),
533− );
534−
535−export const googlePresetIdForTool = (toolName: string): string | undefined => {
536− const [first, second] = toolName.split(".");
537− if (first === "admin") {
538− return second === "channels" ? "google-admin-reports" : "google-admin-directory";
539− }
540− return GOOGLE_TOOL_PREFIX_TO_PRESET_ID.get(first ?? "");
541−};
542−
543−export const microsoftPresetIdsForTool = (toolName: string): readonly string[] => {
544− const segments = toolName.split(".");
545− const first = segments[0] ?? "";
546− const ownershipKey = microsoftFirstSegmentOwnershipKey(first);
547− const fromFirstSegment = ownershipKey
548− ? (microsoftToolFirstSegmentPresetIds.get(ownershipKey) ?? [])
549− : [];
550− const workbookOwned =
551− ownershipKey !== undefined &&
552− MICROSOFT_WORKBOOK_FIRST_SEGMENTS.has(ownershipKey) &&
553− segments.some((segment) => segment.toLowerCase().includes("workbook"));
554− return unique([...fromFirstSegment, ...(workbookOwned ? ["excel"] : [])]).filter((presetId) =>
555− MICROSOFT_TOOL_IMPLIED_PRESET_IDS.has(presetId),
556− );
557−};
558−
559−const presetIdsForTool = (pluginId: MonolithPluginId, toolName: string): readonly string[] => {
560− const presetId = pluginId === "google" ? googlePresetIdForTool(toolName) : undefined;
561− return pluginId === "google" ? (presetId ? [presetId] : []) : microsoftPresetIdsForTool(toolName);
562−};
563−
564−const deriveServices = (
565− integration: IntegrationRow,
566− tools: readonly ToolRow[],
567− bootRailCreateToolImpliedServices: boolean,
568−): readonly ServiceTarget[] => {
569− const pluginId = integration.plugin_id as MonolithPluginId;
570− const fromConfig =
571− pluginId === "google"
572− ? googlePresetIdsFromConfig(integration)
573− : microsoftPresetIdsFromConfig(integration);
574− if (pluginId === "microsoft") {
575− const unmigratablePresetIds = microsoftUnmigratablePresetIds(fromConfig);
576− if (unmigratablePresetIds.length > 0) {
577− throw new Error(
578− `Monolith ${tenantHash(integration.tenant)}/${integration.plugin_id}/${integration.slug} references unmigratable-umbrella Microsoft Graph preset(s): ${unmigratablePresetIds.join(", ")}; org needs manual handling`,
579− );
580− }
581− }
582− const fromTools = unique(
583− tools.flatMap((tool) => {
584− const presetIds = presetIdsForTool(pluginId, tool.name);
585− if (
586− pluginId === "microsoft" &&
587− fromConfig.length > 0 &&
588− presetIds.some((presetId) => fromConfig.includes(presetId))
589− ) {
590− return [];
591− }
592− return presetIds;
593− }),
594− );
595− const missingToolPresetIds = fromTools.filter((presetId) => !fromConfig.includes(presetId));
596− if (
597− fromConfig.length > 0 &&
598− missingToolPresetIds.length > 0 &&
599− !bootRailCreateToolImpliedServices
600− ) {
601− throw new Error(
602− `Monolith ${tenantHash(integration.tenant)}/${integration.plugin_id}/${integration.slug} config omits tool-implied service preset(s): ${missingToolPresetIds.join(", ")}`,
603− );
604− }
605− const presetIds =
606− bootRailCreateToolImpliedServices && fromConfig.length > 0
607− ? unique([...fromConfig, ...fromTools])
608− : fromConfig.length > 0
609− ? fromConfig
610− : fromTools;
611− const services = presetIds.flatMap(
612− (presetId) => serviceTargetForPreset(pluginId, presetId) ?? [],
613− );
614− const missingCatalogPresetIds = presetIds.filter(
615− (presetId) => !serviceTargetForPreset(pluginId, presetId),
616− );
617− if (missingCatalogPresetIds.length > 0) {
618− throw new Error(
619− `Monolith ${tenantHash(integration.tenant)}/${integration.plugin_id}/${integration.slug} references unknown service preset(s): ${missingCatalogPresetIds.join(", ")}`,
620− );
621− }
622− if (services.length === 0) {
623− throw new Error(
624− `Monolith ${tenantHash(integration.tenant)}/${integration.plugin_id}/${integration.slug} has no derivable services`,
625− );
626− }
627− return services;
628−};
629−
630−const configForService = (source: IntegrationRow, target: ServiceTarget): unknown => {
631− const config = configRecord(source);
632− const specHash =
633− typeof config.specHash === "string" && config.specHash.length > 0 ? config.specHash : undefined;
634− return {
635− ...(specHash ? { specHash } : {}),
636− ...(target.sourceUrl ? { sourceUrl: target.sourceUrl } : {}),
637− specFormat: target.specFormat,
638− family: target.family,
639− ...(target.authenticationTemplate
640− ? { authenticationTemplate: target.authenticationTemplate }
641− : {}),
642− };
643−};
644−
645−const rowKey = (...parts: readonly string[]): string => parts.join("\u0000");
646−
647−const stableKeyHash = (value: string): string => {
648− let hash = 0xcbf29ce484222325n;
649− const prime = 0x100000001b3n;
650− const mask = 0xffffffffffffffffn;
651− for (let index = 0; index < value.length; index += 1) {
652− hash ^= BigInt(value.charCodeAt(index));
653− hash = (hash * prime) & mask;
654− }
655− return hash.toString(36).padStart(13, "0");
656−};
657−
658−export const operationStorageKey = (integration: string, toolName: string): string =>
659− `op.${stableKeyHash(integration)}.${stableKeyHash(toolName)}`;
660−
661−export const storageDataRecord = (row: Pick<PluginStorageRow, "data">): Record<string, unknown> =>
662− recordFromJsonLike(row.data);
663−
664−const operationToolName = (row: PluginStorageRow): string | undefined => {
665− const value = storageDataRecord(row).toolName;
666− return typeof value === "string" && value.length > 0 ? value : undefined;
667−};
668−
669−const operationIntegration = (row: PluginStorageRow): string | undefined => {
670− const value = storageDataRecord(row).integration;
671− return typeof value === "string" && value.length > 0 ? value : undefined;
672−};
673−
674−const specHashFor = (integration: IntegrationRow): string => {
675− const value = configRecord(integration).specHash;
676− if (typeof value === "string" && value.length > 0) return value;
677− throw new Error(
678− `Monolith ${tenantHash(integration.tenant)}/${integration.plugin_id}/${integration.slug} has no specHash; serving state cannot be migrated`,
679− );
680−};
681−
682−const pluginBlobNamespace = (tenant: string, pluginId: string): string => `o:${tenant}/${pluginId}`;
683−
684−const blobObjectName = (namespace: string, key: string): string => `${namespace}/${key}`;
685−
686−const operationRowsForService = (
687− monolith: IntegrationRow,
688− target: ServiceTarget,
689− rows: readonly PluginStorageRow[],
690−): readonly PluginStorageRow[] =>
691− rows.filter((row) => {
692− if (row.tenant !== monolith.tenant) return false;
693− if (row.plugin_id !== monolith.plugin_id) return false;
694− if (row.collection !== "operation") return false;
695− if (operationIntegration(row) !== monolith.slug) return false;
696− const toolName = operationToolName(row);
697− if (!toolName) return false;
698− if (isIntentionallyDroppedTool(monolith.plugin_id as MonolithPluginId, toolName)) return false;
699− return serviceForMatchedTool(monolith.plugin_id as MonolithPluginId, toolName, [
700− target,
701− ]).includes(target.slug);
702− });
703−
704−const operationRowsForMonolith = (
705− monolith: IntegrationRow,
706− rows: readonly PluginStorageRow[],
707−): readonly PluginStorageRow[] =>
708− rows.filter((row) => {
709− if (row.tenant !== monolith.tenant) return false;
710− if (row.plugin_id !== monolith.plugin_id) return false;
711− if (row.collection !== "operation") return false;
712− return operationIntegration(row) === monolith.slug;
713− });
714−
715−const isConfigOnlyMonolith = (input: {
716− readonly tools: readonly ToolRow[];
717− readonly connections: readonly ConnectionRow[];
718− readonly operations: readonly PluginStorageRow[];
719−}): boolean =>
720− input.tools.length === 0 && input.connections.length === 0 && input.operations.length === 0;
721−
722−const isIntentionallyDroppedTool = (pluginId: MonolithPluginId, toolName: string): boolean =>
723− pluginId === "google" && toolName.startsWith("oauth2.");
724−
725−const serviceForMatchedTool = (
726− pluginId: MonolithPluginId,
727− toolName: string,
728− services: readonly ServiceTarget[],
729−): readonly string[] => {
730− const presetIds = presetIdsForTool(pluginId, toolName);
731− return presetIds.flatMap((presetId) => {
732− const slug = serviceSlugForPreset(pluginId, presetId);
733− return slug && services.some((service) => service.slug === slug) ? [slug] : [];
734− });
735−};
736−
737−const allServiceSlugs = (services: readonly ServiceTarget[]): readonly string[] =>
738− services.map((service) => service.slug);
739−
740−const unassignedToolNames = (
741− pluginId: MonolithPluginId,
742− tools: readonly { readonly name: string }[],
743− services: readonly ServiceTarget[],
744−): readonly string[] =>
745− unique(
746− tools.flatMap((tool) => {
747− if (isIntentionallyDroppedTool(pluginId, tool.name)) return [];
748− return serviceForMatchedTool(pluginId, tool.name, services).length === 0 ? [tool.name] : [];
749− }),
750− );
751−
752−const assertNoUnassignedRows = (
753− monolith: IntegrationRow,
754− tools: readonly ToolRow[],
755− operations: readonly PluginStorageRow[],
756− services: readonly ServiceTarget[],
757−): void => {
758− const pluginId = monolith.plugin_id as MonolithPluginId;
759− const unassignedTools = unassignedToolNames(pluginId, tools, services);
760− if (unassignedTools.length > 0) {
761− throw new Error(
762− `Monolith ${tenantHash(monolith.tenant)}/${monolith.plugin_id}/${monolith.slug} has tool row(s) with no target service: ${unassignedTools.slice(0, 20).join(", ")}`,
763− );
764− }
765− const operationTools = operations.flatMap((row) => {
766− const toolName = operationToolName(row);
767− return toolName ? [{ name: toolName }] : [];
768− });
769− const unassignedOperations = unassignedToolNames(pluginId, operationTools, services);
770− if (unassignedOperations.length > 0) {
771− throw new Error(
772− `Monolith ${tenantHash(monolith.tenant)}/${monolith.plugin_id}/${monolith.slug} has operation row(s) with no target service: ${unassignedOperations.slice(0, 20).join(", ")}`,
773− );
774− }
775−};
776−
777−const serviceSlugsForToolPattern = (
778− pluginId: MonolithPluginId,
779− toolSegments: readonly string[],
780− services: readonly ServiceTarget[],
781−): readonly string[] => {
782− const firstToolSegment = toolSegments[0];
783− if (!firstToolSegment || firstToolSegment === "*") return allServiceSlugs(services);
784− const presetIds = presetIdsForTool(pluginId, toolSegments.join("."));
785− return presetIds.flatMap((presetId) => {
786− const slug = serviceSlugForPreset(pluginId, presetId);
787− return slug && services.some((service) => service.slug === slug) ? [slug] : [];
788− });
789−};
790−
791−const serviceSlugsForPolicyPattern = (
792− policy: ToolPolicyRow,
793− monolith: IntegrationRow,
794− services: readonly ServiceTarget[],
795−): readonly string[] => {
796− const tail = withoutToolsPrefix(policy.pattern);
797− const segments = tail.split(".");
798− const rest = segments.slice(1);
799− const pluginId = monolith.plugin_id as MonolithPluginId;
800− if (rest.length === 0) return allServiceSlugs(services);
801−
802− if (rest[0] === "org" || rest[0] === "user") {
803− return serviceSlugsForToolPattern(pluginId, rest.slice(2), services);
804− }
805−
806− if (rest[0] === "*") {
807− if (rest.length <= 2) return allServiceSlugs(services);
808− return serviceSlugsForToolPattern(pluginId, rest.slice(2), services);
809− }
810−
811− return serviceSlugsForToolPattern(pluginId, rest, services);
812−};
813−
814−const rewritePatternIntegration = (pattern: string, targetSlug: string): string => {
815− const hasTools = pattern.startsWith("tools.");
816− const tail = hasTools ? pattern.slice("tools.".length) : pattern;
817− const segments = tail.split(".");
818− segments[0] = targetSlug;
819− return withOriginalToolsPrefix(pattern, segments.join("."));
820−};
821−
822−const policyMatches = (pattern: string, tool: ToolRow): boolean => {
823− const normalized = withoutToolsPrefix(pattern);
824− return matchPattern(normalized, toolAddress(tool));
825−};
826−
827−const rewritePolicy = (
828− policy: ToolPolicyRow,
829− monolith: IntegrationRow,
830− services: readonly ServiceTarget[],
831− orphanPolicyMode: MigrationInput["orphanPolicyMode"],
832−): PlannedPolicyRewrite => {
833− const patternIntegration = integrationPatternSegment(policy.pattern).integration;
834− if (patternIntegration !== monolith.slug) {
835− throw new Error(
836− `Policy ${policy.id} for org ${tenantHash(policy.tenant)} does not target ${monolith.slug}`,
837− );
838− }
839−
840− const serviceSlugs = unique(serviceSlugsForPolicyPattern(policy, monolith, services));
841− const orphanPolicyRetarget =
842− orphanPolicyMode === "retarget_all" &&
843− (policy.action === "block" || policy.action === "require_approval")
844− ? allServiceSlugs(services)
845− : [];
846−
847− if (serviceSlugs.length === 0 && orphanPolicyRetarget.length === 0) {
848− throw new Error(
849− `Policy ${policy.id} (${policy.pattern}) for org ${tenantHash(policy.tenant)} would be dropped; no target service could be derived`,
850− );
851− }
852− const matchedServices = serviceSlugs.length > 0 ? serviceSlugs : orphanPolicyRetarget;
853−
854− return {
855− policy,
856− action: "rewrite",
857− afterPatterns: matchedServices.map((slug) => rewritePatternIntegration(policy.pattern, slug)),
858− matchedServices,
859− };
860−};
861−
862−export interface NeverWidenResult {
863− readonly ok: boolean;
864− readonly checkedPolicies: number;
865− readonly widened: readonly {
866− readonly policyId: string;
867− readonly beforePattern: string;
868− readonly afterPatterns: readonly string[];
869− readonly extraAddresses: readonly string[];
870− }[];
871− readonly narrowed: readonly {
872− readonly policyId: string;
873− readonly beforePattern: string;
874− readonly missingServices: readonly string[];
875− }[];
876−}
877−
878−export const verifyPolicyRewriteNeverWidens = (
879− plan: MigrationPlan,
880− input: Pick<MigrationInput, "tools">,
881−): NeverWidenResult => {
882− const widened: {
883− readonly policyId: string;
884− readonly beforePattern: string;
885− readonly afterPatterns: readonly string[];
886− readonly extraAddresses: readonly string[];
887− }[] = [];
888− const narrowed: {
889− readonly policyId: string;
890− readonly beforePattern: string;
891− readonly missingServices: readonly string[];
892− }[] = [];
893− let checkedPolicies = 0;
894−
895− for (const org of plan.orgs) {
896− const orgTools = input.tools.filter((tool) => tool.tenant === org.tenant);
897− for (const policy of org.policies) {
898− if (policy.action !== "rewrite") continue;
899− checkedPolicies += 1;
900− if (policy.policy.action === "block" || policy.policy.action === "require_approval") {
901− const afterServices = new Set(
902− policy.afterPatterns.map(
903− (afterPattern) => integrationPatternSegment(afterPattern).integration,
904− ),
905− );
906− const missingServices = policy.matchedServices.filter((service) => {
907− if (!afterServices.has(service)) return true;
908− const expectedPattern = rewritePatternIntegration(policy.policy.pattern, service);
909− return !policy.afterPatterns.includes(expectedPattern);
910− });
911− if (missingServices.length > 0) {
912− narrowed.push({
913− policyId: policy.policy.id,
914− beforePattern: policy.policy.pattern,
915− missingServices,
916− });
917− }
918− }
919− const before = new Set(
920− orgTools
921− .filter((tool) => policyMatches(policy.policy.pattern, tool))
922− .flatMap((tool) => {
923− const monolith = org.deleteMonoliths.find((row) => row.slug === tool.integration);
924− if (!monolith) return [];
925− const services = policy.matchedServices.filter((slug) =>
926− serviceForMatchedTool(
927− monolith.plugin_id as MonolithPluginId,
928− tool.name,
929− org.integrations.map((i) => i.target),
930− ).includes(slug),
931− );
932− return services.map((slug) => `${slug}.${tool.owner}.${tool.connection}.${tool.name}`);
933− }),
934− );
935− const after = new Set(
936− orgTools.flatMap((tool) => {
937− const monolith = org.deleteMonoliths.find((row) => row.slug === tool.integration);
938− if (!monolith) return [];
939− const toolServices = serviceForMatchedTool(
940− monolith.plugin_id as MonolithPluginId,
941− tool.name,
942− org.integrations.map((integration) => integration.target),
943− );
944− return policy.afterPatterns
945− .filter((afterPattern) => {
946− const targetSlug = integrationPatternSegment(afterPattern).integration;
947− if (!toolServices.includes(targetSlug)) return false;
948− const targetTool = { ...tool, integration: targetSlug };
949− return policyMatches(afterPattern, targetTool);
950− })
951− .map((afterPattern) => {
952− const targetSlug = integrationPatternSegment(afterPattern).integration;
953− return `${targetSlug}.${tool.owner}.${tool.connection}.${tool.name}`;
954− });
955− }),
956− );
957− const extra = [...after].filter((address) => !before.has(address));
958− if (extra.length > 0) {
959− widened.push({
960− policyId: policy.policy.id,
961− beforePattern: policy.policy.pattern,
962− afterPatterns: policy.afterPatterns,
963− extraAddresses: extra.slice(0, 20),
964− });
965− }
966− }
967− }
968−
969− return {
970− ok: widened.length === 0 && narrowed.length === 0,
971− checkedPolicies,
972− widened,
973− narrowed,
974− };
975−};
976−
977−export const planMigration = (input: MigrationInput): MigrationPlan => {
978− const completed = new Set(input.completedTenants ?? []);
979− const monoliths = input.integrations.filter(
980− (row) => row.plugin_id === "google" || row.plugin_id === "microsoft",
981− );
982− const tenants = [...unique(monoliths.map((row) => row.tenant))].sort();
983− const trafficLastTenant = input.trafficLastTenant;
984− const orderedTenants = trafficLastTenant
985− ? [
986− ...tenants.filter((tenant) => tenant !== trafficLastTenant),
987− ...tenants.filter((tenant) => tenant === trafficLastTenant),
988− ]
989− : tenants;
990−
991− const integrationExists = new Set(input.integrations.map((row) => rowKey(row.tenant, row.slug)));
992− const blobBackend = input.blobBackend ?? "database";
993− const connectionExists = new Set(
994− input.connections.map((row) =>
995− rowKey(row.tenant, row.owner, row.subject, row.integration, row.name),
996− ),
997− );
998−
999− const orgs = orderedTenants.map((tenant): OrgPlan => {
1000− const orgMonoliths = monoliths.filter((row) => row.tenant === tenant);
1001− const orgTools = input.tools.filter((row) => row.tenant === tenant);
1002− const orgStorage = input.pluginStorage?.filter((row) => row.tenant === tenant) ?? [];
1003− const orgBlobs = input.blobs?.filter((row) => row.namespace.startsWith(`o:${tenant}/`)) ?? [];
1004− const orgConnections: PlannedConnection[] = [];
1005− const orgPolicies: PlannedPolicyRewrite[] = [];
1006− const orgBlobCopies: PlannedBlobCopy[] = [];
1007− const plannedBlobCopyKeys = new Set<string>();
1008− const hardErrors: string[] = [];
1009− const plannedIntegrationsBySlug = new Map<string, PlannedIntegration>();
1010− let clonedToolRows = 0;
1011− let operationsToBuild = 0;
1012−
1013− for (const monolith of orgMonoliths) {
1014− const monolithTools = orgTools.filter((tool) => tool.integration === monolith.slug);
1015− const monolithConnections = input.connections.filter(
1016− (connection) => connection.tenant === tenant && connection.integration === monolith.slug,
1017− );
1018− const monolithOperations = operationRowsForMonolith(monolith, orgStorage);
1019− let services: readonly ServiceTarget[];
1020− try {
1021− services = deriveServices(
1022− monolith,
1023− monolithTools,
1024− input.bootRailCreateToolImpliedServices ?? false,
1025− );
1026− } catch (error) {
1027− if (!input.collectPolicyErrors) throw error;
1028− hardErrors.push(error instanceof Error ? error.message : String(error));
1029− continue;
1030− }
1031− if (
1032− isConfigOnlyMonolith({
1033− tools: monolithTools,
1034− connections: monolithConnections,
1035− operations: monolithOperations,
1036− })
1037− ) {
1038− continue;
1039− }
1040− try {
1041− assertNoUnassignedRows(monolith, monolithTools, monolithOperations, services);
1042− } catch (error) {
1043− if (!input.collectPolicyErrors) throw error;
1044− hardErrors.push(error instanceof Error ? error.message : String(error));
1045− continue;
1046− }
1047− const specHash = specHashFor(monolith);
1048− const namespace = pluginBlobNamespace(tenant, monolith.plugin_id);
1049− const targetNamespace = pluginBlobNamespace(tenant, "openapi");
1050− const specBlobPresent =
1051− (blobBackend === "external" && input.assumeExternalBlobSourcePresent === true) ||
1052− orgBlobs.some((blob) => blob.namespace === namespace && blob.key === `spec/${specHash}`);
1053− const defsBlobPresent =
1054− (blobBackend === "external" && input.assumeExternalBlobSourcePresent === true) ||
1055− orgBlobs.some((blob) => blob.namespace === namespace && blob.key === `defs/${specHash}`);
1056− for (const key of [`spec/${specHash}`, `defs/${specHash}`]) {
1057− const copyKey = rowKey(namespace, targetNamespace, key);
1058− if (plannedBlobCopyKeys.has(copyKey)) continue;
1059− plannedBlobCopyKeys.add(copyKey);
1060− const sourcePresent =
1061− (blobBackend === "external" && input.assumeExternalBlobSourcePresent === true) ||
1062− orgBlobs.some((blob) => blob.namespace === namespace && blob.key === key);
1063− const targetPresent =
1064− blobBackend === "external"
1065− ? false
1066− : orgBlobs.some((blob) => blob.namespace === targetNamespace && blob.key === key);
1067− if (!sourcePresent) {
1068− hardErrors.push(
1069− `Missing source blob ${namespace}/${key} for migrated OpenAPI service namespace ${targetNamespace}`,
1070− );
1071− }
1072− orgBlobCopies.push({
1073− source: monolith,
1074− specHash,
1075− key,
1076− sourceNamespace: namespace,
1077− targetNamespace,
1078− backend: blobBackend,
1079− sourcePresent,
1080− targetPresent,
1081− sourceObjectName: blobObjectName(namespace, key),
1082− targetObjectName: blobObjectName(targetNamespace, key),
1083− });
1084− }
1085− for (const target of services) {
1086− const serviceOperations = operationRowsForService(monolith, target, orgStorage);
1087− const operationToolNames = unique(
1088− serviceOperations.flatMap((row) => operationToolName(row) ?? []),
1089− );
1090− operationsToBuild += operationToolNames.length;
1091− const contribution = {
1092− source: monolith,
1093− specHash,
1094− operationsToBuild: operationToolNames.length,
1095− operationToolNames,
1096− specBlobPresent,
1097− defsBlobPresent,
1098− };
1099− const existing = plannedIntegrationsBySlug.get(target.slug);
1100− if (existing) {
1101− const sourceContributions = [...existing.sourceContributions, contribution];
1102− const winner = sourceContributions.reduce((best, candidate) =>
1103− candidate.operationsToBuild > best.operationsToBuild ? candidate : best,
1104− );
1105− const mergedOperationToolNames = unique(
1106− sourceContributions.flatMap((item) => item.operationToolNames),
1107− );
1108− const next: PlannedIntegration = {
1109− ...existing,
1110− sourceContributions,
1111− source: winner.source,
1112− config: configForService(winner.source, target),
1113− servingState: {
1114− specHash: winner.specHash,
1115− specSource: `${winner.source.plugin_id}/${winner.source.slug}`,
1116− blobBackend,
1117− specBlobPresent: winner.specBlobPresent,
1118− defsBlobPresent: winner.defsBlobPresent,
1119− operationsToBuild: mergedOperationToolNames.length,
1120− operationToolNames: mergedOperationToolNames,
1121− expectedZeroOperations: false,
1122− },
1123− };
1124− plannedIntegrationsBySlug.set(target.slug, next);
1125− } else {
1126− const exists = integrationExists.has(rowKey(tenant, target.slug));
1127− const planned: PlannedIntegration = {
1128− source: monolith,
1129− sourceContributions: [contribution],
1130− target,
1131− action: exists ? "skip_existing" : "create",
1132− config: configForService(monolith, target),
1133− ...(target.healthCheck ? { healthCheck: target.healthCheck } : {}),
1134− servingState: {
1135− specHash,
1136− specSource: `${monolith.plugin_id}/${monolith.slug}`,
1137− blobBackend,
1138− specBlobPresent,
1139− defsBlobPresent,
1140− operationsToBuild: operationToolNames.length,
1141− operationToolNames,
1142− expectedZeroOperations: false,
1143− },
1144− };
1145− plannedIntegrationsBySlug.set(target.slug, planned);
1146− }
1147− }
1148−
1149− for (const connection of monolithConnections) {
1150− for (const target of services) {
1151− const key = rowKey(
1152− tenant,
1153− connection.owner,
1154− connection.subject,
1155− target.slug,
1156− connection.name,
1157− );
1158− const exists = connectionExists.has(key);
1159− orgConnections.push({
1160− source: connection,
1161− targetIntegration: target.slug,
1162− action: exists ? "skip_existing" : "clone",
1163− tokenReuse: "copy_item_ids_and_oauth_columns",
1164− });
1165− if (!exists) {
1166− connectionExists.add(key);
1167− }
1168− clonedToolRows += monolithTools.filter((tool) =>
1169− serviceForMatchedTool(monolith.plugin_id as MonolithPluginId, tool.name, [
1170− target,
1171− ]).includes(target.slug),
1172− ).length;
1173− }
1174− }
1175−
1176− const candidatePolicies = input.policies.filter((policy) => policy.tenant === tenant);
1177− for (const policy of candidatePolicies) {
1178− const patternIntegration = integrationPatternSegment(policy.pattern).integration;
1179− if (patternIntegration !== monolith.slug) continue;
1180− try {
1181− orgPolicies.push(rewritePolicy(policy, monolith, services, input.orphanPolicyMode));
1182− } catch (error) {
1183− if (!input.collectPolicyErrors) throw error;
1184− hardErrors.push(error instanceof Error ? error.message : String(error));
1185− }
1186− }
1187− }
1188−
1189− return {
1190− tenant,
1191− tenantHash: tenantHash(tenant),
1192− completed: completed.has(tenant),
1193− integrations: [...plannedIntegrationsBySlug.values()],
1194− connections: orgConnections,
1195− policies: orgPolicies,
1196− blobCopies: orgBlobCopies,
1197− deleteMonoliths: orgMonoliths,
1198− clonedToolRows,
1199− operationsToBuild,
1200− hardErrors,
1201− };
1202− });
1203−
1204− const activeOrgs = orgs.filter((org) => !org.completed && org.hardErrors.length === 0);
1205− const summary = {
1206− orgs: orgs.length,
1207− completedOrgs: orgs.filter((org) => org.completed).length,
1208− integrationsCreate: activeOrgs
1209− .flatMap((org) => org.integrations)
1210− .filter((row) => row.action === "create").length,
1211− integrationsSkipExisting: activeOrgs
1212− .flatMap((org) => org.integrations)
1213− .filter((row) => row.action === "skip_existing").length,
1214− connectionsClone: activeOrgs
1215− .flatMap((org) => org.connections)
1216− .filter((row) => row.action === "clone").length,
1217− connectionsSkipExisting: activeOrgs
1218− .flatMap((org) => org.connections)
1219− .filter((row) => row.action === "skip_existing").length,
1220− policiesRewrite: activeOrgs
1221− .flatMap((org) => org.policies)
1222− .filter((row) => row.action === "rewrite").length,
1223− policiesSkip: 0,
1224− policyRowsAfter: activeOrgs
1225− .flatMap((org) => org.policies)
1226− .reduce((sum, row) => sum + row.afterPatterns.length, 0),
1227− monolithDeletes: activeOrgs.flatMap((org) => org.deleteMonoliths).length,
1228− clonedToolRows: activeOrgs.reduce((sum, org) => sum + org.clonedToolRows, 0),
1229− operationsToBuild: activeOrgs.reduce((sum, org) => sum + org.operationsToBuild, 0),
1230− integrationsMissingSpecBlob: activeOrgs
1231− .flatMap((org) => org.integrations)
1232− .filter((row) => !row.servingState.specBlobPresent).length,
1233− integrationsMissingDefsBlob: activeOrgs
1234− .flatMap((org) => org.integrations)
1235− .filter((row) => !row.servingState.defsBlobPresent).length,
1236− hardErrorOrgs: orgs.filter((org) => !org.completed && org.hardErrors.length > 0).length,
1237− policyHardErrors: orgs.reduce((sum, org) => sum + org.hardErrors.length, 0),
1238− };
1239− return { orgs, summary };
1240−};
1241−
1242−const printableJson = (value: unknown): string =>
1243− JSON.stringify(value, (_key, inner) => (inner === undefined ? null : inner), 2);
1244−
1245−export const renderOrgDiff = (org: OrgPlan): string => {
1246− const lines: string[] = [];
1247− lines.push(`# Org ${org.tenantHash}`);
1248− lines.push("");
1249− if (org.completed) {
1250− lines.push("Already completed, no changes planned.");
1251− lines.push("");
1252− return lines.join("\n");
1253− }
1254− if (org.hardErrors.length > 0) {
1255− lines.push("## Hard Errors");
1256− lines.push("Apply is blocked for this org until these errors are handled.");
1257− for (const error of org.hardErrors) {
1258− lines.push(`- ${error}`);
1259− }
1260− lines.push("");
1261− }
1262− lines.push("## Integrations");
1263− for (const row of org.integrations) {
1264− lines.push(
1265− `- ${row.action}: ${row.source.plugin_id}/${row.source.slug} (${row.source.name ?? "unnamed"}) -> ${row.target.slug} (${row.target.name})`,
1266− );
1267− lines.push(
1268− ` serving: operations to build: ${row.servingState.operationsToBuild} / spec source: ${row.servingState.specSource} / blob backend: ${row.servingState.blobBackend} / specHash: ${row.servingState.specHash} / spec blob: ${row.servingState.specBlobPresent ? "present" : "missing"} / defs blob: ${row.servingState.defsBlobPresent ? "present" : "missing"}`,
1269− );
1270− if (row.sourceContributions.length > 1) {
1271− for (const contribution of row.sourceContributions) {
1272− lines.push(
1273− ` contribution: ${contribution.source.plugin_id}/${contribution.source.slug} / operations: ${contribution.operationsToBuild} / specHash: ${contribution.specHash ?? "none"}`,
1274− );
1275− }
1276− }
1277− lines.push(` config: ${printableJson(row.config).replaceAll("\n", "\n ")}`);
1278− }
1279− for (const row of org.deleteMonoliths) {
1280− lines.push(
1281− `- delete monolith in apply mode: ${row.plugin_id}/${row.slug} (${row.name ?? "unnamed"})`,
1282− );
1283− }
1284− lines.push("");
1285− lines.push("## Blob Copies");
1286− for (const row of org.blobCopies) {
1287− lines.push(
1288− `- ${row.backend}: ${row.sourceObjectName} -> ${row.targetObjectName} / source: ${row.sourcePresent ? "present" : "missing"} / target: ${row.targetPresent ? "present" : "missing"}`,
1289− );
1290− }
1291− lines.push("");
1292− lines.push("## Connections");
1293− for (const row of org.connections) {
1294− lines.push(
1295− `- ${row.action}: ${row.source.integration}.${row.source.owner}.${row.source.name} -> ${row.targetIntegration}.${row.source.owner}.${row.source.name} (${row.tokenReuse})`,
1296− );
1297− }
1298− lines.push("");
1299− lines.push("## Policies");
1300− for (const row of org.policies) {
1301− lines.push(`- rewrite ${row.policy.id}: ${row.policy.pattern}`);
1302− for (const after of row.afterPatterns) {
1303− lines.push(` -> ${after}`);
1304− }
1305− }
1306− lines.push("");
1307− lines.push("## Internal Rows");
1308− lines.push(`- tool rows cloned in apply mode: ${org.clonedToolRows}`);
1309− lines.push(`- operation rows copied in apply mode: ${org.operationsToBuild}`);
1310− lines.push("");
1311− return lines.join("\n");
1312−};
1313−
1314−export const renderSummary = (plan: MigrationPlan): string => {
1315− const s = plan.summary;
1316− return [
1317− `orgs=${s.orgs}`,
1318− `completed_orgs=${s.completedOrgs}`,
1319− `integrations_create=${s.integrationsCreate}`,
1320− `integrations_skip_existing=${s.integrationsSkipExisting}`,
1321− `connections_clone=${s.connectionsClone}`,
1322− `connections_skip_existing=${s.connectionsSkipExisting}`,
1323− `policies_rewrite=${s.policiesRewrite}`,
1324− `policy_rows_after=${s.policyRowsAfter}`,
1325− `policies_skip=${s.policiesSkip}`,
1326− `monolith_deletes=${s.monolithDeletes}`,
1327− `tool_rows_clone=${s.clonedToolRows}`,
1328− `operation_rows_build=${s.operationsToBuild}`,
1329− `integrations_missing_spec_blob=${s.integrationsMissingSpecBlob}`,
1330− `integrations_missing_defs_blob=${s.integrationsMissingDefsBlob}`,
1331− `hard_error_orgs=${s.hardErrorOrgs}`,
1332− `policy_hard_errors=${s.policyHardErrors}`,
1333− ].join("\n");
1334−};
scripts/migration/service-split-postgres-cli.ts+1−1
@@ -22,7 +22,7 @@ import { · 21 unmodified lines
2222 type PluginStorageRow,
2323 type ToolPolicyRow,
2424 type ToolRow,
25−} from "./service-split-planner";
25+} from "@executor-js/plugin-provider-service-split/planner";
2626
2727 export interface SqlClient {
2828 readonly unsafe: <T extends readonly object[] = readonly Record<string, unknown>[]>(
scripts/migration/service-split-sqlite.ts+0−551
@@ -1,551 +0,0 @@
1−import { Effect } from "effect";
2−import {
3− DataMigrationError,
4− type SqliteDataMigrationClient,
5−} from "../../packages/core/sdk/src/sqlite-data-migrations";
6−
7−import {
8− operationStorageKey,
9− planMigration,
10− storageDataRecord,
11− tenantHash,
12− type BlobRow,
13− type ConnectionRow,
14− type IntegrationRow,
15− type MigrationInput,
16− type OrgPlan,
17− type PluginStorageRow,
18− type ToolPolicyRow,
19− type ToolRow,
20−} from "./service-split-planner";
21−
22−const MIGRATION_NAME = "2026-07-08-provider-service-split";
23−const LEDGER_TABLE = "provider_service_split_org_migration";
24−
25−const operationKeyPrefix = (integration: string): string =>
26− `${operationStorageKey(integration, "").split(".").slice(0, 2).join(".")}.%`;
27−
28−const execute = (
29− client: SqliteDataMigrationClient,
30− stmt: string | { readonly sql: string; readonly args: readonly unknown[] },
31−) =>
32− Effect.tryPromise({
33− try: () => client.execute(stmt),
34− catch: (cause) => new DataMigrationError({ migration: MIGRATION_NAME, cause }),
35− });
36−
37−const parseJsonLike = (value: unknown): unknown => {
38− if (typeof value !== "string") return value;
39− try {
40− return JSON.parse(value) as unknown;
41− } catch {
42− return value;
43− }
44−};
45−
46−const scrubJson = (value: unknown): unknown => {
47− if (Array.isArray(value)) return value.map(scrubJson);
48− if (value && typeof value === "object") {
49− return Object.fromEntries(
50− Object.entries(value)
51− .filter(([, inner]) => inner !== undefined)
52− .map(([key, inner]) => [key, scrubJson(inner)]),
53− );
54− }
55− return value;
56−};
57−
58−const stableId = (...parts: readonly string[]): string => {
59− const text = parts.join("\u0000");
60− let hash = 0;
61− for (let index = 0; index < text.length; index += 1) {
62− hash = (hash * 31 + text.charCodeAt(index)) >>> 0;
63− }
64− return `service_split_${hash.toString(36)}`;
65−};
66−
67−const tableExists = (client: SqliteDataMigrationClient, table: string) =>
68− execute(client, {
69− sql: "SELECT name FROM sqlite_master WHERE type = 'table' AND name = ?",
70− args: [table],
71− }).pipe(Effect.map((result) => result.rows.length > 0));
72−
73−const ensureLedgerTable = (client: SqliteDataMigrationClient) =>
74− execute(
75− client,
76− `CREATE TABLE IF NOT EXISTS ${LEDGER_TABLE} (
77− tenant text PRIMARY KEY NOT NULL,
78− time_completed integer NOT NULL
79− )`,
80− );
81−
82−const readCompletedTenants = (client: SqliteDataMigrationClient) =>
83− execute(client, `SELECT tenant FROM ${LEDGER_TABLE} ORDER BY tenant`).pipe(
84− Effect.map((result) => result.rows.map((row) => String(row.tenant))),
85− );
86−
87−const stampCompletedTenant = (client: SqliteDataMigrationClient, tenant: string) =>
88− execute(client, {
89− sql: `INSERT INTO ${LEDGER_TABLE} (tenant, time_completed)
90− VALUES (?, ?)
91− ON CONFLICT(tenant) DO UPDATE SET time_completed = excluded.time_completed`,
92− args: [tenant, Date.now()],
93− });
94−
95−const readDatabaseInput = (
96− client: SqliteDataMigrationClient,
97− options: {
98− readonly blobBackend?: MigrationInput["blobBackend"];
99− readonly blobs?: readonly BlobRow[];
100− } = {},
101−): Effect.Effect<MigrationInput, DataMigrationError> =>
102− Effect.gen(function* () {
103− yield* ensureLedgerTable(client);
104− const completedTenants = yield* readCompletedTenants(client);
105− const integrations = yield* execute(
106− client,
107− `SELECT tenant, slug, plugin_id, name, description, config, health_check,
108− config_revised_at, can_remove, can_refresh, CAST(created_at AS TEXT) AS created_at,
109− CAST(updated_at AS TEXT) AS updated_at, row_id
110− FROM integration
111− WHERE plugin_id IN ('google', 'microsoft')
112− OR slug IN ('google', 'microsoft')
113− OR tenant IN (
114− SELECT tenant FROM integration
115− WHERE plugin_id IN ('google', 'microsoft')
116− )
117− ORDER BY tenant, slug`,
118− );
119− const monolithTenants = new Set(
120− integrations.rows
121− .filter((row) => row.plugin_id === "google" || row.plugin_id === "microsoft")
122− .map((row) => String(row.tenant)),
123− );
124− if (monolithTenants.size === 0) {
125− return {
126− integrations: [],
127− connections: [],
128− tools: [],
129− pluginStorage: [],
130− blobs: options.blobs ?? [],
131− policies: [],
132− completedTenants,
133− };
134− }
135−
136− const connections = yield* execute(
137− client,
138− `SELECT tenant, owner, subject, integration, name, template, provider, item_ids,
139− identity_label, description, last_health, tools_synced_at, oauth_client,
140− oauth_client_owner, refresh_item_id, expires_at, oauth_scope, oauth_token_url,
141− provider_state, CAST(created_at AS TEXT) AS created_at,
142− CAST(updated_at AS TEXT) AS updated_at, row_id
143− FROM connection
144− ORDER BY tenant, integration, owner, subject, name`,
145− );
146− const tools = yield* execute(
147− client,
148− `SELECT tenant, owner, subject, integration, connection, plugin_id, name, description,
149− input_schema, output_schema, annotations, CAST(created_at AS TEXT) AS created_at,
150− CAST(updated_at AS TEXT) AS updated_at, row_id
151− FROM tool
152− ORDER BY tenant, integration, connection, name`,
153− );
154− const pluginStorage = yield* execute(
155− client,
156− `SELECT tenant, owner, subject, plugin_id, collection, key, data,
157− CAST(created_at AS TEXT) AS created_at, CAST(updated_at AS TEXT) AS updated_at, row_id
158− FROM plugin_storage
159− WHERE plugin_id IN ('google', 'microsoft')
160− AND collection = 'operation'
161− ORDER BY tenant, plugin_id, collection, key`,
162− );
163− const blobs = yield* execute(
164− client,
165− `SELECT id, namespace, key
166− FROM blob
167− WHERE key LIKE 'spec/%' OR key LIKE 'defs/%'
168− ORDER BY namespace, key`,
169− );
170− const policies = yield* execute(
171− client,
172− `SELECT tenant, owner, subject, id, pattern, action, position,
173− CAST(created_at AS TEXT) AS created_at, CAST(updated_at AS TEXT) AS updated_at, row_id
174− FROM tool_policy
175− ORDER BY tenant, owner, subject, position, id`,
176− );
177−
178− const tenantFilter = (row: Record<string, unknown>) => monolithTenants.has(String(row.tenant));
179− return {
180− integrations: integrations.rows.filter(tenantFilter).map(
181− (row): IntegrationRow => ({
182− tenant: String(row.tenant),
183− slug: String(row.slug),
184− plugin_id: String(row.plugin_id),
185− name: typeof row.name === "string" ? row.name : null,
186− description: typeof row.description === "string" ? row.description : null,
187− config: parseJsonLike(row.config),
188− health_check: parseJsonLike(row.health_check),
189− config_revised_at: row.config_revised_at as string | number | bigint | null,
190− can_remove: Boolean(row.can_remove),
191− can_refresh: Boolean(row.can_refresh),
192− created_at: String(row.created_at),
193− updated_at: String(row.updated_at),
194− row_id: String(row.row_id),
195− }),
196− ),
197− connections: connections.rows.filter(tenantFilter).map(
198− (row): ConnectionRow => ({
199− tenant: String(row.tenant),
200− owner: String(row.owner),
201− subject: String(row.subject),
202− integration: String(row.integration),
203− name: String(row.name),
204− template: String(row.template),
205− provider: String(row.provider),
206− item_ids: parseJsonLike(row.item_ids),
207− identity_label: typeof row.identity_label === "string" ? row.identity_label : null,
208− description: typeof row.description === "string" ? row.description : null,
209− last_health: parseJsonLike(row.last_health),
210− tools_synced_at: row.tools_synced_at as string | number | bigint | null,
211− oauth_client: typeof row.oauth_client === "string" ? row.oauth_client : null,
212− oauth_client_owner:
213− typeof row.oauth_client_owner === "string" ? row.oauth_client_owner : null,
214− refresh_item_id: typeof row.refresh_item_id === "string" ? row.refresh_item_id : null,
215− expires_at: row.expires_at as string | number | bigint | null,
216− oauth_scope: typeof row.oauth_scope === "string" ? row.oauth_scope : null,
217− oauth_token_url: typeof row.oauth_token_url === "string" ? row.oauth_token_url : null,
218− provider_state: parseJsonLike(row.provider_state),
219− created_at: String(row.created_at),
220− updated_at: String(row.updated_at),
221− row_id: String(row.row_id),
222− }),
223− ),
224− tools: tools.rows.filter(tenantFilter).map(
225− (row): ToolRow => ({
226− tenant: String(row.tenant),
227− owner: String(row.owner),
228− subject: String(row.subject),
229− integration: String(row.integration),
230− connection: String(row.connection),
231− plugin_id: String(row.plugin_id),
232− name: String(row.name),
233− description: typeof row.description === "string" ? row.description : undefined,
234− input_schema: parseJsonLike(row.input_schema),
235− output_schema: parseJsonLike(row.output_schema),
236− annotations: parseJsonLike(row.annotations),
237− created_at: String(row.created_at),
238− updated_at: String(row.updated_at),
239− row_id: String(row.row_id),
240− }),
241− ),
242− pluginStorage: pluginStorage.rows.filter(tenantFilter).map(
243− (row): PluginStorageRow => ({
244− tenant: String(row.tenant),
245− owner: String(row.owner),
246− subject: String(row.subject),
247− plugin_id: String(row.plugin_id),
248− collection: String(row.collection),
249− key: String(row.key),
250− data: parseJsonLike(row.data),
251− created_at: String(row.created_at),
252− updated_at: String(row.updated_at),
253− row_id: String(row.row_id),
254− }),
255− ),
256− blobs: [
257− ...blobs.rows.map(
258− (row): BlobRow => ({
259− id: String(row.id),
260− namespace: String(row.namespace),
261− key: String(row.key),
262− }),
263− ),
264− ...(options.blobs ?? []),
265− ],
266− policies: policies.rows.filter(tenantFilter).map(
267− (row): ToolPolicyRow => ({
268− tenant: String(row.tenant),
269− owner: String(row.owner),
270− subject: String(row.subject),
271− id: String(row.id),
272− pattern: String(row.pattern),
273− action: String(row.action),
274− position: String(row.position),
275− created_at: String(row.created_at),
276− updated_at: String(row.updated_at),
277− row_id: String(row.row_id),
278− }),
279− ),
280− blobBackend: options.blobBackend ?? "database",
281− orphanPolicyMode: "retarget_all",
282− collectPolicyErrors: true,
283− bootRailCreateToolImpliedServices: true,
284− completedTenants,
285− };
286− });
287−
288−const applyOrg = (
289− client: SqliteDataMigrationClient,
290− org: OrgPlan,
291−): Effect.Effect<void, DataMigrationError> =>
292− Effect.gen(function* () {
293− if (org.completed) return;
294− if (org.hardErrors.length > 0) return;
295− const now = Date.now();
296−
297− for (const copy of org.blobCopies.filter((item) => item.backend === "database")) {
298− const source = yield* execute(client, {
299− sql: "SELECT value FROM blob WHERE namespace = ? AND key = ? LIMIT 1",
300− args: [copy.sourceNamespace, copy.key],
301− });
302− const sourceRow = source.rows[0];
303− if (!sourceRow || typeof sourceRow.value !== "string") {
304− return yield* new DataMigrationError({
305− migration: MIGRATION_NAME,
306− cause: `Missing source blob ${copy.sourceNamespace}/${copy.key}`,
307− });
308− }
309− yield* execute(client, {
310− sql: `INSERT OR IGNORE INTO blob (namespace, key, value, row_id, id)
311− VALUES (?, ?, ?, ?, ?)`,
312− args: [
313− copy.targetNamespace,
314− copy.key,
315− sourceRow.value,
316− stableId("blob", org.tenant, copy.targetNamespace, copy.key),
317− JSON.stringify([copy.targetNamespace, copy.key]),
318− ],
319− });
320− }
321−
322− for (const row of org.integrations.filter((item) => item.action === "create")) {
323− yield* execute(client, {
324− sql: `INSERT OR IGNORE INTO integration
325− (slug, plugin_id, name, description, config, health_check, config_revised_at,
326− can_remove, can_refresh, created_at, updated_at, row_id, tenant)
327− VALUES (?, ?, ?, ?, ?, ?, NULL, 1, 1, ?, ?, ?, ?)`,
328− args: [
329− row.target.slug,
330− row.target.pluginId,
331− row.target.name,
332− row.target.description,
333− JSON.stringify(scrubJson(row.config)),
334− row.healthCheck ? JSON.stringify(scrubJson(row.healthCheck)) : null,
335− now,
336− now,
337− stableId("integration", org.tenant, row.target.slug),
338− org.tenant,
339− ],
340− });
341− }
342−
343− for (const row of org.connections.filter((item) => item.action === "clone")) {
344− yield* execute(client, {
345− sql: `INSERT OR IGNORE INTO connection
346− (integration, name, template, provider, item_ids, identity_label, description,
347− last_health, tools_synced_at, oauth_client, oauth_client_owner, refresh_item_id,
348− expires_at, oauth_scope, oauth_token_url, provider_state, created_at, updated_at,
349− row_id, tenant, owner, subject)
350− SELECT ?, name, template, provider, item_ids, identity_label, description,
351− NULL, NULL, oauth_client, oauth_client_owner, refresh_item_id, expires_at,
352− oauth_scope, oauth_token_url, provider_state, created_at, ?, ?, tenant, owner, subject
353− FROM connection
354− WHERE tenant = ? AND owner = ? AND subject = ? AND integration = ? AND name = ?`,
355− args: [
356− row.targetIntegration,
357− now,
358− stableId(
359− "connection",
360− org.tenant,
361− row.source.owner,
362− row.source.subject,
363− row.targetIntegration,
364− row.source.name,
365− ),
366− org.tenant,
367− row.source.owner,
368− row.source.subject,
369− row.source.integration,
370− row.source.name,
371− ],
372− });
373− }
374−
375− for (const integration of org.integrations) {
376− for (const contribution of integration.sourceContributions) {
377− for (const toolName of contribution.operationToolNames) {
378− const operation = yield* execute(client, {
379− sql: `SELECT data, created_at, updated_at
380− FROM plugin_storage
381− WHERE tenant = ?
382− AND owner = 'org'
383− AND subject = ''
384− AND plugin_id = ?
385− AND collection = 'operation'
386− AND (
387− key = ?
388− OR (
389− json_extract(data, '$.integration') = ?
390− AND json_extract(data, '$.toolName') = ?
391− )
392− )
393− LIMIT 1`,
394− args: [
395− org.tenant,
396− contribution.source.plugin_id,
397− operationStorageKey(contribution.source.slug, toolName),
398− contribution.source.slug,
399− toolName,
400− ],
401− });
402− const source = operation.rows[0];
403− if (!source) {
404− return yield* new DataMigrationError({
405− migration: MIGRATION_NAME,
406− cause: new Error(
407− `Missing operation row for ${tenantHash(org.tenant)}/${contribution.source.slug}/${toolName}`,
408− ),
409− });
410− }
411− yield* execute(client, {
412− sql: `INSERT INTO plugin_storage
413− (plugin_id, collection, key, data, created_at, updated_at, row_id, tenant, owner, subject)
414− VALUES (?, 'operation', ?, ?, ?, ?, ?, ?, 'org', '')
415− ON CONFLICT(tenant, owner, subject, plugin_id, collection, key)
416− DO UPDATE SET data = excluded.data, updated_at = excluded.updated_at`,
417− args: [
418− integration.target.pluginId,
419− operationStorageKey(integration.target.slug, toolName),
420− JSON.stringify(
421− scrubJson({
422− ...storageDataRecord({ data: parseJsonLike(source.data) }),
423− integration: integration.target.slug,
424− toolName,
425− }),
426− ),
427− source.created_at,
428− now,
429− stableId("operation", org.tenant, integration.target.slug, toolName),
430− org.tenant,
431− ],
432− });
433− yield* execute(client, {
434− sql: `INSERT OR IGNORE INTO tool
435− (integration, connection, plugin_id, name, description, input_schema, output_schema,
436− annotations, created_at, updated_at, row_id, tenant, owner, subject)
437− SELECT ?, connection, ?, name, description, input_schema, output_schema,
438− annotations, created_at, ?, ? || '_' || row_id, tenant, owner, subject
439− FROM tool
440− WHERE tenant = ? AND integration = ? AND name = ?`,
441− args: [
442− integration.target.slug,
443− integration.target.pluginId,
444− now,
445− stableId("tool", org.tenant, integration.target.slug),
446− org.tenant,
447− contribution.source.slug,
448− toolName,
449− ],
450− });
451− }
452− }
453− }
454−
455− for (const policy of org.policies) {
456− yield* execute(client, {
457− sql: "DELETE FROM tool_policy WHERE tenant = ? AND owner = ? AND subject = ? AND id = ?",
458− args: [org.tenant, policy.policy.owner, policy.policy.subject, policy.policy.id],
459− });
460− for (const [index, pattern] of policy.afterPatterns.entries()) {
461− yield* execute(client, {
462− sql: `INSERT OR IGNORE INTO tool_policy
463− (id, pattern, action, position, created_at, updated_at, row_id, tenant, owner, subject)
464− VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
465− args: [
466− policy.afterPatterns.length === 1
467− ? policy.policy.id
468− : `${policy.policy.id}_${index + 1}`,
469− pattern,
470− policy.policy.action,
471− policy.policy.position,
472− now,
473− now,
474− stableId("policy", org.tenant, policy.policy.id, pattern),
475− org.tenant,
476− policy.policy.owner,
477− policy.policy.subject,
478− ],
479− });
480− }
481− }
482−
483− for (const monolith of org.deleteMonoliths) {
484− yield* execute(client, {
485− sql: "DELETE FROM tool WHERE tenant = ? AND integration = ?",
486− args: [org.tenant, monolith.slug],
487− });
488− yield* execute(client, {
489− sql: "DELETE FROM connection WHERE tenant = ? AND integration = ?",
490− args: [org.tenant, monolith.slug],
491− });
492− yield* execute(client, {
493− sql: `DELETE FROM plugin_storage
494− WHERE tenant = ?
495− AND owner = 'org'
496− AND subject = ''
497− AND plugin_id = ?
498− AND collection = 'operation'
499− AND (
500− json_extract(data, '$.integration') = ?
501− OR key LIKE ?
502− )`,
503− args: [org.tenant, monolith.plugin_id, monolith.slug, operationKeyPrefix(monolith.slug)],
504− });
505− yield* execute(client, {
506− sql: "DELETE FROM integration WHERE tenant = ? AND slug = ? AND plugin_id = ?",
507− args: [org.tenant, monolith.slug, monolith.plugin_id],
508− });
509− }
510− });
511−
512−export const runSqliteProviderServiceSplitMigration = (
513− client: SqliteDataMigrationClient,
514− options: {
515− readonly beforeStampOrg?: (org: OrgPlan) => Effect.Effect<void, DataMigrationError>;
516− readonly blobBackend?: MigrationInput["blobBackend"];
517− readonly blobs?: readonly BlobRow[];
518− } = {},
519−): Effect.Effect<number, DataMigrationError> =>
520− Effect.gen(function* () {
521− if (!(yield* tableExists(client, "integration"))) return 0;
522− const input = yield* readDatabaseInput(client, options);
523− const plan = planMigration(input);
524− const moved = plan.orgs.filter((org) => !org.completed && org.hardErrors.length === 0).length;
525− if (moved === 0) return 0;
526−
527− for (const org of plan.orgs) {
528− if (org.completed) continue;
529− if (org.hardErrors.length > 0) {
530− console.warn(
531− `provider-service-split: skipped org ${org.tenantHash}: ${org.hardErrors.join("; ")}`,
532− );
533− continue;
534− }
535− yield* execute(client, "BEGIN");
536− const applyOne = Effect.gen(function* () {
537− yield* applyOrg(client, org);
538− if (options.beforeStampOrg) yield* options.beforeStampOrg(org);
539− yield* stampCompletedTenant(client, org.tenant);
540− yield* execute(client, "COMMIT");
541− });
542− yield* applyOne.pipe(Effect.tapError(() => execute(client, "ROLLBACK").pipe(Effect.ignore)));
543− }
544− return moved;
545− });
546−
547−export const providerServiceSplitDataMigration = {
548− name: MIGRATION_NAME,
549− run: (client: SqliteDataMigrationClient) =>
550− runSqliteProviderServiceSplitMigration(client).pipe(Effect.asVoid),
551−};