History

Package provider service split migration

Rhys Sullivan committed 4 months ago

Commit df7ca6e1 parent a75506527 files · +303 −159Browse files

Showing the changes to scripts (3 of 27 files). Show all

Unified · Split

3 files
  1. scripts/migration/service-split-planner.ts +0 −1334
  2. scripts/migration/service-split-postgres-cli.ts +1 −1
  3. scripts/migration/service-split-sqlite.ts +0 −551
scripts/migration/service-split-planner.ts+0−1334
@@ -1,1334 +0,0 @@
1import { createHash } from "node:crypto";
2
3import {
4 googleCatalog,
5 googlePresetForDiscoveryUrl,
6} from "../../packages/plugins/google/src/sdk/presets";
7import {
8 MICROSOFT_GRAPH_DEFAULT_PRESET_IDS,
9 microsoftCatalog,
10} from "../../packages/plugins/microsoft/src/sdk/presets";
11import type { OpenApiPreset } from "../../packages/plugins/openapi/src/sdk/presets";
12
13type MonolithPluginId = "google" | "microsoft";
14
15const normalizeUrl = (url: string): string => {
16 const trimmed = url.trim();
17 if (!URL.canParse(trimmed)) return trimmed.replace(/\/$/, "");
18 const parsed = new URL(trimmed);
19 parsed.hash = "";
20 parsed.searchParams.sort();
21 return parsed.toString().replace(/\/$/, "");
22};
23
24const googleCatalogByDiscoveryUrl = new Map(
25 googleCatalog.flatMap((preset) =>
26 preset.url ? [[normalizeUrl(preset.url), preset] as const] : [],
27 ),
28);
29
30const matchPattern = (pattern: string, toolId: string): boolean => {
31 if (pattern === "*") return true;
32 const patternSegments = pattern.split(".");
33 const toolSegments = toolId.split(".");
34 for (let index = 0; index < patternSegments.length; index += 1) {
35 const segment = patternSegments[index]!;
36 if (segment === "*") {
37 if (index === patternSegments.length - 1) return toolSegments.length >= index;
38 if (index >= toolSegments.length) return false;
39 continue;
40 }
41 if (index >= toolSegments.length || toolSegments[index] !== segment) return false;
42 }
43 return patternSegments.length === toolSegments.length;
44};
45
46export type PluginId = "openapi";
47
48export interface IntegrationRow {
49 readonly tenant: string;
50 readonly slug: string;
51 readonly plugin_id: string;
52 readonly name: string | null;
53 readonly description: string | null;
54 readonly config: unknown;
55 readonly health_check?: unknown;
56 readonly config_revised_at?: string | number | bigint | null;
57 readonly can_remove: boolean;
58 readonly can_refresh: boolean;
59 readonly created_at: string;
60 readonly updated_at: string;
61 readonly row_id: string;
62}
63
64export interface ConnectionRow {
65 readonly tenant: string;
66 readonly owner: string;
67 readonly subject: string;
68 readonly integration: string;
69 readonly name: string;
70 readonly template: string;
71 readonly provider: string;
72 readonly item_ids: unknown;
73 readonly identity_label: string | null;
74 readonly description?: string | null;
75 readonly last_health?: unknown;
76 readonly tools_synced_at?: string | number | bigint | null;
77 readonly oauth_client: string | null;
78 readonly oauth_client_owner: string | null;
79 readonly refresh_item_id: string | null;
80 readonly expires_at: string | number | bigint | null;
81 readonly oauth_scope: string | null;
82 readonly oauth_token_url?: string | null;
83 readonly provider_state: unknown;
84 readonly created_at: string;
85 readonly updated_at: string;
86 readonly row_id: string;
87}
88
89export interface ToolRow {
90 readonly tenant: string;
91 readonly owner: string;
92 readonly subject: string;
93 readonly integration: string;
94 readonly connection: string;
95 readonly plugin_id: string;
96 readonly name: string;
97 readonly description?: string;
98 readonly input_schema?: unknown;
99 readonly output_schema?: unknown;
100 readonly annotations?: unknown;
101 readonly created_at?: string;
102 readonly updated_at?: string;
103 readonly row_id: string;
104}
105
106export interface PluginStorageRow {
107 readonly tenant: string;
108 readonly owner: string;
109 readonly subject: string;
110 readonly plugin_id: string;
111 readonly collection: string;
112 readonly key: string;
113 readonly data: unknown;
114 readonly created_at: string;
115 readonly updated_at: string;
116 readonly row_id: string;
117}
118
119export interface BlobRow {
120 readonly id: string;
121 readonly namespace: string;
122 readonly key: string;
123}
124
125export interface ToolPolicyRow {
126 readonly tenant: string;
127 readonly owner: string;
128 readonly subject: string;
129 readonly id: string;
130 readonly pattern: string;
131 readonly action: string;
132 readonly position: string;
133 readonly created_at: string;
134 readonly updated_at: string;
135 readonly row_id: string;
136}
137
138export interface MigrationInput {
139 readonly integrations: readonly IntegrationRow[];
140 readonly connections: readonly ConnectionRow[];
141 readonly tools: readonly ToolRow[];
142 readonly pluginStorage?: readonly PluginStorageRow[];
143 readonly blobs?: readonly BlobRow[];
144 readonly policies: readonly ToolPolicyRow[];
145 readonly completedTenants?: readonly string[];
146 readonly trafficLastTenant?: string;
147 readonly collectPolicyErrors?: boolean;
148 readonly orphanPolicyMode?: "hard_error" | "retarget_all";
149 readonly blobBackend?: "database" | "external";
150 readonly assumeExternalBlobSourcePresent?: boolean;
151 readonly bootRailCreateToolImpliedServices?: boolean;
152}
153
154export interface ServiceTarget {
155 readonly family: MonolithPluginId;
156 readonly pluginId: PluginId;
157 readonly presetId: string;
158 readonly slug: string;
159 readonly name: string;
160 readonly description: string;
161 readonly sourceUrl?: string;
162 readonly specFormat: "google-discovery" | "microsoft-graph";
163 readonly authenticationTemplate?: readonly unknown[];
164 readonly healthCheck?: unknown;
165}
166
167export interface PlannedIntegration {
168 readonly source: Pick<IntegrationRow, "tenant" | "slug" | "plugin_id" | "name">;
169 readonly sourceContributions: readonly {
170 readonly source: IntegrationRow;
171 readonly specHash?: string;
172 readonly operationsToBuild: number;
173 readonly operationToolNames: readonly string[];
174 readonly specBlobPresent: boolean;
175 readonly defsBlobPresent: boolean;
176 }[];
177 readonly target: ServiceTarget;
178 readonly action: "create" | "skip_existing";
179 readonly config: unknown;
180 readonly healthCheck?: unknown;
181 readonly servingState: {
182 readonly specHash?: string;
183 readonly specSource: string;
184 readonly blobBackend: "database" | "external";
185 readonly specBlobPresent: boolean;
186 readonly defsBlobPresent: boolean;
187 readonly operationsToBuild: number;
188 readonly operationToolNames: readonly string[];
189 readonly expectedZeroOperations: boolean;
190 };
191}
192
193export interface PlannedBlobCopy {
194 readonly source: Pick<IntegrationRow, "tenant" | "slug" | "plugin_id">;
195 readonly specHash: string;
196 readonly key: string;
197 readonly sourceNamespace: string;
198 readonly targetNamespace: string;
199 readonly backend: "database" | "external";
200 readonly sourcePresent: boolean;
201 readonly targetPresent: boolean;
202 readonly sourceObjectName: string;
203 readonly targetObjectName: string;
204}
205
206export interface PlannedConnection {
207 readonly source: Pick<ConnectionRow, "tenant" | "owner" | "subject" | "integration" | "name">;
208 readonly targetIntegration: string;
209 readonly action: "clone" | "skip_existing";
210 readonly tokenReuse: "copy_item_ids_and_oauth_columns";
211}
212
213export interface PlannedPolicyRewrite {
214 readonly policy: Pick<
215 ToolPolicyRow,
216 "tenant" | "owner" | "subject" | "id" | "pattern" | "action" | "position"
217 >;
218 readonly action: "rewrite";
219 readonly afterPatterns: readonly string[];
220 readonly matchedServices: readonly string[];
221}
222
223export interface OrgPlan {
224 readonly tenant: string;
225 readonly tenantHash: string;
226 readonly completed: boolean;
227 readonly integrations: readonly PlannedIntegration[];
228 readonly connections: readonly PlannedConnection[];
229 readonly policies: readonly PlannedPolicyRewrite[];
230 readonly blobCopies: readonly PlannedBlobCopy[];
231 readonly deleteMonoliths: readonly Pick<
232 IntegrationRow,
233 "tenant" | "slug" | "plugin_id" | "name"
234 >[];
235 readonly clonedToolRows: number;
236 readonly operationsToBuild: number;
237 readonly hardErrors: readonly string[];
238}
239
240export interface MigrationPlan {
241 readonly orgs: readonly OrgPlan[];
242 readonly summary: {
243 readonly orgs: number;
244 readonly completedOrgs: number;
245 readonly integrationsCreate: number;
246 readonly integrationsSkipExisting: number;
247 readonly connectionsClone: number;
248 readonly connectionsSkipExisting: number;
249 readonly policiesRewrite: number;
250 readonly policiesSkip: number;
251 readonly policyRowsAfter: number;
252 readonly monolithDeletes: number;
253 readonly clonedToolRows: number;
254 readonly operationsToBuild: number;
255 readonly integrationsMissingSpecBlob: number;
256 readonly integrationsMissingDefsBlob: number;
257 readonly hardErrorOrgs: number;
258 readonly policyHardErrors: number;
259 };
260}
261
262const GOOGLE_IDENTITY_DISCOVERY_URL = "https://www.googleapis.com/discovery/v1/apis/oauth2/v2/rest";
263
264const GOOGLE_TOOL_PREFIX_TO_PRESET_ID: ReadonlyMap<string, string> = new Map([
265 ["calendar", "google-calendar"],
266 ["gmail", "google-gmail"],
267 ["sheets", "google-sheets"],
268 ["drive", "google-drive"],
269 ["docs", "google-docs"],
270 ["slides", "google-slides"],
271 ["forms", "google-forms"],
272 ["tasks", "google-tasks"],
273 ["people", "google-people"],
274 ["photoslibrary", "google-photos-library"],
275 ["photospicker", "google-photos-picker"],
276 ["chat", "google-chat"],
277 ["keep", "google-keep"],
278 ["youtube", "google-youtube-data"],
279 ["searchconsole", "google-search-console"],
280 ["webmasters", "google-search-console"],
281 ["classroom", "google-classroom"],
282 ["directory", "google-admin-directory"],
283 ["reports", "google-admin-reports"],
284 ["script", "google-apps-script"],
285 ["bigquery", "google-bigquery"],
286 ["cloudresourcemanager", "google-cloud-resource-manager"],
287]);
288
289const unique = <T>(values: Iterable<T>): readonly T[] => [...new Set(values)];
290
291export const tenantHash = (tenant: string): string =>
292 createHash("sha256").update(tenant).digest("hex").slice(0, 12);
293
294const isRecord = (value: unknown): value is Record<string, unknown> =>
295 typeof value === "object" && value !== null && !Array.isArray(value);
296
297const recordFromJsonLike = (value: unknown): Record<string, unknown> => {
298 if (isRecord(value)) return value;
299 if (typeof value !== "string") return {};
300 try {
301 const parsed = JSON.parse(value) as unknown;
302 return isRecord(parsed) ? parsed : {};
303 } catch {
304 return {};
305 }
306};
307
308const stringArray = (value: unknown): readonly string[] =>
309 Array.isArray(value) ? value.filter((item): item is string => typeof item === "string") : [];
310
311const configRecord = (integration: IntegrationRow): Record<string, unknown> =>
312 recordFromJsonLike(integration.config);
313
314const toolAddress = (
315 tool: Pick<ToolRow, "integration" | "owner" | "connection" | "name">,
316): string => `${tool.integration}.${tool.owner}.${tool.connection}.${tool.name}`;
317
318const withoutToolsPrefix = (pattern: string): string =>
319 pattern.startsWith("tools.") ? pattern.slice("tools.".length) : pattern;
320
321const withOriginalToolsPrefix = (original: string, rewrittenTail: string): string =>
322 original.startsWith("tools.") ? `tools.${rewrittenTail}` : rewrittenTail;
323
324const integrationPatternSegment = (
325 pattern: string,
326): { readonly prefix: boolean; readonly integration: string } => {
327 const prefix = pattern.startsWith("tools.");
328 const tail = prefix ? pattern.slice("tools.".length) : pattern;
329 return { prefix, integration: tail.split(".")[0] ?? "" };
330};
331
332const googleCatalogById: ReadonlyMap<string, OpenApiPreset> = new Map(
333 googleCatalog.map((preset) => [preset.id, preset]),
334);
335
336const microsoftCatalogByMonolithPresetId: ReadonlyMap<string, OpenApiPreset> = new Map(
337 microsoftCatalog.map((preset) => [preset.id.replace(/^microsoft-/, ""), preset]),
338);
339
340const MICROSOFT_UNMIGRATABLE_UMBRELLA_PRESET_IDS = new Set(["me-surface", "users", "groups"]);
341const MICROSOFT_TOOL_IMPLIED_PRESET_IDS = new Set(MICROSOFT_GRAPH_DEFAULT_PRESET_IDS);
342const microsoftCatalogBackedPresetIds = new Set(
343 microsoftCatalog
344 .filter((preset) => preset.defaultSlug && preset.defaultSlug.length > 0)
345 .map((preset) => preset.id.replace(/^microsoft-/, "")),
346);
347
348export const microsoftToolFirstSegmentPresetIds: ReadonlyMap<string, readonly string[]> = new Map([
349 ["chats", ["teams-chat"]],
350 ["chatsChat", ["teams-chat"]],
351 ["communications", ["meetings-calls"]],
352 ["communicationsCall", ["meetings-calls"]],
353 ["communicationsOnlineMeeting", ["meetings-calls"]],
354 ["communicationsPresence", ["meetings-calls"]],
355 ["drives", ["files"]],
356 ["drivesDrive", ["files"]],
357 ["drivesDriveItem", ["files"]],
358 ["groupsDrive", ["files"]],
359 ["groupsDrives", ["files"]],
360 ["groupsOnenote", ["onenote"]],
361 ["groupsTeam", ["teams-channels"]],
362 ["meCalendar", ["calendar"]],
363 ["meCalendarGroups", ["calendar"]],
364 ["meCalendars", ["calendar"]],
365 ["meCalendarView", ["calendar"]],
366 ["meChat", ["teams-chat"]],
367 ["meChats", ["teams-chat"]],
368 ["meContact", ["contacts"]],
369 ["meContactFolder", ["contacts"]],
370 ["meContactFolders", ["contacts"]],
371 ["meContacts", ["contacts"]],
372 ["meDrive", ["files"]],
373 ["meDrives", ["files"]],
374 ["meEvent", ["calendar"]],
375 ["meEvents", ["calendar"]],
376 ["meFindMeetingTimes", ["calendar"]],
377 ["meFollowedSites", ["files"]],
378 ["meGetMailTips", ["mail"]],
379 ["meInferenceClassification", ["mail"]],
380 ["meJoinedTeams", ["teams-channels"]],
381 ["meMailFolder", ["mail"]],
382 ["meMailFolders", ["mail"]],
383 ["meMailboxSettings", ["mail"]],
384 ["meMessage", ["mail"]],
385 ["meMessages", ["mail"]],
386 ["meOnlineMeeting", ["meetings-calls"]],
387 ["meOnlineMeetings", ["meetings-calls"]],
388 ["meOnenote", ["onenote"]],
389 ["meOutlook", ["mail"]],
390 ["meOutlookUser", ["mail"]],
391 ["mePeople", ["contacts"]],
392 ["mePerson", ["contacts"]],
393 ["mePhoto", ["profile"]],
394 ["meProfilePhoto", ["profile"]],
395 ["meReminderView", ["calendar"]],
396 ["meSendMail", ["mail"]],
397 ["meSite", ["files"]],
398 ["meTeam", ["teams-channels"]],
399 ["meTodo", ["tasks"]],
400 ["meUser", ["profile"]],
401 ["meUserActions", ["profile"]],
402 ["meUserFunctions", ["profile"]],
403 ["shares", ["files"]],
404 ["sites", ["sites"]],
405 ["sitesOnenote", ["onenote"]],
406 ["sitesSite", ["sites"]],
407 ["teams", ["teams-channels"]],
408 ["teamsChannel", ["teams-channels"]],
409 ["teamsTeam", ["teams-channels"]],
410 ["teamsTemplates", ["teams-channels"]],
411 ["teamwork", ["teams-channels"]],
412 ["usersCalendar", ["calendar"]],
413 ["usersCalendarGroups", ["calendar"]],
414 ["usersCalendars", ["calendar"]],
415 ["usersCalendarView", ["calendar"]],
416 ["usersContact", ["contacts"]],
417 ["usersContactFolder", ["contacts"]],
418 ["usersContactFolders", ["contacts"]],
419 ["usersContacts", ["contacts"]],
420 ["usersDrive", ["files"]],
421 ["usersDrives", ["files"]],
422 ["usersEvent", ["calendar"]],
423 ["usersEvents", ["calendar"]],
424 ["usersFindMeetingTimes", ["calendar"]],
425 ["usersMailFolder", ["mail"]],
426 ["usersMailFolders", ["mail"]],
427 ["usersMessage", ["mail"]],
428 ["usersMessages", ["mail"]],
429 ["usersOnlineMeeting", ["meetings-calls"]],
430 ["usersOnlineMeetings", ["meetings-calls"]],
431 ["usersOnenote", ["onenote"]],
432 ["usersOutlook", ["mail"]],
433 ["usersPeople", ["contacts"]],
434 ["usersPerson", ["contacts"]],
435 ["usersReminderView", ["calendar"]],
436 ["usersSendMail", ["mail"]],
437 ["usersTodo", ["tasks"]],
438 ["usersUser", ["profile"]],
439 ["usersUserActions", ["profile"]],
440 ["usersUserFunctions", ["profile"]],
441]);
442
443const MICROSOFT_WORKBOOK_FIRST_SEGMENTS = new Set([
444 "drivesDriveItem",
445 "groupsDrive",
446 "meDrive",
447 "usersDrive",
448]);
449
450const microsoftFirstSegmentOwnershipKey = (firstSegment: string): string | undefined =>
451 [...microsoftToolFirstSegmentPresetIds.keys()]
452 .filter(
453 (key) =>
454 firstSegment === key ||
455 (firstSegment.startsWith(key) &&
456 firstSegment[key.length]?.toUpperCase() === firstSegment[key.length]),
457 )
458 .sort((left, right) => right.length - left.length)[0];
459
460const serviceSlugForPreset = (family: MonolithPluginId, presetId: string): string | undefined => {
461 const preset =
462 family === "google"
463 ? googleCatalogById.get(presetId)
464 : microsoftCatalogByMonolithPresetId.get(presetId);
465 return preset?.defaultSlug;
466};
467
468const serviceTargetForPreset = (
469 family: MonolithPluginId,
470 presetId: string,
471): ServiceTarget | undefined => {
472 if (family === "google") {
473 const preset = googleCatalogById.get(presetId);
474 if (!preset) return undefined;
475 return {
476 family,
477 pluginId: "openapi",
478 presetId,
479 slug: preset.defaultSlug ?? preset.id,
480 name: preset.name,
481 description: preset.summary,
482 ...(preset.url ? { sourceUrl: preset.url } : {}),
483 specFormat: "google-discovery",
484 ...(preset.authTemplate ? { authenticationTemplate: preset.authTemplate } : {}),
485 ...(preset.healthCheck ? { healthCheck: preset.healthCheck } : {}),
486 };
487 }
488 const preset = microsoftCatalogByMonolithPresetId.get(presetId);
489 if (!preset) return undefined;
490 return {
491 family,
492 pluginId: "openapi",
493 presetId,
494 slug: preset.defaultSlug ?? preset.id,
495 name: preset.name,
496 description: preset.summary,
497 ...(preset.url ? { sourceUrl: preset.url } : {}),
498 specFormat: "microsoft-graph",
499 ...(preset.authTemplate ? { authenticationTemplate: preset.authTemplate } : {}),
500 ...(preset.healthCheck ? { healthCheck: preset.healthCheck } : {}),
501 };
502};
503
504const googlePresetIdsFromConfig = (integration: IntegrationRow): readonly string[] => {
505 const urls = stringArray(configRecord(integration).googleDiscoveryUrls);
506 return unique(
507 urls.flatMap((url) => {
508 if (url === GOOGLE_IDENTITY_DISCOVERY_URL || url.includes("/oauth2/")) return [];
509 const preset =
510 googleCatalogByDiscoveryUrl.get(normalizeUrl(url)) ?? googlePresetForDiscoveryUrl(url);
511 return preset ? [preset.id] : [];
512 }),
513 );
514};
515
516const microsoftPresetIdsFromConfig = (integration: IntegrationRow): readonly string[] => {
517 const config = configRecord(integration);
518 const configured = [
519 ...stringArray(config.microsoftGraphPresetIds),
520 ...stringArray(config.microsoftGraphScopePresetIds),
521 ];
522 if (configured.length > 0) return unique(configured);
523 throw new Error(
524 `Microsoft monolith ${tenantHash(integration.tenant)}/${integration.slug} has no stored microsoftGraphPresetIds; refusing to fabricate ${MICROSOFT_GRAPH_DEFAULT_PRESET_IDS.length} default workloads`,
525 );
526};
527
528const microsoftUnmigratablePresetIds = (presetIds: readonly string[]): readonly string[] =>
529 presetIds.filter(
530 (presetId) =>
531 MICROSOFT_UNMIGRATABLE_UMBRELLA_PRESET_IDS.has(presetId) ||
532 !microsoftCatalogBackedPresetIds.has(presetId),
533 );
534
535export const googlePresetIdForTool = (toolName: string): string | undefined => {
536 const [first, second] = toolName.split(".");
537 if (first === "admin") {
538 return second === "channels" ? "google-admin-reports" : "google-admin-directory";
539 }
540 return GOOGLE_TOOL_PREFIX_TO_PRESET_ID.get(first ?? "");
541};
542
543export const microsoftPresetIdsForTool = (toolName: string): readonly string[] => {
544 const segments = toolName.split(".");
545 const first = segments[0] ?? "";
546 const ownershipKey = microsoftFirstSegmentOwnershipKey(first);
547 const fromFirstSegment = ownershipKey
548 ? (microsoftToolFirstSegmentPresetIds.get(ownershipKey) ?? [])
549 : [];
550 const workbookOwned =
551 ownershipKey !== undefined &&
552 MICROSOFT_WORKBOOK_FIRST_SEGMENTS.has(ownershipKey) &&
553 segments.some((segment) => segment.toLowerCase().includes("workbook"));
554 return unique([...fromFirstSegment, ...(workbookOwned ? ["excel"] : [])]).filter((presetId) =>
555 MICROSOFT_TOOL_IMPLIED_PRESET_IDS.has(presetId),
556 );
557};
558
559const presetIdsForTool = (pluginId: MonolithPluginId, toolName: string): readonly string[] => {
560 const presetId = pluginId === "google" ? googlePresetIdForTool(toolName) : undefined;
561 return pluginId === "google" ? (presetId ? [presetId] : []) : microsoftPresetIdsForTool(toolName);
562};
563
564const deriveServices = (
565 integration: IntegrationRow,
566 tools: readonly ToolRow[],
567 bootRailCreateToolImpliedServices: boolean,
568): readonly ServiceTarget[] => {
569 const pluginId = integration.plugin_id as MonolithPluginId;
570 const fromConfig =
571 pluginId === "google"
572 ? googlePresetIdsFromConfig(integration)
573 : microsoftPresetIdsFromConfig(integration);
574 if (pluginId === "microsoft") {
575 const unmigratablePresetIds = microsoftUnmigratablePresetIds(fromConfig);
576 if (unmigratablePresetIds.length > 0) {
577 throw new Error(
578 `Monolith ${tenantHash(integration.tenant)}/${integration.plugin_id}/${integration.slug} references unmigratable-umbrella Microsoft Graph preset(s): ${unmigratablePresetIds.join(", ")}; org needs manual handling`,
579 );
580 }
581 }
582 const fromTools = unique(
583 tools.flatMap((tool) => {
584 const presetIds = presetIdsForTool(pluginId, tool.name);
585 if (
586 pluginId === "microsoft" &&
587 fromConfig.length > 0 &&
588 presetIds.some((presetId) => fromConfig.includes(presetId))
589 ) {
590 return [];
591 }
592 return presetIds;
593 }),
594 );
595 const missingToolPresetIds = fromTools.filter((presetId) => !fromConfig.includes(presetId));
596 if (
597 fromConfig.length > 0 &&
598 missingToolPresetIds.length > 0 &&
599 !bootRailCreateToolImpliedServices
600 ) {
601 throw new Error(
602 `Monolith ${tenantHash(integration.tenant)}/${integration.plugin_id}/${integration.slug} config omits tool-implied service preset(s): ${missingToolPresetIds.join(", ")}`,
603 );
604 }
605 const presetIds =
606 bootRailCreateToolImpliedServices && fromConfig.length > 0
607 ? unique([...fromConfig, ...fromTools])
608 : fromConfig.length > 0
609 ? fromConfig
610 : fromTools;
611 const services = presetIds.flatMap(
612 (presetId) => serviceTargetForPreset(pluginId, presetId) ?? [],
613 );
614 const missingCatalogPresetIds = presetIds.filter(
615 (presetId) => !serviceTargetForPreset(pluginId, presetId),
616 );
617 if (missingCatalogPresetIds.length > 0) {
618 throw new Error(
619 `Monolith ${tenantHash(integration.tenant)}/${integration.plugin_id}/${integration.slug} references unknown service preset(s): ${missingCatalogPresetIds.join(", ")}`,
620 );
621 }
622 if (services.length === 0) {
623 throw new Error(
624 `Monolith ${tenantHash(integration.tenant)}/${integration.plugin_id}/${integration.slug} has no derivable services`,
625 );
626 }
627 return services;
628};
629
630const configForService = (source: IntegrationRow, target: ServiceTarget): unknown => {
631 const config = configRecord(source);
632 const specHash =
633 typeof config.specHash === "string" && config.specHash.length > 0 ? config.specHash : undefined;
634 return {
635 ...(specHash ? { specHash } : {}),
636 ...(target.sourceUrl ? { sourceUrl: target.sourceUrl } : {}),
637 specFormat: target.specFormat,
638 family: target.family,
639 ...(target.authenticationTemplate
640 ? { authenticationTemplate: target.authenticationTemplate }
641 : {}),
642 };
643};
644
645const rowKey = (...parts: readonly string[]): string => parts.join("\u0000");
646
647const stableKeyHash = (value: string): string => {
648 let hash = 0xcbf29ce484222325n;
649 const prime = 0x100000001b3n;
650 const mask = 0xffffffffffffffffn;
651 for (let index = 0; index < value.length; index += 1) {
652 hash ^= BigInt(value.charCodeAt(index));
653 hash = (hash * prime) & mask;
654 }
655 return hash.toString(36).padStart(13, "0");
656};
657
658export const operationStorageKey = (integration: string, toolName: string): string =>
659 `op.${stableKeyHash(integration)}.${stableKeyHash(toolName)}`;
660
661export const storageDataRecord = (row: Pick<PluginStorageRow, "data">): Record<string, unknown> =>
662 recordFromJsonLike(row.data);
663
664const operationToolName = (row: PluginStorageRow): string | undefined => {
665 const value = storageDataRecord(row).toolName;
666 return typeof value === "string" && value.length > 0 ? value : undefined;
667};
668
669const operationIntegration = (row: PluginStorageRow): string | undefined => {
670 const value = storageDataRecord(row).integration;
671 return typeof value === "string" && value.length > 0 ? value : undefined;
672};
673
674const specHashFor = (integration: IntegrationRow): string => {
675 const value = configRecord(integration).specHash;
676 if (typeof value === "string" && value.length > 0) return value;
677 throw new Error(
678 `Monolith ${tenantHash(integration.tenant)}/${integration.plugin_id}/${integration.slug} has no specHash; serving state cannot be migrated`,
679 );
680};
681
682const pluginBlobNamespace = (tenant: string, pluginId: string): string => `o:${tenant}/${pluginId}`;
683
684const blobObjectName = (namespace: string, key: string): string => `${namespace}/${key}`;
685
686const operationRowsForService = (
687 monolith: IntegrationRow,
688 target: ServiceTarget,
689 rows: readonly PluginStorageRow[],
690): readonly PluginStorageRow[] =>
691 rows.filter((row) => {
692 if (row.tenant !== monolith.tenant) return false;
693 if (row.plugin_id !== monolith.plugin_id) return false;
694 if (row.collection !== "operation") return false;
695 if (operationIntegration(row) !== monolith.slug) return false;
696 const toolName = operationToolName(row);
697 if (!toolName) return false;
698 if (isIntentionallyDroppedTool(monolith.plugin_id as MonolithPluginId, toolName)) return false;
699 return serviceForMatchedTool(monolith.plugin_id as MonolithPluginId, toolName, [
700 target,
701 ]).includes(target.slug);
702 });
703
704const operationRowsForMonolith = (
705 monolith: IntegrationRow,
706 rows: readonly PluginStorageRow[],
707): readonly PluginStorageRow[] =>
708 rows.filter((row) => {
709 if (row.tenant !== monolith.tenant) return false;
710 if (row.plugin_id !== monolith.plugin_id) return false;
711 if (row.collection !== "operation") return false;
712 return operationIntegration(row) === monolith.slug;
713 });
714
715const isConfigOnlyMonolith = (input: {
716 readonly tools: readonly ToolRow[];
717 readonly connections: readonly ConnectionRow[];
718 readonly operations: readonly PluginStorageRow[];
719}): boolean =>
720 input.tools.length === 0 && input.connections.length === 0 && input.operations.length === 0;
721
722const isIntentionallyDroppedTool = (pluginId: MonolithPluginId, toolName: string): boolean =>
723 pluginId === "google" && toolName.startsWith("oauth2.");
724
725const serviceForMatchedTool = (
726 pluginId: MonolithPluginId,
727 toolName: string,
728 services: readonly ServiceTarget[],
729): readonly string[] => {
730 const presetIds = presetIdsForTool(pluginId, toolName);
731 return presetIds.flatMap((presetId) => {
732 const slug = serviceSlugForPreset(pluginId, presetId);
733 return slug && services.some((service) => service.slug === slug) ? [slug] : [];
734 });
735};
736
737const allServiceSlugs = (services: readonly ServiceTarget[]): readonly string[] =>
738 services.map((service) => service.slug);
739
740const unassignedToolNames = (
741 pluginId: MonolithPluginId,
742 tools: readonly { readonly name: string }[],
743 services: readonly ServiceTarget[],
744): readonly string[] =>
745 unique(
746 tools.flatMap((tool) => {
747 if (isIntentionallyDroppedTool(pluginId, tool.name)) return [];
748 return serviceForMatchedTool(pluginId, tool.name, services).length === 0 ? [tool.name] : [];
749 }),
750 );
751
752const assertNoUnassignedRows = (
753 monolith: IntegrationRow,
754 tools: readonly ToolRow[],
755 operations: readonly PluginStorageRow[],
756 services: readonly ServiceTarget[],
757): void => {
758 const pluginId = monolith.plugin_id as MonolithPluginId;
759 const unassignedTools = unassignedToolNames(pluginId, tools, services);
760 if (unassignedTools.length > 0) {
761 throw new Error(
762 `Monolith ${tenantHash(monolith.tenant)}/${monolith.plugin_id}/${monolith.slug} has tool row(s) with no target service: ${unassignedTools.slice(0, 20).join(", ")}`,
763 );
764 }
765 const operationTools = operations.flatMap((row) => {
766 const toolName = operationToolName(row);
767 return toolName ? [{ name: toolName }] : [];
768 });
769 const unassignedOperations = unassignedToolNames(pluginId, operationTools, services);
770 if (unassignedOperations.length > 0) {
771 throw new Error(
772 `Monolith ${tenantHash(monolith.tenant)}/${monolith.plugin_id}/${monolith.slug} has operation row(s) with no target service: ${unassignedOperations.slice(0, 20).join(", ")}`,
773 );
774 }
775};
776
777const serviceSlugsForToolPattern = (
778 pluginId: MonolithPluginId,
779 toolSegments: readonly string[],
780 services: readonly ServiceTarget[],
781): readonly string[] => {
782 const firstToolSegment = toolSegments[0];
783 if (!firstToolSegment || firstToolSegment === "*") return allServiceSlugs(services);
784 const presetIds = presetIdsForTool(pluginId, toolSegments.join("."));
785 return presetIds.flatMap((presetId) => {
786 const slug = serviceSlugForPreset(pluginId, presetId);
787 return slug && services.some((service) => service.slug === slug) ? [slug] : [];
788 });
789};
790
791const serviceSlugsForPolicyPattern = (
792 policy: ToolPolicyRow,
793 monolith: IntegrationRow,
794 services: readonly ServiceTarget[],
795): readonly string[] => {
796 const tail = withoutToolsPrefix(policy.pattern);
797 const segments = tail.split(".");
798 const rest = segments.slice(1);
799 const pluginId = monolith.plugin_id as MonolithPluginId;
800 if (rest.length === 0) return allServiceSlugs(services);
801
802 if (rest[0] === "org" || rest[0] === "user") {
803 return serviceSlugsForToolPattern(pluginId, rest.slice(2), services);
804 }
805
806 if (rest[0] === "*") {
807 if (rest.length <= 2) return allServiceSlugs(services);
808 return serviceSlugsForToolPattern(pluginId, rest.slice(2), services);
809 }
810
811 return serviceSlugsForToolPattern(pluginId, rest, services);
812};
813
814const rewritePatternIntegration = (pattern: string, targetSlug: string): string => {
815 const hasTools = pattern.startsWith("tools.");
816 const tail = hasTools ? pattern.slice("tools.".length) : pattern;
817 const segments = tail.split(".");
818 segments[0] = targetSlug;
819 return withOriginalToolsPrefix(pattern, segments.join("."));
820};
821
822const policyMatches = (pattern: string, tool: ToolRow): boolean => {
823 const normalized = withoutToolsPrefix(pattern);
824 return matchPattern(normalized, toolAddress(tool));
825};
826
827const rewritePolicy = (
828 policy: ToolPolicyRow,
829 monolith: IntegrationRow,
830 services: readonly ServiceTarget[],
831 orphanPolicyMode: MigrationInput["orphanPolicyMode"],
832): PlannedPolicyRewrite => {
833 const patternIntegration = integrationPatternSegment(policy.pattern).integration;
834 if (patternIntegration !== monolith.slug) {
835 throw new Error(
836 `Policy ${policy.id} for org ${tenantHash(policy.tenant)} does not target ${monolith.slug}`,
837 );
838 }
839
840 const serviceSlugs = unique(serviceSlugsForPolicyPattern(policy, monolith, services));
841 const orphanPolicyRetarget =
842 orphanPolicyMode === "retarget_all" &&
843 (policy.action === "block" || policy.action === "require_approval")
844 ? allServiceSlugs(services)
845 : [];
846
847 if (serviceSlugs.length === 0 && orphanPolicyRetarget.length === 0) {
848 throw new Error(
849 `Policy ${policy.id} (${policy.pattern}) for org ${tenantHash(policy.tenant)} would be dropped; no target service could be derived`,
850 );
851 }
852 const matchedServices = serviceSlugs.length > 0 ? serviceSlugs : orphanPolicyRetarget;
853
854 return {
855 policy,
856 action: "rewrite",
857 afterPatterns: matchedServices.map((slug) => rewritePatternIntegration(policy.pattern, slug)),
858 matchedServices,
859 };
860};
861
862export interface NeverWidenResult {
863 readonly ok: boolean;
864 readonly checkedPolicies: number;
865 readonly widened: readonly {
866 readonly policyId: string;
867 readonly beforePattern: string;
868 readonly afterPatterns: readonly string[];
869 readonly extraAddresses: readonly string[];
870 }[];
871 readonly narrowed: readonly {
872 readonly policyId: string;
873 readonly beforePattern: string;
874 readonly missingServices: readonly string[];
875 }[];
876}
877
878export const verifyPolicyRewriteNeverWidens = (
879 plan: MigrationPlan,
880 input: Pick<MigrationInput, "tools">,
881): NeverWidenResult => {
882 const widened: {
883 readonly policyId: string;
884 readonly beforePattern: string;
885 readonly afterPatterns: readonly string[];
886 readonly extraAddresses: readonly string[];
887 }[] = [];
888 const narrowed: {
889 readonly policyId: string;
890 readonly beforePattern: string;
891 readonly missingServices: readonly string[];
892 }[] = [];
893 let checkedPolicies = 0;
894
895 for (const org of plan.orgs) {
896 const orgTools = input.tools.filter((tool) => tool.tenant === org.tenant);
897 for (const policy of org.policies) {
898 if (policy.action !== "rewrite") continue;
899 checkedPolicies += 1;
900 if (policy.policy.action === "block" || policy.policy.action === "require_approval") {
901 const afterServices = new Set(
902 policy.afterPatterns.map(
903 (afterPattern) => integrationPatternSegment(afterPattern).integration,
904 ),
905 );
906 const missingServices = policy.matchedServices.filter((service) => {
907 if (!afterServices.has(service)) return true;
908 const expectedPattern = rewritePatternIntegration(policy.policy.pattern, service);
909 return !policy.afterPatterns.includes(expectedPattern);
910 });
911 if (missingServices.length > 0) {
912 narrowed.push({
913 policyId: policy.policy.id,
914 beforePattern: policy.policy.pattern,
915 missingServices,
916 });
917 }
918 }
919 const before = new Set(
920 orgTools
921 .filter((tool) => policyMatches(policy.policy.pattern, tool))
922 .flatMap((tool) => {
923 const monolith = org.deleteMonoliths.find((row) => row.slug === tool.integration);
924 if (!monolith) return [];
925 const services = policy.matchedServices.filter((slug) =>
926 serviceForMatchedTool(
927 monolith.plugin_id as MonolithPluginId,
928 tool.name,
929 org.integrations.map((i) => i.target),
930 ).includes(slug),
931 );
932 return services.map((slug) => `${slug}.${tool.owner}.${tool.connection}.${tool.name}`);
933 }),
934 );
935 const after = new Set(
936 orgTools.flatMap((tool) => {
937 const monolith = org.deleteMonoliths.find((row) => row.slug === tool.integration);
938 if (!monolith) return [];
939 const toolServices = serviceForMatchedTool(
940 monolith.plugin_id as MonolithPluginId,
941 tool.name,
942 org.integrations.map((integration) => integration.target),
943 );
944 return policy.afterPatterns
945 .filter((afterPattern) => {
946 const targetSlug = integrationPatternSegment(afterPattern).integration;
947 if (!toolServices.includes(targetSlug)) return false;
948 const targetTool = { ...tool, integration: targetSlug };
949 return policyMatches(afterPattern, targetTool);
950 })
951 .map((afterPattern) => {
952 const targetSlug = integrationPatternSegment(afterPattern).integration;
953 return `${targetSlug}.${tool.owner}.${tool.connection}.${tool.name}`;
954 });
955 }),
956 );
957 const extra = [...after].filter((address) => !before.has(address));
958 if (extra.length > 0) {
959 widened.push({
960 policyId: policy.policy.id,
961 beforePattern: policy.policy.pattern,
962 afterPatterns: policy.afterPatterns,
963 extraAddresses: extra.slice(0, 20),
964 });
965 }
966 }
967 }
968
969 return {
970 ok: widened.length === 0 && narrowed.length === 0,
971 checkedPolicies,
972 widened,
973 narrowed,
974 };
975};
976
977export const planMigration = (input: MigrationInput): MigrationPlan => {
978 const completed = new Set(input.completedTenants ?? []);
979 const monoliths = input.integrations.filter(
980 (row) => row.plugin_id === "google" || row.plugin_id === "microsoft",
981 );
982 const tenants = [...unique(monoliths.map((row) => row.tenant))].sort();
983 const trafficLastTenant = input.trafficLastTenant;
984 const orderedTenants = trafficLastTenant
985 ? [
986 ...tenants.filter((tenant) => tenant !== trafficLastTenant),
987 ...tenants.filter((tenant) => tenant === trafficLastTenant),
988 ]
989 : tenants;
990
991 const integrationExists = new Set(input.integrations.map((row) => rowKey(row.tenant, row.slug)));
992 const blobBackend = input.blobBackend ?? "database";
993 const connectionExists = new Set(
994 input.connections.map((row) =>
995 rowKey(row.tenant, row.owner, row.subject, row.integration, row.name),
996 ),
997 );
998
999 const orgs = orderedTenants.map((tenant): OrgPlan => {
1000 const orgMonoliths = monoliths.filter((row) => row.tenant === tenant);
1001 const orgTools = input.tools.filter((row) => row.tenant === tenant);
1002 const orgStorage = input.pluginStorage?.filter((row) => row.tenant === tenant) ?? [];
1003 const orgBlobs = input.blobs?.filter((row) => row.namespace.startsWith(`o:${tenant}/`)) ?? [];
1004 const orgConnections: PlannedConnection[] = [];
1005 const orgPolicies: PlannedPolicyRewrite[] = [];
1006 const orgBlobCopies: PlannedBlobCopy[] = [];
1007 const plannedBlobCopyKeys = new Set<string>();
1008 const hardErrors: string[] = [];
1009 const plannedIntegrationsBySlug = new Map<string, PlannedIntegration>();
1010 let clonedToolRows = 0;
1011 let operationsToBuild = 0;
1012
1013 for (const monolith of orgMonoliths) {
1014 const monolithTools = orgTools.filter((tool) => tool.integration === monolith.slug);
1015 const monolithConnections = input.connections.filter(
1016 (connection) => connection.tenant === tenant && connection.integration === monolith.slug,
1017 );
1018 const monolithOperations = operationRowsForMonolith(monolith, orgStorage);
1019 let services: readonly ServiceTarget[];
1020 try {
1021 services = deriveServices(
1022 monolith,
1023 monolithTools,
1024 input.bootRailCreateToolImpliedServices ?? false,
1025 );
1026 } catch (error) {
1027 if (!input.collectPolicyErrors) throw error;
1028 hardErrors.push(error instanceof Error ? error.message : String(error));
1029 continue;
1030 }
1031 if (
1032 isConfigOnlyMonolith({
1033 tools: monolithTools,
1034 connections: monolithConnections,
1035 operations: monolithOperations,
1036 })
1037 ) {
1038 continue;
1039 }
1040 try {
1041 assertNoUnassignedRows(monolith, monolithTools, monolithOperations, services);
1042 } catch (error) {
1043 if (!input.collectPolicyErrors) throw error;
1044 hardErrors.push(error instanceof Error ? error.message : String(error));
1045 continue;
1046 }
1047 const specHash = specHashFor(monolith);
1048 const namespace = pluginBlobNamespace(tenant, monolith.plugin_id);
1049 const targetNamespace = pluginBlobNamespace(tenant, "openapi");
1050 const specBlobPresent =
1051 (blobBackend === "external" && input.assumeExternalBlobSourcePresent === true) ||
1052 orgBlobs.some((blob) => blob.namespace === namespace && blob.key === `spec/${specHash}`);
1053 const defsBlobPresent =
1054 (blobBackend === "external" && input.assumeExternalBlobSourcePresent === true) ||
1055 orgBlobs.some((blob) => blob.namespace === namespace && blob.key === `defs/${specHash}`);
1056 for (const key of [`spec/${specHash}`, `defs/${specHash}`]) {
1057 const copyKey = rowKey(namespace, targetNamespace, key);
1058 if (plannedBlobCopyKeys.has(copyKey)) continue;
1059 plannedBlobCopyKeys.add(copyKey);
1060 const sourcePresent =
1061 (blobBackend === "external" && input.assumeExternalBlobSourcePresent === true) ||
1062 orgBlobs.some((blob) => blob.namespace === namespace && blob.key === key);
1063 const targetPresent =
1064 blobBackend === "external"
1065 ? false
1066 : orgBlobs.some((blob) => blob.namespace === targetNamespace && blob.key === key);
1067 if (!sourcePresent) {
1068 hardErrors.push(
1069 `Missing source blob ${namespace}/${key} for migrated OpenAPI service namespace ${targetNamespace}`,
1070 );
1071 }
1072 orgBlobCopies.push({
1073 source: monolith,
1074 specHash,
1075 key,
1076 sourceNamespace: namespace,
1077 targetNamespace,
1078 backend: blobBackend,
1079 sourcePresent,
1080 targetPresent,
1081 sourceObjectName: blobObjectName(namespace, key),
1082 targetObjectName: blobObjectName(targetNamespace, key),
1083 });
1084 }
1085 for (const target of services) {
1086 const serviceOperations = operationRowsForService(monolith, target, orgStorage);
1087 const operationToolNames = unique(
1088 serviceOperations.flatMap((row) => operationToolName(row) ?? []),
1089 );
1090 operationsToBuild += operationToolNames.length;
1091 const contribution = {
1092 source: monolith,
1093 specHash,
1094 operationsToBuild: operationToolNames.length,
1095 operationToolNames,
1096 specBlobPresent,
1097 defsBlobPresent,
1098 };
1099 const existing = plannedIntegrationsBySlug.get(target.slug);
1100 if (existing) {
1101 const sourceContributions = [...existing.sourceContributions, contribution];
1102 const winner = sourceContributions.reduce((best, candidate) =>
1103 candidate.operationsToBuild > best.operationsToBuild ? candidate : best,
1104 );
1105 const mergedOperationToolNames = unique(
1106 sourceContributions.flatMap((item) => item.operationToolNames),
1107 );
1108 const next: PlannedIntegration = {
1109 ...existing,
1110 sourceContributions,
1111 source: winner.source,
1112 config: configForService(winner.source, target),
1113 servingState: {
1114 specHash: winner.specHash,
1115 specSource: `${winner.source.plugin_id}/${winner.source.slug}`,
1116 blobBackend,
1117 specBlobPresent: winner.specBlobPresent,
1118 defsBlobPresent: winner.defsBlobPresent,
1119 operationsToBuild: mergedOperationToolNames.length,
1120 operationToolNames: mergedOperationToolNames,
1121 expectedZeroOperations: false,
1122 },
1123 };
1124 plannedIntegrationsBySlug.set(target.slug, next);
1125 } else {
1126 const exists = integrationExists.has(rowKey(tenant, target.slug));
1127 const planned: PlannedIntegration = {
1128 source: monolith,
1129 sourceContributions: [contribution],
1130 target,
1131 action: exists ? "skip_existing" : "create",
1132 config: configForService(monolith, target),
1133 ...(target.healthCheck ? { healthCheck: target.healthCheck } : {}),
1134 servingState: {
1135 specHash,
1136 specSource: `${monolith.plugin_id}/${monolith.slug}`,
1137 blobBackend,
1138 specBlobPresent,
1139 defsBlobPresent,
1140 operationsToBuild: operationToolNames.length,
1141 operationToolNames,
1142 expectedZeroOperations: false,
1143 },
1144 };
1145 plannedIntegrationsBySlug.set(target.slug, planned);
1146 }
1147 }
1148
1149 for (const connection of monolithConnections) {
1150 for (const target of services) {
1151 const key = rowKey(
1152 tenant,
1153 connection.owner,
1154 connection.subject,
1155 target.slug,
1156 connection.name,
1157 );
1158 const exists = connectionExists.has(key);
1159 orgConnections.push({
1160 source: connection,
1161 targetIntegration: target.slug,
1162 action: exists ? "skip_existing" : "clone",
1163 tokenReuse: "copy_item_ids_and_oauth_columns",
1164 });
1165 if (!exists) {
1166 connectionExists.add(key);
1167 }
1168 clonedToolRows += monolithTools.filter((tool) =>
1169 serviceForMatchedTool(monolith.plugin_id as MonolithPluginId, tool.name, [
1170 target,
1171 ]).includes(target.slug),
1172 ).length;
1173 }
1174 }
1175
1176 const candidatePolicies = input.policies.filter((policy) => policy.tenant === tenant);
1177 for (const policy of candidatePolicies) {
1178 const patternIntegration = integrationPatternSegment(policy.pattern).integration;
1179 if (patternIntegration !== monolith.slug) continue;
1180 try {
1181 orgPolicies.push(rewritePolicy(policy, monolith, services, input.orphanPolicyMode));
1182 } catch (error) {
1183 if (!input.collectPolicyErrors) throw error;
1184 hardErrors.push(error instanceof Error ? error.message : String(error));
1185 }
1186 }
1187 }
1188
1189 return {
1190 tenant,
1191 tenantHash: tenantHash(tenant),
1192 completed: completed.has(tenant),
1193 integrations: [...plannedIntegrationsBySlug.values()],
1194 connections: orgConnections,
1195 policies: orgPolicies,
1196 blobCopies: orgBlobCopies,
1197 deleteMonoliths: orgMonoliths,
1198 clonedToolRows,
1199 operationsToBuild,
1200 hardErrors,
1201 };
1202 });
1203
1204 const activeOrgs = orgs.filter((org) => !org.completed && org.hardErrors.length === 0);
1205 const summary = {
1206 orgs: orgs.length,
1207 completedOrgs: orgs.filter((org) => org.completed).length,
1208 integrationsCreate: activeOrgs
1209 .flatMap((org) => org.integrations)
1210 .filter((row) => row.action === "create").length,
1211 integrationsSkipExisting: activeOrgs
1212 .flatMap((org) => org.integrations)
1213 .filter((row) => row.action === "skip_existing").length,
1214 connectionsClone: activeOrgs
1215 .flatMap((org) => org.connections)
1216 .filter((row) => row.action === "clone").length,
1217 connectionsSkipExisting: activeOrgs
1218 .flatMap((org) => org.connections)
1219 .filter((row) => row.action === "skip_existing").length,
1220 policiesRewrite: activeOrgs
1221 .flatMap((org) => org.policies)
1222 .filter((row) => row.action === "rewrite").length,
1223 policiesSkip: 0,
1224 policyRowsAfter: activeOrgs
1225 .flatMap((org) => org.policies)
1226 .reduce((sum, row) => sum + row.afterPatterns.length, 0),
1227 monolithDeletes: activeOrgs.flatMap((org) => org.deleteMonoliths).length,
1228 clonedToolRows: activeOrgs.reduce((sum, org) => sum + org.clonedToolRows, 0),
1229 operationsToBuild: activeOrgs.reduce((sum, org) => sum + org.operationsToBuild, 0),
1230 integrationsMissingSpecBlob: activeOrgs
1231 .flatMap((org) => org.integrations)
1232 .filter((row) => !row.servingState.specBlobPresent).length,
1233 integrationsMissingDefsBlob: activeOrgs
1234 .flatMap((org) => org.integrations)
1235 .filter((row) => !row.servingState.defsBlobPresent).length,
1236 hardErrorOrgs: orgs.filter((org) => !org.completed && org.hardErrors.length > 0).length,
1237 policyHardErrors: orgs.reduce((sum, org) => sum + org.hardErrors.length, 0),
1238 };
1239 return { orgs, summary };
1240};
1241
1242const printableJson = (value: unknown): string =>
1243 JSON.stringify(value, (_key, inner) => (inner === undefined ? null : inner), 2);
1244
1245export const renderOrgDiff = (org: OrgPlan): string => {
1246 const lines: string[] = [];
1247 lines.push(`# Org ${org.tenantHash}`);
1248 lines.push("");
1249 if (org.completed) {
1250 lines.push("Already completed, no changes planned.");
1251 lines.push("");
1252 return lines.join("\n");
1253 }
1254 if (org.hardErrors.length > 0) {
1255 lines.push("## Hard Errors");
1256 lines.push("Apply is blocked for this org until these errors are handled.");
1257 for (const error of org.hardErrors) {
1258 lines.push(`- ${error}`);
1259 }
1260 lines.push("");
1261 }
1262 lines.push("## Integrations");
1263 for (const row of org.integrations) {
1264 lines.push(
1265 `- ${row.action}: ${row.source.plugin_id}/${row.source.slug} (${row.source.name ?? "unnamed"}) -> ${row.target.slug} (${row.target.name})`,
1266 );
1267 lines.push(
1268 ` serving: operations to build: ${row.servingState.operationsToBuild} / spec source: ${row.servingState.specSource} / blob backend: ${row.servingState.blobBackend} / specHash: ${row.servingState.specHash} / spec blob: ${row.servingState.specBlobPresent ? "present" : "missing"} / defs blob: ${row.servingState.defsBlobPresent ? "present" : "missing"}`,
1269 );
1270 if (row.sourceContributions.length > 1) {
1271 for (const contribution of row.sourceContributions) {
1272 lines.push(
1273 ` contribution: ${contribution.source.plugin_id}/${contribution.source.slug} / operations: ${contribution.operationsToBuild} / specHash: ${contribution.specHash ?? "none"}`,
1274 );
1275 }
1276 }
1277 lines.push(` config: ${printableJson(row.config).replaceAll("\n", "\n ")}`);
1278 }
1279 for (const row of org.deleteMonoliths) {
1280 lines.push(
1281 `- delete monolith in apply mode: ${row.plugin_id}/${row.slug} (${row.name ?? "unnamed"})`,
1282 );
1283 }
1284 lines.push("");
1285 lines.push("## Blob Copies");
1286 for (const row of org.blobCopies) {
1287 lines.push(
1288 `- ${row.backend}: ${row.sourceObjectName} -> ${row.targetObjectName} / source: ${row.sourcePresent ? "present" : "missing"} / target: ${row.targetPresent ? "present" : "missing"}`,
1289 );
1290 }
1291 lines.push("");
1292 lines.push("## Connections");
1293 for (const row of org.connections) {
1294 lines.push(
1295 `- ${row.action}: ${row.source.integration}.${row.source.owner}.${row.source.name} -> ${row.targetIntegration}.${row.source.owner}.${row.source.name} (${row.tokenReuse})`,
1296 );
1297 }
1298 lines.push("");
1299 lines.push("## Policies");
1300 for (const row of org.policies) {
1301 lines.push(`- rewrite ${row.policy.id}: ${row.policy.pattern}`);
1302 for (const after of row.afterPatterns) {
1303 lines.push(` -> ${after}`);
1304 }
1305 }
1306 lines.push("");
1307 lines.push("## Internal Rows");
1308 lines.push(`- tool rows cloned in apply mode: ${org.clonedToolRows}`);
1309 lines.push(`- operation rows copied in apply mode: ${org.operationsToBuild}`);
1310 lines.push("");
1311 return lines.join("\n");
1312};
1313
1314export const renderSummary = (plan: MigrationPlan): string => {
1315 const s = plan.summary;
1316 return [
1317 `orgs=${s.orgs}`,
1318 `completed_orgs=${s.completedOrgs}`,
1319 `integrations_create=${s.integrationsCreate}`,
1320 `integrations_skip_existing=${s.integrationsSkipExisting}`,
1321 `connections_clone=${s.connectionsClone}`,
1322 `connections_skip_existing=${s.connectionsSkipExisting}`,
1323 `policies_rewrite=${s.policiesRewrite}`,
1324 `policy_rows_after=${s.policyRowsAfter}`,
1325 `policies_skip=${s.policiesSkip}`,
1326 `monolith_deletes=${s.monolithDeletes}`,
1327 `tool_rows_clone=${s.clonedToolRows}`,
1328 `operation_rows_build=${s.operationsToBuild}`,
1329 `integrations_missing_spec_blob=${s.integrationsMissingSpecBlob}`,
1330 `integrations_missing_defs_blob=${s.integrationsMissingDefsBlob}`,
1331 `hard_error_orgs=${s.hardErrorOrgs}`,
1332 `policy_hard_errors=${s.policyHardErrors}`,
1333 ].join("\n");
1334};
scripts/migration/service-split-postgres-cli.ts+1−1
@@ -22,7 +22,7 @@ import { · 21 unmodified lines
22 type PluginStorageRow,22 type PluginStorageRow,
23 type ToolPolicyRow,23 type ToolPolicyRow,
24 type ToolRow,24 type ToolRow,
25} from "./service-split-planner";25} from "@executor-js/plugin-provider-service-split/planner";
26 26
27export interface SqlClient {27export interface SqlClient {
28 readonly unsafe: <T extends readonly object[] = readonly Record<string, unknown>[]>(28 readonly unsafe: <T extends readonly object[] = readonly Record<string, unknown>[]>(
scripts/migration/service-split-sqlite.ts+0−551
@@ -1,551 +0,0 @@
1import { Effect } from "effect";
2import {
3 DataMigrationError,
4 type SqliteDataMigrationClient,
5} from "../../packages/core/sdk/src/sqlite-data-migrations";
6
7import {
8 operationStorageKey,
9 planMigration,
10 storageDataRecord,
11 tenantHash,
12 type BlobRow,
13 type ConnectionRow,
14 type IntegrationRow,
15 type MigrationInput,
16 type OrgPlan,
17 type PluginStorageRow,
18 type ToolPolicyRow,
19 type ToolRow,
20} from "./service-split-planner";
21
22const MIGRATION_NAME = "2026-07-08-provider-service-split";
23const LEDGER_TABLE = "provider_service_split_org_migration";
24
25const operationKeyPrefix = (integration: string): string =>
26 `${operationStorageKey(integration, "").split(".").slice(0, 2).join(".")}.%`;
27
28const execute = (
29 client: SqliteDataMigrationClient,
30 stmt: string | { readonly sql: string; readonly args: readonly unknown[] },
31) =>
32 Effect.tryPromise({
33 try: () => client.execute(stmt),
34 catch: (cause) => new DataMigrationError({ migration: MIGRATION_NAME, cause }),
35 });
36
37const parseJsonLike = (value: unknown): unknown => {
38 if (typeof value !== "string") return value;
39 try {
40 return JSON.parse(value) as unknown;
41 } catch {
42 return value;
43 }
44};
45
46const scrubJson = (value: unknown): unknown => {
47 if (Array.isArray(value)) return value.map(scrubJson);
48 if (value && typeof value === "object") {
49 return Object.fromEntries(
50 Object.entries(value)
51 .filter(([, inner]) => inner !== undefined)
52 .map(([key, inner]) => [key, scrubJson(inner)]),
53 );
54 }
55 return value;
56};
57
58const stableId = (...parts: readonly string[]): string => {
59 const text = parts.join("\u0000");
60 let hash = 0;
61 for (let index = 0; index < text.length; index += 1) {
62 hash = (hash * 31 + text.charCodeAt(index)) >>> 0;
63 }
64 return `service_split_${hash.toString(36)}`;
65};
66
67const tableExists = (client: SqliteDataMigrationClient, table: string) =>
68 execute(client, {
69 sql: "SELECT name FROM sqlite_master WHERE type = 'table' AND name = ?",
70 args: [table],
71 }).pipe(Effect.map((result) => result.rows.length > 0));
72
73const ensureLedgerTable = (client: SqliteDataMigrationClient) =>
74 execute(
75 client,
76 `CREATE TABLE IF NOT EXISTS ${LEDGER_TABLE} (
77 tenant text PRIMARY KEY NOT NULL,
78 time_completed integer NOT NULL
79 )`,
80 );
81
82const readCompletedTenants = (client: SqliteDataMigrationClient) =>
83 execute(client, `SELECT tenant FROM ${LEDGER_TABLE} ORDER BY tenant`).pipe(
84 Effect.map((result) => result.rows.map((row) => String(row.tenant))),
85 );
86
87const stampCompletedTenant = (client: SqliteDataMigrationClient, tenant: string) =>
88 execute(client, {
89 sql: `INSERT INTO ${LEDGER_TABLE} (tenant, time_completed)
90 VALUES (?, ?)
91 ON CONFLICT(tenant) DO UPDATE SET time_completed = excluded.time_completed`,
92 args: [tenant, Date.now()],
93 });
94
95const readDatabaseInput = (
96 client: SqliteDataMigrationClient,
97 options: {
98 readonly blobBackend?: MigrationInput["blobBackend"];
99 readonly blobs?: readonly BlobRow[];
100 } = {},
101): Effect.Effect<MigrationInput, DataMigrationError> =>
102 Effect.gen(function* () {
103 yield* ensureLedgerTable(client);
104 const completedTenants = yield* readCompletedTenants(client);
105 const integrations = yield* execute(
106 client,
107 `SELECT tenant, slug, plugin_id, name, description, config, health_check,
108 config_revised_at, can_remove, can_refresh, CAST(created_at AS TEXT) AS created_at,
109 CAST(updated_at AS TEXT) AS updated_at, row_id
110 FROM integration
111 WHERE plugin_id IN ('google', 'microsoft')
112 OR slug IN ('google', 'microsoft')
113 OR tenant IN (
114 SELECT tenant FROM integration
115 WHERE plugin_id IN ('google', 'microsoft')
116 )
117 ORDER BY tenant, slug`,
118 );
119 const monolithTenants = new Set(
120 integrations.rows
121 .filter((row) => row.plugin_id === "google" || row.plugin_id === "microsoft")
122 .map((row) => String(row.tenant)),
123 );
124 if (monolithTenants.size === 0) {
125 return {
126 integrations: [],
127 connections: [],
128 tools: [],
129 pluginStorage: [],
130 blobs: options.blobs ?? [],
131 policies: [],
132 completedTenants,
133 };
134 }
135
136 const connections = yield* execute(
137 client,
138 `SELECT tenant, owner, subject, integration, name, template, provider, item_ids,
139 identity_label, description, last_health, tools_synced_at, oauth_client,
140 oauth_client_owner, refresh_item_id, expires_at, oauth_scope, oauth_token_url,
141 provider_state, CAST(created_at AS TEXT) AS created_at,
142 CAST(updated_at AS TEXT) AS updated_at, row_id
143 FROM connection
144 ORDER BY tenant, integration, owner, subject, name`,
145 );
146 const tools = yield* execute(
147 client,
148 `SELECT tenant, owner, subject, integration, connection, plugin_id, name, description,
149 input_schema, output_schema, annotations, CAST(created_at AS TEXT) AS created_at,
150 CAST(updated_at AS TEXT) AS updated_at, row_id
151 FROM tool
152 ORDER BY tenant, integration, connection, name`,
153 );
154 const pluginStorage = yield* execute(
155 client,
156 `SELECT tenant, owner, subject, plugin_id, collection, key, data,
157 CAST(created_at AS TEXT) AS created_at, CAST(updated_at AS TEXT) AS updated_at, row_id
158 FROM plugin_storage
159 WHERE plugin_id IN ('google', 'microsoft')
160 AND collection = 'operation'
161 ORDER BY tenant, plugin_id, collection, key`,
162 );
163 const blobs = yield* execute(
164 client,
165 `SELECT id, namespace, key
166 FROM blob
167 WHERE key LIKE 'spec/%' OR key LIKE 'defs/%'
168 ORDER BY namespace, key`,
169 );
170 const policies = yield* execute(
171 client,
172 `SELECT tenant, owner, subject, id, pattern, action, position,
173 CAST(created_at AS TEXT) AS created_at, CAST(updated_at AS TEXT) AS updated_at, row_id
174 FROM tool_policy
175 ORDER BY tenant, owner, subject, position, id`,
176 );
177
178 const tenantFilter = (row: Record<string, unknown>) => monolithTenants.has(String(row.tenant));
179 return {
180 integrations: integrations.rows.filter(tenantFilter).map(
181 (row): IntegrationRow => ({
182 tenant: String(row.tenant),
183 slug: String(row.slug),
184 plugin_id: String(row.plugin_id),
185 name: typeof row.name === "string" ? row.name : null,
186 description: typeof row.description === "string" ? row.description : null,
187 config: parseJsonLike(row.config),
188 health_check: parseJsonLike(row.health_check),
189 config_revised_at: row.config_revised_at as string | number | bigint | null,
190 can_remove: Boolean(row.can_remove),
191 can_refresh: Boolean(row.can_refresh),
192 created_at: String(row.created_at),
193 updated_at: String(row.updated_at),
194 row_id: String(row.row_id),
195 }),
196 ),
197 connections: connections.rows.filter(tenantFilter).map(
198 (row): ConnectionRow => ({
199 tenant: String(row.tenant),
200 owner: String(row.owner),
201 subject: String(row.subject),
202 integration: String(row.integration),
203 name: String(row.name),
204 template: String(row.template),
205 provider: String(row.provider),
206 item_ids: parseJsonLike(row.item_ids),
207 identity_label: typeof row.identity_label === "string" ? row.identity_label : null,
208 description: typeof row.description === "string" ? row.description : null,
209 last_health: parseJsonLike(row.last_health),
210 tools_synced_at: row.tools_synced_at as string | number | bigint | null,
211 oauth_client: typeof row.oauth_client === "string" ? row.oauth_client : null,
212 oauth_client_owner:
213 typeof row.oauth_client_owner === "string" ? row.oauth_client_owner : null,
214 refresh_item_id: typeof row.refresh_item_id === "string" ? row.refresh_item_id : null,
215 expires_at: row.expires_at as string | number | bigint | null,
216 oauth_scope: typeof row.oauth_scope === "string" ? row.oauth_scope : null,
217 oauth_token_url: typeof row.oauth_token_url === "string" ? row.oauth_token_url : null,
218 provider_state: parseJsonLike(row.provider_state),
219 created_at: String(row.created_at),
220 updated_at: String(row.updated_at),
221 row_id: String(row.row_id),
222 }),
223 ),
224 tools: tools.rows.filter(tenantFilter).map(
225 (row): ToolRow => ({
226 tenant: String(row.tenant),
227 owner: String(row.owner),
228 subject: String(row.subject),
229 integration: String(row.integration),
230 connection: String(row.connection),
231 plugin_id: String(row.plugin_id),
232 name: String(row.name),
233 description: typeof row.description === "string" ? row.description : undefined,
234 input_schema: parseJsonLike(row.input_schema),
235 output_schema: parseJsonLike(row.output_schema),
236 annotations: parseJsonLike(row.annotations),
237 created_at: String(row.created_at),
238 updated_at: String(row.updated_at),
239 row_id: String(row.row_id),
240 }),
241 ),
242 pluginStorage: pluginStorage.rows.filter(tenantFilter).map(
243 (row): PluginStorageRow => ({
244 tenant: String(row.tenant),
245 owner: String(row.owner),
246 subject: String(row.subject),
247 plugin_id: String(row.plugin_id),
248 collection: String(row.collection),
249 key: String(row.key),
250 data: parseJsonLike(row.data),
251 created_at: String(row.created_at),
252 updated_at: String(row.updated_at),
253 row_id: String(row.row_id),
254 }),
255 ),
256 blobs: [
257 ...blobs.rows.map(
258 (row): BlobRow => ({
259 id: String(row.id),
260 namespace: String(row.namespace),
261 key: String(row.key),
262 }),
263 ),
264 ...(options.blobs ?? []),
265 ],
266 policies: policies.rows.filter(tenantFilter).map(
267 (row): ToolPolicyRow => ({
268 tenant: String(row.tenant),
269 owner: String(row.owner),
270 subject: String(row.subject),
271 id: String(row.id),
272 pattern: String(row.pattern),
273 action: String(row.action),
274 position: String(row.position),
275 created_at: String(row.created_at),
276 updated_at: String(row.updated_at),
277 row_id: String(row.row_id),
278 }),
279 ),
280 blobBackend: options.blobBackend ?? "database",
281 orphanPolicyMode: "retarget_all",
282 collectPolicyErrors: true,
283 bootRailCreateToolImpliedServices: true,
284 completedTenants,
285 };
286 });
287
288const applyOrg = (
289 client: SqliteDataMigrationClient,
290 org: OrgPlan,
291): Effect.Effect<void, DataMigrationError> =>
292 Effect.gen(function* () {
293 if (org.completed) return;
294 if (org.hardErrors.length > 0) return;
295 const now = Date.now();
296
297 for (const copy of org.blobCopies.filter((item) => item.backend === "database")) {
298 const source = yield* execute(client, {
299 sql: "SELECT value FROM blob WHERE namespace = ? AND key = ? LIMIT 1",
300 args: [copy.sourceNamespace, copy.key],
301 });
302 const sourceRow = source.rows[0];
303 if (!sourceRow || typeof sourceRow.value !== "string") {
304 return yield* new DataMigrationError({
305 migration: MIGRATION_NAME,
306 cause: `Missing source blob ${copy.sourceNamespace}/${copy.key}`,
307 });
308 }
309 yield* execute(client, {
310 sql: `INSERT OR IGNORE INTO blob (namespace, key, value, row_id, id)
311 VALUES (?, ?, ?, ?, ?)`,
312 args: [
313 copy.targetNamespace,
314 copy.key,
315 sourceRow.value,
316 stableId("blob", org.tenant, copy.targetNamespace, copy.key),
317 JSON.stringify([copy.targetNamespace, copy.key]),
318 ],
319 });
320 }
321
322 for (const row of org.integrations.filter((item) => item.action === "create")) {
323 yield* execute(client, {
324 sql: `INSERT OR IGNORE INTO integration
325 (slug, plugin_id, name, description, config, health_check, config_revised_at,
326 can_remove, can_refresh, created_at, updated_at, row_id, tenant)
327 VALUES (?, ?, ?, ?, ?, ?, NULL, 1, 1, ?, ?, ?, ?)`,
328 args: [
329 row.target.slug,
330 row.target.pluginId,
331 row.target.name,
332 row.target.description,
333 JSON.stringify(scrubJson(row.config)),
334 row.healthCheck ? JSON.stringify(scrubJson(row.healthCheck)) : null,
335 now,
336 now,
337 stableId("integration", org.tenant, row.target.slug),
338 org.tenant,
339 ],
340 });
341 }
342
343 for (const row of org.connections.filter((item) => item.action === "clone")) {
344 yield* execute(client, {
345 sql: `INSERT OR IGNORE INTO connection
346 (integration, name, template, provider, item_ids, identity_label, description,
347 last_health, tools_synced_at, oauth_client, oauth_client_owner, refresh_item_id,
348 expires_at, oauth_scope, oauth_token_url, provider_state, created_at, updated_at,
349 row_id, tenant, owner, subject)
350 SELECT ?, name, template, provider, item_ids, identity_label, description,
351 NULL, NULL, oauth_client, oauth_client_owner, refresh_item_id, expires_at,
352 oauth_scope, oauth_token_url, provider_state, created_at, ?, ?, tenant, owner, subject
353 FROM connection
354 WHERE tenant = ? AND owner = ? AND subject = ? AND integration = ? AND name = ?`,
355 args: [
356 row.targetIntegration,
357 now,
358 stableId(
359 "connection",
360 org.tenant,
361 row.source.owner,
362 row.source.subject,
363 row.targetIntegration,
364 row.source.name,
365 ),
366 org.tenant,
367 row.source.owner,
368 row.source.subject,
369 row.source.integration,
370 row.source.name,
371 ],
372 });
373 }
374
375 for (const integration of org.integrations) {
376 for (const contribution of integration.sourceContributions) {
377 for (const toolName of contribution.operationToolNames) {
378 const operation = yield* execute(client, {
379 sql: `SELECT data, created_at, updated_at
380 FROM plugin_storage
381 WHERE tenant = ?
382 AND owner = 'org'
383 AND subject = ''
384 AND plugin_id = ?
385 AND collection = 'operation'
386 AND (
387 key = ?
388 OR (
389 json_extract(data, '$.integration') = ?
390 AND json_extract(data, '$.toolName') = ?
391 )
392 )
393 LIMIT 1`,
394 args: [
395 org.tenant,
396 contribution.source.plugin_id,
397 operationStorageKey(contribution.source.slug, toolName),
398 contribution.source.slug,
399 toolName,
400 ],
401 });
402 const source = operation.rows[0];
403 if (!source) {
404 return yield* new DataMigrationError({
405 migration: MIGRATION_NAME,
406 cause: new Error(
407 `Missing operation row for ${tenantHash(org.tenant)}/${contribution.source.slug}/${toolName}`,
408 ),
409 });
410 }
411 yield* execute(client, {
412 sql: `INSERT INTO plugin_storage
413 (plugin_id, collection, key, data, created_at, updated_at, row_id, tenant, owner, subject)
414 VALUES (?, 'operation', ?, ?, ?, ?, ?, ?, 'org', '')
415 ON CONFLICT(tenant, owner, subject, plugin_id, collection, key)
416 DO UPDATE SET data = excluded.data, updated_at = excluded.updated_at`,
417 args: [
418 integration.target.pluginId,
419 operationStorageKey(integration.target.slug, toolName),
420 JSON.stringify(
421 scrubJson({
422 ...storageDataRecord({ data: parseJsonLike(source.data) }),
423 integration: integration.target.slug,
424 toolName,
425 }),
426 ),
427 source.created_at,
428 now,
429 stableId("operation", org.tenant, integration.target.slug, toolName),
430 org.tenant,
431 ],
432 });
433 yield* execute(client, {
434 sql: `INSERT OR IGNORE INTO tool
435 (integration, connection, plugin_id, name, description, input_schema, output_schema,
436 annotations, created_at, updated_at, row_id, tenant, owner, subject)
437 SELECT ?, connection, ?, name, description, input_schema, output_schema,
438 annotations, created_at, ?, ? || '_' || row_id, tenant, owner, subject
439 FROM tool
440 WHERE tenant = ? AND integration = ? AND name = ?`,
441 args: [
442 integration.target.slug,
443 integration.target.pluginId,
444 now,
445 stableId("tool", org.tenant, integration.target.slug),
446 org.tenant,
447 contribution.source.slug,
448 toolName,
449 ],
450 });
451 }
452 }
453 }
454
455 for (const policy of org.policies) {
456 yield* execute(client, {
457 sql: "DELETE FROM tool_policy WHERE tenant = ? AND owner = ? AND subject = ? AND id = ?",
458 args: [org.tenant, policy.policy.owner, policy.policy.subject, policy.policy.id],
459 });
460 for (const [index, pattern] of policy.afterPatterns.entries()) {
461 yield* execute(client, {
462 sql: `INSERT OR IGNORE INTO tool_policy
463 (id, pattern, action, position, created_at, updated_at, row_id, tenant, owner, subject)
464 VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
465 args: [
466 policy.afterPatterns.length === 1
467 ? policy.policy.id
468 : `${policy.policy.id}_${index + 1}`,
469 pattern,
470 policy.policy.action,
471 policy.policy.position,
472 now,
473 now,
474 stableId("policy", org.tenant, policy.policy.id, pattern),
475 org.tenant,
476 policy.policy.owner,
477 policy.policy.subject,
478 ],
479 });
480 }
481 }
482
483 for (const monolith of org.deleteMonoliths) {
484 yield* execute(client, {
485 sql: "DELETE FROM tool WHERE tenant = ? AND integration = ?",
486 args: [org.tenant, monolith.slug],
487 });
488 yield* execute(client, {
489 sql: "DELETE FROM connection WHERE tenant = ? AND integration = ?",
490 args: [org.tenant, monolith.slug],
491 });
492 yield* execute(client, {
493 sql: `DELETE FROM plugin_storage
494 WHERE tenant = ?
495 AND owner = 'org'
496 AND subject = ''
497 AND plugin_id = ?
498 AND collection = 'operation'
499 AND (
500 json_extract(data, '$.integration') = ?
501 OR key LIKE ?
502 )`,
503 args: [org.tenant, monolith.plugin_id, monolith.slug, operationKeyPrefix(monolith.slug)],
504 });
505 yield* execute(client, {
506 sql: "DELETE FROM integration WHERE tenant = ? AND slug = ? AND plugin_id = ?",
507 args: [org.tenant, monolith.slug, monolith.plugin_id],
508 });
509 }
510 });
511
512export const runSqliteProviderServiceSplitMigration = (
513 client: SqliteDataMigrationClient,
514 options: {
515 readonly beforeStampOrg?: (org: OrgPlan) => Effect.Effect<void, DataMigrationError>;
516 readonly blobBackend?: MigrationInput["blobBackend"];
517 readonly blobs?: readonly BlobRow[];
518 } = {},
519): Effect.Effect<number, DataMigrationError> =>
520 Effect.gen(function* () {
521 if (!(yield* tableExists(client, "integration"))) return 0;
522 const input = yield* readDatabaseInput(client, options);
523 const plan = planMigration(input);
524 const moved = plan.orgs.filter((org) => !org.completed && org.hardErrors.length === 0).length;
525 if (moved === 0) return 0;
526
527 for (const org of plan.orgs) {
528 if (org.completed) continue;
529 if (org.hardErrors.length > 0) {
530 console.warn(
531 `provider-service-split: skipped org ${org.tenantHash}: ${org.hardErrors.join("; ")}`,
532 );
533 continue;
534 }
535 yield* execute(client, "BEGIN");
536 const applyOne = Effect.gen(function* () {
537 yield* applyOrg(client, org);
538 if (options.beforeStampOrg) yield* options.beforeStampOrg(org);
539 yield* stampCompletedTenant(client, org.tenant);
540 yield* execute(client, "COMMIT");
541 });
542 yield* applyOne.pipe(Effect.tapError(() => execute(client, "ROLLBACK").pipe(Effect.ignore)));
543 }
544 return moved;
545 });
546
547export const providerServiceSplitDataMigration = {
548 name: MIGRATION_NAME,
549 run: (client: SqliteDataMigrationClient) =>
550 runSqliteProviderServiceSplitMigration(client).pipe(Effect.asVoid),
551};