Showing the changes to scripts/migration/service-split-postgres-cli.ts (1 of 8 files). Show all
Unified · Split
| @@ -48,6 +48,7 @@ const outputDir = resolve( · 47 unmodified lines | |||
| 48 | const orgFilter = argValue("--org"); | 48 | const orgFilter = argValue("--org"); |
| 49 | const trafficLastTenant = argValue("--traffic-last-tenant"); | 49 | const trafficLastTenant = argValue("--traffic-last-tenant"); |
| 50 | const databaseUrl = argValue("--database-url") ?? process.env.DATABASE_URL; | 50 | const databaseUrl = argValue("--database-url") ?? process.env.DATABASE_URL; |
| 51 | const r2CopiesDone = hasArg("--r2-copies-done"); | ||
| 51 | export const LEDGER_TABLE = "provider_service_split_org_migration"; | 52 | export const LEDGER_TABLE = "provider_service_split_org_migration"; |
| 52 | 53 | ||
| 53 | const parseInputJson = (path: string): MigrationInput => | 54 | const parseInputJson = (path: string): MigrationInput => |
| @@ -212,6 +213,7 @@ export const readDatabaseInput = async ( · 158 unmodified lines | |||
| 212 | completedTenants, | 213 | completedTenants, |
| 213 | trafficLastTenant, | 214 | trafficLastTenant, |
| 214 | blobBackend, | 215 | blobBackend, |
| 216 | assumeExternalBlobSourcePresent: blobBackend === "external", | ||
| 215 | }; | 217 | }; |
| 216 | }; | 218 | }; |
| 217 | 219 | ||
| @@ -242,10 +244,34 @@ const operationKeyPrefix = (integration: string): string => · 24 unmodified lines | |||
| 242 | export const applyOrg = async (sql: SqlClient, org: OrgPlan): Promise<void> => { | 244 | export const applyOrg = async (sql: SqlClient, org: OrgPlan): Promise<void> => { |
| 243 | if (org.completed) return; | 245 | if (org.completed) return; |
| 244 | if (org.hardErrors.length > 0) { | 246 | if (org.hardErrors.length > 0) { |
| 245 | throw new Error(`org ${org.tenantHash} has unresolved policy hard errors; apply is blocked`); | 247 | throw new Error(`org ${org.tenantHash} has unresolved hard errors; apply is blocked`); |
| 246 | } | 248 | } |
| 247 | await sql.begin(async (tx) => { | 249 | await sql.begin(async (tx) => { |
| 248 | const now = new Date(); | 250 | const now = new Date(); |
| 251 | for (const copy of org.blobCopies.filter((item) => item.backend === "database")) { | ||
| 252 | const [source] = await tx.unsafe<{ readonly value: string }[]>( | ||
| 253 | "SELECT value FROM blob WHERE namespace = $1 AND key = $2 LIMIT 1", | ||
| 254 | [copy.sourceNamespace, copy.key], | ||
| 255 | ); | ||
| 256 | if (!source) { | ||
| 257 | throw new Error(`Missing source blob ${copy.sourceNamespace}/${copy.key}`); | ||
| 258 | } | ||
| 259 | await tx.unsafe( | ||
| 260 | ` | ||
| 261 | INSERT INTO blob (namespace, key, value, row_id, id) | ||
| 262 | VALUES ($1, $2, $3, $4, $5) | ||
| 263 | ON CONFLICT (id) DO NOTHING | ||
| 264 | `, | ||
| 265 | [ | ||
| 266 | copy.targetNamespace, | ||
| 267 | copy.key, | ||
| 268 | source.value, | ||
| 269 | stableId("blob", org.tenant, copy.targetNamespace, copy.key), | ||
| 270 | JSON.stringify([copy.targetNamespace, copy.key]), | ||
| 271 | ], | ||
| 272 | ); | ||
| 273 | } | ||
| 274 | |||
| 249 | for (const row of org.integrations.filter((item) => item.action === "create")) { | 275 | for (const row of org.integrations.filter((item) => item.action === "create")) { |
| 250 | await tx.unsafe( | 276 | await tx.unsafe( |
| 251 | ` | 277 | ` |
| @@ -471,17 +497,35 @@ export const applyOrg = async (sql: SqlClient, org: OrgPlan): Promise<void> => { · 219 unmodified lines | |||
| 471 | const writeDryRun = (planInput: MigrationInput): void => { | 497 | const writeDryRun = (planInput: MigrationInput): void => { |
| 472 | const plan = planMigration({ ...planInput, collectPolicyErrors: true }); | 498 | const plan = planMigration({ ...planInput, collectPolicyErrors: true }); |
| 473 | const neverWiden = verifyPolicyRewriteNeverWidens(plan, planInput); | 499 | const neverWiden = verifyPolicyRewriteNeverWidens(plan, planInput); |
| 500 | const r2Copies = plan.orgs | ||
| 501 | .filter((org) => !org.completed && org.hardErrors.length === 0) | ||
| 502 | .flatMap((org) => | ||
| 503 | org.blobCopies | ||
| 504 | .filter((copy) => copy.backend === "external") | ||
| 505 | .map((copy) => ({ | ||
| 506 | tenantHash: org.tenantHash, | ||
| 507 | tenant: org.tenant, | ||
| 508 | sourceKey: copy.sourceObjectName, | ||
| 509 | destKey: copy.targetObjectName, | ||
| 510 | })), | ||
| 511 | ); | ||
| 474 | mkdirSync(outputDir, { recursive: true }); | 512 | mkdirSync(outputDir, { recursive: true }); |
| 475 | for (const org of plan.orgs) { | 513 | for (const org of plan.orgs) { |
| 476 | writeFileSync(`${outputDir}/org-${org.tenantHash}.md`, renderOrgDiff(org)); | 514 | writeFileSync(`${outputDir}/org-${org.tenantHash}.md`, renderOrgDiff(org)); |
| 477 | } | 515 | } |
| 516 | writeFileSync(`${outputDir}/r2-copy-manifest.json`, `${JSON.stringify(r2Copies, null, 2)}\n`); | ||
| 478 | writeFileSync( | 517 | writeFileSync( |
| 479 | `${outputDir}/summary.md`, | 518 | `${outputDir}/summary.md`, |
| 480 | `${renderSummary(plan)}\npolicy_never_widen=${neverWiden.ok}\npolicy_never_widen_checked=${neverWiden.checkedPolicies}\n`, | 519 | `${renderSummary(plan)}\npolicy_never_widen=${neverWiden.ok}\npolicy_never_widen_checked=${neverWiden.checkedPolicies}\nr2_copy_manifest=${outputDir}/r2-copy-manifest.json\nr2_copy_count=${r2Copies.length}\nr2_copy_command=wrangler r2 object get <bucket>/<sourceKey> | wrangler r2 object put <bucket>/<destKey> --pipe\n`, |
| 481 | ); | 520 | ); |
| 482 | console.log(renderSummary(plan)); | 521 | console.log(renderSummary(plan)); |
| 483 | console.log(`policy_never_widen=${neverWiden.ok}`); | 522 | console.log(`policy_never_widen=${neverWiden.ok}`); |
| 484 | console.log(`policy_never_widen_checked=${neverWiden.checkedPolicies}`); | 523 | console.log(`policy_never_widen_checked=${neverWiden.checkedPolicies}`); |
| 524 | console.log(`r2_copy_manifest=${outputDir}/r2-copy-manifest.json`); | ||
| 525 | console.log(`r2_copy_count=${r2Copies.length}`); | ||
| 526 | console.log( | ||
| 527 | "r2_copy_command=wrangler r2 object get <bucket>/<sourceKey> | wrangler r2 object put <bucket>/<destKey> --pipe", | ||
| 528 | ); | ||
| 485 | console.log(`diff_dir=${outputDir}`); | 529 | console.log(`diff_dir=${outputDir}`); |
| 486 | if (!neverWiden.ok) { | 530 | if (!neverWiden.ok) { |
| 487 | throw new Error( | 531 | throw new Error( |
| @@ -512,7 +556,10 @@ const main = async (): Promise<void> => { · 24 unmodified lines | |||
| 512 | const input = parseInputJson(inputJson); | 556 | const input = parseInputJson(inputJson); |
| 513 | writeDryRun( | 557 | writeDryRun( |
| 514 | filterInputByOrg( | 558 | filterInputByOrg( |
| 515 | { ...input, trafficLastTenant: trafficLastTenant ?? input.trafficLastTenant }, | 559 | { |
| 560 | ...input, | ||
| 561 | trafficLastTenant: trafficLastTenant ?? input.trafficLastTenant, | ||
| 562 | }, | ||
| 516 | orgFilter, | 563 | orgFilter, |
| 517 | ), | 564 | ), |
| 518 | ); | 565 | ); |
| @@ -560,6 +607,14 @@ const main = async (): Promise<void> => { · 41 unmodified lines | |||
| 560 | `Serving operation check failed for ${zeroOperationIntegrations.length} planned integration(s)`, | 607 | `Serving operation check failed for ${zeroOperationIntegrations.length} planned integration(s)`, |
| 561 | ); | 608 | ); |
| 562 | } | 609 | } |
| 610 | const requiredR2Copies = plan.orgs | ||
| 611 | .filter((org) => !org.completed && org.hardErrors.length === 0) | ||
| 612 | .flatMap((org) => org.blobCopies.filter((copy) => copy.backend === "external")); | ||
| 613 | if (requiredR2Copies.length > 0 && !r2CopiesDone) { | ||
| 614 | throw new Error( | ||
| 615 | `External R2 blob copies are required before apply (${requiredR2Copies.length}); run dry-run, execute r2-copy-manifest.json, then pass --r2-copies-done`, | ||
| 616 | ); | ||
| 617 | } | ||
| 563 | for (const org of plan.orgs) { | 618 | for (const org of plan.orgs) { |
| 564 | await applyOrg(sql, org); | 619 | await applyOrg(sql, org); |
| 565 | console.log(`applied_org=${tenantHash(org.tenant)}`); | 620 | console.log(`applied_org=${tenantHash(org.tenant)}`); |