Showing the changes to scripts/migration/service-split-postgres-cli.ts (1 of 8 files). Show all
Unified · Split
| @@ -48,6 +48,7 @@ const outputDir = resolve( · 47 unmodified lines | ||
| 48 | 48 | const orgFilter = argValue("--org"); |
| 49 | 49 | const trafficLastTenant = argValue("--traffic-last-tenant"); |
| 50 | 50 | const databaseUrl = argValue("--database-url") ?? process.env.DATABASE_URL; |
| 51 | +const r2CopiesDone = hasArg("--r2-copies-done"); | |
| 51 | 52 | export const LEDGER_TABLE = "provider_service_split_org_migration"; |
| 52 | 53 | |
| 53 | 54 | const parseInputJson = (path: string): MigrationInput => |
| @@ -212,6 +213,7 @@ export const readDatabaseInput = async ( · 158 unmodified lines | ||
| 212 | 213 | completedTenants, |
| 213 | 214 | trafficLastTenant, |
| 214 | 215 | blobBackend, |
| 216 | + assumeExternalBlobSourcePresent: blobBackend === "external", | |
| 215 | 217 | }; |
| 216 | 218 | }; |
| 217 | 219 | |
| @@ -242,10 +244,34 @@ const operationKeyPrefix = (integration: string): string => · 24 unmodified lines | ||
| 242 | 244 | export const applyOrg = async (sql: SqlClient, org: OrgPlan): Promise<void> => { |
| 243 | 245 | if (org.completed) return; |
| 244 | 246 | if (org.hardErrors.length > 0) { |
| 245 | − throw new Error(`org ${org.tenantHash} has unresolved policy hard errors; apply is blocked`); | |
| 247 | + throw new Error(`org ${org.tenantHash} has unresolved hard errors; apply is blocked`); | |
| 246 | 248 | } |
| 247 | 249 | await sql.begin(async (tx) => { |
| 248 | 250 | const now = new Date(); |
| 251 | + for (const copy of org.blobCopies.filter((item) => item.backend === "database")) { | |
| 252 | + const [source] = await tx.unsafe<{ readonly value: string }[]>( | |
| 253 | + "SELECT value FROM blob WHERE namespace = $1 AND key = $2 LIMIT 1", | |
| 254 | + [copy.sourceNamespace, copy.key], | |
| 255 | + ); | |
| 256 | + if (!source) { | |
| 257 | + throw new Error(`Missing source blob ${copy.sourceNamespace}/${copy.key}`); | |
| 258 | + } | |
| 259 | + await tx.unsafe( | |
| 260 | + ` | |
| 261 | + INSERT INTO blob (namespace, key, value, row_id, id) | |
| 262 | + VALUES ($1, $2, $3, $4, $5) | |
| 263 | + ON CONFLICT (id) DO NOTHING | |
| 264 | + `, | |
| 265 | + [ | |
| 266 | + copy.targetNamespace, | |
| 267 | + copy.key, | |
| 268 | + source.value, | |
| 269 | + stableId("blob", org.tenant, copy.targetNamespace, copy.key), | |
| 270 | + JSON.stringify([copy.targetNamespace, copy.key]), | |
| 271 | + ], | |
| 272 | + ); | |
| 273 | + } | |
| 274 | + | |
| 249 | 275 | for (const row of org.integrations.filter((item) => item.action === "create")) { |
| 250 | 276 | await tx.unsafe( |
| 251 | 277 | ` |
| @@ -471,17 +497,35 @@ export const applyOrg = async (sql: SqlClient, org: OrgPlan): Promise<void> => { · 219 unmodified lines | ||
| 471 | 497 | const writeDryRun = (planInput: MigrationInput): void => { |
| 472 | 498 | const plan = planMigration({ ...planInput, collectPolicyErrors: true }); |
| 473 | 499 | const neverWiden = verifyPolicyRewriteNeverWidens(plan, planInput); |
| 500 | + const r2Copies = plan.orgs | |
| 501 | + .filter((org) => !org.completed && org.hardErrors.length === 0) | |
| 502 | + .flatMap((org) => | |
| 503 | + org.blobCopies | |
| 504 | + .filter((copy) => copy.backend === "external") | |
| 505 | + .map((copy) => ({ | |
| 506 | + tenantHash: org.tenantHash, | |
| 507 | + tenant: org.tenant, | |
| 508 | + sourceKey: copy.sourceObjectName, | |
| 509 | + destKey: copy.targetObjectName, | |
| 510 | + })), | |
| 511 | + ); | |
| 474 | 512 | mkdirSync(outputDir, { recursive: true }); |
| 475 | 513 | for (const org of plan.orgs) { |
| 476 | 514 | writeFileSync(`${outputDir}/org-${org.tenantHash}.md`, renderOrgDiff(org)); |
| 477 | 515 | } |
| 516 | + writeFileSync(`${outputDir}/r2-copy-manifest.json`, `${JSON.stringify(r2Copies, null, 2)}\n`); | |
| 478 | 517 | writeFileSync( |
| 479 | 518 | `${outputDir}/summary.md`, |
| 480 | − `${renderSummary(plan)}\npolicy_never_widen=${neverWiden.ok}\npolicy_never_widen_checked=${neverWiden.checkedPolicies}\n`, | |
| 519 | + `${renderSummary(plan)}\npolicy_never_widen=${neverWiden.ok}\npolicy_never_widen_checked=${neverWiden.checkedPolicies}\nr2_copy_manifest=${outputDir}/r2-copy-manifest.json\nr2_copy_count=${r2Copies.length}\nr2_copy_command=wrangler r2 object get <bucket>/<sourceKey> | wrangler r2 object put <bucket>/<destKey> --pipe\n`, | |
| 481 | 520 | ); |
| 482 | 521 | console.log(renderSummary(plan)); |
| 483 | 522 | console.log(`policy_never_widen=${neverWiden.ok}`); |
| 484 | 523 | console.log(`policy_never_widen_checked=${neverWiden.checkedPolicies}`); |
| 524 | + console.log(`r2_copy_manifest=${outputDir}/r2-copy-manifest.json`); | |
| 525 | + console.log(`r2_copy_count=${r2Copies.length}`); | |
| 526 | + console.log( | |
| 527 | + "r2_copy_command=wrangler r2 object get <bucket>/<sourceKey> | wrangler r2 object put <bucket>/<destKey> --pipe", | |
| 528 | + ); | |
| 485 | 529 | console.log(`diff_dir=${outputDir}`); |
| 486 | 530 | if (!neverWiden.ok) { |
| 487 | 531 | throw new Error( |
| @@ -512,7 +556,10 @@ const main = async (): Promise<void> => { · 24 unmodified lines | ||
| 512 | 556 | const input = parseInputJson(inputJson); |
| 513 | 557 | writeDryRun( |
| 514 | 558 | filterInputByOrg( |
| 515 | − { ...input, trafficLastTenant: trafficLastTenant ?? input.trafficLastTenant }, | |
| 559 | + { | |
| 560 | + ...input, | |
| 561 | + trafficLastTenant: trafficLastTenant ?? input.trafficLastTenant, | |
| 562 | + }, | |
| 516 | 563 | orgFilter, |
| 517 | 564 | ), |
| 518 | 565 | ); |
| @@ -560,6 +607,14 @@ const main = async (): Promise<void> => { · 41 unmodified lines | ||
| 560 | 607 | `Serving operation check failed for ${zeroOperationIntegrations.length} planned integration(s)`, |
| 561 | 608 | ); |
| 562 | 609 | } |
| 610 | + const requiredR2Copies = plan.orgs | |
| 611 | + .filter((org) => !org.completed && org.hardErrors.length === 0) | |
| 612 | + .flatMap((org) => org.blobCopies.filter((copy) => copy.backend === "external")); | |
| 613 | + if (requiredR2Copies.length > 0 && !r2CopiesDone) { | |
| 614 | + throw new Error( | |
| 615 | + `External R2 blob copies are required before apply (${requiredR2Copies.length}); run dry-run, execute r2-copy-manifest.json, then pass --r2-copies-done`, | |
| 616 | + ); | |
| 617 | + } | |
| 563 | 618 | for (const org of plan.orgs) { |
| 564 | 619 | await applyOrg(sql, org); |
| 565 | 620 | console.log(`applied_org=${tenantHash(org.tenant)}`); |