Promote a named cloud version, decoupled from upload/migrate

A promote-only workflow_dispatch previously re-ran migrate and upload-cloud (building a fresh, un-canaried version) and then promoted whatever was newest, so advancing a canary could ship code that landed on main after the canary started. Decouple promotion: the promote-cloud job no longer needs upload, and migrate/upload/deploy-marketing run on push only. The version to promote is now an explicit promote_version input (a version id, or the literal latest as an opt-in), and promote-cloud.ts promotes exactly that version. Also guard the script against running with a wrangler config whose worker name is not executor-cloud.

Rhys Sullivan committed 4 months ago

Commit 5d11ea51 parent 4b260753 files · +182 −58Browse files

Unified · Split

3 files

.github/workflows/deploy.yml +49 −14
apps/cloud/DEPLOY.md +42 −16
apps/cloud/scripts/promote-cloud.ts +91 −28

.github/workflows/deploy.yml+49−14

@@ -10,9 +10,14 @@ name: Deploy · 9 unmodified lines
10	10	# blast radius of a promotion (the DO restart it causes) is staged and
11	11	# operator-controlled.
12	12	#
13		−# `promote_percent` (workflow_dispatch input): when set, the promote-cloud job
14		−# runs and promotes the LATEST uploaded version to that percentage. Leave empty
15		−# (the default, e.g. on a push) to upload only.
	13	+# Promotion is DECOUPLED from upload: a `workflow_dispatch` that sets
	14	+# `promote_percent` runs ONLY the promote-cloud job (no migrate, no upload, so
	15	+# it never builds a new version). It promotes the version named by
	16	+# `promote_version` (an explicit version id), which is REQUIRED so that
	17	+# "widen the same canary" is the natural operation and advancing a rollout can
	18	+# never silently promote unreviewed code that landed after the canary. The
	19	+# literal value `latest` is an explicit opt-in for the "promote whatever was
	20	+# uploaded last" case (e.g. right after a merge); it is never the default.
16	21
17	22	on:
18	23	push:
@@ -21,8 +26,18 @@ on: · 2 unmodified lines
21	26	inputs:
22	27	promote_percent:
23	28	description: >-
24		− Promote the latest uploaded cloud version to this percentage of
25		− traffic (1-100). Leave empty to upload only (no promotion).
	29	+ Promote a cloud version to this percentage of traffic (1-100).
	30	+ Leave empty to do nothing (this workflow only uploads on push to
	31	+ main; a manual run is for promotion only).
	32	+ required: false
	33	+ default: ""
	34	+ promote_version:
	35	+ description: >-
	36	+ The cloud version id to promote (from `wrangler versions list`).
	37	+ Required when promote_percent is set. Use the literal `latest` to
	38	+ promote the most recently uploaded version (opt-in; only correct
	39	+ right after the intended merge, never for advancing an existing
	40	+ canary).
26	41	required: false
27	42	default: ""
28	43
@@ -31,10 +46,13 @@ concurrency: · 2 unmodified lines
31	46	cancel-in-progress: false
32	47
33	48	jobs:
	49	+ # Runs only on the upload path (push to main). A promote-only
	50	+ # workflow_dispatch must not re-run migrations.
34	51	migrate:
35	52	name: Migrate database
36	53	runs-on: blacksmith-4vcpu-ubuntu-2404
37	54	environment: production
	55	+ if: github.event_name == 'push'
38	56	steps:
39	57	- uses: actions/checkout@v4
40	58
@@ -75,6 +93,7 @@ jobs: · 34 unmodified lines
75	93	runs-on: blacksmith-4vcpu-ubuntu-2404
76	94	environment: production
77	95	needs: migrate
	96	+ if: github.event_name == 'push'
78	97	steps:
79	98	- uses: actions/checkout@v4
80	99
@@ -109,21 +128,33 @@ jobs: · 28 unmodified lines
109	128	CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }}
110	129	CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }}
111	130
112		− # Promote the LATEST uploaded cloud version to a chosen percentage of live
113		− # traffic. Gated: runs only on a manual `workflow_dispatch` that sets
114		− # `promote_percent`. A promotion moves a Durable Object's version and so
115		− # restarts DOs whose sessions land on the newly promoted version -- keep
116		− # `promote_percent` small for the first step of a rollout, watch telemetry,
117		− # then re-run at a higher percentage. See apps/cloud/DEPLOY.md.
	131	+ # Promote a specific, already-uploaded cloud version to a chosen percentage
	132	+ # of live traffic. DECOUPLED from migrate/upload: it does NOT `needs` them and
	133	+ # runs only on a manual `workflow_dispatch` that sets `promote_percent`, so a
	134	+ # promotion never builds a new version. The version to promote is named
	135	+ # explicitly by `promote_version` (required) so advancing an existing canary
	136	+ # widens THAT version rather than silently promoting whatever landed on main
	137	+ # since. A promotion moves a Durable Object's version and so restarts DOs whose
	138	+ # sessions land on the newly promoted version -- keep `promote_percent` small
	139	+ # for the first step of a rollout, watch telemetry, then re-run at a higher
	140	+ # percentage for the SAME `promote_version`. See apps/cloud/DEPLOY.md.
118	141	promote-cloud:
119	142	name: Promote cloud version
120	143	runs-on: blacksmith-4vcpu-ubuntu-2404
121	144	environment: production
122		− needs: upload-cloud
123	145	if: >-
124	146	github.event_name == 'workflow_dispatch' &&
125	147	inputs.promote_percent != ''
126	148	steps:
	149	+ # GitHub cannot conditionally require an input, so enforce it here: fail
	150	+ # fast (before checkout/build) if promote_version is missing rather than
	151	+ # letting the promote script error after the expensive steps.
	152	+ - name: Require promote_version
	153	+ if: inputs.promote_version == ''
	154	+ run: \|
	155	+ echo "::error::promote_version is required when promote_percent is set. Pass a version id from 'wrangler versions list', or the literal 'latest'."
	156	+ exit 1
	157	+
127	158	- uses: actions/checkout@v4
128	159
129	160	- uses: oven-sh/setup-bun@v2
@@ -150,18 +181,22 @@ jobs: · 20 unmodified lines
150	181	env:
151	182	VITE_PUBLIC_SENTRY_DSN: ${{ secrets.VITE_PUBLIC_SENTRY_DSN }}
152	183
153		− - name: Promote latest uploaded version
154		− run: bun run scripts/promote-cloud.ts --percent "${PROMOTE_PERCENT}"
	184	+ - name: Promote cloud version
	185	+ run: bun run scripts/promote-cloud.ts --percent "${PROMOTE_PERCENT}" --version "${PROMOTE_VERSION}"
155	186	working-directory: apps/cloud
156	187	env:
157	188	PROMOTE_PERCENT: ${{ inputs.promote_percent }}
	189	+ PROMOTE_VERSION: ${{ inputs.promote_version }}
158	190	CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }}
159	191	CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }}
160	192
	193	+ # Marketing deploys on the normal merge-to-main path only. A promote-only
	194	+ # workflow_dispatch targets the cloud worker and must not redeploy marketing.
161	195	deploy-marketing:
162	196	name: Deploy marketing
163	197	runs-on: blacksmith-4vcpu-ubuntu-2404
164	198	environment: production
	199	+ if: github.event_name == 'push'
165	200	steps:
166	201	- uses: actions/checkout@v4
167	202

apps/cloud/DEPLOY.md+42−16

@@ -13,34 +13,60 @@ claims are measured, not assumed. The findings are summarized at the end. · 12 unmodified lines
13	13
14	14	## The flow
15	15
16		−1. Merge to main runs `.github/workflows/deploy.yml`:
	16	+1. Merge to main runs `.github/workflows/deploy.yml` (`push` trigger only):
17	17	- `migrate` applies database migrations.
18	18	- `upload-cloud` runs `wrangler versions upload`. This creates a new Worker
19	19	version but routes no traffic to it. No DOs restart; live MCP
20	20	sessions keep running. This is safe to do on every merge.
21		−2. Promotion is a separate, deliberate step. Re-run the Deploy workflow via
22		− workflow_dispatch with a `promote_percent` input (1-100). The
23		− `promote-cloud` job promotes the latest uploaded version to that
24		− percentage of traffic (`scripts/promote-cloud.ts`, which calls
25		− `wrangler versions deploy`). For a canary, start small (e.g. 10), watch
26		− telemetry, then re-run at a higher percentage; run 100 for a full cutover.
	21	+ - `deploy-marketing` deploys the marketing worker (unchanged behaviour).
	22	+ - Note the uploaded version id from the `upload-cloud` job log (or run
	23	+ `wrangler versions list`): you promote it by id below.
	24	+2. Promotion is a separate, deliberate, DECOUPLED step. Run the Deploy
	25	+ workflow via workflow_dispatch with two inputs:
	26	+ - `promote_percent` (1-100): the traffic percentage for this step.
	27	+ - `promote_version`: the version id to promote (required). Use the
	28	+ literal `latest` only for "promote what I just merged"; for advancing an
	29	+ existing canary always pass the same version id you canaried, so you
	30	+ widen that reviewed version rather than something newer.
	31	+
	32	+ A promote-only dispatch runs only the `promote-cloud` job: it does not
	33	+ re-run `migrate` or `upload-cloud`, so it never builds a new version. The job
	34	+ promotes `promote_version` to `promote_percent` of traffic
	35	+ (`scripts/promote-cloud.ts`, which calls `wrangler versions deploy`).
	36	+
	37	+ Canary walk for one version `V`:
	38	+ - dispatch `promote_percent=10`, `promote_version=V` (10% on `V`);
	39	+ - watch telemetry;
	40	+ - dispatch `promote_percent=50`, `promote_version=V` (widen the same `V`);
	41	+ - dispatch `promote_percent=100`, `promote_version=V` (full cutover).
	42	+
	43	+ Merges that land after you start a canary upload their own versions but do
	44	+ not get promoted until you dispatch with their id: advancing `V` never
	45	+ silently ships unreviewed code.
27	46
28	47	### Promoting from the CLI (manual, out of band)
29	48
30		−From `apps/cloud`, against the generated build config:
	49	+From `apps/cloud`, against the generated build config. `--version` is required;
	50	+it is either an explicit version id or the literal `latest`:
31	51
32	52	```bash
33	53	bun run build
34		−# Canary: latest uploaded version at 10%, current-live at 90%.
35		−bun run scripts/promote-cloud.ts --percent 10
36		−# Advance:
37		−bun run scripts/promote-cloud.ts --percent 50
38		−# Full cutover:
39		−bun run scripts/promote-cloud.ts --percent 100
	54	+wrangler versions list -c dist/server/wrangler.json # find the version id
	55	+# Canary: version <id> at 10%, current-live at 90%.
	56	+bun run scripts/promote-cloud.ts --percent 10 --version <id>
	57	+# Advance (same version id):
	58	+bun run scripts/promote-cloud.ts --percent 50 --version <id>
	59	+# Full cutover (same version id):
	60	+bun run scripts/promote-cloud.ts --percent 100 --version <id>
	61	+# Or, right after a merge, promote the most recent upload explicitly:
	62	+bun run scripts/promote-cloud.ts --percent 10 --version latest
40	63	```
41	64
42		−The script sorts versions by version number, promotes the newest, and holds the
43		−remainder of a partial promotion on the current-live version.
	65	+The script promotes exactly the version you name (`latest` resolves to the
	66	+highest version number) and holds the remainder of a partial promotion on the
	67	+current-live version. It refuses to run unless the resolved wrangler config
	68	+targets the `executor-cloud` worker (set `PROMOTE_CLOUD_ALLOW_NAME` to override
	69	+for a non-production worker such as the deploy lab).
44	70
45	71	## Session version-affinity (zone ruleset, MUST be configured manually)
46	72

apps/cloud/scripts/promote-cloud.ts+91−28

@@ -1,21 +1,32 @@
1	1	/* oxlint-disable executor/no-error-constructor, executor/no-try-catch-or-throw -- boundary: out-of-band deploy-promotion CLI */
2	2
3	3	/**
4		− * Promote the LATEST uploaded executor-cloud Worker version to a chosen
5		− * percentage of live traffic, using Cloudflare's staged gradual-deployment
6		− * model (`wrangler versions deploy`).
	4	+ * Promote a specific, already-uploaded executor-cloud Worker version to a
	5	+ * chosen percentage of live traffic, using Cloudflare's staged
	6	+ * gradual-deployment model (`wrangler versions deploy`).
7	7	*
8	8	* This is the second half of the upload -> staged-promotion deploy flow
9	9	* (see apps/cloud/DEPLOY.md and .github/workflows/deploy.yml). Merges upload a
10	10	* version but route no traffic to it; this script promotes it deliberately.
11	11	*
12		− * Behaviour:
13		− * --percent 100 promote the latest uploaded version to 100% (full cutover).
14		− * --percent 10 split traffic 10% latest / 90% current-live version, for a
15		− * canary step. Re-run at a higher percentage to advance the
16		− * rollout; the DOs whose sessions land on the newly promoted
17		− * version restart, so keep early steps small and watch
18		− * telemetry between steps.
	12	+ * The version to promote is chosen EXPLICITLY, never inferred silently:
	13	+ * --version <id> promote exactly that version id (from
	14	+ * `wrangler versions list`). This is what advances an
	15	+ * existing canary: re-run at a higher --percent for the SAME
	16	+ * --version so you widen the version you already reviewed,
	17	+ * not whatever landed on main since.
	18	+ * --version latest promote the most recently uploaded version. This is an
	19	+ * explicit opt-in for "promote what I just merged"; it is
	20	+ * NEVER the default, so advancing a rollout can never
	21	+ * accidentally promote unreviewed code.
	22	+ *
	23	+ * Percentage:
	24	+ * --percent 100 promote the chosen version to 100% (full cutover).
	25	+ * --percent 10 split traffic 10% chosen / 90% current-live version, for a
	26	+ * canary step. Re-run at a higher percentage (same version)
	27	+ * to advance; the DOs whose sessions land on the newly
	28	+ * promoted version restart, so keep early steps small and
	29	+ * watch telemetry between steps.
19	30	*
20	31	* IMPORTANT: a promotion that changes a Durable Object's assigned version
21	32	* restarts that DO (validated empirically, DEPLOY.md experiments 3b/5). Stage
@@ -25,11 +36,36 @@ · 3 unmodified lines
25	36	* in the CI env). Config is the generated dist config the build produced.
26	37	*/
27	38	import { spawnSync } from "node:child_process";
	39	+import { readFileSync } from "node:fs";
28	40
29	41	// The generated dist config the build produced. Overridable via
30	42	// PROMOTE_CLOUD_CONFIG for local/lab testing against a non-prod worker.
31	43	const CONFIG = process.env.PROMOTE_CLOUD_CONFIG ?? "dist/server/wrangler.json";
32	44
	45	+// Target-worker guard: refuse to run against anything but executor-cloud unless
	46	+// the operator explicitly names the worker they expect via
	47	+// PROMOTE_CLOUD_ALLOW_NAME (for the deploy lab / a non-prod worker). A promotion
	48	+// routes live traffic and restarts DOs, so a misconfigured config that points at
	49	+// the wrong worker must fail loudly, not promote the wrong thing. `--yes` does
	50	+// NOT bypass this: it only skips wrangler's own interactive confirmation.
	51	+const EXPECTED_WORKER_NAME = process.env.PROMOTE_CLOUD_ALLOW_NAME ?? "executor-cloud";
	52	+{
	53	+ let configName: unknown;
	54	+ try {
	55	+ configName = (JSON.parse(readFileSync(CONFIG, "utf8")) as { name?: unknown }).name;
	56	+ } catch (error) {
	57	+ throw new Error(
	58	+ `could not read wrangler config ${CONFIG} to verify the target worker: ${String(error)}`,
	59	+ );
	60	+ }
	61	+ if (configName !== EXPECTED_WORKER_NAME) {
	62	+ throw new Error(
	63	+ `refusing to promote: wrangler config ${CONFIG} targets worker ${JSON.stringify(configName)}, expected ${JSON.stringify(EXPECTED_WORKER_NAME)}. ` +
	64	+ `Set PROMOTE_CLOUD_ALLOW_NAME to override for a non-production worker.`,
	65	+ );
	66	+ }
	67	+}
	68	+
33	69	const args = process.argv.slice(2);
34	70	const argValue = (name: string): string \| undefined => {
35	71	const i = args.indexOf(name);
@@ -45,6 +81,16 @@ if (!Number.isInteger(percent) \|\| percent < 1 \|\| percent > 100) { · 9 unmodified lines
45	81	throw new Error(`--percent must be an integer 1-100, got: ${percentRaw}`);
46	82	}
47	83
	84	+// The version to promote is required and explicit: an id, or the literal
	85	+// `latest` to opt in to "promote the most recently uploaded version".
	86	+const versionArg = argValue("--version") ?? process.env.PROMOTE_VERSION;
	87	+if (!versionArg) {
	88	+ throw new Error(
	89	+ "--version is required: pass an explicit version id (from `wrangler versions list`), " +
	90	+ "or the literal `latest` to promote the most recently uploaded version.",
	91	+ );
	92	+}
	93	+
48	94	const wrangler = (wranglerArgs: readonly string[]): string => {
49	95	const result = spawnSync("bun", ["run", "wrangler", ...wranglerArgs], {
50	96	encoding: "utf8",
@@ -67,21 +113,38 @@ type DeploymentStatus = { · 16 unmodified lines
67	113	readonly versions: ReadonlyArray<{ readonly version_id: string; readonly percentage: number }>;
68	114	};
69	115
70		−// `versions list --json` order is not guaranteed, so sort explicitly by
71		−// version number (monotonic per upload) descending to get the newest.
72	116	const versions = JSON.parse(
73	117	wrangler(["versions", "list", "-c", CONFIG, "--json"]),
74	118	) as VersionListEntry[];
75	119	if (versions.length === 0) {
76	120	throw new Error("no uploaded versions found");
77	121	}
78		−const sorted = [...versions].sort((a, b) => {
79		− if (a.number !== b.number) return b.number - a.number;
80		− const at = a.metadata?.created_on ?? "";
81		− const bt = b.metadata?.created_on ?? "";
82		− return bt.localeCompare(at);
83		−});
84		−const latest = sorted[0];
	122	+
	123	+// Resolve the target version EXPLICITLY. `latest` is an opt-in that sorts by
	124	+// version number (monotonic per upload) descending, since `versions list`
	125	+// order is not guaranteed. Any other value is treated as a version id and must
	126	+// match an uploaded version exactly, so a typo or a since-deleted id fails
	127	+// loudly rather than promoting the wrong version.
	128	+let target: VersionListEntry;
	129	+if (versionArg === "latest") {
	130	+ const sorted = [...versions].sort((a, b) => {
	131	+ if (a.number !== b.number) return b.number - a.number;
	132	+ const at = a.metadata?.created_on ?? "";
	133	+ const bt = b.metadata?.created_on ?? "";
	134	+ return bt.localeCompare(at);
	135	+ });
	136	+ target = sorted[0];
	137	+} else {
	138	+ const found = versions.find((v) => v.id === versionArg);
	139	+ if (!found) {
	140	+ throw new Error(
	141	+ `version ${versionArg} not found among uploaded versions. ` +
	142	+ `Run \`wrangler versions list -c ${CONFIG}\` to see valid ids, ` +
	143	+ "or pass `latest` to promote the most recent upload.",
	144	+ );
	145	+ }
	146	+ target = found;
	147	+}
85	148
86	149	// The version(s) currently serving live traffic, so a partial promotion holds
87	150	// the remainder on the current-live version rather than an arbitrary one.
@@ -90,24 +153,24 @@ const status = JSON.parse( · 2 unmodified lines
90	153	) as DeploymentStatus;
91	154	// The version to hold the remainder of a partial promotion: the current-live
92	155	// version carrying the most traffic that is NOT the one we are promoting. If
93		−// the only current version IS the latest, there is nothing to split against.
	156	+// the only current version IS the target, there is nothing to split against.
94	157	const currentLive = [...status.versions]
95		− .filter((v) => v.version_id !== latest.id)
	158	+ .filter((v) => v.version_id !== target.id)
96	159	.sort((a, b) => b.percentage - a.percentage)[0]?.version_id;
97	160
98		−if (latest.id === currentLive && percent === 100) {
99		− console.log(`Latest version ${latest.id} is already live at 100%; nothing to promote.`);
	161	+if (target.id === currentLive && percent === 100) {
	162	+ console.log(`Version ${target.id} is already live at 100%; nothing to promote.`);
100	163	process.exit(0);
101	164	}
102	165
103	166	const deployArgs = ["versions", "deploy", "-c", CONFIG, "--yes"];
104		−if (percent === 100 \|\| !currentLive \|\| currentLive === latest.id) {
105		− deployArgs.push(`${latest.id}@100%`);
	167	+if (percent === 100 \|\| !currentLive \|\| currentLive === target.id) {
	168	+ deployArgs.push(`${target.id}@100%`);
106	169	} else {
107		− // Staged split: latest@percent, current-live@remainder.
108		− deployArgs.push(`${latest.id}@${percent}%`, `${currentLive}@${100 - percent}%`);
	170	+ // Staged split: target@percent, current-live@remainder.
	171	+ deployArgs.push(`${target.id}@${percent}%`, `${currentLive}@${100 - percent}%`);
109	172	}
110	173
111		−console.log(`Promoting ${latest.id} (version #${latest.number}) at ${percent}%...`);
	174	+console.log(`Promoting ${target.id} (version #${target.number}) at ${percent}%...`);
112	175	console.log(wrangler(deployArgs));
113	176	console.log("Done. Monitor telemetry, then re-run at a higher percentage to advance the rollout.");