ci: Generate the Collab deployment workflow (#49306)

This PR updates the Collab deployment workflow to be generated with `cargo xtask workflows`. Release Notes: - N/A

Marshall Bowers committed 8 months ago

Commit 958798d1 parent 05940764 files · +311 −130Browse files

Unified · Split

4 files

.github/workflows/deploy_collab.yml +136 −130
tooling/xtask/src/tasks/workflows.rs +2 −0
tooling/xtask/src/tasks/workflows/deploy_collab.rs +171 −0
tooling/xtask/src/tasks/workflows/vars.rs +2 −0

.github/workflows/deploy_collab.yml+136−130

@@ -1,146 +1,152 @@
1	name: Publish Collab Server Image	1	# Generated from xtask::workflows::deploy_collab
2		2	# Rebuild with `cargo xtask workflows`.
		3	name: deploy_collab
		4	env:
		5	DOCKER_BUILDKIT: '1'
3	on:	6	on:
4	push:	7	push:
5	tags:	8	tags:
6	- collab-production	9	- collab-production
7	- collab-staging
8
9	env:
10	DOCKER_BUILDKIT: 1
11
12	jobs:	10	jobs:
13	style:	11	style:
		12	if: (github.repository_owner == 'zed-industries' \|\| github.repository_owner == 'zed-extensions')
14	name: Check formatting and Clippy lints	13	name: Check formatting and Clippy lints
15	if: github.repository_owner == 'zed-industries'	14	runs-on: namespace-profile-16x32-ubuntu-2204
16	runs-on:
17	- namespace-profile-16x32-ubuntu-2204
18	steps:	15	steps:
19	- name: Checkout repo	16	- name: steps::checkout_repo
20	uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4	17	uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
21	with:	18	with:
22	clean: false	19	clean: false
23	fetch-depth: 0	20	fetch-depth: 0
24		21	- name: steps::setup_cargo_config
25	- name: Run style checks	22	run: \|
26	uses: ./.github/actions/check_style	23	mkdir -p ./../.cargo
27		24	cp ./.cargo/ci-config.toml ./../.cargo/config.toml
28	- name: Run clippy	25	- name: steps::cache_rust_dependencies_namespace
29	run: ./script/clippy	26	uses: namespacelabs/nscloud-cache-action@v1
30		27	with:
		28	cache: rust
		29	path: ~/.rustup
		30	- name: steps::cargo_fmt
		31	run: cargo fmt --all -- --check
		32	- name: steps::clippy
		33	run: ./script/clippy
31	tests:	34	tests:
		35	needs:
		36	- style
32	name: Run tests	37	name: Run tests
33	runs-on:	38	runs-on: namespace-profile-16x32-ubuntu-2204
34	- namespace-profile-16x32-ubuntu-2204
35	needs: style
36	steps:	39	steps:
37	- name: Checkout repo	40	- name: steps::checkout_repo
38	uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4	41	uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
39	with:	42	with:
40	clean: false	43	clean: false
41	fetch-depth: 0	44	fetch-depth: 0
42		45	- name: steps::setup_cargo_config
43	- name: Install cargo nextest	46	run: \|
44	uses: taiki-e/install-action@nextest	47	mkdir -p ./../.cargo
45		48	cp ./.cargo/ci-config.toml ./../.cargo/config.toml
46	- name: Limit target directory size	49	- name: steps::cache_rust_dependencies_namespace
47	shell: bash -euxo pipefail {0}	50	uses: namespacelabs/nscloud-cache-action@v1
48	run: script/clear-target-dir-if-larger-than 300	51	with:
49		52	cache: rust
50	- name: Run tests	53	path: ~/.rustup
51	shell: bash -euxo pipefail {0}	54	- name: steps::cargo_install_nextest
52	run: cargo nextest run --package collab --no-fail-fast	55	uses: taiki-e/install-action@nextest
53		56	- name: steps::clear_target_dir_if_large
		57	run: ./script/clear-target-dir-if-larger-than 250
		58	- name: deploy_collab::tests::run_collab_tests
		59	run: cargo nextest run --package collab --no-fail-fast
		60	services:
		61	postgres:
		62	image: postgres:15
		63	env:
		64	POSTGRES_HOST_AUTH_METHOD: trust
		65	ports:
		66	- 5432:5432
		67	options: --health-cmd pg_isready --health-interval 500ms --health-timeout 5s --health-retries 10
54	publish:	68	publish:
55	name: Publish collab server image
56	needs:	69	needs:
57	- style	70	- style
58	- tests	71	- tests
59	runs-on:	72	name: Publish collab server image
60	- namespace-profile-16x32-ubuntu-2204	73	runs-on: namespace-profile-16x32-ubuntu-2204
61	steps:	74	steps:
62	- name: Install doctl	75	- name: deploy_collab::publish::install_doctl
63	uses: digitalocean/action-doctl@v2	76	uses: digitalocean/action-doctl@v2
64	with:	77	with:
65	token: ${{ secrets.DIGITALOCEAN_ACCESS_TOKEN }}	78	token: ${{ secrets.DIGITALOCEAN_ACCESS_TOKEN }}
66		79	- name: deploy_collab::publish::sign_into_registry
67	- name: Sign into DigitalOcean docker registry	80	run: doctl registry login
68	run: doctl registry login	81	- name: steps::checkout_repo
69		82	uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
70	- name: Checkout repo	83	with:
71	uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4	84	clean: false
72	with:	85	- name: deploy_collab::publish::build_docker_image
73	clean: false	86	run: \|
74		87	docker build -f Dockerfile-collab \
75	- name: Build docker image	88	--build-arg "GITHUB_SHA=$GITHUB_SHA" \
76	run: \|	89	--tag "registry.digitalocean.com/zed/collab:$GITHUB_SHA" \
77	docker build -f Dockerfile-collab \	90	.
78	--build-arg "GITHUB_SHA=$GITHUB_SHA" \	91	- name: deploy_collab::publish::publish_docker_image
79	--tag "registry.digitalocean.com/zed/collab:$GITHUB_SHA" \	92	run: docker push "registry.digitalocean.com/zed/collab:${GITHUB_SHA}"
80	.	93	- name: deploy_collab::publish::prune_docker_system
81		94	run: docker system prune --filter 'until=72h' -f
82	- name: Publish docker image
83	run: docker push "registry.digitalocean.com/zed/collab:${GITHUB_SHA}"
84
85	- name: Prune Docker system
86	run: docker system prune --filter 'until=72h' -f
87
88	deploy:	95	deploy:
89	name: Deploy new server image
90	needs:	96	needs:
91	- publish	97	- publish
92	runs-on:	98	name: Deploy new server image
93	- namespace-profile-16x32-ubuntu-2204	99	runs-on: namespace-profile-16x32-ubuntu-2204
94
95	steps:	100	steps:
96	- name: Checkout repo	101	- name: steps::checkout_repo
97	uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4	102	uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
98	with:	103	with:
99	clean: false	104	clean: false
100		105	- name: deploy_collab::deploy::install_doctl
101	- name: Install doctl	106	uses: digitalocean/action-doctl@v2
102	uses: digitalocean/action-doctl@v2	107	with:
103	with:	108	token: ${{ secrets.DIGITALOCEAN_ACCESS_TOKEN }}
104	token: ${{ secrets.DIGITALOCEAN_ACCESS_TOKEN }}	109	- name: deploy_collab::deploy::sign_into_kubernetes
105		110	run: \|
106	- name: Sign into Kubernetes	111	doctl kubernetes cluster kubeconfig save --expiry-seconds 600 ${{ secrets.CLUSTER_NAME }}
107	run: doctl kubernetes cluster kubeconfig save --expiry-seconds 600 ${{ secrets.CLUSTER_NAME }}	112	- name: deploy_collab::deploy::start_rollout
108		113	run: \|
109	- name: Start rollout	114	set -eu
110	run: \|	115	if [[ $GITHUB_REF_NAME = "collab-production" ]]; then
111	set -eu	116	export ZED_KUBE_NAMESPACE=production
112	if [[ $GITHUB_REF_NAME = "collab-production" ]]; then	117	export ZED_COLLAB_LOAD_BALANCER_SIZE_UNIT=10
113	export ZED_KUBE_NAMESPACE=production	118	export ZED_API_LOAD_BALANCER_SIZE_UNIT=2
114	export ZED_COLLAB_LOAD_BALANCER_SIZE_UNIT=10	119	elif [[ $GITHUB_REF_NAME = "collab-staging" ]]; then
115	export ZED_API_LOAD_BALANCER_SIZE_UNIT=2	120	export ZED_KUBE_NAMESPACE=staging
116	elif [[ $GITHUB_REF_NAME = "collab-staging" ]]; then	121	export ZED_COLLAB_LOAD_BALANCER_SIZE_UNIT=1
117	export ZED_KUBE_NAMESPACE=staging	122	export ZED_API_LOAD_BALANCER_SIZE_UNIT=1
118	export ZED_COLLAB_LOAD_BALANCER_SIZE_UNIT=1	123	else
119	export ZED_API_LOAD_BALANCER_SIZE_UNIT=1	124	echo "cowardly refusing to deploy from an unknown branch"
120	else	125	exit 1
121	echo "cowardly refusing to deploy from an unknown branch"	126	fi
122	exit 1	127
123	fi	128	echo "Deploying collab:$GITHUB_SHA to $ZED_KUBE_NAMESPACE"
124		129
125	echo "Deploying collab:$GITHUB_SHA to $ZED_KUBE_NAMESPACE"	130	source script/lib/deploy-helpers.sh
126		131	export_vars_for_environment $ZED_KUBE_NAMESPACE
127	source script/lib/deploy-helpers.sh	132
128	export_vars_for_environment $ZED_KUBE_NAMESPACE	133	ZED_DO_CERTIFICATE_ID="$(doctl compute certificate list --format ID --no-header)"
129		134	export ZED_DO_CERTIFICATE_ID
130	ZED_DO_CERTIFICATE_ID="$(doctl compute certificate list --format ID --no-header)"	135	export ZED_IMAGE_ID="registry.digitalocean.com/zed/collab:${GITHUB_SHA}"
131	export ZED_DO_CERTIFICATE_ID	136
132	export ZED_IMAGE_ID="registry.digitalocean.com/zed/collab:${GITHUB_SHA}"	137	export ZED_SERVICE_NAME=collab
133		138	export ZED_LOAD_BALANCER_SIZE_UNIT=$ZED_COLLAB_LOAD_BALANCER_SIZE_UNIT
134	export ZED_SERVICE_NAME=collab	139	export DATABASE_MAX_CONNECTIONS=850
135	export ZED_LOAD_BALANCER_SIZE_UNIT=$ZED_COLLAB_LOAD_BALANCER_SIZE_UNIT	140	envsubst < crates/collab/k8s/collab.template.yml \| kubectl apply -f -
136	export DATABASE_MAX_CONNECTIONS=850	141	kubectl -n "$ZED_KUBE_NAMESPACE" rollout status deployment/$ZED_SERVICE_NAME --watch
137	envsubst < crates/collab/k8s/collab.template.yml \| kubectl apply -f -	142	echo "deployed ${ZED_SERVICE_NAME} to ${ZED_KUBE_NAMESPACE}"
138	kubectl -n "$ZED_KUBE_NAMESPACE" rollout status deployment/$ZED_SERVICE_NAME --watch	143
139	echo "deployed ${ZED_SERVICE_NAME} to ${ZED_KUBE_NAMESPACE}"	144	export ZED_SERVICE_NAME=api
140		145	export ZED_LOAD_BALANCER_SIZE_UNIT=$ZED_API_LOAD_BALANCER_SIZE_UNIT
141	export ZED_SERVICE_NAME=api	146	export DATABASE_MAX_CONNECTIONS=60
142	export ZED_LOAD_BALANCER_SIZE_UNIT=$ZED_API_LOAD_BALANCER_SIZE_UNIT	147	envsubst < crates/collab/k8s/collab.template.yml \| kubectl apply -f -
143	export DATABASE_MAX_CONNECTIONS=60	148	kubectl -n "$ZED_KUBE_NAMESPACE" rollout status deployment/$ZED_SERVICE_NAME --watch
144	envsubst < crates/collab/k8s/collab.template.yml \| kubectl apply -f -	149	echo "deployed ${ZED_SERVICE_NAME} to ${ZED_KUBE_NAMESPACE}"
145	kubectl -n "$ZED_KUBE_NAMESPACE" rollout status deployment/$ZED_SERVICE_NAME --watch	150	defaults:
146	echo "deployed ${ZED_SERVICE_NAME} to ${ZED_KUBE_NAMESPACE}"	151	run:
		152	shell: bash -euxo pipefail {0}

tooling/xtask/src/tasks/workflows.rs+2−0

 mod cherry_pick;
 mod compare_perf;
 mod danger;
+mod deploy_collab;
 mod extension_bump;
 mod extension_tests;
 mod extension_workflow_rollout;
         WorkflowFile::zed(cherry_pick::cherry_pick),
         WorkflowFile::zed(compare_perf::compare_perf),
         WorkflowFile::zed(danger::danger),
+        WorkflowFile::zed(deploy_collab::deploy_collab),
         WorkflowFile::zed(extension_bump::extension_bump),
         WorkflowFile::zed(extension_tests::extension_tests),
         WorkflowFile::zed(extension_workflow_rollout::extension_workflow_rollout),

tooling/xtask/src/tasks/workflows/deploy_collab.rs+171−0

@@ -0,0 +1,171 @@
	1	use gh_workflow::{Container, Event, Port, Push, Run, Step, Use, Workflow};
	2	use indoc::{formatdoc, indoc};
	3
	4	use crate::tasks::workflows::{
	5	runners::{self, Platform},
	6	steps::{self, CommonJobConditions, NamedJob, dependant_job, named},
	7	vars,
	8	};
	9
	10	pub(crate) fn deploy_collab() -> Workflow {
	11	let style = style();
	12	let tests = tests(&[&style]);
	13	let publish = publish(&[&style, &tests]);
	14	let deploy = deploy(&[&publish]);
	15
	16	named::workflow()
	17	.on(Event::default().push(Push::default().add_tag("collab-production")))
	18	.add_env(("DOCKER_BUILDKIT", "1"))
	19	.add_job(style.name, style.job)
	20	.add_job(tests.name, tests.job)
	21	.add_job(publish.name, publish.job)
	22	.add_job(deploy.name, deploy.job)
	23	}
	24
	25	fn style() -> NamedJob {
	26	named::job(
	27	dependant_job(&[])
	28	.name("Check formatting and Clippy lints")
	29	.with_repository_owner_guard()
	30	.runs_on(runners::LINUX_XL)
	31	.add_step(steps::checkout_repo().add_with(("fetch-depth", 0)))
	32	.add_step(steps::setup_cargo_config(Platform::Linux))
	33	.add_step(steps::cache_rust_dependencies_namespace())
	34	.add_step(steps::cargo_fmt())
	35	.add_step(steps::clippy(Platform::Linux)),
	36	)
	37	}
	38
	39	fn tests(deps: &[&NamedJob]) -> NamedJob {
	40	fn run_collab_tests() -> Step<Run> {
	41	named::bash("cargo nextest run --package collab --no-fail-fast")
	42	}
	43
	44	named::job(
	45	dependant_job(deps)
	46	.name("Run tests")
	47	.runs_on(runners::LINUX_XL)
	48	.add_service(
	49	"postgres",
	50	Container::new("postgres:15")
	51	.add_env(("POSTGRES_HOST_AUTH_METHOD", "trust"))
	52	.ports(vec![Port::Name("5432:5432".into())])
	53	.options(
	54	"--health-cmd pg_isready \
	55	--health-interval 500ms \
	56	--health-timeout 5s \
	57	--health-retries 10",
	58	),
	59	)
	60	.add_step(steps::checkout_repo().add_with(("fetch-depth", 0)))
	61	.add_step(steps::setup_cargo_config(Platform::Linux))
	62	.add_step(steps::cache_rust_dependencies_namespace())
	63	.add_step(steps::cargo_install_nextest())
	64	.add_step(steps::clear_target_dir_if_large(Platform::Linux))
	65	.add_step(run_collab_tests()),
	66	)
	67	}
	68
	69	fn publish(deps: &[&NamedJob]) -> NamedJob {
	70	fn install_doctl() -> Step<Use> {
	71	named::uses("digitalocean", "action-doctl", "v2")
	72	.add_with(("token", vars::DIGITALOCEAN_ACCESS_TOKEN))
	73	}
	74
	75	fn sign_into_registry() -> Step<Run> {
	76	named::bash("doctl registry login")
	77	}
	78
	79	fn build_docker_image() -> Step<Run> {
	80	named::bash(indoc! {r#"
	81	docker build -f Dockerfile-collab \
	82	--build-arg "GITHUB_SHA=$GITHUB_SHA" \
	83	--tag "registry.digitalocean.com/zed/collab:$GITHUB_SHA" \
	84	.
	85	"#})
	86	}
	87
	88	fn publish_docker_image() -> Step<Run> {
	89	named::bash(r#"docker push "registry.digitalocean.com/zed/collab:${GITHUB_SHA}""#)
	90	}
	91
	92	fn prune_docker_system() -> Step<Run> {
	93	named::bash("docker system prune --filter 'until=72h' -f")
	94	}
	95
	96	named::job(
	97	dependant_job(deps)
	98	.name("Publish collab server image")
	99	.runs_on(runners::LINUX_XL)
	100	.add_step(install_doctl())
	101	.add_step(sign_into_registry())
	102	.add_step(steps::checkout_repo())
	103	.add_step(build_docker_image())
	104	.add_step(publish_docker_image())
	105	.add_step(prune_docker_system()),
	106	)
	107	}
	108
	109	fn deploy(deps: &[&NamedJob]) -> NamedJob {
	110	fn install_doctl() -> Step<Use> {
	111	named::uses("digitalocean", "action-doctl", "v2")
	112	.add_with(("token", vars::DIGITALOCEAN_ACCESS_TOKEN))
	113	}
	114
	115	fn sign_into_kubernetes() -> Step<Run> {
	116	named::bash(formatdoc! {r#"
	117	doctl kubernetes cluster kubeconfig save --expiry-seconds 600 {cluster_name}
	118	"#, cluster_name = vars::CLUSTER_NAME})
	119	}
	120
	121	fn start_rollout() -> Step<Run> {
	122	named::bash(indoc! {r#"
	123	set -eu
	124	if [[ $GITHUB_REF_NAME = "collab-production" ]]; then
	125	export ZED_KUBE_NAMESPACE=production
	126	export ZED_COLLAB_LOAD_BALANCER_SIZE_UNIT=10
	127	export ZED_API_LOAD_BALANCER_SIZE_UNIT=2
	128	elif [[ $GITHUB_REF_NAME = "collab-staging" ]]; then
	129	export ZED_KUBE_NAMESPACE=staging
	130	export ZED_COLLAB_LOAD_BALANCER_SIZE_UNIT=1
	131	export ZED_API_LOAD_BALANCER_SIZE_UNIT=1
	132	else
	133	echo "cowardly refusing to deploy from an unknown branch"
	134	exit 1
	135	fi
	136
	137	echo "Deploying collab:$GITHUB_SHA to $ZED_KUBE_NAMESPACE"
	138
	139	source script/lib/deploy-helpers.sh
	140	export_vars_for_environment $ZED_KUBE_NAMESPACE
	141
	142	ZED_DO_CERTIFICATE_ID="$(doctl compute certificate list --format ID --no-header)"
	143	export ZED_DO_CERTIFICATE_ID
	144	export ZED_IMAGE_ID="registry.digitalocean.com/zed/collab:${GITHUB_SHA}"
	145
	146	export ZED_SERVICE_NAME=collab
	147	export ZED_LOAD_BALANCER_SIZE_UNIT=$ZED_COLLAB_LOAD_BALANCER_SIZE_UNIT
	148	export DATABASE_MAX_CONNECTIONS=850
	149	envsubst < crates/collab/k8s/collab.template.yml \| kubectl apply -f -
	150	kubectl -n "$ZED_KUBE_NAMESPACE" rollout status deployment/$ZED_SERVICE_NAME --watch
	151	echo "deployed ${ZED_SERVICE_NAME} to ${ZED_KUBE_NAMESPACE}"
	152
	153	export ZED_SERVICE_NAME=api
	154	export ZED_LOAD_BALANCER_SIZE_UNIT=$ZED_API_LOAD_BALANCER_SIZE_UNIT
	155	export DATABASE_MAX_CONNECTIONS=60
	156	envsubst < crates/collab/k8s/collab.template.yml \| kubectl apply -f -
	157	kubectl -n "$ZED_KUBE_NAMESPACE" rollout status deployment/$ZED_SERVICE_NAME --watch
	158	echo "deployed ${ZED_SERVICE_NAME} to ${ZED_KUBE_NAMESPACE}"
	159	"#})
	160	}
	161
	162	named::job(
	163	dependant_job(deps)
	164	.name("Deploy new server image")
	165	.runs_on(runners::LINUX_XL)
	166	.add_step(steps::checkout_repo())
	167	.add_step(install_doctl())
	168	.add_step(sign_into_kubernetes())
	169	.add_step(start_rollout()),
	170	)
	171	}

tooling/xtask/src/tasks/workflows/vars.rs+2−0

 secret!(AZURE_SIGNING_CLIENT_SECRET);
 secret!(AZURE_SIGNING_TENANT_ID);
 secret!(CACHIX_AUTH_TOKEN);
+secret!(CLUSTER_NAME);
+secret!(DIGITALOCEAN_ACCESS_TOKEN);
 secret!(DIGITALOCEAN_SPACES_ACCESS_KEY);
 secret!(DIGITALOCEAN_SPACES_SECRET_KEY);
 secret!(GITHUB_TOKEN);