History

ci: Generate the Collab deployment workflow (#49306)

This PR updates the Collab deployment workflow to be generated with `cargo xtask workflows`. Release Notes: - N/A

Marshall Bowers committed 8 months ago

Commit 958798d1 parent 05940764 files · +311 −130Browse files

Unified · Split

4 files
  1. .github/workflows/deploy_collab.yml +136 −130
  2. tooling/xtask/src/tasks/workflows.rs +2 −0
  3. tooling/xtask/src/tasks/workflows/deploy_collab.rs +171 −0
  4. tooling/xtask/src/tasks/workflows/vars.rs +2 −0
.github/workflows/deploy_collab.yml+136−130
@@ -1,146 +1,152 @@
1−name: Publish Collab Server Image
2−
1+# Generated from xtask::workflows::deploy_collab
2+# Rebuild with `cargo xtask workflows`.
3+name: deploy_collab
4+env:
5+ DOCKER_BUILDKIT: '1'
36 on:
47 push:
58 tags:
6− - collab-production
7− - collab-staging
8−
9−env:
10− DOCKER_BUILDKIT: 1
11−
9+ - collab-production
1210 jobs:
1311 style:
12+ if: (github.repository_owner == 'zed-industries' || github.repository_owner == 'zed-extensions')
1413 name: Check formatting and Clippy lints
15− if: github.repository_owner == 'zed-industries'
16− runs-on:
17− - namespace-profile-16x32-ubuntu-2204
14+ runs-on: namespace-profile-16x32-ubuntu-2204
1815 steps:
19− - name: Checkout repo
20− uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
21− with:
22− clean: false
23− fetch-depth: 0
24−
25− - name: Run style checks
26− uses: ./.github/actions/check_style
27−
28− - name: Run clippy
29− run: ./script/clippy
30−
16+ - name: steps::checkout_repo
17+ uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
18+ with:
19+ clean: false
20+ fetch-depth: 0
21+ - name: steps::setup_cargo_config
22+ run: |
23+ mkdir -p ./../.cargo
24+ cp ./.cargo/ci-config.toml ./../.cargo/config.toml
25+ - name: steps::cache_rust_dependencies_namespace
26+ uses: namespacelabs/nscloud-cache-action@v1
27+ with:
28+ cache: rust
29+ path: ~/.rustup
30+ - name: steps::cargo_fmt
31+ run: cargo fmt --all -- --check
32+ - name: steps::clippy
33+ run: ./script/clippy
3134 tests:
35+ needs:
36+ - style
3237 name: Run tests
33− runs-on:
34− - namespace-profile-16x32-ubuntu-2204
35− needs: style
38+ runs-on: namespace-profile-16x32-ubuntu-2204
3639 steps:
37− - name: Checkout repo
38− uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
39− with:
40− clean: false
41− fetch-depth: 0
42−
43− - name: Install cargo nextest
44− uses: taiki-e/install-action@nextest
45−
46− - name: Limit target directory size
47− shell: bash -euxo pipefail {0}
48− run: script/clear-target-dir-if-larger-than 300
49−
50− - name: Run tests
51− shell: bash -euxo pipefail {0}
52− run: cargo nextest run --package collab --no-fail-fast
53−
40+ - name: steps::checkout_repo
41+ uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
42+ with:
43+ clean: false
44+ fetch-depth: 0
45+ - name: steps::setup_cargo_config
46+ run: |
47+ mkdir -p ./../.cargo
48+ cp ./.cargo/ci-config.toml ./../.cargo/config.toml
49+ - name: steps::cache_rust_dependencies_namespace
50+ uses: namespacelabs/nscloud-cache-action@v1
51+ with:
52+ cache: rust
53+ path: ~/.rustup
54+ - name: steps::cargo_install_nextest
55+ uses: taiki-e/install-action@nextest
56+ - name: steps::clear_target_dir_if_large
57+ run: ./script/clear-target-dir-if-larger-than 250
58+ - name: deploy_collab::tests::run_collab_tests
59+ run: cargo nextest run --package collab --no-fail-fast
60+ services:
61+ postgres:
62+ image: postgres:15
63+ env:
64+ POSTGRES_HOST_AUTH_METHOD: trust
65+ ports:
66+ - 5432:5432
67+ options: --health-cmd pg_isready --health-interval 500ms --health-timeout 5s --health-retries 10
5468 publish:
55− name: Publish collab server image
5669 needs:
57− - style
58− - tests
59− runs-on:
60− - namespace-profile-16x32-ubuntu-2204
70+ - style
71+ - tests
72+ name: Publish collab server image
73+ runs-on: namespace-profile-16x32-ubuntu-2204
6174 steps:
62− - name: Install doctl
63− uses: digitalocean/action-doctl@v2
64− with:
65− token: ${{ secrets.DIGITALOCEAN_ACCESS_TOKEN }}
66−
67− - name: Sign into DigitalOcean docker registry
68− run: doctl registry login
69−
70− - name: Checkout repo
71− uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
72− with:
73− clean: false
74−
75− - name: Build docker image
76− run: |
77− docker build -f Dockerfile-collab \
78− --build-arg "GITHUB_SHA=$GITHUB_SHA" \
79− --tag "registry.digitalocean.com/zed/collab:$GITHUB_SHA" \
80− .
81−
82− - name: Publish docker image
83− run: docker push "registry.digitalocean.com/zed/collab:${GITHUB_SHA}"
84−
85− - name: Prune Docker system
86− run: docker system prune --filter 'until=72h' -f
87−
75+ - name: deploy_collab::publish::install_doctl
76+ uses: digitalocean/action-doctl@v2
77+ with:
78+ token: ${{ secrets.DIGITALOCEAN_ACCESS_TOKEN }}
79+ - name: deploy_collab::publish::sign_into_registry
80+ run: doctl registry login
81+ - name: steps::checkout_repo
82+ uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
83+ with:
84+ clean: false
85+ - name: deploy_collab::publish::build_docker_image
86+ run: |
87+ docker build -f Dockerfile-collab \
88+ --build-arg "GITHUB_SHA=$GITHUB_SHA" \
89+ --tag "registry.digitalocean.com/zed/collab:$GITHUB_SHA" \
90+ .
91+ - name: deploy_collab::publish::publish_docker_image
92+ run: docker push "registry.digitalocean.com/zed/collab:${GITHUB_SHA}"
93+ - name: deploy_collab::publish::prune_docker_system
94+ run: docker system prune --filter 'until=72h' -f
8895 deploy:
89− name: Deploy new server image
9096 needs:
91− - publish
92− runs-on:
93− - namespace-profile-16x32-ubuntu-2204
94−
97+ - publish
98+ name: Deploy new server image
99+ runs-on: namespace-profile-16x32-ubuntu-2204
95100 steps:
96− - name: Checkout repo
97− uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
98− with:
99− clean: false
100−
101− - name: Install doctl
102− uses: digitalocean/action-doctl@v2
103− with:
104− token: ${{ secrets.DIGITALOCEAN_ACCESS_TOKEN }}
105−
106− - name: Sign into Kubernetes
107− run: doctl kubernetes cluster kubeconfig save --expiry-seconds 600 ${{ secrets.CLUSTER_NAME }}
108−
109− - name: Start rollout
110− run: |
111− set -eu
112− if [[ $GITHUB_REF_NAME = "collab-production" ]]; then
113− export ZED_KUBE_NAMESPACE=production
114− export ZED_COLLAB_LOAD_BALANCER_SIZE_UNIT=10
115− export ZED_API_LOAD_BALANCER_SIZE_UNIT=2
116− elif [[ $GITHUB_REF_NAME = "collab-staging" ]]; then
117− export ZED_KUBE_NAMESPACE=staging
118− export ZED_COLLAB_LOAD_BALANCER_SIZE_UNIT=1
119− export ZED_API_LOAD_BALANCER_SIZE_UNIT=1
120− else
121− echo "cowardly refusing to deploy from an unknown branch"
122− exit 1
123− fi
124−
125− echo "Deploying collab:$GITHUB_SHA to $ZED_KUBE_NAMESPACE"
126−
127− source script/lib/deploy-helpers.sh
128− export_vars_for_environment $ZED_KUBE_NAMESPACE
129−
130− ZED_DO_CERTIFICATE_ID="$(doctl compute certificate list --format ID --no-header)"
131− export ZED_DO_CERTIFICATE_ID
132− export ZED_IMAGE_ID="registry.digitalocean.com/zed/collab:${GITHUB_SHA}"
133−
134− export ZED_SERVICE_NAME=collab
135− export ZED_LOAD_BALANCER_SIZE_UNIT=$ZED_COLLAB_LOAD_BALANCER_SIZE_UNIT
136− export DATABASE_MAX_CONNECTIONS=850
137− envsubst < crates/collab/k8s/collab.template.yml | kubectl apply -f -
138− kubectl -n "$ZED_KUBE_NAMESPACE" rollout status deployment/$ZED_SERVICE_NAME --watch
139− echo "deployed ${ZED_SERVICE_NAME} to ${ZED_KUBE_NAMESPACE}"
140−
141− export ZED_SERVICE_NAME=api
142− export ZED_LOAD_BALANCER_SIZE_UNIT=$ZED_API_LOAD_BALANCER_SIZE_UNIT
143− export DATABASE_MAX_CONNECTIONS=60
144− envsubst < crates/collab/k8s/collab.template.yml | kubectl apply -f -
145− kubectl -n "$ZED_KUBE_NAMESPACE" rollout status deployment/$ZED_SERVICE_NAME --watch
146− echo "deployed ${ZED_SERVICE_NAME} to ${ZED_KUBE_NAMESPACE}"
101+ - name: steps::checkout_repo
102+ uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
103+ with:
104+ clean: false
105+ - name: deploy_collab::deploy::install_doctl
106+ uses: digitalocean/action-doctl@v2
107+ with:
108+ token: ${{ secrets.DIGITALOCEAN_ACCESS_TOKEN }}
109+ - name: deploy_collab::deploy::sign_into_kubernetes
110+ run: |
111+ doctl kubernetes cluster kubeconfig save --expiry-seconds 600 ${{ secrets.CLUSTER_NAME }}
112+ - name: deploy_collab::deploy::start_rollout
113+ run: |
114+ set -eu
115+ if [[ $GITHUB_REF_NAME = "collab-production" ]]; then
116+ export ZED_KUBE_NAMESPACE=production
117+ export ZED_COLLAB_LOAD_BALANCER_SIZE_UNIT=10
118+ export ZED_API_LOAD_BALANCER_SIZE_UNIT=2
119+ elif [[ $GITHUB_REF_NAME = "collab-staging" ]]; then
120+ export ZED_KUBE_NAMESPACE=staging
121+ export ZED_COLLAB_LOAD_BALANCER_SIZE_UNIT=1
122+ export ZED_API_LOAD_BALANCER_SIZE_UNIT=1
123+ else
124+ echo "cowardly refusing to deploy from an unknown branch"
125+ exit 1
126+ fi
127+
128+ echo "Deploying collab:$GITHUB_SHA to $ZED_KUBE_NAMESPACE"
129+
130+ source script/lib/deploy-helpers.sh
131+ export_vars_for_environment $ZED_KUBE_NAMESPACE
132+
133+ ZED_DO_CERTIFICATE_ID="$(doctl compute certificate list --format ID --no-header)"
134+ export ZED_DO_CERTIFICATE_ID
135+ export ZED_IMAGE_ID="registry.digitalocean.com/zed/collab:${GITHUB_SHA}"
136+
137+ export ZED_SERVICE_NAME=collab
138+ export ZED_LOAD_BALANCER_SIZE_UNIT=$ZED_COLLAB_LOAD_BALANCER_SIZE_UNIT
139+ export DATABASE_MAX_CONNECTIONS=850
140+ envsubst < crates/collab/k8s/collab.template.yml | kubectl apply -f -
141+ kubectl -n "$ZED_KUBE_NAMESPACE" rollout status deployment/$ZED_SERVICE_NAME --watch
142+ echo "deployed ${ZED_SERVICE_NAME} to ${ZED_KUBE_NAMESPACE}"
143+
144+ export ZED_SERVICE_NAME=api
145+ export ZED_LOAD_BALANCER_SIZE_UNIT=$ZED_API_LOAD_BALANCER_SIZE_UNIT
146+ export DATABASE_MAX_CONNECTIONS=60
147+ envsubst < crates/collab/k8s/collab.template.yml | kubectl apply -f -
148+ kubectl -n "$ZED_KUBE_NAMESPACE" rollout status deployment/$ZED_SERVICE_NAME --watch
149+ echo "deployed ${ZED_SERVICE_NAME} to ${ZED_KUBE_NAMESPACE}"
150+defaults:
151+ run:
152+ shell: bash -euxo pipefail {0}
tooling/xtask/src/tasks/workflows.rs+2−0
@@ -10,6 +10,7 @@ mod bump_patch_version; · 9 unmodified lines
1010 mod cherry_pick;
1111 mod compare_perf;
1212 mod danger;
13+mod deploy_collab;
1314 mod extension_bump;
1415 mod extension_tests;
1516 mod extension_workflow_rollout;
@@ -133,6 +134,7 @@ pub fn run_workflows(_: GenerateWorkflowArgs) -> Result<()> { · 117 unmodified lines
133134 WorkflowFile::zed(cherry_pick::cherry_pick),
134135 WorkflowFile::zed(compare_perf::compare_perf),
135136 WorkflowFile::zed(danger::danger),
137+ WorkflowFile::zed(deploy_collab::deploy_collab),
136138 WorkflowFile::zed(extension_bump::extension_bump),
137139 WorkflowFile::zed(extension_tests::extension_tests),
138140 WorkflowFile::zed(extension_workflow_rollout::extension_workflow_rollout),
tooling/xtask/src/tasks/workflows/deploy_collab.rs+171−0
@@ -0,0 +1,171 @@
1+use gh_workflow::{Container, Event, Port, Push, Run, Step, Use, Workflow};
2+use indoc::{formatdoc, indoc};
3+
4+use crate::tasks::workflows::{
5+ runners::{self, Platform},
6+ steps::{self, CommonJobConditions, NamedJob, dependant_job, named},
7+ vars,
8+};
9+
10+pub(crate) fn deploy_collab() -> Workflow {
11+ let style = style();
12+ let tests = tests(&[&style]);
13+ let publish = publish(&[&style, &tests]);
14+ let deploy = deploy(&[&publish]);
15+
16+ named::workflow()
17+ .on(Event::default().push(Push::default().add_tag("collab-production")))
18+ .add_env(("DOCKER_BUILDKIT", "1"))
19+ .add_job(style.name, style.job)
20+ .add_job(tests.name, tests.job)
21+ .add_job(publish.name, publish.job)
22+ .add_job(deploy.name, deploy.job)
23+}
24+
25+fn style() -> NamedJob {
26+ named::job(
27+ dependant_job(&[])
28+ .name("Check formatting and Clippy lints")
29+ .with_repository_owner_guard()
30+ .runs_on(runners::LINUX_XL)
31+ .add_step(steps::checkout_repo().add_with(("fetch-depth", 0)))
32+ .add_step(steps::setup_cargo_config(Platform::Linux))
33+ .add_step(steps::cache_rust_dependencies_namespace())
34+ .add_step(steps::cargo_fmt())
35+ .add_step(steps::clippy(Platform::Linux)),
36+ )
37+}
38+
39+fn tests(deps: &[&NamedJob]) -> NamedJob {
40+ fn run_collab_tests() -> Step<Run> {
41+ named::bash("cargo nextest run --package collab --no-fail-fast")
42+ }
43+
44+ named::job(
45+ dependant_job(deps)
46+ .name("Run tests")
47+ .runs_on(runners::LINUX_XL)
48+ .add_service(
49+ "postgres",
50+ Container::new("postgres:15")
51+ .add_env(("POSTGRES_HOST_AUTH_METHOD", "trust"))
52+ .ports(vec![Port::Name("5432:5432".into())])
53+ .options(
54+ "--health-cmd pg_isready \
55+ --health-interval 500ms \
56+ --health-timeout 5s \
57+ --health-retries 10",
58+ ),
59+ )
60+ .add_step(steps::checkout_repo().add_with(("fetch-depth", 0)))
61+ .add_step(steps::setup_cargo_config(Platform::Linux))
62+ .add_step(steps::cache_rust_dependencies_namespace())
63+ .add_step(steps::cargo_install_nextest())
64+ .add_step(steps::clear_target_dir_if_large(Platform::Linux))
65+ .add_step(run_collab_tests()),
66+ )
67+}
68+
69+fn publish(deps: &[&NamedJob]) -> NamedJob {
70+ fn install_doctl() -> Step<Use> {
71+ named::uses("digitalocean", "action-doctl", "v2")
72+ .add_with(("token", vars::DIGITALOCEAN_ACCESS_TOKEN))
73+ }
74+
75+ fn sign_into_registry() -> Step<Run> {
76+ named::bash("doctl registry login")
77+ }
78+
79+ fn build_docker_image() -> Step<Run> {
80+ named::bash(indoc! {r#"
81+ docker build -f Dockerfile-collab \
82+ --build-arg "GITHUB_SHA=$GITHUB_SHA" \
83+ --tag "registry.digitalocean.com/zed/collab:$GITHUB_SHA" \
84+ .
85+ "#})
86+ }
87+
88+ fn publish_docker_image() -> Step<Run> {
89+ named::bash(r#"docker push "registry.digitalocean.com/zed/collab:${GITHUB_SHA}""#)
90+ }
91+
92+ fn prune_docker_system() -> Step<Run> {
93+ named::bash("docker system prune --filter 'until=72h' -f")
94+ }
95+
96+ named::job(
97+ dependant_job(deps)
98+ .name("Publish collab server image")
99+ .runs_on(runners::LINUX_XL)
100+ .add_step(install_doctl())
101+ .add_step(sign_into_registry())
102+ .add_step(steps::checkout_repo())
103+ .add_step(build_docker_image())
104+ .add_step(publish_docker_image())
105+ .add_step(prune_docker_system()),
106+ )
107+}
108+
109+fn deploy(deps: &[&NamedJob]) -> NamedJob {
110+ fn install_doctl() -> Step<Use> {
111+ named::uses("digitalocean", "action-doctl", "v2")
112+ .add_with(("token", vars::DIGITALOCEAN_ACCESS_TOKEN))
113+ }
114+
115+ fn sign_into_kubernetes() -> Step<Run> {
116+ named::bash(formatdoc! {r#"
117+ doctl kubernetes cluster kubeconfig save --expiry-seconds 600 {cluster_name}
118+ "#, cluster_name = vars::CLUSTER_NAME})
119+ }
120+
121+ fn start_rollout() -> Step<Run> {
122+ named::bash(indoc! {r#"
123+ set -eu
124+ if [[ $GITHUB_REF_NAME = "collab-production" ]]; then
125+ export ZED_KUBE_NAMESPACE=production
126+ export ZED_COLLAB_LOAD_BALANCER_SIZE_UNIT=10
127+ export ZED_API_LOAD_BALANCER_SIZE_UNIT=2
128+ elif [[ $GITHUB_REF_NAME = "collab-staging" ]]; then
129+ export ZED_KUBE_NAMESPACE=staging
130+ export ZED_COLLAB_LOAD_BALANCER_SIZE_UNIT=1
131+ export ZED_API_LOAD_BALANCER_SIZE_UNIT=1
132+ else
133+ echo "cowardly refusing to deploy from an unknown branch"
134+ exit 1
135+ fi
136+
137+ echo "Deploying collab:$GITHUB_SHA to $ZED_KUBE_NAMESPACE"
138+
139+ source script/lib/deploy-helpers.sh
140+ export_vars_for_environment $ZED_KUBE_NAMESPACE
141+
142+ ZED_DO_CERTIFICATE_ID="$(doctl compute certificate list --format ID --no-header)"
143+ export ZED_DO_CERTIFICATE_ID
144+ export ZED_IMAGE_ID="registry.digitalocean.com/zed/collab:${GITHUB_SHA}"
145+
146+ export ZED_SERVICE_NAME=collab
147+ export ZED_LOAD_BALANCER_SIZE_UNIT=$ZED_COLLAB_LOAD_BALANCER_SIZE_UNIT
148+ export DATABASE_MAX_CONNECTIONS=850
149+ envsubst < crates/collab/k8s/collab.template.yml | kubectl apply -f -
150+ kubectl -n "$ZED_KUBE_NAMESPACE" rollout status deployment/$ZED_SERVICE_NAME --watch
151+ echo "deployed ${ZED_SERVICE_NAME} to ${ZED_KUBE_NAMESPACE}"
152+
153+ export ZED_SERVICE_NAME=api
154+ export ZED_LOAD_BALANCER_SIZE_UNIT=$ZED_API_LOAD_BALANCER_SIZE_UNIT
155+ export DATABASE_MAX_CONNECTIONS=60
156+ envsubst < crates/collab/k8s/collab.template.yml | kubectl apply -f -
157+ kubectl -n "$ZED_KUBE_NAMESPACE" rollout status deployment/$ZED_SERVICE_NAME --watch
158+ echo "deployed ${ZED_SERVICE_NAME} to ${ZED_KUBE_NAMESPACE}"
159+ "#})
160+ }
161+
162+ named::job(
163+ dependant_job(deps)
164+ .name("Deploy new server image")
165+ .runs_on(runners::LINUX_XL)
166+ .add_step(steps::checkout_repo())
167+ .add_step(install_doctl())
168+ .add_step(sign_into_kubernetes())
169+ .add_step(start_rollout()),
170+ )
171+}
tooling/xtask/src/tasks/workflows/vars.rs+2−0
@@ -30,6 +30,8 @@ secret!(AZURE_SIGNING_CLIENT_ID); · 29 unmodified lines
3030 secret!(AZURE_SIGNING_CLIENT_SECRET);
3131 secret!(AZURE_SIGNING_TENANT_ID);
3232 secret!(CACHIX_AUTH_TOKEN);
33+secret!(CLUSTER_NAME);
34+secret!(DIGITALOCEAN_ACCESS_TOKEN);
3335 secret!(DIGITALOCEAN_SPACES_ACCESS_KEY);
3436 secret!(DIGITALOCEAN_SPACES_SECRET_KEY);
3537 secret!(GITHUB_TOKEN);