Automate blue/green updates for the browser/API

.cloudbuild/bluegreen-flip.sh

@@ -0,0 +1,43 @@
+#!/bin/bash
+
+set -euo pipefail
+
+# Ensure that our cloudbuild has everything we need
+for cmd in git yq; do
+    if ! command -v $cmd &> /dev/null; then
+        echo "Error: $cmd is not installed." >&2
+        exit 1
+    fi
+done
+
+
+mkdir -p /root/.ssh && chmod 0700 /root/.ssh
+echo "$DEPLOY_KEY" > /root/.ssh/id_rsa
+chmod 400 /root/.ssh/id_rsa
+ssh-keyscan -t rsa github.com > /root/.ssh/known_hosts
+git clone [email protected]:broadinstitute/gnomad-deployments.git
+
+cd gnomad-deployments/gnomad-browser/prod-deflector
+
+CURRENT_DEPLOYMENT=$(yq '.spec.selector.deployment' < gnomad-bluegreen.service.yaml)
+
+echo "Current deployment is: $CURRENT_DEPLOYMENT"
+
+if [[ "$CURRENT_DEPLOYMENT" != "blue" && "$CURRENT_DEPLOYMENT" != "green" ]]; then
+    echo "Error: Current deployment is not either green or blue, unknown state. Exiting."
+    exit 1
+fi
+
+if [[ "$CURRENT_DEPLOYMENT" == "blue" ]]; then
+    TARGET_DEPLOYMENT="green"
+else
+    TARGET_DEPLOYMENT="blue"
+fi
+
+yq -i ".spec.selector.deployment = \"${TARGET_DEPLOYMENT}\"" gnomad-bluegreen.service.yaml
+
+git add gnomad-bluegreen.service.yaml
+git -c user.name="TGG Automation" -c user.email="[email protected]" commit -m "Updating gnomad-browser active service to $TARGET_DEPLOYMENT"
+git show
+
+git push origin main

.cloudbuild/bluegreen-update.sh

@@ -0,0 +1,66 @@
+#!/bin/bash
+
+set -euo pipefail
+
+# Ensure that our cloudbuild has everything we need
+for cmd in git yq kubectl gcloud kustomize curl; do
+    if ! command -v $cmd &> /dev/null; then
+        echo "Error: $cmd is not installed." >&2
+        exit 1
+    fi
+done
+
+####
+# Retrieve current active blue/green and verify that it matches what's currently defined in git
+####
+
+gcloud container clusters get-credentials --dns-endpoint --zone $CLOUDSDK_COMPUTE_ZONE $CLOUDSDK_CONTAINER_CLUSTER
+
+CURRENT_DEPLOYMENT=$(kubectl get service gnomad-browser-bluegreen -o jsonpath='{.spec.selector.deployment}')
+
+echo "current deployment: $CURRENT_DEPLOYMENT"
+
+mkdir -p /root/.ssh && chmod 0700 /root/.ssh
+echo "$DEPLOY_KEY" > /root/.ssh/id_rsa
+chmod 400 /root/.ssh/id_rsa
+ssh-keyscan -t rsa github.com > /root/.ssh/known_hosts
+git clone [email protected]:broadinstitute/gnomad-deployments.git
+
+cd gnomad-deployments/gnomad-browser
+
+INTENDED_DEPLOYMENT=$(yq '.spec.selector.deployment' < prod-deflector/gnomad-bluegreen.service.yaml)
+
+echo "intended current deployment: $INTENDED_DEPLOYMENT"
+
+# If current and intended deployments are not either blue or green, something has gone wrong.
+if [[ "$CURRENT_DEPLOYMENT" != "blue" && "$CURRENT_DEPLOYMENT" != "green" ]] || [[ "$INTENDED_DEPLOYMENT" != "blue" && "$INTENDED_DEPLOYMENT" != "green" ]]; then
+    echo "Error: Deployments must be either 'blue' or 'green'."
+    exit 1
+fi
+
+# Determine inactive deployment target
+if [ "$CURRENT_DEPLOYMENT" == "$INTENDED_DEPLOYMENT" ]; then
+    if [ "$CURRENT_DEPLOYMENT" == "blue" ]; then
+        TARGET_DEPLOYMENT="green"
+    else
+        TARGET_DEPLOYMENT="blue"
+    fi
+else
+    echo "The current deployment and the intended deployment don't match. Exiting"
+    exit 1
+fi
+
+echo "target/inactive deployment is $TARGET_DEPLOYMENT"
+
+####
+# Update image tags in the inactive deployment and push
+####
+pushd $TARGET_DEPLOYMENT
+kustomize --stack-trace edit set image "gnomad-api=us-docker.pkg.dev/${REPO_PROJECT}/gnomad/gnomad-api:${DOCKER_TAG}"
+kustomize --stack-trace edit set image "gnomad-browser=us-docker.pkg.dev/${REPO_PROJECT}/gnomad/gnomad-browser:${DOCKER_TAG}"
+popd
+
+git add $TARGET_DEPLOYMENT
+git -c user.name="TGG Automation" -c user.email="[email protected]" commit -m "Updating gnomad-browser $TARGET_DEPLOYMENT deployments to image tag: $DOCKER_TAG"
+git show
+git push origin main

.cloudbuild/browser-flip-bluegreen.cloudbuild.yaml

@@ -0,0 +1,12 @@
+# Note: You can find a definition for the deploykit image in https://github.com/broadinstitute/tgg-sre/tree/main/dockerfiles/deployment-toolkit
+steps:
+  - name: 'us-docker.pkg.dev/${PROJECT_ID}/gnomad/tgg-deploykit:v1.0.0'
+    entrypoint: 'bash'
+    args: ['./bluegreen-flip.sh']
+    dir: '.cloudbuild'
+    secretEnv:
+      - 'DEPLOY_KEY'
+availableSecrets:
+  secretManager:
+    - versionName: projects/$PROJECT_ID/secrets/GITHUB_DEPLOY_KEY/versions/latest
+      env: 'DEPLOY_KEY'

.cloudbuild/browser.cloudbuild.yaml

@@ -0,0 +1,65 @@
+steps:
+  - name: 'ubuntu'
+    entrypoint: 'bash'
+    args:
+      - -c
+      - |
+        echo $$GNOMAD_BUILD_ENV > browser/build.env
+    secretEnv:
+      - 'GNOMAD_BUILD_ENV'
+  - name: 'gcr.io/cloud-builders/docker'
+    args:
+      [
+        'build',
+        '-t',
+        'us-docker.pkg.dev/${PROJECT_ID}/gnomad/gnomad-api:$SHORT_SHA',
+        '-t',
+        'us-docker.pkg.dev/${PROJECT_ID}/gnomad/gnomad-api:${_BUILD_TAG}',
+        '-f',
+        'deploy/dockerfiles/browser/api.dockerfile',
+        '.',
+      ]
+  - name: 'gcr.io/cloud-builders/docker'
+    args: ['push', 'us-docker.pkg.dev/${PROJECT_ID}/gnomad/gnomad-api:$SHORT_SHA']
+  - name: 'gcr.io/cloud-builders/docker'
+    args: ['push', 'us-docker.pkg.dev/${PROJECT_ID}/gnomad/gnomad-api:${_BUILD_TAG}']
+
+  - name: 'gcr.io/cloud-builders/docker'
+    args:
+      [
+        'build',
+        '-t',
+        'us-docker.pkg.dev/${PROJECT_ID}/gnomad/gnomad-browser:$SHORT_SHA',
+        '-t',
+        'us-docker.pkg.dev/${PROJECT_ID}/gnomad/gnomad-browser:${_BUILD_TAG}',
+        '-f',
+        'deploy/dockerfiles/browser/browser.dockerfile',
+        '.',
+      ]
+  - name: 'gcr.io/cloud-builders/docker'
+    args: ['push', 'us-docker.pkg.dev/${PROJECT_ID}/gnomad/gnomad-browser:$SHORT_SHA']
+  - name: 'gcr.io/cloud-builders/docker'
+    args: ['push', 'us-docker.pkg.dev/${PROJECT_ID}/gnomad/gnomad-browser:${_BUILD_TAG}']
+
+  # Note: You can find a definition for the deploykit image in https://github.com/broadinstitute/tgg-sre/tree/main/dockerfiles/deployment-toolkit
+  - name: 'us-docker.pkg.dev/${PROJECT_ID}/gnomad/tgg-deploykit:v1.0.0'
+    entrypoint: 'bash'
+    args: ['./bluegreen-update.sh']
+    dir: '.cloudbuild'
+    env:
+      - 'CLOUDSDK_COMPUTE_ZONE=us-east1-b'
+      - 'CLOUDSDK_CONTAINER_CLUSTER=gnomad-v4'
+      - 'REPO_PROJECT=$PROJECT_ID'
+      - 'DOCKER_TAG=$SHORT_SHA'
+    secretEnv:
+      - 'DEPLOY_KEY'
+availableSecrets:
+  secretManager:
+    - versionName: projects/$PROJECT_ID/secrets/GITHUB_DEPLOY_KEY/versions/latest
+      env: 'DEPLOY_KEY'
+    - versionName: projects/$PROJECT_ID/secrets/GNOMAD_BUILD_ENV/versions/1
+      env: 'GNOMAD_BUILD_ENV'
+options:
+  dynamicSubstitutions: true
+substitutions:
+  _BUILD_TAG: '${_BRANCH_FOR_IMAGE_NAME}-${BUILD_ID}'