build-deploy-pudl.yml

name: build-deploy-pudl
on:
  workflow_dispatch:
  push:
    tags:
      - "v20*"
  schedule:
    - cron: "0 6 * * 1-5" # Weekdays at midnight on MST

env:
  GCP_BILLING_PROJECT: ${{ secrets.GCP_BILLING_PROJECT }}
  GITHUB_REF: ${{ github.ref_name }} # This is changed to dev if running on a schedule
  GCE_INSTANCE: pudl-deployment-tag # This is changed to pudl-deployment-dev if running on a schedule
  GCE_INSTANCE_ZONE: ${{ secrets.GCE_INSTANCE_ZONE }}
  GCS_OUTPUT_BUCKET: gs://nightly-build-outputs.catalyst.coop

jobs:
  build_and_deploy_pudl:
    name: Build Docker image, push to Docker Hub and deploy to a GCE VM
    runs-on: ubuntu-latest
    permissions:
      contents: read
      id-token: write
    steps:
      - name: Use pudl-deployment-dev vm and dev branch if running on a schedule
        if: ${{ (github.event_name == 'schedule') }}
        run: |
          echo "This action was triggered by a schedule." && echo "GCE_INSTANCE=pudl-deployment-dev" >> $GITHUB_ENV && echo "GITHUB_REF=dev" >> $GITHUB_ENV

      - name: Checkout Repository
        uses: actions/checkout@v4
        with:
          ref: ${{ env.GITHUB_REF }}

      - name: Get HEAD of the branch (main or dev)
        run: |
          echo "ACTION_SHA=$(git rev-parse HEAD)" >> $GITHUB_ENV
          echo "SHORT_SHA=$(git rev-parse --short HEAD)" >> $GITHUB_ENV

      - name: Print action vars
        run: |
          echo "ACTION_SHA: $ACTION_SHA" && \
          echo "GITHUB_REF: $GITHUB_REF" && \
          echo "GCE_INSTANCE: $GCE_INSTANCE"

      - name: Docker Metadata
        id: docker_metadata
        uses: docker/metadata-action@v5.3.0
        with:
          images: catalystcoop/pudl-etl
          flavor: |
            latest=auto
          tags: |
            type=raw,value=${{ env.GITHUB_REF }}
            type=ref,event=tag

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3.0.0

      - name: Login to DockerHub
        if: github.event_name != 'pull_request'
        uses: docker/login-action@v3.0.0
        with:
          username: ${{ secrets.DOCKERHUB_USERNAME }}
          password: ${{ secrets.DOCKERHUB_TOKEN }}

      - name: Build image and push to Docker Hub
        uses: docker/build-push-action@v5.1.0
        with:
          context: .
          file: docker/Dockerfile
          push: ${{ github.event_name != 'pull_request' }}
          tags: ${{ steps.docker_metadata.outputs.tags }}
          labels: ${{ steps.docker_metadata.outputs.labels }}
          cache-from: type=gha
          cache-to: type=gha,mode=max

      - id: "auth"
        uses: "google-github-actions/auth@v2"
        with:
          workload_identity_provider: "projects/345950277072/locations/global/workloadIdentityPools/gh-actions-pool/providers/gh-actions-provider"
          service_account: "deploy-pudl-github-action@catalyst-cooperative-pudl.iam.gserviceaccount.com"

      # Setup gcloud CLI
      - name: Set up Cloud SDK
        uses: google-github-actions/setup-gcloud@v2

      - name: Determine commit information
        run: |-
          echo "COMMIT_BRANCH=$(gitrev-parse --abbrev-ref HEAD)" >> $GITHUB_ENV
          echo "COMMIT_TIME=$(git log -1 --format=%cd --date=format:%Y-%m-%d-%H%M)" >> $GITHUB_ENV

      # Deploy PUDL image to GCE
      - name: Deploy
        env:
          DAGSTER_PG_PASSWORD: ${{ secrets.DAGSTER_PG_PASSWORD }}
        run: |-
          gcloud compute instances add-metadata "$GCE_INSTANCE" \
            --zone "$GCE_INSTANCE_ZONE" \
            --metadata-from-file startup-script=./docker/vm_startup_script.sh
          gcloud compute instances update-container "$GCE_INSTANCE" \
            --zone "$GCE_INSTANCE_ZONE" \
            --container-image "docker.io/catalystcoop/pudl-etl:${{ env.GITHUB_REF }}" \
            --container-command "micromamba" \
            --container-arg="run" \
            --container-arg="--prefix" \
            --container-arg="/home/mambauser/env" \
            --container-arg="--attach" \
            --container-arg='' \
            --container-arg="bash" \
            --container-arg="./docker/gcp_pudl_etl.sh" \
            --container-env-file="./docker/.env" \
            --container-env ACTION_SHA=$ACTION_SHA \
            --container-env GITHUB_REF=${{ env.GITHUB_REF }} \
            --container-env GITHUB_ACTION_TRIGGER=${{ github.event_name }} \
            --container-env SLACK_TOKEN=${{ secrets.PUDL_DEPLOY_SLACK_TOKEN }} \
            --container-env GCE_INSTANCE=${{ env.GCE_INSTANCE }} \
            --container-env GCE_INSTANCE_ZONE=${{ env.GCE_INSTANCE_ZONE }} \
            --container-env GCP_BILLING_PROJECT=${{ secrets.GCP_BILLING_PROJECT }} \
            --container-env AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }} \
            --container-env AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }} \
            --container-env AWS_DEFAULT_REGION=${{ secrets.AWS_DEFAULT_REGION }} \
            --container-env DAGSTER_PG_USERNAME="postgres" \
            --container-env DAGSTER_PG_PASSWORD="$DAGSTER_PG_PASSWORD" \
            --container-env DAGSTER_PG_HOST="104.154.182.24" \
            --container-env DAGSTER_PG_DB="dagster-storage" \
            --container-env FLY_ACCESS_TOKEN=${{ secrets.FLY_ACCESS_TOKEN }} \
            --container-env PUDL_SETTINGS_YML="/home/mambauser/src/pudl/package_data/settings/etl_full.yml" \
            --container-env PUDL_GCS_OUTPUT=${{ env.GCS_OUTPUT_BUCKET }}/${{ env.COMMIT_TIME }}-${{ env.SHORT_SHA }}-${{ env.COMMIT_BRANCH }}

      # Start the VM
      - name: Start the deploy-pudl-vm
        run: gcloud compute instances start "$GCE_INSTANCE" --zone="$GCE_INSTANCE_ZONE"

      - name: Post to a pudl-deployments channel
        id: slack
        uses: slackapi/slack-github-action@v1.24.0
        with:
          channel-id: "C03FHB9N0PQ"
          slack-message: "build-deploy-pudl status: ${{ job.status }}\n${{ env.COMMIT_TIME}}-${{ env.SHORT_SHA }}-${{ env.COMMIT_BRANCH }}"
        env:
          channel-id: "C03FHB9N0PQ"
          SLACK_BOT_TOKEN: ${{ secrets.PUDL_DEPLOY_SLACK_TOKEN }}