build-deploy-pudl #555
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: build-deploy-pudl | |
| on: | |
| workflow_dispatch: | |
| push: | |
| tags: | |
| - "v20*" | |
| schedule: | |
| - cron: "0 6 * * 1-5" # Weekdays at midnight on MST | |
| env: | |
| GCP_BILLING_PROJECT: ${{ secrets.GCP_BILLING_PROJECT }} | |
| GITHUB_REF: ${{ github.ref_name }} # This is changed to dev if running on a schedule | |
| GCE_INSTANCE: pudl-deployment-tag # This is changed to pudl-deployment-dev if running on a schedule | |
| GCE_INSTANCE_ZONE: ${{ secrets.GCE_INSTANCE_ZONE }} | |
| GCS_OUTPUT_BUCKET: gs://nightly-build-outputs.catalyst.coop | |
| jobs: | |
| build_and_deploy_pudl: | |
| name: Build Docker image, push to Docker Hub and deploy to a GCE VM | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| id-token: write | |
| steps: | |
| - name: Use pudl-deployment-dev vm and dev branch if running on a schedule | |
| if: ${{ (github.event_name == 'schedule') }} | |
| run: | | |
| echo "This action was triggered by a schedule." && echo "GCE_INSTANCE=pudl-deployment-dev" >> $GITHUB_ENV && echo "GITHUB_REF=dev" >> $GITHUB_ENV | |
| - name: Checkout Repository | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ env.GITHUB_REF }} | |
| - name: Get HEAD of the branch (main or dev) | |
| run: | | |
| echo "ACTION_SHA=$(git rev-parse HEAD)" >> $GITHUB_ENV | |
| echo "SHORT_SHA=$(git rev-parse --short HEAD)" >> $GITHUB_ENV | |
| - name: Print action vars | |
| run: | | |
| echo "ACTION_SHA: $ACTION_SHA" && \ | |
| echo "GITHUB_REF: $GITHUB_REF" && \ | |
| echo "GCE_INSTANCE: $GCE_INSTANCE" | |
| - name: Docker Metadata | |
| id: docker_metadata | |
| uses: docker/[email protected] | |
| with: | |
| images: catalystcoop/pudl-etl | |
| flavor: | | |
| latest=auto | |
| tags: | | |
| type=raw,value=${{ env.GITHUB_REF }} | |
| type=ref,event=tag | |
| - name: Set up Docker Buildx | |
| uses: docker/[email protected] | |
| - name: Login to DockerHub | |
| if: github.event_name != 'pull_request' | |
| uses: docker/[email protected] | |
| with: | |
| username: ${{ secrets.DOCKERHUB_USERNAME }} | |
| password: ${{ secrets.DOCKERHUB_TOKEN }} | |
| - name: Build image and push to Docker Hub | |
| uses: docker/[email protected] | |
| with: | |
| context: . | |
| file: docker/Dockerfile | |
| push: ${{ github.event_name != 'pull_request' }} | |
| tags: ${{ steps.docker_metadata.outputs.tags }} | |
| labels: ${{ steps.docker_metadata.outputs.labels }} | |
| cache-from: type=gha | |
| cache-to: type=gha,mode=max | |
| - id: "auth" | |
| uses: "google-github-actions/auth@v2" | |
| with: | |
| workload_identity_provider: "projects/345950277072/locations/global/workloadIdentityPools/gh-actions-pool/providers/gh-actions-provider" | |
| service_account: "deploy-pudl-github-action@catalyst-cooperative-pudl.iam.gserviceaccount.com" | |
| # Setup gcloud CLI | |
| - name: Set up Cloud SDK | |
| uses: google-github-actions/setup-gcloud@v1 | |
| - name: Determine commit information | |
| run: |- | |
| echo "COMMIT_BRANCH=$(gitrev-parse --abbrev-ref HEAD)" >> $GITHUB_ENV | |
| echo "COMMIT_TIME=$(git log -1 --format=%cd --date=format:%Y-%m-%d-%H%M)" >> $GITHUB_ENV | |
| # Deploy PUDL image to GCE | |
| - name: Deploy | |
| env: | |
| DAGSTER_PG_PASSWORD: ${{ secrets.DAGSTER_PG_PASSWORD }} | |
| run: |- | |
| gcloud compute instances add-metadata "$GCE_INSTANCE" \ | |
| --zone "$GCE_INSTANCE_ZONE" \ | |
| --metadata-from-file startup-script=./docker/vm_startup_script.sh | |
| gcloud compute instances update-container "$GCE_INSTANCE" \ | |
| --zone "$GCE_INSTANCE_ZONE" \ | |
| --container-image "docker.io/catalystcoop/pudl-etl:${{ env.GITHUB_REF }}" \ | |
| --container-command "micromamba" \ | |
| --container-arg="run" \ | |
| --container-arg="--prefix" \ | |
| --container-arg="/home/mambauser/env" \ | |
| --container-arg="--attach" \ | |
| --container-arg='' \ | |
| --container-arg="bash" \ | |
| --container-arg="./docker/gcp_pudl_etl.sh" \ | |
| --container-env-file="./docker/.env" \ | |
| --container-env ACTION_SHA=$ACTION_SHA \ | |
| --container-env GITHUB_REF=${{ env.GITHUB_REF }} \ | |
| --container-env GITHUB_ACTION_TRIGGER=${{ github.event_name }} \ | |
| --container-env SLACK_TOKEN=${{ secrets.PUDL_DEPLOY_SLACK_TOKEN }} \ | |
| --container-env GCE_INSTANCE=${{ env.GCE_INSTANCE }} \ | |
| --container-env GCE_INSTANCE_ZONE=${{ env.GCE_INSTANCE_ZONE }} \ | |
| --container-env GCP_BILLING_PROJECT=${{ secrets.GCP_BILLING_PROJECT }} \ | |
| --container-env AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }} \ | |
| --container-env AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }} \ | |
| --container-env AWS_DEFAULT_REGION=${{ secrets.AWS_DEFAULT_REGION }} \ | |
| --container-env DAGSTER_PG_USERNAME="postgres" \ | |
| --container-env DAGSTER_PG_PASSWORD="$DAGSTER_PG_PASSWORD" \ | |
| --container-env DAGSTER_PG_HOST="104.154.182.24" \ | |
| --container-env DAGSTER_PG_DB="dagster-storage" \ | |
| --container-env FLY_ACCESS_TOKEN=${{ secrets.FLY_ACCESS_TOKEN }} \ | |
| --container-env PUDL_SETTINGS_YML="/home/mambauser/src/pudl/package_data/settings/etl_full.yml" \ | |
| --container-env PUDL_GCS_OUTPUT=${{ env.GCS_OUTPUT_BUCKET }}/${{ env.COMMIT_TIME }}-${{ env.SHORT_SHA }}-${{ env.COMMIT_BRANCH }} | |
| # Start the VM | |
| - name: Start the deploy-pudl-vm | |
| run: gcloud compute instances start "$GCE_INSTANCE" --zone="$GCE_INSTANCE_ZONE" | |
| - name: Post to a pudl-deployments channel | |
| id: slack | |
| uses: slackapi/[email protected] | |
| with: | |
| channel-id: "C03FHB9N0PQ" | |
| slack-message: "build-deploy-pudl status: ${{ job.status }}\n${{ env.COMMIT_TIME}}-${{ env.SHORT_SHA }}-${{ env.COMMIT_BRANCH }}" | |
| env: | |
| channel-id: "C03FHB9N0PQ" | |
| SLACK_BOT_TOKEN: ${{ secrets.PUDL_DEPLOY_SLACK_TOKEN }} |