From 329202294d8587f18f1a47ed0723264cfcd703f5 Mon Sep 17 00:00:00 2001
From: alangenfeld <alex@elementl.com>
Date: Mon, 8 Apr 2024 15:00:09 -0500
Subject: [PATCH] DbtProject support in serverless template

---
 github/serverless/dbt/branch_deployments.yml | 37 ++++++++++++++------
 github/serverless/dbt/deploy.yml             | 37 ++++++++++++++------
 2 files changed, 54 insertions(+), 20 deletions(-)

diff --git a/github/serverless/dbt/branch_deployments.yml b/github/serverless/dbt/branch_deployments.yml
index 624b751d..00c7fd96 100644
--- a/github/serverless/dbt/branch_deployments.yml
+++ b/github/serverless/dbt/branch_deployments.yml
@@ -33,25 +33,42 @@ jobs:
         with:
           dagster_cloud_file: $DAGSTER_CLOUD_FILE
 
-      - name: Checkout for Python Executable Deploy
+      - name: Checkout
         if: steps.prerun.outputs.result == 'pex-deploy'
         uses: actions/checkout@v3
         with:
           ref: ${{ github.head_ref }}
           path: project-repo
 
-      - name: Parse dbt project and package with Dagster project
+      # Validate dagster_cloud.yaml and the connection to dagster.cloud
+      - name: Validate configuration
+        id: ci-validate
+        if: steps.prerun.outputs.result == 'pex-deploy'
+        uses: dagster-io/dagster-cloud-action/actions/utils/dagster-cloud-cli@v0.1
+        with:
+          command: "ci check --project-dir ${{ env.DAGSTER_PROJECT_DIR }} --dagster-cloud-yaml-path ${{ env.DAGSTER_CLOUD_YAML_PATH }}"
+
+      # Parse dagster_cloud.yaml, detect if this is branch deployment and initialize the build session
+      - name: Initialize build session
+        id: ci-init
+        if: steps.prerun.outputs.result == 'pex-deploy'
+        uses: dagster-io/dagster-cloud-action/actions/utils/ci-init@v0.1
+        with:
+          project_dir: ${{ env.DAGSTER_PROJECT_DIR }}
+          dagster_cloud_yaml_path: ${{ env.DAGSTER_CLOUD_YAML_PATH }}
+          # The full deployment name. If this run is for a PR, this value is ignored and a branch
+          # deployment is used.
+          deployment: 'prod'
+
+      - name: Prepare DBT project for deployment
         if: steps.prerun.outputs.result == 'pex-deploy'
+        # --upgrade-strategy eager picks up newer packages that are required for things to work
         run: |
           pip install pip --upgrade
-          cd ${{ env.DBT_PROJECT_DIR }}/${{ env.DAGSTER_PROJECT_NAME }}
-          pip install . MarkupSafe==2.0.1 'click>8.1.0' 'Jinja2>3.0.0'
-          pip install pyOpenSSL --upgrade
-          cd -
-          rsync -avz --exclude ${{ env.DAGSTER_PROJECT_NAME }} --exclude .git ${{ env.DBT_PROJECT_DIR }}/ ${{ env.DBT_PACKAGE_DATA_DIR }}
-          dbt deps --project-dir ${{ env.DBT_PACKAGE_DATA_DIR }} --profiles-dir ${{ env.DBT_PACKAGE_DATA_DIR }}
-          dbt parse --project-dir ${{ env.DBT_PACKAGE_DATA_DIR }} --profiles-dir ${{ env.DBT_PACKAGE_DATA_DIR }}
-          rm ${{ env.DBT_PACKAGE_DATA_DIR }}/target/partial_parse.msgpack
+          cd ${{ env.ROOT_DIR }}/${{ env.DAGSTER_PROJECT_NAME }}
+          pip install . --upgrade --upgrade-strategy eager
+          dagster-dbt project prepare-for-deployment --file ${{ env.DAGSTER_PROJECT_NAME }}/project.py
+          # dagster-cloud ci dbt-project manage-state --file ${{ env.DAGSTER_PROJECT_NAME }}/project.py
         shell: bash
         env:
           DAGSTER_PROJECT_NAME: 'dagster_dbt_scaffold'
diff --git a/github/serverless/dbt/deploy.yml b/github/serverless/dbt/deploy.yml
index fafb0d72..d114ab19 100644
--- a/github/serverless/dbt/deploy.yml
+++ b/github/serverless/dbt/deploy.yml
@@ -35,25 +35,42 @@ jobs:
         with:
           dagster_cloud_file: $DAGSTER_CLOUD_FILE
 
-      - name: Checkout for Python Executable Deploy
+      - name: Checkout
         if: steps.prerun.outputs.result == 'pex-deploy'
         uses: actions/checkout@v3
         with:
           ref: ${{ github.head_ref }}
           path: project-repo
 
-      - name: Parse dbt project and package with Dagster project
+      # Validate dagster_cloud.yaml and the connection to dagster.cloud
+      - name: Validate configuration
+        id: ci-validate
+        if: steps.prerun.outputs.result == 'pex-deploy'
+        uses: dagster-io/dagster-cloud-action/actions/utils/dagster-cloud-cli@v0.1
+        with:
+          command: "ci check --project-dir ${{ env.DAGSTER_PROJECT_DIR }} --dagster-cloud-yaml-path ${{ env.DAGSTER_CLOUD_YAML_PATH }}"
+
+      # Parse dagster_cloud.yaml, detect if this is branch deployment and initialize the build session
+      - name: Initialize build session
+        id: ci-init
+        if: steps.prerun.outputs.result == 'pex-deploy'
+        uses: dagster-io/dagster-cloud-action/actions/utils/ci-init@v0.1
+        with:
+          project_dir: ${{ env.DAGSTER_PROJECT_DIR }}
+          dagster_cloud_yaml_path: ${{ env.DAGSTER_CLOUD_YAML_PATH }}
+          # The full deployment name. If this run is for a PR, this value is ignored and a branch
+          # deployment is used.
+          deployment: 'prod'
+
+      - name: Prepare DBT project for deployment
         if: steps.prerun.outputs.result == 'pex-deploy'
+        # --upgrade-strategy eager picks up newer packages that are required for things to work
         run: |
           pip install pip --upgrade
-          cd ${{ env.DBT_PROJECT_DIR }}/${{ env.DAGSTER_PROJECT_NAME }}
-          pip install . MarkupSafe==2.0.1 'click>8.1.0' 'Jinja2>3.0.0'
-          pip install pyOpenSSL --upgrade
-          cd -
-          rsync -avz --exclude ${{ env.DAGSTER_PROJECT_NAME }} --exclude .git ${{ env.DBT_PROJECT_DIR }}/ ${{ env.DBT_PACKAGE_DATA_DIR }}
-          dbt deps --project-dir ${{ env.DBT_PACKAGE_DATA_DIR }} --profiles-dir ${{ env.DBT_PACKAGE_DATA_DIR }}
-          dbt parse --project-dir ${{ env.DBT_PACKAGE_DATA_DIR }} --profiles-dir ${{ env.DBT_PACKAGE_DATA_DIR }}
-          rm ${{ env.DBT_PACKAGE_DATA_DIR }}/target/partial_parse.msgpack
+          cd ${{ env.ROOT_DIR }}/${{ env.DAGSTER_PROJECT_NAME }}
+          pip install . --upgrade --upgrade-strategy eager
+          dagster-dbt project prepare-for-deployment --file ${{ env.DAGSTER_PROJECT_NAME }}/project.py
+          # dagster-cloud ci dbt-project manage-state --file ${{ env.DAGSTER_PROJECT_NAME }}/project.py
         shell: bash
         env:
           DAGSTER_PROJECT_NAME: 'dagster_dbt_scaffold'