Skip to content

Commit

Permalink
added macro for limit 0 to fix fabric bug; added prov attribution tes…
Browse files Browse the repository at this point in the history 
…ting;
  • Loading branch information
@thutuva
thutuva committed Oct 16, 2024
1 parent c98dc0d commit 6b03972
Show file tree
Hide file tree
Showing 2 changed files with 278 additions and 59 deletions.
298 changes: 241 additions & 57 deletions .github/workflows/outside_contributor_dbt_v1.8.6.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,29 +36,24 @@ env:
DBT_SNOWFLAKE_CI_WAREHOUSE: ${{ secrets.DBT_SNOWFLAKE_CI_WAREHOUSE }}

jobs:
####### BigQuery
bigquery_clinical_and_claims_enabled:
ci_testing_start:
runs-on: ubuntu-latest

steps:
- uses: actions/checkout@v2
with:
ref: refs/pull/${{ github.event.inputs.prNumber }}/merge

- name: Send Slack alert that PR has been opened
- name: Send Slack alert that PR has been opened and CI is starting
id: slack
uses: slackapi/[email protected]
with:
# For posting a rich message using Block Kit
payload: |
{
"text": "CI Testing is starting on: ${{ github.event.pull_request.html_url || github.event.head_commit.url }}",
"text": "CI Testing is starting on an outside contributor's PR: ${{ github.event.pull_request.html_url || github.event.head_commit.url }}",
"blocks": [
{
"type": "section",
"text": {
"type": "mrkdwn",
"text": "CI Testing is starting on: ${{ github.event.pull_request.html_url || github.event.head_commit.url }}"
"text": "CI Testing is starting on an outside contributor's PR: ${{ github.event.pull_request.html_url || github.event.head_commit.url }}"
}
}
]
Expand All @@ -67,6 +62,15 @@ jobs:
SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK

####### BigQuery
bigquery_clinical_and_claims_enabled:
runs-on: ubuntu-latest

steps:
- uses: actions/checkout@v2
with:
ref: refs/pull/${{ github.event.inputs.prNumber }}/merge

- name: Set up Python
uses: actions/setup-python@v4
with:
Expand Down Expand Up @@ -122,32 +126,6 @@ jobs:
needs: bigquery_clinical_and_claims_enabled

steps:
- uses: actions/checkout@v2
with:
ref: refs/pull/${{ github.event.inputs.prNumber }}/merge

- name: Send Slack alert that PR has been opened
id: slack
uses: slackapi/[email protected]
with:
# For posting a rich message using Block Kit
payload: |
{
"text": "CI Testing is starting on: ${{ github.event.pull_request.html_url || github.event.head_commit.url }}",
"blocks": [
{
"type": "section",
"text": {
"type": "mrkdwn",
"text": "CI Testing is starting on: ${{ github.event.pull_request.html_url || github.event.head_commit.url }}"
}
}
]
}
env:
SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK

- name: Set up Python
uses: actions/setup-python@v4
with:
Expand Down Expand Up @@ -207,27 +185,64 @@ jobs:
with:
ref: refs/pull/${{ github.event.inputs.prNumber }}/merge

- name: Send Slack alert that PR has been opened
id: slack
uses: slackapi/[email protected]
- name: Set up Python
uses: actions/setup-python@v4
with:
# For posting a rich message using Block Kit
payload: |
{
"text": "CI Testing is starting on: ${{ github.event.pull_request.html_url || github.event.head_commit.url }}",
"blocks": [
{
"type": "section",
"text": {
"type": "mrkdwn",
"text": "CI Testing is starting on: ${{ github.event.pull_request.html_url || github.event.head_commit.url }}"
}
}
]
}
env:
SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK
python-version: '3.9'

- name: Install dbt-core and BigQuery adapter
run: |
python -m pip install --upgrade pip
pip install dbt-core==1.8.6 dbt-bigquery
- name: Create dbt profiles.yml for BigQuery
run: |
mkdir -p ./profiles/bigquery
echo "default:
outputs:
dev:
type: bigquery
method: service-account
project: dev-ci-testing
keyfile: ./creds.json
dataset: connector
threads: 4
timeout_seconds: 300
priority: interactive
target: dev" > ./profiles/bigquery/profiles.yml
working-directory: ci_testing

- name: Create BigQuery credentials file
run: |
echo "${{ secrets.TUVA_BIGQUERY_TOKEN }}" | base64 --decode > ./creds.json
working-directory: ci_testing

- name: dbt-deps
run: dbt deps --profiles-dir ./profiles/bigquery
working-directory: ci_testing

- name: dbt-debug
run: dbt debug --profiles-dir ./profiles/bigquery
working-directory: ci_testing

- name: dbt-build
run: |
dbt build --full-refresh --profiles-dir ./profiles/bigquery --vars '{"input_database": "dev-ci-testing","input_schema": "input_layer","clinical_enabled": true}'
working-directory: ci_testing

- name: Get the result
if: ${{ always() }}
run: echo "${{ steps.dbt-build.outputs.result }}"
shell: bash

bigquery_claims_prov_attribution_enabled:
runs-on: ubuntu-latest
needs: bigquery_clinical_enabled

steps:
- uses: actions/checkout@v2
with:
ref: refs/pull/${{ github.event.inputs.prNumber }}/merge

- name: Set up Python
uses: actions/setup-python@v4
Expand Down Expand Up @@ -271,7 +286,7 @@ jobs:

- name: dbt-build
run: |
dbt build --full-refresh --profiles-dir ./profiles/bigquery --vars '{"input_database": "dev-ci-testing","input_schema": "input_layer","clinical_enabled": true}'
dbt build --full-refresh --profiles-dir ./profiles/bigquery --vars '{"input_database": "dev-ci-testing","input_schema": "input_layer","claims_enabled": true,"provider_attribution_enabled":true}'
working-directory: ci_testing

- name: Get the result
Expand Down Expand Up @@ -410,6 +425,51 @@ jobs:
if: ${{ always() }}
run: echo "${{ steps.dbt-build.outputs.result }}"
shell: bash

fabric_claims_prov_attribution_enabled:
runs-on: ubuntu-latest
needs: fabric_clinical_and_claims_enabled

steps:
- uses: actions/checkout@v2
with:
ref: refs/pull/${{ github.event.inputs.prNumber }}/merge

- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: '3.9'

- name: Install ODBC Driver 18 for SQL Server
run: |
curl https://packages.microsoft.com/keys/microsoft.asc | sudo apt-key add -
curl https://packages.microsoft.com/config/ubuntu/$(lsb_release -rs)/prod.list | sudo tee /etc/apt/sources.list.d/msprod.list
sudo apt-get update
sudo ACCEPT_EULA=Y apt-get install -y msodbcsql18
- name: Install dbt-core and Fabric adapter
run: |
python -m pip install --upgrade pip
pip install dbt-core==1.8.6 dbt-fabric
- name: dbt-deps
run: dbt deps --profiles-dir ./profiles/fabric
working-directory: ci_testing

- name: dbt-debug
run: dbt debug --profiles-dir ./profiles/fabric
working-directory: ci_testing

- name: dbt-build
run: |
dbt build --full-refresh --profiles-dir ./profiles/fabric --vars '{"input_database":"dev_ci_testing","input_schema":"input_layer","claims_enabled":true,"provider_attribution_enabled":true}'
working-directory: ci_testing

- name: Get the result
if: ${{ always() }}
run: echo "${{ steps.dbt-build.outputs.result }}"
shell: bash

####### Redshift
redshift_clinical_and_claims_enabled:
runs-on: ubuntu-latest
Expand Down Expand Up @@ -521,6 +581,43 @@ jobs:
run: echo "${{ steps.dbt-build.outputs.result }}"
shell: bash

redshift_claims_prov_attribution_enabled:
runs-on: ubuntu-latest
needs: redshift_clinical_and_claims_enabled

steps:
- uses: actions/checkout@v2
with:
ref: refs/pull/${{ github.event.inputs.prNumber }}/merge

- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: '3.9'

- name: Install dbt-core and Redshift adapter
run: |
python -m pip install --upgrade pip
pip install dbt-core==1.8.6 dbt-redshift
- name: dbt-deps
run: dbt deps --profiles-dir ./profiles/redshift
working-directory: ci_testing

- name: dbt-debug
run: dbt debug --profiles-dir ./profiles/redshift
working-directory: ci_testing

- name: dbt-build
run: |
dbt build --full-refresh --profiles-dir ./profiles/redshift --vars '{"input_database":"dev_ci_testing","input_schema":"input_layer","claims_enabled":true,"provider_attribution_enabled":true}'
working-directory: ci_testing

- name: Get the result
if: ${{ always() }}
run: echo "${{ steps.dbt-build.outputs.result }}"
shell: bash

####### Snowflake
snowflake_clinical_and_claims_enabled:
runs-on: ubuntu-latest
Expand Down Expand Up @@ -632,20 +729,61 @@ jobs:
run: echo "${{ steps.dbt-build.outputs.result }}"
shell: bash

post_status:
snowflake_claims_prov_attribution_enabled:
runs-on: ubuntu-latest
needs: snowflake_clinical_and_claims_enabled

steps:
- uses: actions/checkout@v2
with:
ref: refs/pull/${{ github.event.inputs.prNumber }}/merge

- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: '3.9'

- name: Install dbt-core and Snowflake adapter
run: |
python -m pip install --upgrade pip
pip install dbt-core==1.8.6 dbt-snowflake
- name: dbt-deps
run: dbt deps --profiles-dir ./profiles/snowflake
working-directory: ci_testing

- name: dbt-debug
run: dbt debug --profiles-dir ./profiles/snowflake
working-directory: ci_testing

- name: dbt-build
run: |
dbt build --full-refresh --profiles-dir ./profiles/snowflake --vars '{"input_database":"dev_ci_testing","input_schema":"input_layer","claims_enabled":true,"provider_attribution_enabled":true}'
working-directory: ci_testing

- name: Get the result
if: ${{ always() }}
run: echo "${{ steps.dbt-build.outputs.result }}"
shell: bash
####### Post message to the PR with the status of each job (i.e. success or failure)
post_status_to_PR:
needs: [
bigquery_clinical_and_claims_enabled,
bigquery_claims_enabled,
bigquery_clinical_enabled,
bigquery_claims_prov_attribution_enabled,
fabric_clinical_and_claims_enabled,
fabric_claims_enabled,
fabric_clinical_enabled,
fabric_claims_prov_attribution_enabled,
redshift_clinical_and_claims_enabled,
redshift_claims_enabled,
redshift_clinical_enabled,
redshift_claims_prov_attribution_enabled,
snowflake_clinical_and_claims_enabled,
snowflake_claims_enabled,
snowflake_clinical_enabled
snowflake_claims_prov_attribution_enabled,
]
if: always()
runs-on: ubuntu-latest
Expand All @@ -672,3 +810,49 @@ jobs:
gh pr comment $PR_ID --body "$PR_COMMENT"
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}

####### Post message to the PR with the status of each job (i.e. success or failure)
ci_testing_end:
runs-on: ubuntu-latest
needs: [
bigquery_clinical_and_claims_enabled,
bigquery_claims_enabled,
bigquery_clinical_enabled,
bigquery_claims_prov_attribution_enabled,
fabric_clinical_and_claims_enabled,
fabric_claims_enabled,
fabric_clinical_enabled,
fabric_claims_prov_attribution_enabled,
redshift_clinical_and_claims_enabled,
redshift_claims_enabled,
redshift_clinical_enabled,
redshift_claims_prov_attribution_enabled,
snowflake_clinical_and_claims_enabled,
snowflake_claims_enabled,
snowflake_clinical_enabled
snowflake_claims_prov_attribution_enabled,
]
if: always()
steps:

- name: Send Slack alert that CI is complete
id: slack
uses: slackapi/[email protected]
with:
# For posting a rich message using Block Kit
payload: |
{
"text": "CI testing is done on: ${{ github.event.pull_request.html_url || github.event.head_commit.url }}",
"blocks": [
{
"type": "section",
"text": {
"type": "mrkdwn",
"text": "CI testing is done on: ${{ github.event.pull_request.html_url || github.event.head_commit.url }}\n\n*Workflow has finished with the following statuses:*\n\n• *BigQuery Clinical and Claims:* ${{ needs.bigquery_clinical_and_claims_enabled.result }}\n• *Fabric Clinical and Claims:* ${{ needs.fabric_clinical_and_claims_enabled.result }}\n• *Redshift Clinical and Claims:* ${{ needs.redshift_clinical_and_claims_enabled.result }}\n• *Snowflake Clinical and Claims:* ${{ needs.snowflake_clinical_and_claims_enabled.result }}\n• *Snowflake Claims:* ${{ needs.snowflake_claims_enabled.result }}\n• *Snowflake Clinical:* ${{ needs.snowflake_clinical_enabled.result }}"
}
}
]
}
env:
SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK
Loading

0 comments on commit 6b03972

Please sign in to comment.