From dbe09b2ca8ef2a0b6db0c058d3dc8e908eb39af4 Mon Sep 17 00:00:00 2001 From: Amisha Singla Date: Tue, 29 Oct 2024 14:52:37 -0500 Subject: [PATCH 01/15] Add workflow to calculate schema changes --- .../update_data_schema_changelog.yml | 66 +++++++++++++++++++ scripts/update_data_schema_changelog.sh | 27 ++++++++ 2 files changed, 93 insertions(+) create mode 100644 .github/workflows/update_data_schema_changelog.yml create mode 100644 scripts/update_data_schema_changelog.sh diff --git a/.github/workflows/update_data_schema_changelog.yml b/.github/workflows/update_data_schema_changelog.yml new file mode 100644 index 00000000..36c67a6f --- /dev/null +++ b/.github/workflows/update_data_schema_changelog.yml @@ -0,0 +1,66 @@ +name: Update Data Schema Changelog + +on: + push: + branches: + - patch/add-data-changelog + +concurrency: + group: ${{ github.workflow }}-${{ github.ref_protected == 'true' && github.sha || github.ref }}-{{ github.event_name }} + cancel-in-progress: true + +env: + IS_RECENCY_AIRFLOW_TASK: "false" + +jobs: + build: + runs-on: ubuntu-latest + # if: github.event.pull_request.merged == true + steps: + - name: Checkout Repository + uses: actions/checkout@v2 + + - name: Authenticate to crypto-stellar GCP + uses: "google-github-actions/auth@v2" + with: + project_id: hubble-261722 + credentials_json: "${{ secrets.CREDS_PROD_HUBBLE }}" + + - name: Set up Google Cloud SDK + run: | + echo "Installing Google Cloud SDK..." + echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] http://packages.cloud.google.com/apt cloud-sdk main" | sudo tee -a /etc/apt/sources.list.d/google-cloud-sdk.list + curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key --keyring /usr/share/keyrings/cloud.google.gpg add - + sudo apt-get update && sudo apt-get install -y google-cloud-sdk + + - name: Create new branch + id: create_branch + run: | + git config --local user.email "action@github.com" + git config --local user.name "GitHub Action" + BRANCH_NAME="update-data-schema-changelog-${{ github.run_id }}" + git checkout -b $BRANCH_NAME + echo "::set-output name=branch::$BRANCH_NAME" + + - name: Run Bash Script + run: | + cd $GITHUB_WORKSPACE + output=$(. scripts/update_data_schema_changelog.sh) + echo "$output" > CHANGELOG_DATA_SCHEMA.md + + - name: Commit changes + run: | + git add CHANGELOG_DATA_SCHEMA.md + git commit -m "Update data schema changelog" || echo "No changes to commit" + + - name: Push branch + run: | + git push origin ${{ steps.create_branch.outputs.branch }} + + - name: Create Pull Request + run: | + gh pr create -B master -H ${{ steps.create_branch.outputs.branch }} \ + --title 'Merge ${{ steps.create_branch.outputs.branch }} into master' \ + --body 'Created by GitHub action' + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/scripts/update_data_schema_changelog.sh b/scripts/update_data_schema_changelog.sh new file mode 100644 index 00000000..95a95ac5 --- /dev/null +++ b/scripts/update_data_schema_changelog.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +# result=$(bq query --format=prettyjson --nouse_legacy_sql \ +# 'SELECT +# date(detected_at) as date +# , database_name +# , schema_name +# , table_name +# , sub_type +# , ARRAY_AGG(column_name) as columns +# FROM +# `hubble-261722`.elementary.alerts_schema_changes +# GROUP BY +# 1, 2, 3, 4, 5 +# ORDER BY 1 DESC, 2 DESC, 3 DESC, 4 ASC +# ') + +result=$(bq query --nouse_legacy_sql 'SELECT count(*) from `hubble-261722`.crypto_stellar_internal_2.country_code') +echo "# Changes in data schema" +echo "$result" + +# echo "" + +# echo "| Date | Database Name | Schema Name | Table Name | Sub Type | Columns |" +# echo "|------------|---------------|-------------|--------------------------------|--------------|--------------------------|" + +# echo "$result" | jq -r '.[] | "| \(.date) | \(.database_name) | \(.schema_name) | \(.table_name) | \(.sub_type) | \(.columns | join(", ")) |"' From 9359a4ca3ec04a015fc61762d8790b03d42d61f2 Mon Sep 17 00:00:00 2001 From: Amisha Singla Date: Tue, 29 Oct 2024 15:00:37 -0500 Subject: [PATCH 02/15] use the actual table --- scripts/update_data_schema_changelog.sh | 30 ++++++++++++------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/scripts/update_data_schema_changelog.sh b/scripts/update_data_schema_changelog.sh index 95a95ac5..29535e00 100644 --- a/scripts/update_data_schema_changelog.sh +++ b/scripts/update_data_schema_changelog.sh @@ -1,21 +1,21 @@ #!/bin/bash -# result=$(bq query --format=prettyjson --nouse_legacy_sql \ -# 'SELECT -# date(detected_at) as date -# , database_name -# , schema_name -# , table_name -# , sub_type -# , ARRAY_AGG(column_name) as columns -# FROM -# `hubble-261722`.elementary.alerts_schema_changes -# GROUP BY -# 1, 2, 3, 4, 5 -# ORDER BY 1 DESC, 2 DESC, 3 DESC, 4 ASC -# ') +result=$(bq query --nouse_legacy_sql \ +'SELECT + date(detected_at) as date + , database_name + , schema_name + , table_name + , sub_type + , ARRAY_AGG(column_name) as columns + FROM + `hubble-261722`.elementary.alerts_schema_changes + GROUP BY + 1, 2, 3, 4, 5 + ORDER BY 1 DESC, 2 DESC, 3 DESC, 4 ASC +') -result=$(bq query --nouse_legacy_sql 'SELECT count(*) from `hubble-261722`.crypto_stellar_internal_2.country_code') +# result=$(bq query --nouse_legacy_sql 'SELECT count(*) from `hubble-261722`.crypto_stellar_internal_2.country_code') echo "# Changes in data schema" echo "$result" From 770f21b3180866919630a4d6b3077269c5c925ed Mon Sep 17 00:00:00 2001 From: Amisha Singla Date: Tue, 29 Oct 2024 15:02:43 -0500 Subject: [PATCH 03/15] slightly better format --- scripts/update_data_schema_changelog.sh | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/scripts/update_data_schema_changelog.sh b/scripts/update_data_schema_changelog.sh index 29535e00..29bec855 100644 --- a/scripts/update_data_schema_changelog.sh +++ b/scripts/update_data_schema_changelog.sh @@ -1,6 +1,6 @@ #!/bin/bash -result=$(bq query --nouse_legacy_sql \ +result=$(bq query --format=prettyjson --nouse_legacy_sql \ 'SELECT date(detected_at) as date , database_name @@ -15,13 +15,11 @@ result=$(bq query --nouse_legacy_sql \ ORDER BY 1 DESC, 2 DESC, 3 DESC, 4 ASC ') -# result=$(bq query --nouse_legacy_sql 'SELECT count(*) from `hubble-261722`.crypto_stellar_internal_2.country_code') echo "# Changes in data schema" -echo "$result" -# echo "" +echo "" -# echo "| Date | Database Name | Schema Name | Table Name | Sub Type | Columns |" -# echo "|------------|---------------|-------------|--------------------------------|--------------|--------------------------|" +echo "| Date | Database Name | Schema Name | Table Name | Sub Type | Columns |" +echo "|------------|---------------|-------------|--------------------------------|--------------|--------------------------|" -# echo "$result" | jq -r '.[] | "| \(.date) | \(.database_name) | \(.schema_name) | \(.table_name) | \(.sub_type) | \(.columns | join(", ")) |"' +echo "$result" | jq -r '.[] | "| \(.date) | \(.database_name) | \(.schema_name) | \(.table_name) | \(.sub_type) | \(.columns | join(", ")) |"' From 8db5b45cc2a84e29504c2574e81dc072e58c8312 Mon Sep 17 00:00:00 2001 From: Amisha Singla Date: Tue, 29 Oct 2024 15:07:15 -0500 Subject: [PATCH 04/15] Check if it would create a new PR for no diff --- CHANGELOG_DATA_SCHEMA.md | 19 +++++++++++++++++++ scripts/update_data_schema_changelog.sh | 1 + 2 files changed, 20 insertions(+) create mode 100644 CHANGELOG_DATA_SCHEMA.md diff --git a/CHANGELOG_DATA_SCHEMA.md b/CHANGELOG_DATA_SCHEMA.md new file mode 100644 index 00000000..5383a42a --- /dev/null +++ b/CHANGELOG_DATA_SCHEMA.md @@ -0,0 +1,19 @@ +# Changes in data schema + +| Date | Database Name | Schema Name | Table Name | Sub Type | Columns | +|------------|---------------|-------------|--------------------------------|--------------|--------------------------| +| 2024-09-12 | HUBBLE-261722 | SDF_MARTS | ASSET_STATS_AGG | column_added | airflow_start_ts | +| 2024-09-12 | HUBBLE-261722 | SDF_MARTS | DIM_DATES | column_added | airflow_start_ts | +| 2024-09-12 | HUBBLE-261722 | SDF_MARTS | DIM_MGI_WALLETS | column_added | airflow_start_ts | +| 2024-09-12 | HUBBLE-261722 | SDF_MARTS | FCT_MGI_CASHFLOW | column_added | airflow_start_ts | +| 2024-09-12 | HUBBLE-261722 | SDF_MARTS | LIQUIDITY_POOLS_VALUE | column_added | airflow_start_ts | +| 2024-09-12 | HUBBLE-261722 | SDF_MARTS | LIQUIDITY_POOLS_VALUE_HISTORY | column_added | airflow_start_ts | +| 2024-09-12 | HUBBLE-261722 | SDF_MARTS | LIQUIDITY_POOL_TRADE_VOLUME | column_added | airflow_start_ts | +| 2024-09-12 | HUBBLE-261722 | SDF_MARTS | LIQUIDITY_PROVIDERS | column_added | airflow_start_ts | +| 2024-09-12 | HUBBLE-261722 | SDF_MARTS | MGI_MONTHLY_USD_BALANCE | column_added | airflow_start_ts | +| 2024-09-12 | HUBBLE-261722 | SDF_MARTS | MGI_NETWORK_STATS_AGG | column_added | airflow_start_ts | +| 2024-09-12 | HUBBLE-261722 | SDF_MARTS | NETWORK_STATS_AGG | column_added | airflow_start_ts | +| 2024-09-12 | HUBBLE-261722 | SDF_MARTS | OHLC_EXCHANGE_FACT | column_added | airflow_start_ts | +| 2024-09-12 | HUBBLE-261722 | SDF_MARTS | PARTNERSHIP_ASSETS__ACCOUNT_HOLDERS_ACTIVITY_FACT | column_added | airflow_start_ts | +| 2024-09-12 | HUBBLE-261722 | SDF_MARTS | PARTNERSHIP_ASSETS__ASSET_ACTIVITY_FACT | column_added | airflow_start_ts | +| 2024-09-12 | HUBBLE-261722 | SDF_MARTS | PARTNERSHIP_ASSETS__MOST_ACTIVE_FACT | column_added | airflow_start_ts | diff --git a/scripts/update_data_schema_changelog.sh b/scripts/update_data_schema_changelog.sh index 29bec855..683e4bbf 100644 --- a/scripts/update_data_schema_changelog.sh +++ b/scripts/update_data_schema_changelog.sh @@ -23,3 +23,4 @@ echo "| Date | Database Name | Schema Name | Table Name echo "|------------|---------------|-------------|--------------------------------|--------------|--------------------------|" echo "$result" | jq -r '.[] | "| \(.date) | \(.database_name) | \(.schema_name) | \(.table_name) | \(.sub_type) | \(.columns | join(", ")) |"' +echo "" From b6e285558d9c984bd76eca793a6bab1e9d57de95 Mon Sep 17 00:00:00 2001 From: Amisha Singla Date: Tue, 29 Oct 2024 15:11:06 -0500 Subject: [PATCH 05/15] Push branch only when there are changes --- .github/workflows/update_data_schema_changelog.yml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/.github/workflows/update_data_schema_changelog.yml b/.github/workflows/update_data_schema_changelog.yml index 36c67a6f..6fd957b7 100644 --- a/.github/workflows/update_data_schema_changelog.yml +++ b/.github/workflows/update_data_schema_changelog.yml @@ -51,13 +51,21 @@ jobs: - name: Commit changes run: | git add CHANGELOG_DATA_SCHEMA.md - git commit -m "Update data schema changelog" || echo "No changes to commit" + if git commit -m "Update data schema changelog"; then + echo "Changes committed." + echo "::set-output name=changes_committed::true" + else + echo "No changes to commit." + echo "::set-output name=changes_committed::false" + fi - name: Push branch + if: steps.commit_changes.outputs.changes_committed == 'true' run: | git push origin ${{ steps.create_branch.outputs.branch }} - name: Create Pull Request + if: steps.commit_changes.outputs.changes_committed == 'true' run: | gh pr create -B master -H ${{ steps.create_branch.outputs.branch }} \ --title 'Merge ${{ steps.create_branch.outputs.branch }} into master' \ From 54aa097b354f1ee1a88938490d8aa7105e6a57c9 Mon Sep 17 00:00:00 2001 From: Amisha Singla Date: Tue, 29 Oct 2024 15:48:55 -0500 Subject: [PATCH 06/15] Refactor --- ... => update_dbt_marts_schema_changelog.yml} | 10 +++---- CHANGELOG_DATA_SCHEMA.md | 19 -------------- changelog/dbt_marts.md | 0 scripts/update_data_schema_changelog.sh | 26 ------------------- scripts/update_dbt_marts_schema_changelog.sh | 24 +++++++++++++++++ 5 files changed, 29 insertions(+), 50 deletions(-) rename .github/workflows/{update_data_schema_changelog.yml => update_dbt_marts_schema_changelog.yml} (90%) delete mode 100644 CHANGELOG_DATA_SCHEMA.md create mode 100644 changelog/dbt_marts.md delete mode 100644 scripts/update_data_schema_changelog.sh create mode 100644 scripts/update_dbt_marts_schema_changelog.sh diff --git a/.github/workflows/update_data_schema_changelog.yml b/.github/workflows/update_dbt_marts_schema_changelog.yml similarity index 90% rename from .github/workflows/update_data_schema_changelog.yml rename to .github/workflows/update_dbt_marts_schema_changelog.yml index 6fd957b7..d8b2eae4 100644 --- a/.github/workflows/update_data_schema_changelog.yml +++ b/.github/workflows/update_dbt_marts_schema_changelog.yml @@ -1,4 +1,4 @@ -name: Update Data Schema Changelog +name: Update changelog for DBT marts on: push: @@ -45,13 +45,13 @@ jobs: - name: Run Bash Script run: | cd $GITHUB_WORKSPACE - output=$(. scripts/update_data_schema_changelog.sh) - echo "$output" > CHANGELOG_DATA_SCHEMA.md + output=$(. scripts/update_dbt_marts_schema_changelog.sh) + echo "$output" > changelog/dbt_marts.md - name: Commit changes run: | - git add CHANGELOG_DATA_SCHEMA.md - if git commit -m "Update data schema changelog"; then + git add changelog/dbt_marts.md + if git commit -m "Update changelog for DBT marts"; then echo "Changes committed." echo "::set-output name=changes_committed::true" else diff --git a/CHANGELOG_DATA_SCHEMA.md b/CHANGELOG_DATA_SCHEMA.md deleted file mode 100644 index 5383a42a..00000000 --- a/CHANGELOG_DATA_SCHEMA.md +++ /dev/null @@ -1,19 +0,0 @@ -# Changes in data schema - -| Date | Database Name | Schema Name | Table Name | Sub Type | Columns | -|------------|---------------|-------------|--------------------------------|--------------|--------------------------| -| 2024-09-12 | HUBBLE-261722 | SDF_MARTS | ASSET_STATS_AGG | column_added | airflow_start_ts | -| 2024-09-12 | HUBBLE-261722 | SDF_MARTS | DIM_DATES | column_added | airflow_start_ts | -| 2024-09-12 | HUBBLE-261722 | SDF_MARTS | DIM_MGI_WALLETS | column_added | airflow_start_ts | -| 2024-09-12 | HUBBLE-261722 | SDF_MARTS | FCT_MGI_CASHFLOW | column_added | airflow_start_ts | -| 2024-09-12 | HUBBLE-261722 | SDF_MARTS | LIQUIDITY_POOLS_VALUE | column_added | airflow_start_ts | -| 2024-09-12 | HUBBLE-261722 | SDF_MARTS | LIQUIDITY_POOLS_VALUE_HISTORY | column_added | airflow_start_ts | -| 2024-09-12 | HUBBLE-261722 | SDF_MARTS | LIQUIDITY_POOL_TRADE_VOLUME | column_added | airflow_start_ts | -| 2024-09-12 | HUBBLE-261722 | SDF_MARTS | LIQUIDITY_PROVIDERS | column_added | airflow_start_ts | -| 2024-09-12 | HUBBLE-261722 | SDF_MARTS | MGI_MONTHLY_USD_BALANCE | column_added | airflow_start_ts | -| 2024-09-12 | HUBBLE-261722 | SDF_MARTS | MGI_NETWORK_STATS_AGG | column_added | airflow_start_ts | -| 2024-09-12 | HUBBLE-261722 | SDF_MARTS | NETWORK_STATS_AGG | column_added | airflow_start_ts | -| 2024-09-12 | HUBBLE-261722 | SDF_MARTS | OHLC_EXCHANGE_FACT | column_added | airflow_start_ts | -| 2024-09-12 | HUBBLE-261722 | SDF_MARTS | PARTNERSHIP_ASSETS__ACCOUNT_HOLDERS_ACTIVITY_FACT | column_added | airflow_start_ts | -| 2024-09-12 | HUBBLE-261722 | SDF_MARTS | PARTNERSHIP_ASSETS__ASSET_ACTIVITY_FACT | column_added | airflow_start_ts | -| 2024-09-12 | HUBBLE-261722 | SDF_MARTS | PARTNERSHIP_ASSETS__MOST_ACTIVE_FACT | column_added | airflow_start_ts | diff --git a/changelog/dbt_marts.md b/changelog/dbt_marts.md new file mode 100644 index 00000000..e69de29b diff --git a/scripts/update_data_schema_changelog.sh b/scripts/update_data_schema_changelog.sh deleted file mode 100644 index 683e4bbf..00000000 --- a/scripts/update_data_schema_changelog.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/bash - -result=$(bq query --format=prettyjson --nouse_legacy_sql \ -'SELECT - date(detected_at) as date - , database_name - , schema_name - , table_name - , sub_type - , ARRAY_AGG(column_name) as columns - FROM - `hubble-261722`.elementary.alerts_schema_changes - GROUP BY - 1, 2, 3, 4, 5 - ORDER BY 1 DESC, 2 DESC, 3 DESC, 4 ASC -') - -echo "# Changes in data schema" - -echo "" - -echo "| Date | Database Name | Schema Name | Table Name | Sub Type | Columns |" -echo "|------------|---------------|-------------|--------------------------------|--------------|--------------------------|" - -echo "$result" | jq -r '.[] | "| \(.date) | \(.database_name) | \(.schema_name) | \(.table_name) | \(.sub_type) | \(.columns | join(", ")) |"' -echo "" diff --git a/scripts/update_dbt_marts_schema_changelog.sh b/scripts/update_dbt_marts_schema_changelog.sh new file mode 100644 index 00000000..ae44e735 --- /dev/null +++ b/scripts/update_dbt_marts_schema_changelog.sh @@ -0,0 +1,24 @@ +#!/bin/bash + +result=$(bq query --format=prettyjson --nouse_legacy_sql \ +'SELECT + date(detected_at) as date + , table_name + , sub_type as operation + , ARRAY_AGG(column_name) as columns + FROM + `hubble-261722`.elementary.alerts_schema_changes + GROUP BY + 1, 2, 3 + ORDER BY 1 DESC, 2 ASC +') + +echo "# Changes in DBT marts schema" + +echo "" + +echo "| Date | Table Name | Operation | Columns |" +echo "|------------|---------------------------------|---------------|--------------------------|" + +echo "$result" | jq -r '.[] | "| \(.date) | \(.table_name) | \(.operation) | \(.columns | join(", ")) |"' +echo "" From 3055b2b32df80350c9f61471857aec4f07102d96 Mon Sep 17 00:00:00 2001 From: Amisha Singla Date: Tue, 29 Oct 2024 15:59:05 -0500 Subject: [PATCH 07/15] use github output --- .github/workflows/update_dbt_marts_schema_changelog.yml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/workflows/update_dbt_marts_schema_changelog.yml b/.github/workflows/update_dbt_marts_schema_changelog.yml index d8b2eae4..fe598bd5 100644 --- a/.github/workflows/update_dbt_marts_schema_changelog.yml +++ b/.github/workflows/update_dbt_marts_schema_changelog.yml @@ -49,16 +49,21 @@ jobs: echo "$output" > changelog/dbt_marts.md - name: Commit changes + id: commit_changes run: | git add changelog/dbt_marts.md if git commit -m "Update changelog for DBT marts"; then echo "Changes committed." - echo "::set-output name=changes_committed::true" + echo "changes_committed=true" >> $GITHUB_OUTPUT else echo "No changes to commit." - echo "::set-output name=changes_committed::false" + echo "changes_committed=false" >> $GITHUB_OUTPUT fi + - name: Use that secret output (protected by a mask) + run: | + echo "the secret number is ${{ steps.commit_changes.outputs.changes_committed }}" + - name: Push branch if: steps.commit_changes.outputs.changes_committed == 'true' run: | From d59b89f84ab7e78a85d4ba22034f837090d63d88 Mon Sep 17 00:00:00 2001 From: Amisha Singla Date: Tue, 29 Oct 2024 16:02:14 -0500 Subject: [PATCH 08/15] Test no diff works as expected --- changelog/dbt_marts.md | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/changelog/dbt_marts.md b/changelog/dbt_marts.md index e69de29b..165488ea 100644 --- a/changelog/dbt_marts.md +++ b/changelog/dbt_marts.md @@ -0,0 +1,19 @@ +# Changes in DBT marts schema + +| Date | Table Name | Operation | Columns | +|------------|---------------------------------|---------------|--------------------------| +| 2024-09-12 | ASSET_STATS_AGG | column_added | airflow_start_ts | +| 2024-09-12 | DIM_DATES | column_added | airflow_start_ts | +| 2024-09-12 | DIM_MGI_WALLETS | column_added | airflow_start_ts | +| 2024-09-12 | FCT_MGI_CASHFLOW | column_added | airflow_start_ts | +| 2024-09-12 | LIQUIDITY_POOLS_VALUE | column_added | airflow_start_ts | +| 2024-09-12 | LIQUIDITY_POOLS_VALUE_HISTORY | column_added | airflow_start_ts | +| 2024-09-12 | LIQUIDITY_POOL_TRADE_VOLUME | column_added | airflow_start_ts | +| 2024-09-12 | LIQUIDITY_PROVIDERS | column_added | airflow_start_ts | +| 2024-09-12 | MGI_MONTHLY_USD_BALANCE | column_added | airflow_start_ts | +| 2024-09-12 | MGI_NETWORK_STATS_AGG | column_added | airflow_start_ts | +| 2024-09-12 | NETWORK_STATS_AGG | column_added | airflow_start_ts | +| 2024-09-12 | OHLC_EXCHANGE_FACT | column_added | airflow_start_ts | +| 2024-09-12 | PARTNERSHIP_ASSETS__ACCOUNT_HOLDERS_ACTIVITY_FACT | column_added | airflow_start_ts | +| 2024-09-12 | PARTNERSHIP_ASSETS__ASSET_ACTIVITY_FACT | column_added | airflow_start_ts | +| 2024-09-12 | PARTNERSHIP_ASSETS__MOST_ACTIVE_FACT | column_added | airflow_start_ts | From 8b98baa706183cecd1428ed2f6672447ac940934 Mon Sep 17 00:00:00 2001 From: Amisha Singla Date: Tue, 29 Oct 2024 16:08:26 -0500 Subject: [PATCH 09/15] Update workflow to run on schedule as testing is done --- .github/workflows/update_dbt_marts_schema_changelog.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/update_dbt_marts_schema_changelog.yml b/.github/workflows/update_dbt_marts_schema_changelog.yml index fe598bd5..a81c1763 100644 --- a/.github/workflows/update_dbt_marts_schema_changelog.yml +++ b/.github/workflows/update_dbt_marts_schema_changelog.yml @@ -1,9 +1,8 @@ name: Update changelog for DBT marts on: - push: - branches: - - patch/add-data-changelog + schedule: + - cron: '0 23 * * 1-5' # This will run at 11 PM UTC, which is 5 PM CST concurrency: group: ${{ github.workflow }}-${{ github.ref_protected == 'true' && github.sha || github.ref }}-{{ github.event_name }} @@ -15,7 +14,6 @@ env: jobs: build: runs-on: ubuntu-latest - # if: github.event.pull_request.merged == true steps: - name: Checkout Repository uses: actions/checkout@v2 From 48e8c775eecd1d44bbec83cc7a73e727c498aa5a Mon Sep 17 00:00:00 2001 From: Amisha Singla Date: Tue, 29 Oct 2024 16:15:04 -0500 Subject: [PATCH 10/15] lint --- .../update_dbt_marts_schema_changelog.yml | 2 +- changelog/dbt_marts.md | 19 ------------------- scripts/update_dbt_marts_schema_changelog.sh | 4 ++-- 3 files changed, 3 insertions(+), 22 deletions(-) delete mode 100644 changelog/dbt_marts.md diff --git a/.github/workflows/update_dbt_marts_schema_changelog.yml b/.github/workflows/update_dbt_marts_schema_changelog.yml index a81c1763..356e1844 100644 --- a/.github/workflows/update_dbt_marts_schema_changelog.yml +++ b/.github/workflows/update_dbt_marts_schema_changelog.yml @@ -2,7 +2,7 @@ name: Update changelog for DBT marts on: schedule: - - cron: '0 23 * * 1-5' # This will run at 11 PM UTC, which is 5 PM CST + - cron: "0 23 * * 1-5" # This will run at 11 PM UTC, which is 5 PM CST concurrency: group: ${{ github.workflow }}-${{ github.ref_protected == 'true' && github.sha || github.ref }}-{{ github.event_name }} diff --git a/changelog/dbt_marts.md b/changelog/dbt_marts.md deleted file mode 100644 index 165488ea..00000000 --- a/changelog/dbt_marts.md +++ /dev/null @@ -1,19 +0,0 @@ -# Changes in DBT marts schema - -| Date | Table Name | Operation | Columns | -|------------|---------------------------------|---------------|--------------------------| -| 2024-09-12 | ASSET_STATS_AGG | column_added | airflow_start_ts | -| 2024-09-12 | DIM_DATES | column_added | airflow_start_ts | -| 2024-09-12 | DIM_MGI_WALLETS | column_added | airflow_start_ts | -| 2024-09-12 | FCT_MGI_CASHFLOW | column_added | airflow_start_ts | -| 2024-09-12 | LIQUIDITY_POOLS_VALUE | column_added | airflow_start_ts | -| 2024-09-12 | LIQUIDITY_POOLS_VALUE_HISTORY | column_added | airflow_start_ts | -| 2024-09-12 | LIQUIDITY_POOL_TRADE_VOLUME | column_added | airflow_start_ts | -| 2024-09-12 | LIQUIDITY_PROVIDERS | column_added | airflow_start_ts | -| 2024-09-12 | MGI_MONTHLY_USD_BALANCE | column_added | airflow_start_ts | -| 2024-09-12 | MGI_NETWORK_STATS_AGG | column_added | airflow_start_ts | -| 2024-09-12 | NETWORK_STATS_AGG | column_added | airflow_start_ts | -| 2024-09-12 | OHLC_EXCHANGE_FACT | column_added | airflow_start_ts | -| 2024-09-12 | PARTNERSHIP_ASSETS__ACCOUNT_HOLDERS_ACTIVITY_FACT | column_added | airflow_start_ts | -| 2024-09-12 | PARTNERSHIP_ASSETS__ASSET_ACTIVITY_FACT | column_added | airflow_start_ts | -| 2024-09-12 | PARTNERSHIP_ASSETS__MOST_ACTIVE_FACT | column_added | airflow_start_ts | diff --git a/scripts/update_dbt_marts_schema_changelog.sh b/scripts/update_dbt_marts_schema_changelog.sh index ae44e735..78530ee7 100644 --- a/scripts/update_dbt_marts_schema_changelog.sh +++ b/scripts/update_dbt_marts_schema_changelog.sh @@ -1,7 +1,7 @@ #!/bin/bash result=$(bq query --format=prettyjson --nouse_legacy_sql \ -'SELECT + 'SELECT date(detected_at) as date , table_name , sub_type as operation @@ -20,5 +20,5 @@ echo "" echo "| Date | Table Name | Operation | Columns |" echo "|------------|---------------------------------|---------------|--------------------------|" -echo "$result" | jq -r '.[] | "| \(.date) | \(.table_name) | \(.operation) | \(.columns | join(", ")) |"' +echo "$result" | jq -r '.[] | "| \(.date) | \(.table_name | ljust(50)) | \(.sub_type) | \(.columns | join(", ")) |"' echo "" From e481aed34d921ec1c5b141218754caa374e05b3f Mon Sep 17 00:00:00 2001 From: Amisha Singla Date: Tue, 29 Oct 2024 16:16:34 -0500 Subject: [PATCH 11/15] test again --- .github/workflows/update_dbt_marts_schema_changelog.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/update_dbt_marts_schema_changelog.yml b/.github/workflows/update_dbt_marts_schema_changelog.yml index 356e1844..c14a01ce 100644 --- a/.github/workflows/update_dbt_marts_schema_changelog.yml +++ b/.github/workflows/update_dbt_marts_schema_changelog.yml @@ -1,6 +1,9 @@ name: Update changelog for DBT marts on: + push: + branches: + - patch/add-data-changelog schedule: - cron: "0 23 * * 1-5" # This will run at 11 PM UTC, which is 5 PM CST From 0d1bb39a0048aaa21bf63a76676ec60f93347f50 Mon Sep 17 00:00:00 2001 From: Amisha Singla Date: Tue, 29 Oct 2024 16:24:01 -0500 Subject: [PATCH 12/15] fix script --- scripts/update_dbt_marts_schema_changelog.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/scripts/update_dbt_marts_schema_changelog.sh b/scripts/update_dbt_marts_schema_changelog.sh index 78530ee7..b141e856 100644 --- a/scripts/update_dbt_marts_schema_changelog.sh +++ b/scripts/update_dbt_marts_schema_changelog.sh @@ -20,5 +20,9 @@ echo "" echo "| Date | Table Name | Operation | Columns |" echo "|------------|---------------------------------|---------------|--------------------------|" -echo "$result" | jq -r '.[] | "| \(.date) | \(.table_name | ljust(50)) | \(.sub_type) | \(.columns | join(", ")) |"' +echo "$result" | jq -r '.[] | + "\(.date) | \(.table_name) | \(.operation) | \(.columns | join(", "))" ' | while IFS= read -r line; do + IFS='|' read -ra fields <<< "$line" + printf "| %-10s | %-60s | %-15s | %s |\n" "${fields[0]}" "${fields[1]}" "${fields[2]}" "${fields[3]}" +done echo "" From 2059018d8b4325915c97ab057c84a759a22d24bd Mon Sep 17 00:00:00 2001 From: Amisha Singla Date: Tue, 29 Oct 2024 16:25:37 -0500 Subject: [PATCH 13/15] Add empty file --- changelog/dbt_marts.md | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 changelog/dbt_marts.md diff --git a/changelog/dbt_marts.md b/changelog/dbt_marts.md new file mode 100644 index 00000000..e69de29b From ef8ff2d56f43a3d4f6c83e60f47b985d7c2cecfe Mon Sep 17 00:00:00 2001 From: Amisha Singla Date: Tue, 29 Oct 2024 16:33:19 -0500 Subject: [PATCH 14/15] test lints --- .github/workflows/update_dbt_marts_schema_changelog.yml | 2 +- .sqlfluffignore | 1 + scripts/update_dbt_marts_schema_changelog.sh | 6 +----- 3 files changed, 3 insertions(+), 6 deletions(-) diff --git a/.github/workflows/update_dbt_marts_schema_changelog.yml b/.github/workflows/update_dbt_marts_schema_changelog.yml index c14a01ce..900ad26d 100644 --- a/.github/workflows/update_dbt_marts_schema_changelog.yml +++ b/.github/workflows/update_dbt_marts_schema_changelog.yml @@ -53,7 +53,7 @@ jobs: id: commit_changes run: | git add changelog/dbt_marts.md - if git commit -m "Update changelog for DBT marts"; then + if git commit --no-verify -m "Update changelog for DBT marts"; then echo "Changes committed." echo "changes_committed=true" >> $GITHUB_OUTPUT else diff --git a/.sqlfluffignore b/.sqlfluffignore index 05ccce3b..7736f89e 100644 --- a/.sqlfluffignore +++ b/.sqlfluffignore @@ -1 +1,2 @@ dags/ddls/queries +changelog/dbt_marts.md diff --git a/scripts/update_dbt_marts_schema_changelog.sh b/scripts/update_dbt_marts_schema_changelog.sh index b141e856..78530ee7 100644 --- a/scripts/update_dbt_marts_schema_changelog.sh +++ b/scripts/update_dbt_marts_schema_changelog.sh @@ -20,9 +20,5 @@ echo "" echo "| Date | Table Name | Operation | Columns |" echo "|------------|---------------------------------|---------------|--------------------------|" -echo "$result" | jq -r '.[] | - "\(.date) | \(.table_name) | \(.operation) | \(.columns | join(", "))" ' | while IFS= read -r line; do - IFS='|' read -ra fields <<< "$line" - printf "| %-10s | %-60s | %-15s | %s |\n" "${fields[0]}" "${fields[1]}" "${fields[2]}" "${fields[3]}" -done +echo "$result" | jq -r '.[] | "| \(.date) | \(.table_name | ljust(50)) | \(.sub_type) | \(.columns | join(", ")) |"' echo "" From 0e1655e9c67be1ef08ac6bcb18d2431adffb3350 Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Tue, 29 Oct 2024 21:34:02 +0000 Subject: [PATCH 15/15] Update changelog for DBT marts --- changelog/dbt_marts.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/changelog/dbt_marts.md b/changelog/dbt_marts.md index e69de29b..e19b5c42 100644 --- a/changelog/dbt_marts.md +++ b/changelog/dbt_marts.md @@ -0,0 +1,4 @@ +# Changes in DBT marts schema + +| Date | Table Name | Operation | Columns | +|------------|---------------------------------|---------------|--------------------------|