Merge branch 'datahub-project:master' into master

acryldata · Mar 7, 2024 · 3f780cd · 3f780cd
2 parents db7144f + e963047
commit 3f780cd
Show file tree

Hide file tree

Showing 17 changed files with 2,001 additions and 776 deletions.
diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
@@ -76,7 +76,17 @@ jobs:
       - name: Gradle build (and test) for NOT metadata ingestion
         if: ${{  matrix.command == 'except_metadata_ingestion' && needs.setup.outputs.backend_change == 'true' }}
         run: |
-          ./gradlew build -x :metadata-ingestion:build -x :metadata-ingestion:check -x docs-website:build -x :metadata-integration:java:spark-lineage:test -x :metadata-io:test -x :metadata-ingestion-modules:airflow-plugin:build -x :metadata-ingestion-modules:airflow-plugin:check -x :datahub-frontend:build -x :datahub-web-react:build --parallel
+          ./gradlew build \
+            -x :metadata-ingestion:build \
+            -x :metadata-ingestion:check \
+            -x docs-website:build \
+            -x :metadata-integration:java:spark-lineage:test \
+            -x :metadata-io:test \
+            -x :metadata-ingestion-modules:airflow-plugin:build \
+            -x :metadata-ingestion-modules:airflow-plugin:check \
+            -x :datahub-frontend:build \
+            -x :datahub-web-react:build \
+            --parallel
       - name: Gradle build (and test) for frontend
         if: ${{  matrix.command == 'frontend' && needs.setup.outputs.frontend_change == 'true' }}
         run: |

diff --git a/.github/workflows/metadata-ingestion.yml b/.github/workflows/metadata-ingestion.yml
@@ -6,13 +6,15 @@ on:
     paths:
       - ".github/workflows/metadata-ingestion.yml"
       - "metadata-ingestion/**"
+      - "metadata-ingestion-modules/**"
       - "metadata-models/**"
   pull_request:
     branches:
       - "**"
     paths:
       - ".github/workflows/metadata-ingestion.yml"
       - "metadata-ingestion/**"
+      - "metadata-ingestion-modules/**"
       - "metadata-models/**"
   release:
     types: [published]

diff --git a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_config.py b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_config.py
@@ -220,7 +220,7 @@ def validate_unsupported_configs(cls, values: Dict) -> Dict:
             and values["stateful_ingestion"].remove_stale_metadata
         )
 
-        # TODO: Allow lineage extraction and profiling irrespective of basic schema extraction,
+        # TODO: Allow profiling irrespective of basic schema extraction,
         # as it seems possible with some refactor
         if not include_technical_schema and any(
             [include_profiles, delete_detection_enabled]