Merge branch 'datahub-project:master' into master

acryldata · Aug 30, 2024 · daa56e3 · daa56e3
2 parents dd911e4 + aa4061a
commit daa56e3
Show file tree

Hide file tree

Showing 202 changed files with 5,131 additions and 1,619 deletions.
diff --git a/.github/actions/ci-optimization/action.yml b/.github/actions/ci-optimization/action.yml
@@ -57,7 +57,8 @@ runs:
             - "metadata-ingestion-modules/**"
             - "metadata-ingestion/**"
             - "metadata-models/**"
-            - "docker/datahub-ingestion**"
+            - "docker/datahub-ingestion-base/**"
+            - "docker/datahub-ingestion/**"
           ingestion-base:
             - "docker/datahub-ingestion-base/**"
           docker:

diff --git a/.github/actions/docker-custom-build-and-push/action.yml b/.github/actions/docker-custom-build-and-push/action.yml
@@ -31,16 +31,21 @@ inputs:
     description: "Main tag to use for the Docker image"
     required: true
   flavor:
-    description: 'Image flavor (e.g., slim, full)'
+    description: "Image flavor (e.g., slim, full)"
     required: false
   target:
     description: "Sets the target stage to build"
     required: false
+  depot-project:
+    # Setting this will use native arm64 docker builds instead of QEMU emulation.
+    # This speeds up builds by 2-3x.
+    description: "Depot project id"
+    required: false
+
 outputs:
   image_tag:
     description: "Docker image tags"
     value: ${{ steps.docker_meta.outputs.tags }}
-  # image_name: ${{ env.DATAHUB_GMS_IMAGE }}
 
 runs:
   using: "composite"
@@ -53,15 +58,27 @@ runs:
         images: ${{ inputs.images }}
         flavor: |
           latest=false
-          suffix=${{ inputs.flavor && format('-{0}', inputs.flavor) || '' }}
         tags: |
           type=raw,value=${{ inputs.image_tag }}
-          type=raw,value=head,enable={{is_default_branch}}
-          type=sha,prefix=,format=short
+          type=raw,value=head,suffix=${{ inputs.flavor && format('-{0}', inputs.flavor) || '' }},enable={{is_default_branch}}
+          type=sha,prefix=,format=short,suffix=${{ inputs.flavor && format('-{0}', inputs.flavor) || '' }}
+
+    - name: Single Tag
+      id: single_tag
+      shell: bash
+      run: |
+        IMAGES="""
+        ${{ inputs.images }}
+        """
+        TAGS="""
+        ${{ inputs.image_tag }}
+        """
+        echo "SINGLE_IMAGE=$(echo $IMAGES | tr '\n' ' ' | awk -F' |,' '{ print $1 }')" >> "$GITHUB_OUTPUT"
+        echo "SINGLE_TAG=$(echo $IMAGES | tr '\n' ' ' | awk -F' |,' '{ print $1 }'):$(echo $TAGS | tr '\n' ' ' | awk -F' |,' '{ print $1 }')" >> "$GITHUB_OUTPUT"
 
     # Code for testing the build when not pushing to Docker Hub.
     - name: Build and Load image for testing (if not publishing)
-      uses: docker/build-push-action@v5
+      uses: docker/build-push-action@v6
       if: ${{ inputs.publish != 'true' }}
       with:
         context: ${{ inputs.context }}
@@ -74,20 +91,11 @@ runs:
         target: ${{ inputs.target }}
         load: true
         push: false
-        cache-from: type=registry,ref=${{ steps.docker_meta.outputs.tags }}
-        cache-to: type=inline
-    - name: Single Tag
-      if: ${{ inputs.publish != 'true' }}
-      shell: bash
-      run: |
-        IMAGES="""
-        ${{ inputs.images }}
-        """
-        TAGS="""
-        ${{ inputs.image_tag }}
-        """
-        echo "SINGLE_TAG=$(echo $IMAGES | tr '\n' ' ' | awk -F' |,' '{ print $1 }'):$(echo $TAGS | tr '\n' ' ' | awk -F' |,' '{ print $1 }')" >> $GITHUB_OUTPUT
-      id: single_tag
+        cache-from: |
+          type=registry,ref=${{ steps.single_tag.outputs.SINGLE_IMAGE }}:head${{ inputs.flavor && format('-{0}', inputs.flavor) || '' }}
+          type=registry,ref=${{ steps.docker_meta.outputs.tags }}
+        cache-to: |
+          type=inline
     - name: Upload image locally for testing (if not publishing)
       uses: ishworkh/docker-image-artifact-upload@v1
       if: ${{ inputs.publish != 'true' }}
@@ -97,19 +105,42 @@ runs:
     # Code for building multi-platform images and pushing to Docker Hub.
     - name: Set up QEMU
       uses: docker/setup-qemu-action@v3
-      if: ${{ inputs.publish == 'true' }}
+      if: ${{ inputs.publish == 'true' && inputs.depot-project == '' }}
     - name: Set up Docker Buildx
       uses: docker/setup-buildx-action@v3
-      if: ${{ inputs.publish == 'true' }}
+      if: ${{ inputs.publish == 'true' && inputs.depot-project == '' }}
+    - name: Setup Depot CLI
+      uses: depot/setup-action@v1
+      if: ${{ inputs.publish == 'true' && inputs.depot-project != '' }}
     - name: Login to DockerHub
       uses: docker/login-action@v3
       if: ${{ inputs.publish == 'true' }}
       with:
         username: ${{ inputs.username }}
         password: ${{ inputs.password }}
+
+    # Depot variant.
     - name: Build and Push Multi-Platform image
-      uses: docker/build-push-action@v5
-      if: ${{ inputs.publish == 'true' }}
+      uses: depot/build-push-action@v1
+      if: ${{ inputs.publish == 'true' && inputs.depot-project != '' }}
+      with:
+        project: ${{ inputs.depot-project }}
+        context: ${{ inputs.context }}
+        file: ${{ inputs.file }}
+        platforms: ${{ inputs.platforms }}
+        build-args: ${{ inputs.build-args }}
+        tags: ${{ steps.docker_meta.outputs.tags }}
+        target: ${{ inputs.target }}
+        push: true
+        cache-from: |
+          type=registry,ref=${{ steps.single_tag.outputs.SINGLE_IMAGE }}:head${{ inputs.flavor && format('-{0}', inputs.flavor) || '' }}
+          type=registry,ref=${{ steps.docker_meta.outputs.tags }}
+        cache-to: |
+          type=inline
+
+    - name: Build and Push Multi-Platform image
+      uses: docker/build-push-action@v6
+      if: ${{ inputs.publish == 'true' && inputs.depot-project == '' }}
       with:
         context: ${{ inputs.context }}
         file: ${{ inputs.file }}
@@ -118,7 +149,10 @@ runs:
         tags: ${{ steps.docker_meta.outputs.tags }}
         target: ${{ inputs.target }}
         push: true
-        cache-from: type=registry,ref=${{ steps.docker_meta.outputs.tags }}
-        cache-to: type=inline
+        cache-from: |
+          type=registry,ref=${{ steps.single_tag.outputs.SINGLE_IMAGE }}:head${{ inputs.flavor && format('-{0}', inputs.flavor) || '' }}
+          type=registry,ref=${{ steps.docker_meta.outputs.tags }}
+        cache-to: |
+          type=inline
 
     # TODO add code for vuln scanning?
diff --git a/.github/workflows/airflow-plugin.yml b/.github/workflows/airflow-plugin.yml
@@ -57,13 +57,13 @@ jobs:
       fail-fast: false
     steps:
       - name: Set up JDK 17
-        uses: actions/setup-java@v3
+        uses: actions/setup-java@v4
         with:
           distribution: "zulu"
           java-version: 17
       - uses: gradle/actions/setup-gradle@v3
       - uses: acryldata/sane-checkout-action@v3
-      - uses: actions/setup-python@v4
+      - uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
           cache: "pip"

diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
@@ -68,12 +68,12 @@ jobs:
       - name: Check out the repo
         uses: acryldata/sane-checkout-action@v3
       - name: Set up JDK 17
-        uses: actions/setup-java@v3
+        uses: actions/setup-java@v4
         with:
           distribution: "zulu"
           java-version: 17
       - uses: gradle/actions/setup-gradle@v3
-      - uses: actions/setup-python@v4
+      - uses: actions/setup-python@v5
         if: ${{ needs.setup.outputs.ingestion_change == 'true' }}
         with:
           python-version: "3.10"
@@ -91,6 +91,8 @@ jobs:
             -x :metadata-ingestion-modules:airflow-plugin:check \
             -x :metadata-ingestion-modules:dagster-plugin:build \
             -x :metadata-ingestion-modules:dagster-plugin:check \
+            -x :metadata-ingestion-modules:prefect-plugin:build \
+            -x :metadata-ingestion-modules:prefect-plugin:check \
             -x :metadata-ingestion-modules:gx-plugin:build \
             -x :metadata-ingestion-modules:gx-plugin:check \
             -x :datahub-frontend:build \
@@ -125,7 +127,7 @@ jobs:
     steps:
       - name: Check out the repo
         uses: acryldata/sane-checkout-action@v3
-      - uses: actions/setup-python@v4
+      - uses: actions/setup-python@v5
         with:
           python-version: "3.10"
       - name: Quickstart Compose Validation
@@ -138,4 +140,4 @@ jobs:
         uses: actions/upload-artifact@v3
         with:
           name: Event File
-          path: ${{ github.event_path }}
+          path: ${{ github.event_path }}
diff --git a/.github/workflows/check-datahub-jars.yml b/.github/workflows/check-datahub-jars.yml
@@ -29,12 +29,12 @@ jobs:
     steps:
       - uses: acryldata/sane-checkout-action@v3
       - name: Set up JDK 17
-        uses: actions/setup-java@v3
+        uses: actions/setup-java@v4
         with:
           distribution: "zulu"
           java-version: 17
       - uses: gradle/actions/setup-gradle@v3
-      - uses: actions/setup-python@v4
+      - uses: actions/setup-python@v5
         with:
           python-version: "3.10"
       - name: check ${{ matrix.command }} jar

diff --git a/.github/workflows/code-checks.yml b/.github/workflows/code-checks.yml
@@ -32,7 +32,7 @@ jobs:
     steps:
       - name: Check out the repo
         uses: acryldata/sane-checkout-action@v3
-      - uses: actions/setup-python@v4
+      - uses: actions/setup-python@v5
         with:
           python-version: "3.10"
       - name: run check ${{ matrix.command }}

diff --git a/.github/workflows/dagster-plugin.yml b/.github/workflows/dagster-plugin.yml
@@ -40,12 +40,12 @@ jobs:
       fail-fast: false
     steps:
       - name: Set up JDK 17
-        uses: actions/setup-java@v3
+        uses: actions/setup-java@v4
         with:
           distribution: "zulu"
           java-version: 17
-      - uses: actions/checkout@v3
-      - uses: actions/setup-python@v4
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
           cache: "pip"