Skip to content

fix(dataProduct) Show entity count excluding soft deleted entities #12551

fix(dataProduct) Show entity count excluding soft deleted entities

fix(dataProduct) Show entity count excluding soft deleted entities #12551

Workflow file for this run

name: Docker Build, Scan, Test
on:
push:
branches:
- master
paths-ignore:
- "docs/**"
- "**.md"
pull_request:
branches:
- master
paths-ignore:
- "docs/**"
- "**.md"
release:
types: [published]
concurrency:
# Using `github.run_id` (unique val) instead of `github.ref` here
# because we don't want to cancel this workflow on master only for PRs
# as that makes reproducing issues easier
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }}
cancel-in-progress: true
env:
DATAHUB_GMS_IMAGE: "linkedin/datahub-gms"
DATAHUB_FRONTEND_IMAGE: "linkedin/datahub-frontend-react"
DATAHUB_MAE_CONSUMER_IMAGE: "linkedin/datahub-mae-consumer"
DATAHUB_MCE_CONSUMER_IMAGE: "linkedin/datahub-mce-consumer"
DATAHUB_KAFKA_SETUP_IMAGE: "linkedin/datahub-kafka-setup"
DATAHUB_ELASTIC_SETUP_IMAGE: "linkedin/datahub-elasticsearch-setup"
DATAHUB_MYSQL_SETUP_IMAGE: "acryldata/datahub-mysql-setup"
DATAHUB_UPGRADE_IMAGE: "acryldata/datahub-upgrade"
jobs:
setup:
runs-on: ubuntu-latest
outputs:
tag: ${{ steps.tag.outputs.tag }}
unique_tag: ${{ steps.tag.outputs.unique_tag }}
publish: ${{ steps.publish.outputs.publish }}
steps:
- name: Checkout
uses: actions/checkout@v3
- name: Compute Tag
id: tag
run: |
source .github/scripts/docker_helpers.sh
echo "tag=$(get_tag)" >> $GITHUB_OUTPUT
echo "unique_tag=$(get_unique_tag)" >> $GITHUB_OUTPUT
- name: Check whether publishing enabled
id: publish
env:
ENABLE_PUBLISH: ${{ secrets.DOCKER_PASSWORD }}
run: |
echo "Enable publish: ${{ env.ENABLE_PUBLISH != '' }}"
echo "publish=${{ env.ENABLE_PUBLISH != '' }}" >> $GITHUB_OUTPUT
gms_build:
name: Build and Push DataHub GMS Docker Image
runs-on: ubuntu-latest
needs: setup
steps:
- name: Check out the repo
uses: actions/checkout@v3
with:
fetch-depth: 0
- name: Pre-build artifacts for docker image
run: |
./gradlew :metadata-service:war:build -x test --parallel
mv ./metadata-service/war/build/libs/war.war .
- name: Build and push
uses: ./.github/actions/docker-custom-build-and-push
with:
images: |
${{ env.DATAHUB_GMS_IMAGE }}
tags: ${{ needs.setup.outputs.tag }}
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
publish: ${{ needs.setup.outputs.publish }}
context: .
file: ./docker/datahub-gms/Dockerfile
platforms: linux/amd64,linux/arm64/v8
gms_scan:
permissions:
contents: read # for actions/checkout to fetch code
security-events: write # for github/codeql-action/upload-sarif to upload SARIF results
actions: read # only required for a private repository by github/codeql-action/upload-sarif to get the Action run status
name: "[Monitoring] Scan GMS images for vulnerabilities"
runs-on: ubuntu-latest
needs: [setup, gms_build]
steps:
- name: Checkout # adding checkout step just to make trivy upload happy
uses: actions/checkout@v3
- name: Download image
uses: ishworkh/docker-image-artifact-download@v1
if: ${{ needs.setup.outputs.publish != 'true' }}
with:
image: ${{ env.DATAHUB_GMS_IMAGE }}:${{ needs.setup.outputs.unique_tag }}
- name: Run Trivy vulnerability scanner
uses: aquasecurity/[email protected]
env:
TRIVY_OFFLINE_SCAN: true
with:
image-ref: ${{ env.DATAHUB_GMS_IMAGE }}:${{ needs.setup.outputs.unique_tag }}
format: "template"
template: "@/contrib/sarif.tpl"
output: "trivy-results.sarif"
severity: "CRITICAL,HIGH"
ignore-unfixed: true
vuln-type: "os,library"
- name: Upload Trivy scan results to GitHub Security tab
uses: github/codeql-action/upload-sarif@v2
with:
sarif_file: "trivy-results.sarif"
mae_consumer_build:
name: Build and Push DataHub MAE Consumer Docker Image
runs-on: ubuntu-latest
needs: setup
steps:
- name: Check out the repo
uses: actions/checkout@v3
with:
fetch-depth: 0
- name: Pre-build artifacts for docker image
run: |
./gradlew :metadata-jobs:mae-consumer-job:build -x test --parallel
mv ./metadata-jobs/mae-consumer-job/build/libs/mae-consumer-job.jar .
- name: Build and push
uses: ./.github/actions/docker-custom-build-and-push
with:
images: |
${{ env.DATAHUB_MAE_CONSUMER_IMAGE }}
tags: ${{ needs.setup.outputs.tag }}
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
publish: ${{ needs.setup.outputs.publish }}
context: .
file: ./docker/datahub-mae-consumer/Dockerfile
platforms: linux/amd64,linux/arm64/v8
mae_consumer_scan:
name: "[Monitoring] Scan MAE consumer images for vulnerabilities"
runs-on: ubuntu-latest
needs: [setup, mae_consumer_build]
permissions:
contents: read # for actions/checkout to fetch code
security-events: write # for github/codeql-action/upload-sarif to upload SARIF results
actions: read # only required for a private repository by github/codeql-action/upload-sarif to get the Action run status
steps:
- name: Checkout # adding checkout step just to make trivy upload happy
uses: actions/checkout@v3
- name: Download image
uses: ishworkh/docker-image-artifact-download@v1
if: ${{ needs.setup.outputs.publish != 'true' }}
with:
image: ${{ env.DATAHUB_MAE_CONSUMER_IMAGE }}:${{ needs.setup.outputs.unique_tag }}
- name: Run Trivy vulnerability scanner
uses: aquasecurity/[email protected]
env:
TRIVY_OFFLINE_SCAN: true
with:
image-ref: ${{ env.DATAHUB_MAE_CONSUMER_IMAGE }}:${{ needs.setup.outputs.unique_tag }}
format: "template"
template: "@/contrib/sarif.tpl"
output: "trivy-results.sarif"
severity: "CRITICAL,HIGH"
ignore-unfixed: true
vuln-type: "os,library"
- name: Upload Trivy scan results to GitHub Security tab
uses: github/codeql-action/upload-sarif@v2
with:
sarif_file: "trivy-results.sarif"
mce_consumer_build:
name: Build and Push DataHub MCE Consumer Docker Image
runs-on: ubuntu-latest
needs: setup
steps:
- name: Check out the repo
uses: actions/checkout@v3
with:
fetch-depth: 0
- name: Pre-build artifacts for docker image
run: |
./gradlew :metadata-jobs:mce-consumer-job:build -x test --parallel
mv ./metadata-jobs/mce-consumer-job/build/libs/mce-consumer-job.jar .
- name: Build and push
uses: ./.github/actions/docker-custom-build-and-push
with:
images: |
${{ env.DATAHUB_MCE_CONSUMER_IMAGE }}
tags: ${{ needs.setup.outputs.tag }}
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
publish: ${{ needs.setup.outputs.publish }}
context: .
file: ./docker/datahub-mce-consumer/Dockerfile
platforms: linux/amd64,linux/arm64/v8
mce_consumer_scan:
name: "[Monitoring] Scan MCE consumer images for vulnerabilities"
runs-on: ubuntu-latest
needs: [setup, mce_consumer_build]
permissions:
contents: read # for actions/checkout to fetch code
security-events: write # for github/codeql-action/upload-sarif to upload SARIF results
actions: read # only required for a private repository by github/codeql-action/upload-sarif to get the Action run status
steps:
- name: Checkout # adding checkout step just to make trivy upload happy
uses: actions/checkout@v3
- name: Download image
uses: ishworkh/docker-image-artifact-download@v1
if: ${{ needs.setup.outputs.publish != 'true' }}
with:
image: ${{ env.DATAHUB_MCE_CONSUMER_IMAGE }}:${{ needs.setup.outputs.unique_tag }}
- name: Run Trivy vulnerability scanner
uses: aquasecurity/[email protected]
env:
TRIVY_OFFLINE_SCAN: true
with:
image-ref: ${{ env.DATAHUB_MCE_CONSUMER_IMAGE }}:${{ needs.setup.outputs.unique_tag }}
format: "template"
template: "@/contrib/sarif.tpl"
output: "trivy-results.sarif"
severity: "CRITICAL,HIGH"
ignore-unfixed: true
vuln-type: "os,library"
- name: Upload Trivy scan results to GitHub Security tab
uses: github/codeql-action/upload-sarif@v2
with:
sarif_file: "trivy-results.sarif"
datahub_upgrade_build:
name: Build and Push DataHub Upgrade Docker Image
runs-on: ubuntu-latest
needs: setup
steps:
- name: Check out the repo
uses: actions/checkout@v3
with:
fetch-depth: 0
- name: Pre-build artifacts for docker image
run: |
./gradlew :datahub-upgrade:build -x test --parallel
mv ./datahub-upgrade/build/libs/datahub-upgrade.jar .
- name: Build and push
uses: ./.github/actions/docker-custom-build-and-push
with:
images: |
${{ env.DATAHUB_UPGRADE_IMAGE }}
tags: ${{ needs.setup.outputs.tag }}
username: ${{ secrets.ACRYL_DOCKER_USERNAME }}
password: ${{ secrets.ACRYL_DOCKER_PASSWORD }}
publish: ${{ needs.setup.outputs.publish }}
context: .
file: ./docker/datahub-upgrade/Dockerfile
platforms: linux/amd64,linux/arm64/v8
datahub_upgrade_scan:
name: "[Monitoring] Scan DataHub Upgrade images for vulnerabilities"
runs-on: ubuntu-latest
needs: [setup, datahub_upgrade_build]
permissions:
contents: read # for actions/checkout to fetch code
security-events: write # for github/codeql-action/upload-sarif to upload SARIF results
actions: read # only required for a private repository by github/codeql-action/upload-sarif to get the Action run status
steps:
- name: Checkout # adding checkout step just to make trivy upload happy
uses: actions/checkout@v3
- name: Download image
uses: ishworkh/docker-image-artifact-download@v1
if: ${{ needs.setup.outputs.publish != 'true' }}
with:
image: ${{ env.DATAHUB_UPGRADE_IMAGE }}:${{ needs.setup.outputs.unique_tag }}
- name: Run Trivy vulnerability scanner
uses: aquasecurity/[email protected]
env:
TRIVY_OFFLINE_SCAN: true
with:
image-ref: ${{ env.DATAHUB_UPGRADE_IMAGE }}:${{ needs.setup.outputs.unique_tag }}
format: "template"
template: "@/contrib/sarif.tpl"
output: "trivy-results.sarif"
severity: "CRITICAL,HIGH"
ignore-unfixed: true
vuln-type: "os,library"
- name: Upload Trivy scan results to GitHub Security tab
uses: github/codeql-action/upload-sarif@v2
with:
sarif_file: "trivy-results.sarif"
frontend_build:
name: Build and Push DataHub Frontend Docker Image
runs-on: ubuntu-latest
needs: setup
steps:
- name: Check out the repo
uses: actions/checkout@v3
with:
fetch-depth: 0
- name: Pre-build artifacts for docker image
run: |
./gradlew :datahub-frontend:dist -x test -x yarnTest -x yarnLint --parallel
mv ./datahub-frontend/build/distributions/datahub-frontend-*.zip datahub-frontend.zip
env:
NODE_OPTIONS: "--max-old-space-size=3072"
- name: Build and push
uses: ./.github/actions/docker-custom-build-and-push
with:
images: |
${{ env.DATAHUB_FRONTEND_IMAGE }}
tags: ${{ needs.setup.outputs.tag }}
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
publish: ${{ needs.setup.outputs.publish }}
context: .
file: ./docker/datahub-frontend/Dockerfile
platforms: linux/amd64,linux/arm64/v8
frontend_scan:
name: "[Monitoring] Scan Frontend images for vulnerabilities"
runs-on: ubuntu-latest
needs: [setup, frontend_build]
permissions:
contents: read # for actions/checkout to fetch code
security-events: write # for github/codeql-action/upload-sarif to upload SARIF results
actions: read # only required for a private repository by github/codeql-action/upload-sarif to get the Action run status
steps:
- name: Checkout # adding checkout step just to make trivy upload happy
uses: actions/checkout@v3
- name: Download image
uses: ishworkh/docker-image-artifact-download@v1
if: ${{ needs.setup.outputs.publish != 'true' }}
with:
image: ${{ env.DATAHUB_FRONTEND_IMAGE }}:${{ needs.setup.outputs.unique_tag }}
- name: Run Trivy vulnerability scanner
uses: aquasecurity/[email protected]
env:
TRIVY_OFFLINE_SCAN: true
with:
image-ref: ${{ env.DATAHUB_FRONTEND_IMAGE }}:${{ needs.setup.outputs.unique_tag }}
format: "template"
template: "@/contrib/sarif.tpl"
output: "trivy-results.sarif"
severity: "CRITICAL,HIGH"
ignore-unfixed: true
vuln-type: "os,library"
- name: Upload Trivy scan results to GitHub Security tab
uses: github/codeql-action/upload-sarif@v2
with:
sarif_file: "trivy-results.sarif"
kafka_setup_build:
name: Build and Push DataHub Kafka Setup Docker Image
runs-on: ubuntu-latest
needs: setup
steps:
- name: Check out the repo
uses: actions/checkout@v3
with:
fetch-depth: 0
- name: Build and push
uses: ./.github/actions/docker-custom-build-and-push
with:
images: |
${{ env.DATAHUB_KAFKA_SETUP_IMAGE }}
tags: ${{ needs.setup.outputs.tag }}
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
publish: ${{ needs.setup.outputs.publish }}
context: .
file: ./docker/kafka-setup/Dockerfile
platforms: linux/amd64,linux/arm64/v8
mysql_setup_build:
name: Build and Push DataHub MySQL Setup Docker Image
runs-on: ubuntu-latest
needs: setup
steps:
- name: Check out the repo
uses: actions/checkout@v3
with:
fetch-depth: 0
- name: Build and push
uses: ./.github/actions/docker-custom-build-and-push
with:
images: |
${{ env.DATAHUB_MYSQL_SETUP_IMAGE }}
tags: ${{ needs.setup.outputs.tag }}
username: ${{ secrets.ACRYL_DOCKER_USERNAME }}
password: ${{ secrets.ACRYL_DOCKER_PASSWORD }}
publish: ${{ needs.setup.outputs.publish }}
context: .
file: ./docker/mysql-setup/Dockerfile
platforms: linux/amd64,linux/arm64/v8
elasticsearch_setup_build:
name: Build and Push DataHub Elasticsearch Setup Docker Image
runs-on: ubuntu-latest
needs: setup
steps:
- name: Check out the repo
uses: actions/checkout@v3
with:
fetch-depth: 0
- name: Build and push
uses: ./.github/actions/docker-custom-build-and-push
with:
images: |
${{ env.DATAHUB_ELASTIC_SETUP_IMAGE }}
tags: ${{ needs.setup.outputs.tag }}
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
publish: ${{ needs.setup.outputs.publish }}
context: .
file: ./docker/elasticsearch-setup/Dockerfile
platforms: linux/amd64,linux/arm64/v8
smoke_test:
name: Run Smoke Tests
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
test_strategy: ["no_cypress", "cypress_suite1", "cypress_rest"]
needs:
[
setup,
gms_build,
frontend_build,
kafka_setup_build,
mysql_setup_build,
elasticsearch_setup_build,
mae_consumer_build,
mce_consumer_build,
datahub_upgrade_build,
]
steps:
- name: Check out the repo
uses: actions/checkout@v3
- name: Set up JDK 11
uses: actions/setup-java@v3
with:
distribution: "zulu"
java-version: 11
- uses: actions/setup-python@v4
with:
python-version: "3.7"
cache: "pip"
- name: Install dependencies
run: ./metadata-ingestion/scripts/install_deps.sh
- name: Build datahub cli
run: |
./gradlew :metadata-ingestion:install
- name: Download GMS image
uses: ishworkh/docker-image-artifact-download@v1
if: ${{ needs.setup.outputs.publish != 'true' }}
with:
image: ${{ env.DATAHUB_GMS_IMAGE }}:${{ needs.setup.outputs.unique_tag }}
- name: Download Frontend image
uses: ishworkh/docker-image-artifact-download@v1
if: ${{ needs.setup.outputs.publish != 'true' }}
with:
image: ${{ env.DATAHUB_FRONTEND_IMAGE }}:${{ needs.setup.outputs.unique_tag }}
- name: Download Kafka Setup image
uses: ishworkh/docker-image-artifact-download@v1
if: ${{ needs.setup.outputs.publish != 'true' }}
with:
image: ${{ env.DATAHUB_KAFKA_SETUP_IMAGE }}:${{ needs.setup.outputs.unique_tag }}
- name: Download Mysql Setup image
uses: ishworkh/docker-image-artifact-download@v1
if: ${{ needs.setup.outputs.publish != 'true' }}
with:
image: ${{ env.DATAHUB_MYSQL_SETUP_IMAGE }}:${{ needs.setup.outputs.unique_tag }}
- name: Download Elastic Setup image
uses: ishworkh/docker-image-artifact-download@v1
if: ${{ needs.setup.outputs.publish != 'true' }}
with:
image: ${{ env.DATAHUB_ELASTIC_SETUP_IMAGE }}:${{ needs.setup.outputs.unique_tag }}
- name: Download MCE Consumer image
uses: ishworkh/docker-image-artifact-download@v1
if: ${{ needs.setup.outputs.publish != 'true' }}
with:
image: ${{ env.DATAHUB_MCE_CONSUMER_IMAGE }}:${{ needs.setup.outputs.unique_tag }}
- name: Download MAE Consumer image
uses: ishworkh/docker-image-artifact-download@v1
if: ${{ needs.setup.outputs.publish != 'true' }}
with:
image: ${{ env.DATAHUB_MAE_CONSUMER_IMAGE }}:${{ needs.setup.outputs.unique_tag }}
- name: Download upgrade image
uses: ishworkh/docker-image-artifact-download@v1
if: ${{ needs.setup.outputs.publish != 'true' }}
with:
image: ${{ env.DATAHUB_UPGRADE_IMAGE }}:${{ needs.setup.outputs.unique_tag }}
- name: run quickstart
env:
DATAHUB_TELEMETRY_ENABLED: false
DATAHUB_VERSION: ${{ needs.setup.outputs.unique_tag }}
run: |
./smoke-test/run-quickstart.sh
- name: sleep 60s
run: |
# we are doing this because gms takes time to get ready
# and we don't have a better readiness check when bootstrap is done
sleep 60s
- name: Smoke test
env:
RUN_QUICKSTART: false
DATAHUB_VERSION: ${{ needs.setup.outputs.unique_tag }}
CYPRESS_RECORD_KEY: ${{ secrets.CYPRESS_RECORD_KEY }}
CLEANUP_DATA: "false"
TEST_STRATEGY: ${{ matrix.test_strategy }}
run: |
echo "$DATAHUB_VERSION"
./smoke-test/smoke.sh
- name: store logs
if: failure()
run: |
docker ps -a
docker logs datahub-gms >& gms-${{ matrix.test_strategy }}.log
- name: Upload logs
uses: actions/upload-artifact@v3
if: failure()
with:
name: docker logs
path: "*.log"
- name: Upload screenshots
uses: actions/upload-artifact@v3
if: failure()
with:
name: cypress-snapshots-${{ matrix.test_strategy }}
path: smoke-test/tests/cypress/cypress/screenshots/
- uses: actions/upload-artifact@v3
if: always()
with:
name: Test Results (smoke tests) ${{ matrix.test_strategy }}
path: |
**/build/reports/tests/test/**
**/build/test-results/test/**
**/junit.*.xml
- name: Slack failure notification
if: failure() && github.event_name == 'push'
uses: kpritam/slack-job-status-action@v1
with:
job-status: ${{ job.status }}
slack-bot-token: ${{ secrets.SLACK_BOT_TOKEN }}
channel: github-activities