Merge branch 'master' into ebean-batched-transactions

.github/workflows/airflow-plugin.yml

@@ -0,0 +1,85 @@
+name: Airflow Plugin
+on:
+  push:
+    branches:
+      - master
+    paths:
+      - ".github/workflows/airflow-plugin.yml"
+      - "metadata-ingestion-modules/airflow-plugin/**"
+      - "metadata-ingestion/**"
+      - "metadata-models/**"
+  pull_request:
+    branches:
+      - master
+    paths:
+      - ".github/**"
+      - "metadata-ingestion-modules/airflow-plugin/**"
+      - "metadata-ingestion/**"
+      - "metadata-models/**"
+  release:
+    types: [published]
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  airflow-plugin:
+    runs-on: ubuntu-latest
+    env:
+      SPARK_VERSION: 3.0.3
+      DATAHUB_TELEMETRY_ENABLED: false
+    strategy:
+      matrix:
+        include:
+          - python-version: "3.7"
+            extraPythonRequirement: "apache-airflow~=2.1.0"
+          - python-version: "3.7"
+            extraPythonRequirement: "apache-airflow~=2.2.0"
+          - python-version: "3.10"
+            extraPythonRequirement: "apache-airflow~=2.4.0"
+          - python-version: "3.10"
+            extraPythonRequirement: "apache-airflow~=2.6.0"
+          - python-version: "3.10"
+            extraPythonRequirement: "apache-airflow>2.6.0"
+      fail-fast: false
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+          cache: "pip"
+      - name: Install dependencies
+        run: ./metadata-ingestion/scripts/install_deps.sh
+      - name: Install airflow package and test  (extras ${{ matrix.extraPythonRequirement }})
+        run: ./gradlew -Pextra_pip_requirements='${{ matrix.extraPythonRequirement }}' :metadata-ingestion-modules:airflow-plugin:lint :metadata-ingestion-modules:airflow-plugin:testQuick
+      - name: pip freeze show list installed
+        if: always()
+        run: source metadata-ingestion-modules/airflow-plugin/venv/bin/activate && pip freeze
+      - uses: actions/upload-artifact@v3
+        if: ${{ always() && matrix.python-version == '3.10' && matrix.extraPythonRequirement == 'apache-airflow>2.6.0' }}
+        with:
+          name: Test Results (Airflow Plugin ${{ matrix.python-version}})
+          path: |
+            **/build/reports/tests/test/**
+            **/build/test-results/test/**
+            **/junit.*.xml
+      - name: Upload coverage to Codecov
+        if: always()
+        uses: codecov/codecov-action@v3
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          directory: .
+          fail_ci_if_error: false
+          flags: airflow-${{ matrix.python-version }}-${{ matrix.extraPythonRequirement }}
+          name: pytest-airflow
+          verbose: true
+
+  event-file:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Upload
+        uses: actions/upload-artifact@v3
+        with:
+          name: Event File
+          path: ${{ github.event_path }}

.github/workflows/metadata-ingestion.yml

@@ -25,7 +25,7 @@ jobs:
   metadata-ingestion:
     runs-on: ubuntu-latest
     env:
-      SPARK_VERSION: 3.0.3
+      SPARK_VERSION: 3.3.2
       DATAHUB_TELEMETRY_ENABLED: false
       # TODO: Enable this once the test is fixed.
       # DATAHUB_LOOKML_GIT_TEST_SSH_KEY: ${{ secrets.DATAHUB_LOOKML_GIT_TEST_SSH_KEY }}
@@ -42,9 +42,7 @@ jobs:
           ]
         include:
           - python-version: "3.7"
-            extraPythonRequirement: "sqlalchemy==1.3.24 apache-airflow~=2.2.0"
           - python-version: "3.10"
-            extraPythonRequirement: "sqlalchemy~=1.4.0 apache-airflow>=2.4.0"
       fail-fast: false
     steps:
       - uses: actions/checkout@v3
@@ -56,8 +54,8 @@ jobs:
         run: ./metadata-ingestion/scripts/install_deps.sh
       - name: Install package
         run: ./gradlew :metadata-ingestion:installPackageOnly
-      - name: Run metadata-ingestion tests (extras ${{ matrix.extraPythonRequirement }})
-        run: ./gradlew -Pextra_pip_requirements='${{ matrix.extraPythonRequirement }}' :metadata-ingestion:${{ matrix.command }}
+      - name: Run metadata-ingestion tests
+        run: ./gradlew :metadata-ingestion:${{ matrix.command }}
       - name: pip freeze show list installed
         if: always()
         run: source metadata-ingestion/venv/bin/activate && pip freeze
@@ -80,7 +78,6 @@ jobs:
           name: pytest-${{ matrix.command }}
           verbose: true
 
-
   event-file:
     runs-on: ubuntu-latest
     steps:

.github/workflows/spark-smoke-test.yml

@@ -40,6 +40,8 @@ jobs:
           python-version: "3.7"
       - name: Install dependencies
         run: ./metadata-ingestion/scripts/install_deps.sh
+      - name: Remove images
+        run: docker image prune -a -f || true
       - name: Smoke test
         run: |
           ./gradlew :metadata-integration:java:spark-lineage:integrationTest \

.github/workflows/test-results.yml

@@ -2,7 +2,7 @@ name: Test Results
 
 on:
   workflow_run:
-    workflows: ["build & test", "metadata ingestion"]
+    workflows: ["build & test", "metadata ingestion", "Airflow Plugin"]
     types:
       - completed
 

.gitignore

@@ -70,6 +70,7 @@ metadata-ingestion/generated/**
 # docs
 docs/generated/
 docs-website/versioned_docs/
+docs-website/versioned_sidebars/
 tmp*
 temp/**
 

build.gradle

@@ -1,7 +1,7 @@
 buildscript {
   ext.junitJupiterVersion = '5.6.1'
   // Releases: https://github.com/linkedin/rest.li/blob/master/CHANGELOG.md
-  ext.pegasusVersion = '29.22.16'
+  ext.pegasusVersion = '29.45.0'
   ext.mavenVersion = '3.6.3'
   ext.springVersion = '5.3.29'
   ext.springBootVersion = '2.7.14'
@@ -30,16 +30,16 @@ buildscript {
     classpath 'io.acryl.gradle.plugin:gradle-avro-plugin:0.8.1'
     classpath 'org.springframework.boot:spring-boot-gradle-plugin:' + springBootVersion
     classpath "io.codearte.gradle.nexus:gradle-nexus-staging-plugin:0.30.0"
-    classpath "com.palantir.gradle.gitversion:gradle-git-version:0.12.3"
-    classpath "org.gradle.playframework:gradle-playframework:0.12"
-    classpath "gradle.plugin.org.hidetake:gradle-swagger-generator-plugin:2.18.1"
+    classpath "com.palantir.gradle.gitversion:gradle-git-version:3.0.0"
+    classpath "org.gradle.playframework:gradle-playframework:0.14"
+    classpath "gradle.plugin.org.hidetake:gradle-swagger-generator-plugin:2.19.1"
   }
 }
 
 plugins {
   id 'com.gorylenko.gradle-git-properties' version '2.4.0-rc2'
   id 'com.github.johnrengelman.shadow' version '6.1.0'
-  id "com.palantir.docker" version "0.35.0"
+  id 'com.palantir.docker' version '0.35.0'
   // https://blog.ltgt.net/javax-jakarta-mess-and-gradle-solution/
   // TODO id "org.gradlex.java-ecosystem-capabilities" version "1.0"
 }
@@ -97,7 +97,7 @@ project.ext.externalDependency = [
     'graphqlJavaScalars': 'com.graphql-java:graphql-java-extended-scalars:19.1',
     'gson': 'com.google.code.gson:gson:2.8.9',
     'guice': 'com.google.inject:guice:4.2.3',
-    'guava': 'com.google.guava:guava:27.0.1-jre',
+    'guava': 'com.google.guava:guava:32.1.2-jre',
     'h2': 'com.h2database:h2:2.1.214',
     'hadoopCommon':'org.apache.hadoop:hadoop-common:2.7.2',
     'hadoopMapreduceClient':'org.apache.hadoop:hadoop-mapreduce-client-core:2.7.2',
@@ -241,7 +241,7 @@ configure(subprojects.findAll {! it.name.startsWith('spark-lineage')}) {
 
 subprojects {
 
-  apply plugin: 'maven'
+  apply plugin: 'maven-publish'
   apply plugin: 'com.gorylenko.gradle-git-properties'
 
   gitProperties {
@@ -255,7 +255,7 @@ subprojects {
 
   plugins.withType(JavaPlugin) {
     dependencies {
-      testCompile externalDependency.testng
+      testImplementation externalDependency.testng
       constraints {
         implementation('io.netty:netty-all:4.1.86.Final')
         implementation('org.apache.commons:commons-compress:1.21')

buildSrc/build.gradle

@@ -5,12 +5,12 @@ buildscript {
 }
 
 dependencies {
-  compile('io.acryl:json-schema-avro:0.1.5') {
+  implementation('io.acryl:json-schema-avro:0.1.5') {
     exclude group: 'com.fasterxml.jackson.core', module: 'jackson-databind'
     exclude group: 'com.google.guava', module: 'guava'
   }
-  compile 'com.google.guava:guava:27.0.1-jre'
-  compile 'com.fasterxml.jackson.core:jackson-databind:2.13.5'
-  compile 'com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.13.5'
-  compile 'commons-io:commons-io:2.11.0'
+  implementation 'com.google.guava:guava:32.1.2-jre'
+  implementation 'com.fasterxml.jackson.core:jackson-databind:2.13.5'
+  implementation 'com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.13.5'
+  implementation 'commons-io:commons-io:2.11.0'
 }

buildSrc/src/main/java/io/datahubproject/GenerateJsonSchemaTask.java

@@ -21,10 +21,7 @@
 import java.util.List;
 import java.util.stream.Collectors;
 import org.gradle.api.DefaultTask;
-import org.gradle.api.tasks.CacheableTask;
-import org.gradle.api.tasks.InputDirectory;
-import org.gradle.api.tasks.OutputDirectory;
-import org.gradle.api.tasks.TaskAction;
+import org.gradle.api.tasks.*;
 
 import static com.github.fge.processing.ProcessingUtil.*;
 import static org.apache.commons.io.FilenameUtils.*;
@@ -46,6 +43,7 @@ public void setInputDirectory(String inputDirectory) {
   }
 
   @InputDirectory
+  @PathSensitive(PathSensitivity.NAME_ONLY)
   public String getInputDirectory() {
    return inputDirectory;
   }

datahub-frontend/build.gradle

@@ -1,7 +1,7 @@
 plugins {
   id "io.github.kobylynskyi.graphql.codegen" version "4.1.1"
-  id 'com.palantir.docker'
   id 'scala'
+  id 'com.palantir.docker'
 }
 
 apply from: "../gradle/versioning/versioning.gradle"

datahub-frontend/play.gradle

@@ -4,7 +4,7 @@ apply plugin: "org.gradle.playframework"
 project.ext.httpPort = 9001
 project.ext.playBinaryBaseName = "datahub-frontend"
 
-tasks.withType(PlayRun) {
+runPlay {
   httpPort = project.ext.httpPort
 }
 
@@ -33,8 +33,8 @@ dependencies {
     }
   }
 
-  compile project(":metadata-service:restli-client")
-  compile project(":metadata-service:auth-config")
+  implementation project(":metadata-service:restli-client")
+  implementation project(":metadata-service:auth-config")
 
   implementation externalDependency.jettyJaas
   implementation externalDependency.graphqlJava
@@ -70,15 +70,15 @@ dependencies {
   testImplementation 'no.nav.security:mock-oauth2-server:0.3.1'
   testImplementation 'org.junit-pioneer:junit-pioneer:1.9.1'
   testImplementation externalDependency.junitJupiterApi
-  testRuntime externalDependency.junitJupiterEngine
+  testRuntimeOnly externalDependency.junitJupiterEngine
 
   implementation externalDependency.slf4jApi
   compileOnly externalDependency.lombok
-  runtime externalDependency.guice
-  runtime (externalDependency.playDocs) {
+  runtimeOnly externalDependency.guice
+  runtimeOnly (externalDependency.playDocs) {
     exclude group: 'com.typesafe.akka', module: 'akka-http-core_2.12'
   }
-  runtime externalDependency.playGuice
+  runtimeOnly externalDependency.playGuice
   implementation externalDependency.log4j2Api
 
   implementation externalDependency.logbackClassic

datahub-graphql-core/build.gradle

@@ -4,25 +4,26 @@ plugins {
 apply plugin: 'java'
 
 dependencies {
-    compile project(':metadata-service:restli-client')
-    compile project(':metadata-service:auth-impl')
-    compile project(':metadata-service:auth-config')
-    compile project(':metadata-service:configuration')
-    compile project(':metadata-service:services')
-    compile project(':metadata-io')
-    compile project(':metadata-utils')
+    implementation project(':metadata-service:restli-client')
+    implementation project(':metadata-service:auth-impl')
+    implementation project(':metadata-service:auth-config')
+    implementation project(':metadata-service:configuration')
+    implementation project(':metadata-service:services')
+    implementation project(':metadata-io')
+    implementation project(':metadata-utils')
 
     implementation externalDependency.graphqlJava
     implementation externalDependency.graphqlJavaScalars
-    compile externalDependency.antlr4Runtime
-    compile externalDependency.antlr4
-    compile externalDependency.guava
+    implementation externalDependency.antlr4Runtime
+    implementation externalDependency.antlr4
+    implementation externalDependency.guava
+    implementation externalDependency.opentelemetryAnnotations
 
     implementation externalDependency.slf4jApi
     compileOnly externalDependency.lombok
     annotationProcessor externalDependency.lombok
 
-    testCompile externalDependency.mockito
+    testImplementation externalDependency.mockito
 }
 
 graphqlCodegen {

datahub-upgrade/build.gradle

@@ -12,14 +12,15 @@ ext {
 }
 
 dependencies {
-  compile project(':metadata-io')
-  compile project(':metadata-service:factories')
-  compile project(':metadata-service:restli-client')
-  compile project(':metadata-service:configuration')
+  implementation project(':metadata-io')
+  implementation project(':metadata-service:factories')
+  implementation project(':metadata-service:restli-client')
+  implementation project(':metadata-service:configuration')
+  implementation project(':metadata-dao-impl:kafka-producer')
   implementation externalDependency.charle
 
-  compile externalDependency.javaxInject
-  compile(externalDependency.hadoopClient) {
+  implementation externalDependency.javaxInject
+  implementation(externalDependency.hadoopClient) {
     exclude group: 'net.minidev', module: 'json-smart'
     exclude group: 'com.nimbusds', module: 'nimbus-jose-jwt'
     exclude group: "org.apache.htrace", module: "htrace-core4"
@@ -52,28 +53,28 @@ dependencies {
 
   implementation externalDependency.slf4jApi
   compileOnly externalDependency.lombok
-  compile externalDependency.picocli
-  compile externalDependency.parquet
+  implementation externalDependency.picocli
+  implementation externalDependency.parquet
   implementation externalDependency.protobuf
-  compile externalDependency.springBeans
-  compile externalDependency.springBootAutoconfigure
-  compile externalDependency.springCore
-  compile externalDependency.springKafka
+  implementation externalDependency.springBeans
+  implementation externalDependency.springBootAutoconfigure
+  implementation externalDependency.springCore
+  implementation externalDependency.springKafka
 
-  runtime externalDependency.logbackClassic
-  runtime externalDependency.mariadbConnector
-  runtime externalDependency.mysqlConnector
-  runtime externalDependency.postgresql
+  runtimeOnly externalDependency.logbackClassic
+  runtimeOnly externalDependency.mariadbConnector
+  runtimeOnly externalDependency.mysqlConnector
+  runtimeOnly externalDependency.postgresql
 
   implementation externalDependency.awsMskIamAuth
 
   annotationProcessor externalDependency.lombok
   annotationProcessor externalDependency.picocli
 
   testImplementation externalDependency.springBootTest
-  testCompile externalDependency.mockito
-  testCompile externalDependency.testng
-  testRuntime externalDependency.logbackClassic
+  testImplementation externalDependency.mockito
+  testImplementation externalDependency.testng
+  testRuntimeOnly externalDependency.logbackClassic
 }
 
 bootJar {