Sonarcloud

.github/workflows/sonar-cloud.yml

@@ -0,0 +1,37 @@
+name: SonarCloud
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+    types: [opened, synchronize, reopened]
+jobs:
+  build:
+    name: Build and analyze
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0  # Shallow clones should be disabled for a better relevancy of analysis
+      - name: Set up JDK 17
+        uses: actions/setup-java@v3
+        with:
+          java-version: 17
+          distribution: 'zulu' # Alternative distribution options are available
+      - name: Cache SonarCloud packages
+        uses: actions/cache@v3
+        with:
+          path: ~/.sonar/cache
+          key: ${{ runner.os }}-sonar
+          restore-keys: ${{ runner.os }}-sonar
+      - name: Cache Gradle packages
+        uses: actions/cache@v3
+        with:
+          path: ~/.gradle/caches
+          key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle') }}
+          restore-keys: ${{ runner.os }}-gradle
+      - name: Build and analyze
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}  # Needed to get PR information, if any
+          SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
+        run: ./gradlew test sonar --info

build.gradle

@@ -74,16 +74,21 @@ buildscript {
 }
 
 plugins {
-  id 'com.gorylenko.gradle-git-properties' version '2.4.1'
-  id 'com.github.johnrengelman.shadow' version '8.1.1' apply false
-  id 'com.palantir.docker' version '0.35.0' apply false
-  id 'com.avast.gradle.docker-compose' version '0.17.6'
-  id "com.diffplug.spotless" version "6.23.3"
-  // https://blog.ltgt.net/javax-jakarta-mess-and-gradle-solution/
-  // TODO id "org.gradlex.java-ecosystem-capabilities" version "1.0"
+    id 'com.gorylenko.gradle-git-properties' version '2.4.1'
+    id 'com.github.johnrengelman.shadow' version '8.1.1' apply false
+    id 'com.palantir.docker' version '0.35.0' apply false
+    id 'com.avast.gradle.docker-compose' version '0.17.6'
+    id "com.diffplug.spotless" version "6.23.3"
+    id 'jacoco-report-aggregation'
+    id 'io.github.surpsg.delta-coverage' version "2.5.0"
+    id 'org.sonarqube' version "5.1.0.4882"
+//    id "io.github.gw-kit.delta-coverage" version "3.0.0-RC1"
+
+    // TODO id "org.gradlex.java-ecosystem-capabilities" version "1.0"
 }
 
 apply from: "gradle/docker/docker.gradle"
+apply from: "gradle/coverage/coverage.gradle"
 
 project.ext.spec = [
     'product' : [
@@ -278,9 +283,21 @@ project.ext.externalDependency = [
     'mustache': 'com.github.spullara.mustache.java:compiler:0.9.14'
 ]
 
+sonarqube {
+  properties {
+    property "sonar.sourceEncoding", "UTF-8"
+    property "sonar.projectKey", "datahub-project_datahub"
+    property "sonar.organization", "datahub-project"
+    property "sonar.host.url", "https://sonarcloud.io"
+    property "sonar.coverage.jacoco.xmlReportPaths", "$projectDir.parentFile.path/build/reports/jacoco/test/jacocoTestReport.xml"
+    property "sonar.python.coverage.reportPaths", "https://sonarcloud.io"
+  }
+}
+
 allprojects {
   apply plugin: 'idea'
   apply plugin: 'eclipse'
+  apply plugin: 'jacoco'
   // apply plugin: 'org.gradlex.java-ecosystem-capabilities'
 
   tasks.withType(Test).configureEach { task -> if (task.project.name != "metadata-io") {
@@ -291,6 +308,7 @@ allprojects {
               .any { it.getName().contains("testng") }) {
         useTestNG()
       }
+    finalizedBy jacocoTestReport
     }
   }
 
@@ -372,16 +390,34 @@ subprojects {
   apply plugin: 'maven-publish'
   apply plugin: 'com.gorylenko.gradle-git-properties'
   apply plugin: 'com.diffplug.spotless'
-
-  gitProperties {
-    keys = ['git.commit.id','git.commit.id.describe','git.commit.time']
-    // using any tags (not limited to annotated tags) for "git.commit.id.describe" property
-    // see http://ajoberstar.org/grgit/grgit-describe.html for more info about the describe method and available parameters
-    // 'it' is an instance of org.ajoberstar.grgit.Grgit
-    customProperty 'git.commit.id.describe', { it.describe(tags: true) }
-    failOnNoGitDirectory = false
+  apply plugin: 'jacoco'
+  apply plugin: 'org.sonarqube'
+
+  if (!project.name.contains('docker')
+            && !project.name.contains('web-react')
+            && !project.name.contains('metadata-ingestion')
+            && !project.name.contains('metadata-auth')
+            && !project.name.contains('metadata-dao-impl')
+            && !project.name.contains('metadata-events')
+            && !project.name.contains('metadata-integration')
+            && !project.name.contains('metadata-jobs')
+            && !project.name.contains('metadata-service')
+            && !project.name.contains('smoke-test')
+            && !project.name.contains('docs-website')
+  ){
+        apply plugin: 'io.github.surpsg.delta-coverage'
   }
 
+
+    gitProperties {
+        keys = ['git.commit.id', 'git.commit.id.describe', 'git.commit.time']
+        // using any tags (not limited to annotated tags) for "git.commit.id.describe" property
+        // see http://ajoberstar.org/grgit/grgit-describe.html for more info about the describe method and available parameters
+        // 'it' is an instance of org.ajoberstar.grgit.Grgit
+        customProperty 'git.commit.id.describe', { it.describe(tags: true) }
+        failOnNoGitDirectory = false
+    }
+
   plugins.withType(JavaPlugin).configureEach {
     if (project.name == 'datahub-web-react') {
       return
@@ -402,18 +438,56 @@ subprojects {
       }
     }
 
-    spotless {
-      java {
-        googleJavaFormat()
-        target project.fileTree(project.projectDir) {
-          include 'src/**/*.java'
-          exclude 'src/**/resources/'
-          exclude 'src/**/generated/'
-          exclude 'src/**/mainGeneratedDataTemplate/'
-          exclude 'src/**/mainGeneratedRest/'
-          exclude 'src/renamed/avro/'
-          exclude 'src/test/sample-test-plugins/'
+    jacocoTestReport {
+      reports {
+        xml.required = true
+        csv.required = true
+        html.outputLocation = layout.buildDirectory.dir('jacocoHtml')
+      }
+      dependsOn test // tests are required to run before generating the report
+    }
+
+    deltaCoverageReport {
+      diffSource {
+                git.compareWith('refs/remotes/origin/master')
+                git.useNativeGit.set(true)
+            }
+
+      coverageBinaryFiles = files(projectDir.path + "/build/jacoco/test.exec")
+      srcDirs = files(projectDir.path +"/src")
+      classesDirs = files(projectDir.path + "/build/classes")
+//      violationRules.failIfCoverageLessThan 0.9d
+
+      reports {
+        html.set(true)
+        markdown.set(true)
+      }
+    }
+
+    task openDiffReport() {
+      doLast {
+        java.awt.Desktop.desktop.browse "file://".concat(buildDir.path).concat("/reports/coverage-reports/delta-coverage/html/index.html").toURI()
+      }
+    }
+
+        spotless {
+            java {
+                googleJavaFormat()
+                target project.fileTree(project.projectDir) {
+                    include 'src/**/*.java'
+                    exclude 'src/**/resources/'
+                    exclude 'src/**/generated/'
+                    exclude 'src/**/mainGeneratedDataTemplate/'
+                    exclude 'src/**/mainGeneratedRest/'
+                    exclude 'src/renamed/avro/'
+                    exclude 'src/test/sample-test-plugins/'
+                }
+            }
         }
+
+    sonarqube {
+      properties {
+        property "sonar.coverage.jacoco.xmlReportPaths", "$projectDir.parentFile.path/build/reports/jacoco/test/jacocoTestReport.xml"
       }
     }
 
@@ -434,6 +508,8 @@ subprojects {
         options.release = javaClassVersion(project)
       }
 
+
+
       tasks.withType(JavaCompile).configureEach {
         javaCompiler = javaToolchains.compilerFor {
           languageVersion = JavaLanguageVersion.of(jdkVersion(project))
@@ -486,3 +562,16 @@ subprojects {
     }
   }
 }
+
+reporting {
+  reports {
+    testCodeCoverageReport(JacocoCoverageReport) {
+      testType = TestSuiteType.UNIT_TEST
+
+    }
+  }
+}
+
+tasks.named('check') {
+  dependsOn tasks.named('testCodeCoverageReport', JacocoReport)
+}

datahub-graphql-core/build.gradle

@@ -55,3 +55,24 @@ tasks.withType(Checkstyle) {
 
 compileJava.dependsOn 'graphqlCodegen'
 sourceSets.main.java.srcDir "$projectDir/src/mainGeneratedGraphQL/java"
+
+test{
+    useTestNG()
+    finalizedBy jacocoTestReport
+}
+
+def jacocoExclusions = [
+        '**/graphql/generated/**'
+];
+
+jacocoTestReport {
+
+    afterEvaluate {
+        classDirectories.setFrom(files(classDirectories.files.collect {
+            fileTree(dir: it,
+                    exclude: jacocoExclusions
+            )
+        }))
+    }
+
+}

gradle/coverage/coverage.gradle

@@ -0,0 +1,61 @@
+dependencies {
+    jacocoAggregation project(':datahub-frontend')
+    jacocoAggregation project(':datahub-graphql-core')
+    jacocoAggregation project(':datahub-upgrade')
+    jacocoAggregation project(':entity-registry')
+    jacocoAggregation project(':ingestion-scheduler')
+    jacocoAggregation project(':li-utils')
+    jacocoAggregation project(':metadata-auth')
+    jacocoAggregation project(':metadata-dao-impl')
+    jacocoAggregation project(':metadata-events')
+    jacocoAggregation project(':metadata-integration')
+    jacocoAggregation project(':metadata-io')
+    jacocoAggregation project(':metadata-jobs')
+    jacocoAggregation project(':metadata-models')
+    jacocoAggregation project(':metadata-models-custom')
+    jacocoAggregation project(':metadata-models-validator')
+    jacocoAggregation project(':metadata-operation-context')
+    jacocoAggregation project(':metadata-service')
+    jacocoAggregation project(':metadata-utils')
+    jacocoAggregation project(':mock-entity-registry')
+    jacocoAggregation project(':metadata-auth:auth-api')
+    jacocoAggregation project(':metadata-dao-impl:kafka-producer')
+    jacocoAggregation project(':metadata-events:mxe-avro')
+    jacocoAggregation project(':metadata-events:mxe-registration')
+    jacocoAggregation project(':metadata-events:mxe-schemas')
+    jacocoAggregation project(':metadata-events:mxe-utils-avro')
+    jacocoAggregation project(':metadata-integration:java')
+    jacocoAggregation project(':metadata-io:metadata-io-api')
+    jacocoAggregation project(':metadata-jobs:common')
+    jacocoAggregation project(':metadata-jobs:mce-consumer')
+    jacocoAggregation project(':metadata-jobs:mce-consumer-job')
+    jacocoAggregation project(':metadata-jobs:pe-consumer')
+    jacocoAggregation project(':metadata-service:auth-config')
+    jacocoAggregation project(':metadata-service:auth-filter')
+    jacocoAggregation project(':metadata-service:auth-impl')
+    jacocoAggregation project(':metadata-service:auth-servlet-impl')
+    jacocoAggregation project(':metadata-service:configuration')
+    jacocoAggregation project(':metadata-service:factories')
+    jacocoAggregation project(':metadata-service:graphql-servlet-impl')
+    jacocoAggregation project(':metadata-service:openapi-analytics-servlet')
+    jacocoAggregation project(':metadata-service:openapi-entity-servlet')
+    jacocoAggregation project(':metadata-service:openapi-servlet')
+    jacocoAggregation project(':metadata-service:plugin')
+    jacocoAggregation project(':metadata-service:restli-api')
+    jacocoAggregation project(':metadata-service:restli-client')
+    jacocoAggregation project(':metadata-service:restli-client-api')
+    jacocoAggregation project(':metadata-service:restli-servlet-impl')
+    jacocoAggregation project(':metadata-service:schema-registry-api')
+    jacocoAggregation project(':metadata-service:schema-registry-servlet')
+    jacocoAggregation project(':metadata-service:services')
+    jacocoAggregation project(':metadata-service:servlet')
+    jacocoAggregation project(':metadata-integration:java:acryl-spark-lineage')
+    jacocoAggregation project(':metadata-integration:java:custom-plugin-lib')
+    jacocoAggregation project(':metadata-integration:java:datahub-client')
+    jacocoAggregation project(':metadata-integration:java:datahub-event')
+    jacocoAggregation project(':metadata-integration:java:datahub-protobuf')
+    jacocoAggregation project(':metadata-integration:java:examples')
+    jacocoAggregation project(':metadata-integration:java:openlineage-converter')
+    jacocoAggregation project(':metadata-service:openapi-entity-servlet:generators')
+    jacocoAggregation project(':metadata-service:openapi-servlet:models')
+}