build_rules.gradle

/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * License); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an AS IS BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

// This file contains common build rules that are to be applied
// to all projects and also a set of methods called applyXYZNature which
// sets up common build rules for sub-projects with the same needs.
//
// The supported list of natures are:
//  * Java   - Configures plugins commonly found in Java projects
//  * Go     - Configures plugins commonly found in Go projects
//  * Docker - Configures plugins commonly used to build Docker containers
//  * Grpc   - Configures plugins commonly used to generate source from protos
//  * Avro   - Configures plugins commonly used to generate source from Avro specifications
//
// For example, see applyJavaNature below.

println "Applying build_rules.gradle to $project.name"

/*************************************************************************************************/
// Apply common properties/repositories and tasks to all projects.

// Gradle determines project uniqueness based upon the project group and its name.
// We use the project.path as the group name to make this mapping unique since
// we have a few projects with the same name.
group = project.path
version = "2.5.0-SNAPSHOT"

// Define the default set of repositories for all builds.
repositories {
  maven { url offlineRepositoryRoot }

  // To run gradle in offline mode, one must first invoke
  // 'updateOfflineRepository' to create an offline repo
  // inside the root project directory. See the application
  // of the offline repo plugin within build_rules.gradle
  // for further details.
  if (gradle.startParameter.isOffline()) {
    return
  }

  mavenLocal()
  mavenCentral()
  jcenter()

  // Spring for resolving pentaho dependency.
  maven { url "https://repo.spring.io/plugins-release/" }

  // Release staging repository
  maven { url "https://oss.sonatype.org/content/repositories/staging/" }

  // Apache nightly snapshots
  maven { url "https://repository.apache.org/snapshots" }

  // Apache release snapshots
  maven { url "https://repository.apache.org/content/repositories/releases" }
}

// Apply a plugin which enables configuring projects imported into Intellij.
apply plugin: "idea"

// Provide code coverage
// TODO: Should this only apply to Java projects?
apply plugin: "jacoco"

// Apply a plugin which provides tasks for dependency / property / task reports.
// See https://docs.gradle.org/current/userguide/project_reports_plugin.html
// for further details. This is typically very useful to look at the "htmlDependencyReport"
// when attempting to resolve dependency issues.
apply plugin: "project-report"

// Apply a task dependency visualization plugin which creates a ".dot" file in the build directory
// giving the task dependencies for the current build. Unfortunately this creates a ".dot" file
// in each sub-projects report output directory.
// See https://github.com/mmalohlava/gradle-visteg for further details.
apply plugin: "cz.malohlava.visteg"

// Apply a plugin which provides the 'updateOfflineRepository' task that creates an offline
// repository. This offline repository satisfies all Gradle build dependencies and Java
// project dependencies. The offline repository is placed within $rootDir/offline-repo
// but can be overridden by specifying the 'offlineRepositoryRoot' Gradle option.
// Note that parallel build must be disabled when executing 'updateOfflineRepository'
// by specifying '-Dorg.gradle.parallel=false', see
// https://github.com/mdietrichstein/gradle-offline-dependencies-plugin/issues/3
apply plugin: "io.pry.gradle.offline_dependencies"

offlineDependencies {
  repositories {
    maven { url offlineRepositoryRoot }
    mavenLocal()
    mavenCentral()
    jcenter()
    maven { url "https://plugins.gradle.org/m2/" }
    maven { url "http://repo.spring.io/plugins-release" }
  }

  includeSources = false
  includeJavadocs = false
  includeIvyXmls = false
}

/*************************************************************************************************/

// Returns a string representing the relocated path to be used with the shadow plugin when
// given a suffix such as "com.google.common".
ext.getJavaRelocatedPath = { String suffix ->
  return "org.apache.beam"
    + (project.path + ":" + project.name).replace(":", ".").replace("-", "_")
    + ".repackaged."
    + suffix;
}

// A class defining the set of configurable properties accepted by applyJavaNature
class JavaNatureConfiguration {
  double javaVersion = 1.8        // Controls the JDK source language and target compatibility
  boolean enableFindbugs = true   // Controls whether the findbugs plugin is enabled and configured
  boolean enableShadow = true     // Controls whether the shadow plugin is enabled and configured
  String artifactId = null        // Sets the maven publication artifact id
}

// Configures a project with a default set of plugins that should apply to all Java projects.
//
// Users should invoke this method using Groovy map syntax. For example:
// applyJavaNature(javaVersion: 1.8)
//
// See JavaNatureConfiguration for the set of accepted properties.
//
// The following plugins are enabled:
//  * java
//  * maven
//  * net.ltgt.apt (plugin to configure annotation processing tool)
//  * propdeps (provide optional and provided dependency configurations)
//  * propdeps-maven (configure Maven pom generation to understand optional and provided dependency configurations)
//  * checkstyle
//  * findbugs
//  * shadow
//  * com.diffplug.gradle.spotless (code style plugin)
//
// Dependency Management for Java Projects
// ---------------------------------------
//
// By default, the shadow plugin is enabled to perform shading of commonly found dependencies.
// Because of this it is important that dependencies are added to the correct configuration.
// Dependencies should fall into one of these four configurations:
//  * compile     - Required during compilation or runtime of the main source set.
//                  This configuration represents all dependencies that much also be shaded away
//                  otherwise the generated Maven pom will be missing this dependency.
//  * shadow      - Required during compilation or runtime of the main source set.
//                  Will become a runtime dependency of the generated Maven pom.
//  * testCompile - Required during compilation or runtime of the test source set.
//                  This must be shaded away in the shaded test jar.
//  * shadowTest  - Required during compilation or runtime of the test source set.
//                  TODO: Figure out whether this should be a test scope dependency
//                  of the generated Maven pom.
//
// When creating a cross-project dependency between two Java projects, one should only rely on the shaded configurations.
// This allows for compilation/test execution to occur against the final artifact that will be provided to users.
// This is by done by referencing the "shadow" or "shadowTest" configuration as so:
//   dependencies {
//     shadow project(path: "other:java:project1", configuration: "shadow")
//     shadowTest project(path: "other:java:project2", configuration: "shadowTest")
//   }
// This will ensure the correct set of transitive dependencies from those projects are correctly added to the
// main and test source set runtimes.

ext.applyJavaNature = {
  println "applyJavaNature with " + (it ? "$it" : "default configuration") + " for project $project.name"
  // Use the implicit it parameter of the closure to handle zero argument or one argument map calls.
  JavaNatureConfiguration configuration = it ? it as JavaNatureConfiguration : new JavaNatureConfiguration()
  apply plugin: "maven-publish"
  apply plugin: "java"

  // Configure the Java compiler source language and target compatibility levels. Also ensure that
  // we configure the Java compiler to use UTF-8.
  sourceCompatibility = configuration.javaVersion
  targetCompatibility = configuration.javaVersion
  tasks.withType(JavaCompile) {
    options.encoding = "UTF-8"
    options.compilerArgs += ["-Xlint:all","-Xlint:-options","-Xlint:-cast","-Xlint:-deprecation","-Xlint:-processing","-Xlint:-rawtypes","-Xlint:-serial","-Xlint:-try","-Xlint:-unchecked","-Xlint:-varargs","-parameters"]
  }

  // Configure the default test tasks set of tests executed
  // to match the equivalent set that is executed by the maven-surefire-plugin.
  // See http://maven.apache.org/components/surefire/maven-surefire-plugin/test-mojo.html
  test {
    include "**/Test*.class"
    include "**/*Test.class"
    include "**/*Tests.class"
    include "**/*TestCase.class"
  }

  // Configure all test tasks to use JUnit
  tasks.withType(Test) {
    useJUnit { }
  }

  // Ensure that tests are packaged and part of the artifact set.
  task packageTests(type: Jar) {
    classifier = 'tests'
    from sourceSets.test.output
  }
  artifacts.archives packageTests

  // Configures annotation processing for commonly used annotation processors
  // across all Java projects.
  apply plugin: "net.ltgt.apt"
  dependencies {
    // Note that these plugins specifically use the compileOnly and testCompileOnly
    // configurations because they are never required to be shaded or become a
    // dependency of the output.
    def auto_value = "com.google.auto.value:auto-value:1.5.3"
    def auto_service = "com.google.auto.service:auto-service:1.0-rc2"

    compileOnly auto_value
    apt auto_value
    testCompileOnly auto_value
    testApt auto_value

    compileOnly auto_service
    apt auto_service
    testCompileOnly auto_service
    testApt auto_service
  }

  // Add the optional and provided configurations for dependencies
  // TODO: Either remove these plugins and find another way to generate the Maven poms
  // with the correct dependency scopes configured.
  apply plugin: 'propdeps'
  apply plugin: 'propdeps-maven'

  // Configures a checkstyle plugin enforcing a set of rules and also allows for a set of
  // suppressions.
  apply plugin: 'checkstyle'
  tasks.withType(Checkstyle) {
    configFile = project(":").file("sdks/java/build-tools/src/main/resources/beam/checkstyle.xml")
    configProperties = [ "checkstyle.suppressions.file": project(":").file("sdks/java/build-tools/src/main/resources/beam/suppressions.xml") ]
    showViolations = true
    maxErrors = 0
  }
  checkstyle {
    toolVersion = "8.7"
  }

  // Apply the eclipse and apt-eclipse plugins.  This adds the "eclipse" task and
  // connects the apt-eclipse plugin to update the eclipse project files
  // with the instructions needed to run apt within eclipse to handle the AutoValue
  // and additional annotations
  apply plugin: 'eclipse'
  apply plugin: "net.ltgt.apt-eclipse"

  // Enables a plugin which can apply code formatting to source.
  // TODO: Should this plugin be enabled for all projects?
  apply plugin: "com.diffplug.gradle.spotless"

  // Enables a plugin which performs code analysis for common bugs.
  // This plugin is configured to only analyze the "main" source set.
  if (configuration.enableFindbugs) {
    apply plugin: 'findbugs'
    findbugs {
      excludeFilter = rootProject.file('sdks/java/build-tools/src/main/resources/beam/findbugs-filter.xml')
      sourceSets = [sourceSets.main]
    }
    tasks.withType(FindBugs) {
      reports {
        html.enabled = true
        xml.enabled = false
      }
    }
  }

  // Enables a plugin which can perform shading of classes. See the general comments
  // above about dependency management for Java projects and how the shadow plugin
  // is expected to be used for the different Gradle configurations.
  //
  // TODO: Enforce all relocations are always performed to:
  // getJavaRelocatedPath(package_suffix) where package_suffix is something like "com.google.commmon"
  if (configuration.enableShadow) {
    apply plugin: 'com.github.johnrengelman.shadow'

    // Shade guava in all our dependencies.
    shadowJar {
      classifier = "shaded"
      mergeServiceFiles()
      dependencies {
        exclude(".*")
        include(dependency(library.java.guava))
      }
      relocate("com.google.common", getJavaRelocatedPath("com.google.common")) {
        // com.google.common is too generic, need to exclude guava-testlib
        exclude "com.google.common.collect.testing.**"
        exclude "com.google.common.escape.testing.**"
        exclude "com.google.common.testing.**"
        exclude "com.google.common.util.concurrent.testing.**"
      }
    }

    // Create a new configuration 'shadowTest' like 'shadow' for the test scope
    configurations {
      shadow {
        description = "Dependencies for shaded source set 'main'"
      }
      compile.extendsFrom shadow
      shadowTest {
        description = "Dependencies for shaded source set 'test'"
        extendsFrom shadow
      }
      testCompile.extendsFrom shadowTest
    }

    // Ensure that shaded classes are part of the artifact set.
    artifacts.archives shadowJar

    if (configuration.artifactId) {
      // If a publication artifact id is supplied, publish the shadow jar.
      publishing {
        publications {
          mavenJava(MavenPublication) {
            artifact(shadowJar) {
              groupId "org.apache.beam"
              artifactId configuration.artifactId
              // Strip the "shaded" classifier.
              classifier null
              // Set readable name to project description.
              pom.withXml {
                asNode().appendNode('name', description)
              }
            }
          }
        }
      }
    }

    // TODO: Figure out how to create ShadowJar task for ShadowTestJar here
    // that is extendable within each sub-project with any additional includes.
    // This could mirror the "shadowJar" configuration block.
    // Optionally, we could also copy the shading configuration from the main
    // source set and apply it here.
    //
    // Shading the test jar has a lot less need but can still be beneficial for
    // resolving class conflicts for tests during test execution or exposing
    // test libraries for users.
  }

  // Ban these dependencies from all configurations
  configurations.all {
    // guava-jdk5 brings in classes which conflict with guava
    exclude group: "com.google.guava", module: "guava-jdk5"
    // Ban the usage of the JDK tools as a library as this is system dependent
    exclude group: "jdk.tools", module: "jdk.tools"
    // protobuf-lite duplicates classes which conflict with protobuf-java
    exclude group: "com.google.protobuf", module: "protobuf-lite"
    // Exclude these test dependencies because they bundle other common
    // test libraries classes causing version conflicts. Users should rely
    // on using the yyy-core package instead of the yyy-all package.
    exclude group: "org.hamcrest", module: "hamcrest-all"
    exclude group: "org.mockito", module: "mockito-all"
  }

  // Force usage of the libraries defined within our common set found in the root
  // build.gradle instead of using Gradles default dependency resolution mechanism
  // which chooses the latest version available.
  //
  // TODO: Figure out whether we should force all dependency conflict resolution
  // to occur in the "shadow" and "shadowTest" configurations.
  configurations.all {
    resolutionStrategy {
      force library.java.values()
    }
  }

  // These directories for when build actions are delegated to Gradle
  def gradleAptGeneratedMain = "${project.buildDir}/generated/source/apt/main"
  def gradleAptGeneratedTest = "${project.buildDir}/generated/source/apt/test"

  // These directories for when build actions are executed by Idea
  // IntelliJ does not add these source roots (that it owns!) unless hinted
  def ideaRoot = "${project.projectDir}/out"
  def ideaAptGeneratedMain = "${ideaRoot}/production/classes/generated"
  def ideaAptGeneratedTest = "${ideaRoot}/test/classes/generated_test"

  idea {
    module {
      sourceDirs += file(gradleAptGeneratedMain)
      testSourceDirs += file(gradleAptGeneratedTest)

      sourceDirs += file(ideaAptGeneratedMain)
      testSourceDirs += file(ideaAptGeneratedTest)

      generatedSourceDirs += [
        file(gradleAptGeneratedMain),
        file(gradleAptGeneratedTest),
        file(ideaAptGeneratedMain),
        file(ideaAptGeneratedTest)
      ]

    }
  }
}

/*************************************************************************************************/

ext.applyGoNature = {
  println "applyGoNature with " + (it ? "$it" : "default configuration") + " for project $project.name"
  apply plugin: "com.github.blindpirate.gogradle"
  golang {
    goVersion = '1.9'
  }

  repositories {
    golang {
      // Gogradle doesn't like thrift: https://github.com/gogradle/gogradle/issues/183
      root 'git.apache.org/thrift.git'
      emptyDir()
    }
    golang {
      root 'github.com/apache/thrift'
      emptyDir()
    }
  }

  idea {
    module {
      // The gogradle plugin downloads all dependencies into the source tree here,
      // which is a path baked into golang
      excludeDirs += file("${project.path}/vendor")

      // gogradle's private working directory
      excludeDirs += file("${project.path}/.gogradle")
    }
  }
}

/*************************************************************************************************/

ext.applyDockerNature = {
  println "applyDockerNature with " + (it ? "$it" : "default configuration") + " for project $project.name"
  apply plugin: "com.palantir.docker"
  docker {
    noCache true
  }
}

/*************************************************************************************************/

ext.applyGrpcNature = {
  println "applyGrpcNature with " + (it ? "$it" : "default configuration") + " for project $project.name"
  apply plugin: "com.google.protobuf"
  protobuf {
    protoc {
      // The artifact spec for the Protobuf Compiler
      artifact = "com.google.protobuf:protoc:3.2.0"
    }

    // Configure the codegen plugins
    plugins {
      // An artifact spec for a protoc plugin, with "grpc" as
      // the identifier, which can be referred to in the "plugins"
      // container of the "generateProtoTasks" closure.
      grpc {
        artifact = "io.grpc:protoc-gen-grpc-java:1.2.0"
      }
    }

    generateProtoTasks {
      ofSourceSet("main")*.plugins {
        // Apply the "grpc" plugin whose spec is defined above, without
        // options.  Note the braces cannot be omitted, otherwise the
        // plugin will not be added. This is because of the implicit way
        // NamedDomainObjectContainer binds the methods.
        grpc { }
      }
    }
  }

  def generatedProtoMainJavaDir = "${project.buildDir}/generated/source/proto/main/java"
  def generatedProtoTestJavaDir = "${project.buildDir}/generated/source/proto/test/java"
  def generatedGrpcMainJavaDir = "${project.buildDir}/generated/source/proto/main/grpc"
  def generatedGrpcTestJavaDir = "${project.buildDir}/generated/source/proto/test/grpc"
  idea {
    module {
      sourceDirs += file(generatedProtoMainJavaDir)
      generatedSourceDirs += file(generatedProtoMainJavaDir)

      testSourceDirs += file(generatedProtoTestJavaDir)
      generatedSourceDirs += file(generatedProtoTestJavaDir)

      sourceDirs += file(generatedGrpcMainJavaDir)
      generatedSourceDirs += file(generatedGrpcMainJavaDir)

      testSourceDirs += file(generatedGrpcTestJavaDir)
      generatedSourceDirs += file(generatedGrpcTestJavaDir)
    }
  }
}

/*************************************************************************************************/

// TODO: Decide whether this should be inlined into the one project that relies on it
// or be left here.
ext.applyAvroNature = {
  println "applyAvroNature with " + (it ? "$it" : "default configuration") + " for project $project.name"
  apply plugin: "com.commercehub.gradle.plugin.avro"
}

// A class defining the set of configurable properties for createJavaQuickstartValidationTask
class JavaQuickstartConfiguration {
  // Name for the quickstart is required.
  // Used both for the test name runQuickstartJava${name}
  // and also for the script name, quickstart-java-${name}.toLowerCase().
  String name

  // gcpProject sets the gcpProject argument when executing the quickstart.
  String gcpProject

  // gcsBucket sets the gcsProject argument when executing the quickstart.
  String gcsBucket
}

// Creates a task to run the quickstart for a runner.
// Releases version and URL, can be overriden for a RC release with
// ./gradlew :release:runQuickstartJava -Pver=2.3.0 -Prepourl=https://repository.apache.org/content/repositories/orgapachebeam-1027
ext.createJavaQuickstartValidationTask = {
  JavaQuickstartConfiguration config = it as JavaQuickstartConfiguration
  def taskName = "runQuickstartJava${config.name}"
  println "Generating :${taskName}"
  def releaseVersion = project.findProperty('ver') ?: version
  def releaseRepo = project.findProperty('repourl') ?: 'https://repository.apache.org/content/repositories/snapshots'
  def argsNeeded = ["--ver=${releaseVersion}", "--repourl=${releaseRepo}"]
  if (config.gcpProject) {
    argsNeeded.add("--gcpProject=${config.gcpProject}")
  }
  if (config.gcsBucket) {
    argsNeeded.add("--gcsBucket=${config.gcsBucket}")
  }
  project.evaluationDependsOn(':release')
  task "${taskName}" (dependsOn: ':release:classes', type: JavaExec) {
    main = "quickstart-java-${config.name}".toLowerCase()
    classpath = project(':release').sourceSets.main.runtimeClasspath
    args argsNeeded
  }
}