From 8fa672eaf920b43586ca4b5523154942ac4bd2bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?A=C3=A9cio=20Santos?= Date: Fri, 18 Aug 2023 13:42:24 -0400 Subject: [PATCH] Migrate ache/build.gradle to Kotlin DSL --- ache/build.gradle | 184 ------------------------------------------ ache/build.gradle.kts | 175 +++++++++++++++++++++++++++++++++++++++ build.gradle.kts | 6 -- 3 files changed, 175 insertions(+), 190 deletions(-) delete mode 100644 ache/build.gradle create mode 100644 ache/build.gradle.kts diff --git a/ache/build.gradle b/ache/build.gradle deleted file mode 100644 index cc3b96369..000000000 --- a/ache/build.gradle +++ /dev/null @@ -1,184 +0,0 @@ -plugins { - id 'java' - id 'idea' - id 'application' - id 'jacoco' - id 'com.github.kt3k.coveralls' version '2.12.2' -} - -compileJava.options.encoding = "UTF-8" -version = '0.16.0-SNAPSHOT' - -application { - mainClass = 'achecrawler.Main' - applicationDefaultJvmArgs = ["-Dname=ache -XX:+HeapDumpOnOutOfMemoryError"] -} - -dependencies { - // Sub-projects - implementation project(':crawler-commons') - // Logging - implementation libs.slf4j.api - implementation libs.logback.classic - // Commons - implementation(libs.guava) - implementation(libs.commons.lang3) - implementation(libs.commons.compress) - implementation(libs.commons.codec) - implementation(libs.commons.validator) - implementation(libs.crawler.commons) - // CLI - implementation(libs.airline) - // Data serialization - implementation libs.jackson.core.databind - implementation libs.jackson.dataformat.yaml - implementation libs.jackson.dataformat.cbor - implementation libs.kryo.lib - implementation libs.kryo.serializers - // REST server dependencies - implementation libs.javalin - // Metrics and monitoring - implementation libs.dropwizard.metrics.core - implementation libs.dropwizard.metrics.json - implementation libs.dropwizard.metrics.jvm - // Data management and repositories - implementation(libs.rocksdbjni) - implementation(libs.elasticsearch.rest.client) - implementation(libs.kafka.clients) - implementation(libs.webarchive.commons) { - exclude group: 'org.apache.hadoop', module: 'hadoop-core' - exclude group: 'junit', module: 'junit' - } - // Data parsing and extraction - implementation(libs.boilerpipe) - implementation(libs.nekohtml) - implementation(libs.jsoup) - implementation(libs.lucene.analyzers.common) - // HTTP libraries - implementation(libs.okhttp) - implementation(libs.httpclient) - // Others - implementation libs.smile.core - implementation libs.roaringbitmap - // TODO: fill these in from Maven Central instead of that local libs dir - implementation files('libs/langdetect-03-03-2014.jar') // TODO: upgrade to newer version from maven - implementation files('libs/jsonic-1.2.0.jar') // required by by langdetect-03-03-2014.jar - - // Test framework dependencies - testImplementation libs.junit.api - testImplementation libs.junit.params - testRuntimeOnly libs.junit.engine - testImplementation libs.mockito.core - testImplementation libs.assertj.core -} - -// -// Make sure that ache-dashboard is compiled and copied into resources folder -// before the resources are processed and bundled into the JAR file -// -processResources { - dependsOn ':ache-dashboard:install' -} - - -// -// Adds version to final JAR artifact -// -jar { - manifest { - attributes( - "Implementation-Title": project.name, - "Implementation-Version": archiveVersion - ) - } -} - - -// -// Copies config folder into final distribution file -// -tasks.register("copyConfig") { - // copies config directory to the project build directory - def f = file(project.layout.buildDirectory.dir('config').get()) - outputs.dir f - doLast { - copy { - from rootProject.file("config/").absolutePath - into project.layout.buildDirectory.dir('config').get() - exclude "sample_model", "sample_training_data", "sample.seeds" - } - } -} - -application { - // instructs the application plugin to include config directory into the final build - applicationDistribution.from(copyConfig) { - into "config" - } -} - - -// -// Integration for test coverage service: coveralls -// -jacocoTestReport { - reports { - xml.required = true // coveralls plugin depends on xml format report - html.required = true - } - afterEvaluate { - classDirectories.from = files(classDirectories.files.collect { - fileTree( - dir: it, - exclude: ['achecrawler/tools/**'] - ) - }) - } -} - - -// -// Task to compute total size of dependencies: -// gradle depsize -// Taken from: -// - https://gist.github.com/medvedev/968119d7786966d9ed4442ae17aca279 -// - https://stackoverflow.com/questions/22175847/maven-gradle-way-to-calculate-the-total-size-of-a-dependency-with-all-its-transi -// -tasks.register("depsize") { - description = 'Prints dependencies for "default" configuration' - doLast() { - listConfigurationDependencies(configurations.runtimeClasspath) - } -} - -tasks.register("depsize-all-configurations") { - description = 'Prints dependencies for all available configurations' - doLast() { - configurations - .findAll { it.isCanBeResolved() } - .each { listConfigurationDependencies(it) } - } -} - -def listConfigurationDependencies(Configuration configuration) { - def formatStr = "%,10.2f" - - def size = configuration.collect { it.length() / (1024 * 1024) }.sum() - - def out = new StringBuffer() - out << "\nConfiguration name: \"${configuration.name}\"\n" - if (size) { - out << 'Total dependencies size:'.padRight(65) - out << "${String.format(formatStr, size)} Mb\n\n" - - configuration.sort { -it.length() } - .each { - out << "${it.name}".padRight(65) - out << "${String.format(formatStr, (it.length() / 1024))} kb\n" - } - } else { - out << 'No dependencies found' - } - println(out) -} - diff --git a/ache/build.gradle.kts b/ache/build.gradle.kts new file mode 100644 index 000000000..55dfb96a6 --- /dev/null +++ b/ache/build.gradle.kts @@ -0,0 +1,175 @@ +plugins { + id("java") + id("idea") + id("application") + id("jacoco") + id("com.github.kt3k.coveralls") version "2.12.2" +} + +version = "0.16.0-SNAPSHOT" + +tasks.withType() { + options.encoding = "UTF-8" +} + +application { + mainClass = "achecrawler.Main" + applicationDefaultJvmArgs = listOf("-Dname=ache -XX:+HeapDumpOnOutOfMemoryError") +} + +dependencies { + // Sub-projects + implementation(project(":crawler-commons")) + // Logging + implementation(libs.slf4j.api) + implementation(libs.logback.classic) + // Commons + implementation(libs.guava) + implementation(libs.commons.lang3) + implementation(libs.commons.compress) + implementation(libs.commons.codec) + implementation(libs.commons.validator) + implementation(libs.crawler.commons) + // CLI + implementation(libs.airline) + // Data serialization + implementation(libs.jackson.core.databind) + implementation(libs.jackson.dataformat.yaml) + implementation(libs.jackson.dataformat.cbor) + implementation(libs.kryo.lib) + implementation(libs.kryo.serializers) + // REST server dependencies + implementation(libs.javalin) + // Metrics and monitoring + implementation(libs.dropwizard.metrics.core) + implementation(libs.dropwizard.metrics.json) + implementation(libs.dropwizard.metrics.jvm) + // Data management and repositories + implementation(libs.rocksdbjni) + implementation(libs.elasticsearch.rest.client) + implementation(libs.kafka.clients) + implementation(libs.webarchive.commons) { + exclude(group = "org.apache.hadoop", module = "hadoop-core") + exclude(group = "junit", module = "junit") + } + // Data parsing and extraction + implementation(libs.boilerpipe) + implementation(libs.nekohtml) + implementation(libs.jsoup) + implementation(libs.lucene.analyzers.common) + // HTTP libraries + implementation(libs.okhttp) + implementation(libs.httpclient) + // Others + implementation(libs.smile.core) + implementation(libs.roaringbitmap) + // TODO: fill these in from Maven Central instead of that local libs dir + implementation(files("libs/langdetect-03-03-2014.jar")) // TODO: upgrade to newer version from maven + implementation(files("libs/jsonic-1.2.0.jar")) // required by by langdetect-03-03-2014.jar + + // Test framework dependencies + testImplementation(libs.junit.api) + testImplementation(libs.junit.params) + testRuntimeOnly(libs.junit.engine) + testImplementation(libs.mockito.core) + testImplementation(libs.assertj.core) +} + +// +// Make sure that ache-dashboard is compiled and copied into resources folder +// before the resources are processed and bundled into the JAR file +// +tasks.processResources { + dependsOn(":ache-dashboard:install") +} + + + +// +// Adds version to final JAR artifact +// +tasks.named("jar") { + manifest { + attributes( + "Implementation-Title" to project.name, + "Implementation-Version" to archiveVersion + ) + } +} + + +// +// Copies config folder into final distribution file +// +val copyConfigTask = tasks.register("copyConfig") { + // copies config directory to the project build directory + val outputDir = project.layout.buildDirectory.dir("config").get() + outputs.dir(outputDir) + doLast { + copy { + from(rootProject.file("config/").absolutePath) + into(outputDir) + exclude("sample_model", "sample_training_data", "sample.seeds") + } + } +} + +application { + // instructs the application plugin to include config directory into the final build + applicationDistribution.from(copyConfigTask) { + into("config") + } +} + + +// +// Integration for test coverage service: coveralls +// +tasks.jacocoTestReport { + reports { + xml.required.set(true) // coveralls plugin depends on xml format report + html.required.set(true) + } +} + + +// +// Task to compute total size of dependencies: +// gradle depsize +// Taken from: +// - https://gist.github.com/medvedev/968119d7786966d9ed4442ae17aca279 +// - https://stackoverflow.com/questions/22175847/maven-gradle-way-to-calculate-the-total-size-of-a-dependency-with-all-its-transi +// +tasks.register("depsize") { + description = "Prints dependencies for \"default\" configuration" + doLast { + listConfigurationDependencies(configurations.runtimeClasspath.get()) + } +} + +tasks.register("depsize-all-configurations") { + description = "Prints dependencies for all available configurations" + doLast { + configurations + .filter { it.isCanBeResolved } + .forEach { listConfigurationDependencies(it) } + } +} + +fun listConfigurationDependencies(configuration: Configuration) { + val formatStr = "%,10.2f" + val size = configuration.files.sumOf { it.length() / (1024.0 * 1024.0)} + val out = StringBuilder() + out.appendLine("\nConfiguration name: \"${configuration.name}\"") + if (size > 0) { + out.append("Total dependencies size:".padEnd(65)) + out.appendLine(String.format(formatStr, size.toDouble()) + " Mb\n") + configuration.sortedByDescending { it.length() }.forEach { + out.append(it.name.padEnd(65)) + out.appendLine(String.format(formatStr, (it.length() / 1024).toDouble()) + " kb") + } + } else { + out.appendLine("No dependencies found") + } + println(out) +} diff --git a/build.gradle.kts b/build.gradle.kts index 89c0f2b4f..c116f57aa 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -19,9 +19,3 @@ subprojects { } } - - - - - -