From 1b5e7898f47c37294ef559792f30682171f6f6ee Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Wed, 31 May 2023 21:31:35 +0000 Subject: [PATCH 1/5] Add integration test framework (#687) * Add integration test task that uses the OpenSearch gradle plugin Signed-off-by: Ryan Bogan * Added closeTestExtension task to kill testExtension process Signed-off-by: Ryan Bogan * Fix spotless Signed-off-by: Ryan Bogan * Implement resolution strategy to fix dependency conflict Signed-off-by: Ryan Bogan * Minor adjustment Signed-off-by: Ryan Bogan * Change jackson version Signed-off-by: Ryan Bogan * Change jackson dataformat version Signed-off-by: Ryan Bogan * Revert jackson version change Signed-off-by: Ryan Bogan * Add resolution strategy to fix jackson dependency conflict Signed-off-by: Ryan Bogan * Address PR Comments Signed-off-by: Ryan Bogan * Add new implementation to fix error from separate merge Signed-off-by: Ryan Bogan * Address PR Comments Signed-off-by: Ryan Bogan * Address PR Comments Signed-off-by: Ryan Bogan * Revert minor change Signed-off-by: Ryan Bogan * Address PR Comments Signed-off-by: Ryan Bogan * Add reading from yaml for testExtensionCreation Signed-off-by: Ryan Bogan * Spotless Signed-off-by: Ryan Bogan * Minor change Signed-off-by: Ryan Bogan * Debug test commit Signed-off-by: Ryan Bogan * Minor changes Signed-off-by: Ryan Bogan * Remove extra test class Signed-off-by: Ryan Bogan * Add index Signed-off-by: Ryan Bogan * Fix inadvertent change Signed-off-by: Ryan Bogan * Minor change Signed-off-by: Ryan Bogan * Add log4j resolution strategy Signed-off-by: Ryan Bogan * Remove testing println Signed-off-by: Ryan Bogan --------- Signed-off-by: Ryan Bogan Co-authored-by: Sarat Vemulapalli Co-authored-by: Owais Kazi (cherry picked from commit 04d9eab8ca08acca572f258053e05e18809adf4b) Signed-off-by: github-actions[bot] --- DEVELOPER_GUIDE.md | 11 ++ build.gradle | 182 ++++++++++++++++-- .../resources/sample/helloworld-settings.yml | 2 +- .../TransportCommunicationIT.java | 0 src/test/resources/hello-world-extension.yml | 7 + 5 files changed, 186 insertions(+), 16 deletions(-) rename src/test/java/org/opensearch/sdk/{ => integTest}/TransportCommunicationIT.java (100%) create mode 100644 src/test/resources/hello-world-extension.yml diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index 85fc8065..034c3a0a 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -13,6 +13,7 @@ - [Publish OpenSearch SDK for Java to Maven Local](#publish-opensearch-sdk-for-java-to-maven-local) - [Perform a REST Request on the Extension](#perform-a-rest-request-on-the-extension) - [Run Tests](#run-tests) + - [Launching and debugging from an IDE](#launching-and-debugging-from-an-ide) - [Submitting Changes](#submitting-changes) ## Introduction @@ -221,6 +222,16 @@ Run tests : ``` ./gradlew clean build test ``` + +Run integration tests : +``` +./gradlew clean build integTest +``` + +## Launching and debugging from an IDE + +See [Launching and debugging from an IDE in OpenSearch](https://github.com/opensearch-project/OpenSearch/blob/main/TESTING.md#launching-and-debugging-from-an-ide) + ## Generate Artifact In opensearch-sdk-java navigate to build/distributions. Look for tar ball in the form `opensearch-sdk-java-1.0.0-SNAPSHOT.tar`. If not found follow the below steps to create one: diff --git a/build.gradle b/build.gradle index 12e1d3f4..22bb2ee5 100644 --- a/build.gradle +++ b/build.gradle @@ -11,6 +11,26 @@ import java.nio.file.Files import org.gradle.api.internal.tasks.testing.junitplatform.JUnitPlatformTestFramework +import org.opensearch.gradle.test.RestIntegTestTask +import org.opensearch.gradle.testclusters.ExtensionsProperties + +buildscript { + ext { + opensearch_version = System.getProperty("opensearch.version", "3.0.0-SNAPSHOT") + isSnapshot = "true" == System.getProperty("build.snapshot", "true") + buildVersionQualifier = System.getProperty("build.version_qualifier", "") + } + + repositories { + mavenLocal() + mavenCentral() + maven { url "https://aws.oss.sonatype.org/content/repositories/snapshots" } + } + + dependencies { + classpath "org.opensearch.gradle:build-tools:${opensearch_version}" + } +} plugins { id 'java-library' @@ -22,21 +42,51 @@ plugins { } +apply from: 'gradle/formatting.gradle' +apply plugin: 'maven-publish' +apply plugin: 'java' +apply plugin: 'opensearch.opensearchplugin' +apply plugin: 'opensearch.testclusters' + +opensearchplugin { + name 'opensearch-sdk-java' + description 'OpenSearch Software Development Kit' + classname 'org.opensearch.sdk.sample.helloworld.HelloWorldExtension' +} + ext { projectSubstitutions = [:] licenseFile = rootProject.file('LICENSE.txt') noticeFile = rootProject.file('NOTICE.txt') } +dependencyLicenses.enabled = false +thirdPartyAudit.enabled = false +loggerUsageCheck.enabled = false +forbiddenApisMain.enabled = false +forbiddenApisTest.enabled = false +filepermissions.enabled = false +forbiddenPatterns.enabled = false +testingConventions.enabled = false +validateNebulaPom.enabled = false + +def _numNodes = findProperty('numNodes') as Integer ?: 1 +def testPort = 4500 + +allprojects { + project.ext.licenseName = 'The Apache Software License, Version 2.0' + project.ext.licenseUrl = 'http://www.apache.org/licenses/LICENSE-2.0.txt' + group = 'org.opensearch' + version = opensearch_version.tokenize('-')[0] + '.0' + if (buildVersionQualifier) { + version += "-${buildVersionQualifier}" + } + if (isSnapshot) { + version += "-SNAPSHOT" + } +} -apply plugin: 'application' -apply from: 'gradle/formatting.gradle' -apply plugin: 'maven-publish' -// Temporary to keep "gradle run" working -// TODO: change this to an extension designed for testing instead of duplicating a sample -// https://github.com/opensearch-project/opensearch-sdk-java/issues/175 -mainClassName = 'org.opensearch.sdk.sample.helloworld.HelloWorldExtension' group 'org.opensearch.sdk' @@ -49,14 +99,26 @@ java { publishing { publications { - group = "${group}" - version = "${version}" - mavenJava(MavenPublication) { - from components.java - } - sourceCompatibility = 11 - targetCompatibility = 11 - } + // add license information to generated poms + all { + pom { + name = "opensearch-sdk-java" + description = "OpenSearch SDK" + } + pom.withXml { XmlProvider xml -> + Node node = xml.asNode() + node.appendNode('inceptionYear', '2022') + + Node license = node.appendNode('licenses').appendNode('license') + license.appendNode('name', project.licenseName) + license.appendNode('url', project.licenseUrl) + + Node developer = node.appendNode('developers').appendNode('developer') + developer.appendNode('name', 'opensearch-sdk-java') + developer.appendNode('url', 'https://github.com/opensearch-project/opensearch-sdk-java') + } + } + } repositories { maven { @@ -90,6 +152,11 @@ dependencies { def junit5Version = "5.9.3" def junitPlatform = "1.9.3" def jaxbVersion = "2.3.1" + def jakartaVersion = "2.1.1" + def jakartaInjectVersion = "2.0.1" + def javaxVersion = "1" + def guavaFailureAccessVersion = "1.0.1" + def aopallianceVersion = "1.0" api("org.opensearch:opensearch:${opensearchVersion}") implementation("org.apache.logging.log4j:log4j-api:${log4jVersion}") @@ -100,6 +167,7 @@ dependencies { api("org.opensearch.client:opensearch-java:${opensearchJavaClientVersion}") implementation("org.opensearch.plugin:transport-netty4-client:${opensearchVersion}") implementation("io.netty:netty-all:${nettyVersion}") + implementation("jakarta.inject:jakarta.inject-api:${jakartaInjectVersion}") testCompileOnly("junit:junit:${junit4Version}") { exclude module : 'hamcrest' exclude module : 'hamcrest-core' @@ -109,6 +177,13 @@ dependencies { implementation("com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:${jacksonDatabindVersion}") implementation("com.fasterxml.jackson.datatype:jackson-datatype-jsr310:${jacksonDatabindVersion}") implementation("com.fasterxml.jackson.datatype:jackson-datatype-guava:${jacksonDatabindVersion}") + implementation("com.fasterxml.jackson.core:jackson-annotations:${jacksonDatabindVersion}") + + implementation("com.google.guava:guava:${guavaVersion}") + implementation("javax.inject:javax.inject:${javaxVersion}") + implementation("com.google.guava:failureaccess:${guavaFailureAccessVersion}") + implementation("aopalliance:aopalliance:${aopallianceVersion}") + constraints { implementation("com.google.guava:guava:${guavaVersion}") { because 'versions below 30.0 have active CVE' @@ -119,6 +194,15 @@ dependencies { testRuntimeOnly("org.junit.jupiter:junit-jupiter-engine:${junit5Version}") testImplementation("org.opensearch.test:framework:${opensearchVersion}") testRuntimeOnly("org.junit.platform:junit-platform-launcher:${junitPlatform}") + + configurations.all { + resolutionStrategy.force("jakarta.json:jakarta.json-api:${jakartaVersion}") + resolutionStrategy.force("com.fasterxml.jackson.core:jackson-databind:${jacksonDatabindVersion}") + resolutionStrategy.force("com.fasterxml.jackson.core:jackson-core:${jacksonDatabindVersion}") + resolutionStrategy.force("org.apache.logging.log4j:log4j-api:${log4jVersion}") + resolutionStrategy.force("org.apache.logging.log4j:log4j-core:${log4jVersion}") + resolutionStrategy.force("org.apache.logging.log4j:log4j-jul:${log4jVersion}") + } } // this task runs the helloworld sample extension @@ -183,3 +267,71 @@ diffCoverageReport { failOnViolation = true } } + +task integTest(type: RestIntegTestTask) { + description = "Run tests against a cluster" + testClassesDirs = sourceSets.test.output.classesDirs + classpath = sourceSets.test.runtimeClasspath +} +//tasks.named("check").configure { dependsOn(integTest) } + +task startTestExtension(type: Exec) { + commandLine 'bash', '-c', "./gradlew helloWorld &" +} + +integTest { + dependsOn(startTestExtension) + + //TODO: find a way to add permissions for the tests + systemProperty 'tests.security.manager', 'false' + + systemProperty "https", System.getProperty("https") + systemProperty "user", System.getProperty("user") + systemProperty "password", System.getProperty("password") + + // The 'doFirst' delays till execution time. + doFirst { + // Tell the test JVM if the cluster JVM is running under a debugger so that tests can + // use longer timeouts for requests. + def isDebuggingCluster = getDebug() || System.getProperty("test.debug") != null + systemProperty 'cluster.debug', isDebuggingCluster + // Set number of nodes system property to be used in tests + systemProperty 'cluster.number_of_nodes', "1" + // There seems to be an issue when running multi node run or integ tasks with unicast_hosts + // not being written, the waitForAllConditions ensures it's written + getClusters().forEach { cluster -> + cluster.waitForAllConditions() + } + } + + // The --debug-jvm command-line option makes the cluster debuggable; this makes the tests debuggable + if (System.getProperty("test.debug") != null) { + jvmArgs '-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=*:5005' + } +} + +def testExtensionYml = new org.yaml.snakeyaml.Yaml().load(new File("src/test/resources/hello-world-extension.yml").newInputStream()) + +task closeTestExtension (type: Exec) { + commandLine 'bash', '-c', "kill \$(lsof -i:${testExtensionYml.port})" +} + +tasks.named("integTest").configure { finalizedBy(closeTestExtension) } + +testClusters.integTest { + extension(new ExtensionsProperties("${testExtensionYml.name}", "${testExtensionYml.uniqueId}", "${testExtensionYml.hostAddress}", "${testExtensionYml.port}", "${testExtensionYml.version}", "${testExtensionYml.opensearchVersion}", "${testExtensionYml.minimumCompatibleVersion}")) + testDistribution = "ARCHIVE" + // Cluster shrink exception thrown if we try to set numberOfNodes to 1, so only apply if > 1 + if (_numNodes > 1) numberOfNodes = _numNodes + // When running integration tests it doesn't forward the --debug-jvm to the cluster anymore + // i.e. we have to use a custom property to flag when we want to debug elasticsearch JVM + // since we also support multi node integration tests we increase debugPort per node + if (System.getProperty("opensearch.debug") != null) { + def debugPort = 5005 + nodes.forEach { node -> + node.jvmArgs("-agentlib:jdwp=transport=dt_socket,server=n,suspend=y,address=*:${debugPort}") + debugPort += 1 + } + } + +} diff --git a/src/main/resources/sample/helloworld-settings.yml b/src/main/resources/sample/helloworld-settings.yml index 8d46e86b..15cdbdcf 100644 --- a/src/main/resources/sample/helloworld-settings.yml +++ b/src/main/resources/sample/helloworld-settings.yml @@ -1,6 +1,6 @@ extensionName: hello-world hostAddress: 127.0.0.1 -hostPort: 4532 +hostPort: 4500 opensearchAddress: 127.0.0.1 opensearchPort: 9200 #ssl.transport.enabled: true diff --git a/src/test/java/org/opensearch/sdk/TransportCommunicationIT.java b/src/test/java/org/opensearch/sdk/integTest/TransportCommunicationIT.java similarity index 100% rename from src/test/java/org/opensearch/sdk/TransportCommunicationIT.java rename to src/test/java/org/opensearch/sdk/integTest/TransportCommunicationIT.java diff --git a/src/test/resources/hello-world-extension.yml b/src/test/resources/hello-world-extension.yml new file mode 100644 index 00000000..1ccaec91 --- /dev/null +++ b/src/test/resources/hello-world-extension.yml @@ -0,0 +1,7 @@ +name: hello-world +uniqueId: opensearch-sdk-java-1 +hostAddress: '127.0.0.1' +port: '4500' +version: '1.0' +opensearchVersion: '3.0.0' +minimumCompatibleVersion: '3.0.0' From b0ae56be5ceb58a63cbfc1f78f4c568330de1b37 Mon Sep 17 00:00:00 2001 From: Ryan Bogan Date: Mon, 5 Jun 2023 22:15:09 +0000 Subject: [PATCH 2/5] Change versions and resolve conflicts from original PR Signed-off-by: Ryan Bogan --- build.gradle | 4 +++- src/test/resources/hello-world-extension.yml | 4 ++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/build.gradle b/build.gradle index 9799bd80..ae9192f4 100644 --- a/build.gradle +++ b/build.gradle @@ -16,7 +16,7 @@ import org.opensearch.gradle.testclusters.ExtensionsProperties buildscript { ext { - opensearch_version = System.getProperty("opensearch.version", "3.0.0-SNAPSHOT") + opensearch_version = System.getProperty("opensearch.version", "2.+") isSnapshot = "true" == System.getProperty("build.snapshot", "true") buildVersionQualifier = System.getProperty("build.version_qualifier", "") } @@ -202,6 +202,8 @@ dependencies { resolutionStrategy.force("org.apache.logging.log4j:log4j-api:${log4jVersion}") resolutionStrategy.force("org.apache.logging.log4j:log4j-core:${log4jVersion}") resolutionStrategy.force("org.apache.logging.log4j:log4j-jul:${log4jVersion}") + resolutionStrategy.force("org.opensearch:opensearch:${opensearchVersion}") + resolutionStrategy.force("org.opensearch.client:opensearch-rest-client:${opensearchVersion}") } } diff --git a/src/test/resources/hello-world-extension.yml b/src/test/resources/hello-world-extension.yml index 1ccaec91..a93a96a3 100644 --- a/src/test/resources/hello-world-extension.yml +++ b/src/test/resources/hello-world-extension.yml @@ -3,5 +3,5 @@ uniqueId: opensearch-sdk-java-1 hostAddress: '127.0.0.1' port: '4500' version: '1.0' -opensearchVersion: '3.0.0' -minimumCompatibleVersion: '3.0.0' +opensearchVersion: '2.9.0' +minimumCompatibleVersion: '2.9.0' From b85fad405d9e49804ff9594d89dd25a6426902a6 Mon Sep 17 00:00:00 2001 From: Ryan Bogan Date: Mon, 5 Jun 2023 22:32:03 +0000 Subject: [PATCH 3/5] Fix dependency conflicts Signed-off-by: Ryan Bogan --- build.gradle | 2 ++ 1 file changed, 2 insertions(+) diff --git a/build.gradle b/build.gradle index ae9192f4..685a5cf7 100644 --- a/build.gradle +++ b/build.gradle @@ -167,6 +167,7 @@ dependencies { api("org.opensearch.client:opensearch-java:${opensearchJavaClientVersion}") implementation("org.opensearch.plugin:transport-netty4-client:${opensearchVersion}") implementation("io.netty:netty-all:${nettyVersion}") + implementation("io.netty:netty-codec-http2:${nettyVersion}") implementation("jakarta.inject:jakarta.inject-api:${jakartaInjectVersion}") testCompileOnly("junit:junit:${junit4Version}") { exclude module : 'hamcrest' @@ -204,6 +205,7 @@ dependencies { resolutionStrategy.force("org.apache.logging.log4j:log4j-jul:${log4jVersion}") resolutionStrategy.force("org.opensearch:opensearch:${opensearchVersion}") resolutionStrategy.force("org.opensearch.client:opensearch-rest-client:${opensearchVersion}") + resolutionStrategy.force("org.opensearch.test:framework:${opensearchVersion}") } } From b67665a18d757e607f41d71c4e64126d0b62fcf5 Mon Sep 17 00:00:00 2001 From: Ryan Bogan Date: Mon, 5 Jun 2023 22:53:03 +0000 Subject: [PATCH 4/5] Fix snapshot version Signed-off-by: Ryan Bogan --- build.gradle | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/build.gradle b/build.gradle index 685a5cf7..52749b8f 100644 --- a/build.gradle +++ b/build.gradle @@ -16,7 +16,7 @@ import org.opensearch.gradle.testclusters.ExtensionsProperties buildscript { ext { - opensearch_version = System.getProperty("opensearch.version", "2.+") + opensearch_version = System.getProperty("opensearch.version", "2.9.0-SNAPSHOT") isSnapshot = "true" == System.getProperty("build.snapshot", "true") buildVersionQualifier = System.getProperty("build.version_qualifier", "") } @@ -141,7 +141,7 @@ repositories { dependencies { - def opensearchVersion = "2.+" + def opensearchVersion = "${opensearch_version}" def opensearchJavaClientVersion = "2.1.0-SNAPSHOT" def log4jVersion = "2.20.0" def nettyVersion = "4.1.93.Final" @@ -203,9 +203,7 @@ dependencies { resolutionStrategy.force("org.apache.logging.log4j:log4j-api:${log4jVersion}") resolutionStrategy.force("org.apache.logging.log4j:log4j-core:${log4jVersion}") resolutionStrategy.force("org.apache.logging.log4j:log4j-jul:${log4jVersion}") - resolutionStrategy.force("org.opensearch:opensearch:${opensearchVersion}") resolutionStrategy.force("org.opensearch.client:opensearch-rest-client:${opensearchVersion}") - resolutionStrategy.force("org.opensearch.test:framework:${opensearchVersion}") } } From 9377d5fa32167eaed41e6ec6be91865dd9e829ee Mon Sep 17 00:00:00 2001 From: Ryan Bogan Date: Tue, 6 Jun 2023 16:39:58 +0000 Subject: [PATCH 5/5] Combined commit for backporting 'Fix Publishing' PR's Signed-off-by: Ryan Bogan --- build.gradle | 39 +++++++++++---------------------------- 1 file changed, 11 insertions(+), 28 deletions(-) diff --git a/build.gradle b/build.gradle index 52749b8f..b83edc84 100644 --- a/build.gradle +++ b/build.gradle @@ -43,7 +43,6 @@ plugins { apply from: 'gradle/formatting.gradle' -apply plugin: 'maven-publish' apply plugin: 'java' apply plugin: 'opensearch.opensearchplugin' apply plugin: 'opensearch.testclusters' @@ -87,38 +86,15 @@ allprojects { } - - group 'org.opensearch.sdk' version '1.0.0-SNAPSHOT' -java { - withSourcesJar() - withJavadocJar() -} - publishing { publications { - // add license information to generated poms - all { - pom { - name = "opensearch-sdk-java" - description = "OpenSearch SDK" - } - pom.withXml { XmlProvider xml -> - Node node = xml.asNode() - node.appendNode('inceptionYear', '2022') - - Node license = node.appendNode('licenses').appendNode('license') - license.appendNode('name', project.licenseName) - license.appendNode('url', project.licenseUrl) - - Node developer = node.appendNode('developers').appendNode('developer') - developer.appendNode('name', 'opensearch-sdk-java') - developer.appendNode('url', 'https://github.com/opensearch-project/opensearch-sdk-java') - } - } - } + mavenJava(MavenPublication) { + from components.java + } + } repositories { maven { @@ -132,6 +108,10 @@ publishing { } } +tasks.named("publishMavenJavaPublicationToMavenLocal").configure { dependsOn("generatePomFileForNebulaPublication") } +tasks.named("publishNebulaPublicationToMavenLocal").configure { dependsOn("generatePomFileForMavenJavaPublication") } +validateMavenJavaPom.enabled = false + repositories { mavenLocal() mavenCentral() @@ -200,6 +180,9 @@ dependencies { resolutionStrategy.force("jakarta.json:jakarta.json-api:${jakartaVersion}") resolutionStrategy.force("com.fasterxml.jackson.core:jackson-databind:${jacksonDatabindVersion}") resolutionStrategy.force("com.fasterxml.jackson.core:jackson-core:${jacksonDatabindVersion}") + resolutionStrategy.force("com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:${jacksonDatabindVersion}") + resolutionStrategy.force("com.fasterxml.jackson.dataformat:jackson-dataformat-smile:${jacksonDatabindVersion}") + resolutionStrategy.force("com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:${jacksonDatabindVersion}") resolutionStrategy.force("org.apache.logging.log4j:log4j-api:${log4jVersion}") resolutionStrategy.force("org.apache.logging.log4j:log4j-core:${log4jVersion}") resolutionStrategy.force("org.apache.logging.log4j:log4j-jul:${log4jVersion}")