diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index 85fc8065..034c3a0a 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -13,6 +13,7 @@ - [Publish OpenSearch SDK for Java to Maven Local](#publish-opensearch-sdk-for-java-to-maven-local) - [Perform a REST Request on the Extension](#perform-a-rest-request-on-the-extension) - [Run Tests](#run-tests) + - [Launching and debugging from an IDE](#launching-and-debugging-from-an-ide) - [Submitting Changes](#submitting-changes) ## Introduction @@ -221,6 +222,16 @@ Run tests : ``` ./gradlew clean build test ``` + +Run integration tests : +``` +./gradlew clean build integTest +``` + +## Launching and debugging from an IDE + +See [Launching and debugging from an IDE in OpenSearch](https://github.com/opensearch-project/OpenSearch/blob/main/TESTING.md#launching-and-debugging-from-an-ide) + ## Generate Artifact In opensearch-sdk-java navigate to build/distributions. Look for tar ball in the form `opensearch-sdk-java-1.0.0-SNAPSHOT.tar`. If not found follow the below steps to create one: diff --git a/build.gradle b/build.gradle index 30a26de4..072ab29f 100644 --- a/build.gradle +++ b/build.gradle @@ -11,6 +11,26 @@ import java.nio.file.Files import org.gradle.api.internal.tasks.testing.junitplatform.JUnitPlatformTestFramework +import org.opensearch.gradle.test.RestIntegTestTask +import org.opensearch.gradle.testclusters.ExtensionsProperties + +buildscript { + ext { + opensearch_version = System.getProperty("opensearch.version", "3.0.0-SNAPSHOT") + isSnapshot = "true" == System.getProperty("build.snapshot", "true") + buildVersionQualifier = System.getProperty("build.version_qualifier", "") + } + + repositories { + mavenLocal() + mavenCentral() + maven { url "https://aws.oss.sonatype.org/content/repositories/snapshots" } + } + + dependencies { + classpath "org.opensearch.gradle:build-tools:${opensearch_version}" + } +} plugins { id 'java-library' @@ -22,21 +42,51 @@ plugins { } +apply from: 'gradle/formatting.gradle' +apply plugin: 'maven-publish' +apply plugin: 'java' +apply plugin: 'opensearch.opensearchplugin' +apply plugin: 'opensearch.testclusters' + +opensearchplugin { + name 'opensearch-sdk-java' + description 'OpenSearch Software Development Kit' + classname 'org.opensearch.sdk.sample.helloworld.HelloWorldExtension' +} + ext { projectSubstitutions = [:] licenseFile = rootProject.file('LICENSE.txt') noticeFile = rootProject.file('NOTICE.txt') } +dependencyLicenses.enabled = false +thirdPartyAudit.enabled = false +loggerUsageCheck.enabled = false +forbiddenApisMain.enabled = false +forbiddenApisTest.enabled = false +filepermissions.enabled = false +forbiddenPatterns.enabled = false +testingConventions.enabled = false +validateNebulaPom.enabled = false + +def _numNodes = findProperty('numNodes') as Integer ?: 1 +def testPort = 4500 + +allprojects { + project.ext.licenseName = 'The Apache Software License, Version 2.0' + project.ext.licenseUrl = 'http://www.apache.org/licenses/LICENSE-2.0.txt' + group = 'org.opensearch' + version = opensearch_version.tokenize('-')[0] + '.0' + if (buildVersionQualifier) { + version += "-${buildVersionQualifier}" + } + if (isSnapshot) { + version += "-SNAPSHOT" + } +} -apply plugin: 'application' -apply from: 'gradle/formatting.gradle' -apply plugin: 'maven-publish' -// Temporary to keep "gradle run" working -// TODO: change this to an extension designed for testing instead of duplicating a sample -// https://github.com/opensearch-project/opensearch-sdk-java/issues/175 -mainClassName = 'org.opensearch.sdk.sample.helloworld.HelloWorldExtension' group 'org.opensearch.sdk' @@ -49,14 +99,26 @@ java { publishing { publications { - group = "${group}" - version = "${version}" - mavenJava(MavenPublication) { - from components.java - } - sourceCompatibility = 11 - targetCompatibility = 11 - } + // add license information to generated poms + all { + pom { + name = "opensearch-sdk-java" + description = "OpenSearch SDK" + } + pom.withXml { XmlProvider xml -> + Node node = xml.asNode() + node.appendNode('inceptionYear', '2022') + + Node license = node.appendNode('licenses').appendNode('license') + license.appendNode('name', project.licenseName) + license.appendNode('url', project.licenseUrl) + + Node developer = node.appendNode('developers').appendNode('developer') + developer.appendNode('name', 'opensearch-sdk-java') + developer.appendNode('url', 'https://github.com/opensearch-project/opensearch-sdk-java') + } + } + } repositories { maven { @@ -89,6 +151,11 @@ dependencies { def junit5Version = "5.9.3" def junitPlatform = "1.9.3" def jaxbVersion = "2.3.1" + def jakartaVersion = "2.1.1" + def jakartaInjectVersion = "2.0.1" + def javaxVersion = "1" + def guavaFailureAccessVersion = "1.0.1" + def aopallianceVersion = "1.0" api("org.opensearch:opensearch:${opensearchVersion}") implementation("org.apache.logging.log4j:log4j-api:${log4jVersion}") @@ -99,6 +166,7 @@ dependencies { api("org.opensearch.client:opensearch-java:${opensearchVersion}") implementation("org.opensearch.plugin:transport-netty4-client:${opensearchVersion}") implementation("io.netty:netty-all:${nettyVersion}") + implementation("jakarta.inject:jakarta.inject-api:${jakartaInjectVersion}") testCompileOnly("junit:junit:${junit4Version}") { exclude module : 'hamcrest' exclude module : 'hamcrest-core' @@ -108,6 +176,13 @@ dependencies { implementation("com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:${jacksonDatabindVersion}") implementation("com.fasterxml.jackson.datatype:jackson-datatype-jsr310:${jacksonDatabindVersion}") implementation("com.fasterxml.jackson.datatype:jackson-datatype-guava:${jacksonDatabindVersion}") + implementation("com.fasterxml.jackson.core:jackson-annotations:${jacksonDatabindVersion}") + + implementation("com.google.guava:guava:${guavaVersion}") + implementation("javax.inject:javax.inject:${javaxVersion}") + implementation("com.google.guava:failureaccess:${guavaFailureAccessVersion}") + implementation("aopalliance:aopalliance:${aopallianceVersion}") + constraints { implementation("com.google.guava:guava:${guavaVersion}") { because 'versions below 30.0 have active CVE' @@ -118,6 +193,15 @@ dependencies { testRuntimeOnly("org.junit.jupiter:junit-jupiter-engine:${junit5Version}") testImplementation("org.opensearch.test:framework:${opensearchVersion}") testRuntimeOnly("org.junit.platform:junit-platform-launcher:${junitPlatform}") + + configurations.all { + resolutionStrategy.force("jakarta.json:jakarta.json-api:${jakartaVersion}") + resolutionStrategy.force("com.fasterxml.jackson.core:jackson-databind:${jacksonDatabindVersion}") + resolutionStrategy.force("com.fasterxml.jackson.core:jackson-core:${jacksonDatabindVersion}") + resolutionStrategy.force("org.apache.logging.log4j:log4j-api:${log4jVersion}") + resolutionStrategy.force("org.apache.logging.log4j:log4j-core:${log4jVersion}") + resolutionStrategy.force("org.apache.logging.log4j:log4j-jul:${log4jVersion}") + } } // this task runs the helloworld sample extension @@ -182,3 +266,71 @@ diffCoverageReport { failOnViolation = true } } + +task integTest(type: RestIntegTestTask) { + description = "Run tests against a cluster" + testClassesDirs = sourceSets.test.output.classesDirs + classpath = sourceSets.test.runtimeClasspath +} +//tasks.named("check").configure { dependsOn(integTest) } + +task startTestExtension(type: Exec) { + commandLine 'bash', '-c', "./gradlew helloWorld &" +} + +integTest { + dependsOn(startTestExtension) + + //TODO: find a way to add permissions for the tests + systemProperty 'tests.security.manager', 'false' + + systemProperty "https", System.getProperty("https") + systemProperty "user", System.getProperty("user") + systemProperty "password", System.getProperty("password") + + // The 'doFirst' delays till execution time. + doFirst { + // Tell the test JVM if the cluster JVM is running under a debugger so that tests can + // use longer timeouts for requests. + def isDebuggingCluster = getDebug() || System.getProperty("test.debug") != null + systemProperty 'cluster.debug', isDebuggingCluster + // Set number of nodes system property to be used in tests + systemProperty 'cluster.number_of_nodes', "1" + // There seems to be an issue when running multi node run or integ tasks with unicast_hosts + // not being written, the waitForAllConditions ensures it's written + getClusters().forEach { cluster -> + cluster.waitForAllConditions() + } + } + + // The --debug-jvm command-line option makes the cluster debuggable; this makes the tests debuggable + if (System.getProperty("test.debug") != null) { + jvmArgs '-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=*:5005' + } +} + +def testExtensionYml = new org.yaml.snakeyaml.Yaml().load(new File("src/test/resources/hello-world-extension.yml").newInputStream()) + +task closeTestExtension (type: Exec) { + commandLine 'bash', '-c', "kill \$(lsof -i:${testExtensionYml.port})" +} + +tasks.named("integTest").configure { finalizedBy(closeTestExtension) } + +testClusters.integTest { + extension(new ExtensionsProperties("${testExtensionYml.name}", "${testExtensionYml.uniqueId}", "${testExtensionYml.hostAddress}", "${testExtensionYml.port}", "${testExtensionYml.version}", "${testExtensionYml.opensearchVersion}", "${testExtensionYml.minimumCompatibleVersion}")) + testDistribution = "ARCHIVE" + // Cluster shrink exception thrown if we try to set numberOfNodes to 1, so only apply if > 1 + if (_numNodes > 1) numberOfNodes = _numNodes + // When running integration tests it doesn't forward the --debug-jvm to the cluster anymore + // i.e. we have to use a custom property to flag when we want to debug elasticsearch JVM + // since we also support multi node integration tests we increase debugPort per node + if (System.getProperty("opensearch.debug") != null) { + def debugPort = 5005 + nodes.forEach { node -> + node.jvmArgs("-agentlib:jdwp=transport=dt_socket,server=n,suspend=y,address=*:${debugPort}") + debugPort += 1 + } + } + +} diff --git a/src/main/resources/sample/helloworld-settings.yml b/src/main/resources/sample/helloworld-settings.yml index 8d46e86b..15cdbdcf 100644 --- a/src/main/resources/sample/helloworld-settings.yml +++ b/src/main/resources/sample/helloworld-settings.yml @@ -1,6 +1,6 @@ extensionName: hello-world hostAddress: 127.0.0.1 -hostPort: 4532 +hostPort: 4500 opensearchAddress: 127.0.0.1 opensearchPort: 9200 #ssl.transport.enabled: true diff --git a/src/test/java/org/opensearch/sdk/TransportCommunicationIT.java b/src/test/java/org/opensearch/sdk/integTest/TransportCommunicationIT.java similarity index 100% rename from src/test/java/org/opensearch/sdk/TransportCommunicationIT.java rename to src/test/java/org/opensearch/sdk/integTest/TransportCommunicationIT.java diff --git a/src/test/resources/hello-world-extension.yml b/src/test/resources/hello-world-extension.yml new file mode 100644 index 00000000..1ccaec91 --- /dev/null +++ b/src/test/resources/hello-world-extension.yml @@ -0,0 +1,7 @@ +name: hello-world +uniqueId: opensearch-sdk-java-1 +hostAddress: '127.0.0.1' +port: '4500' +version: '1.0' +opensearchVersion: '3.0.0' +minimumCompatibleVersion: '3.0.0'