From 6a7629aadb3bba01a469ac1f967c0c63196aa1c5 Mon Sep 17 00:00:00 2001 From: Ryan Bogan Date: Tue, 18 Apr 2023 18:41:37 +0000 Subject: [PATCH 01/26] Add integration test task that uses the OpenSearch gradle plugin Signed-off-by: Ryan Bogan --- build.gradle | 169 ++++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 153 insertions(+), 16 deletions(-) diff --git a/build.gradle b/build.gradle index 87050bd5..957e7233 100644 --- a/build.gradle +++ b/build.gradle @@ -11,6 +11,26 @@ import java.nio.file.Files import org.gradle.api.internal.tasks.testing.junitplatform.JUnitPlatformTestFramework +import org.opensearch.gradle.test.RestIntegTestTask +import org.opensearch.gradle.testclusters.ExtensionsProperties + +buildscript { + ext { + opensearch_version = System.getProperty("opensearch.version", "3.0.0-SNAPSHOT") + isSnapshot = "true" == System.getProperty("build.snapshot", "true") + buildVersionQualifier = System.getProperty("build.version_qualifier", "") + } + + repositories { + mavenLocal() + mavenCentral() + maven { url "https://aws.oss.sonatype.org/content/repositories/snapshots" } + } + + dependencies { + classpath "org.opensearch.gradle:build-tools:${opensearch_version}" + } +} plugins { id 'java' @@ -22,21 +42,49 @@ plugins { } +apply from: 'gradle/formatting.gradle' +apply plugin: 'maven-publish' +apply plugin: 'opensearch.opensearchplugin' +apply plugin: 'opensearch.testclusters' + +opensearchplugin { + name 'opensearch-sdk-java' + description 'OpenSearch Software Development Kit' + classname 'Sample Class' +} + ext { projectSubstitutions = [:] licenseFile = rootProject.file('LICENSE.txt') noticeFile = rootProject.file('NOTICE.txt') } +dependencyLicenses.enabled = false +thirdPartyAudit.enabled = false +loggerUsageCheck.enabled = false +forbiddenApisMain.enabled = false +forbiddenApisTest.enabled = false +filepermissions.enabled = false +forbiddenPatterns.enabled = false +testingConventions.enabled = false +validateNebulaPom.enabled = false + +def _numNodes = 1 + +allprojects { + project.ext.licenseName = 'The Apache Software License, Version 2.0' + project.ext.licenseUrl = 'http://www.apache.org/licenses/LICENSE-2.0.txt' + group = 'org.opensearch' + version = opensearch_version.tokenize('-')[0] + '.0' + if (buildVersionQualifier) { + version += "-${buildVersionQualifier}" + } + if (isSnapshot) { + version += "-SNAPSHOT" + } +} -apply plugin: 'application' -apply from: 'gradle/formatting.gradle' -apply plugin: 'maven-publish' -// Temporary to keep "gradle run" working -// TODO: change this to an extension designed for testing instead of duplicating a sample -// https://github.com/opensearch-project/opensearch-sdk-java/issues/175 -mainClassName = 'org.opensearch.sdk.sample.helloworld.HelloWorldExtension' group 'org.opensearch.sdk' @@ -45,14 +93,26 @@ version '2.0.0-SNAPSHOT' publishing { publications { - group = "${group}" - version = "${version}" - mavenJava(MavenPublication) { - from components.java - } - sourceCompatibility = 11 - targetCompatibility = 11 - } + // add license information to generated poms + all { + pom { + name = "opensearch-sdk-java" + description = "OpenSearch SDK" + } + pom.withXml { XmlProvider xml -> + Node node = xml.asNode() + node.appendNode('inceptionYear', '2022') + + Node license = node.appendNode('licenses').appendNode('license') + license.appendNode('name', project.licenseName) + license.appendNode('url', project.licenseUrl) + + Node developer = node.appendNode('developers').appendNode('developer') + developer.appendNode('name', 'OpenSearch') + developer.appendNode('url', 'https://github.com/opensearch-project/opensearch-sdk-java') + } + } + } repositories { maven { @@ -76,7 +136,7 @@ repositories { dependencies { def opensearchVersion = "3.0.0-SNAPSHOT" - def log4jVersion = "2.20.0" + def log4jVersion = "2.17.1" def nettyVersion = "4.1.90.Final" def jacksonDatabindVersion = "2.14.2" def guavaVersion = "31.1-jre" @@ -104,6 +164,13 @@ dependencies { implementation("com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:${jacksonDatabindVersion}") implementation("com.fasterxml.jackson.datatype:jackson-datatype-jsr310:${jacksonDatabindVersion}") implementation("com.fasterxml.jackson.datatype:jackson-datatype-guava:${jacksonDatabindVersion}") + implementation("com.fasterxml.jackson.core:jackson-annotations:${jacksonDatabindVersion}") + + implementation("com.google.guava:guava:${guavaVersion}") + implementation('javax.inject:javax.inject:1') + implementation('com.google.guava:failureaccess:1.0.1') + implementation('aopalliance:aopalliance:1.0') + constraints { implementation("com.google.guava:guava:${guavaVersion}") { because 'versions below 30.0 have active CVE' @@ -179,3 +246,73 @@ diffCoverageReport { failOnViolation = true } } + +task integTest(type: RestIntegTestTask) { + description = "Run tests against a cluster" + testClassesDirs = sourceSets.test.output.classesDirs + classpath = sourceSets.test.runtimeClasspath +} +tasks.named("check").configure { dependsOn(integTest) } + +task startTestExtension(type: Exec) { + commandLine 'bash', '-c', "./gradlew helloWorld &" +} + +integTest { + dependsOn(startTestExtension) + systemProperty 'tests.security.manager', 'false' + + systemProperty "https", System.getProperty("https") + systemProperty "user", System.getProperty("user") + systemProperty "password", System.getProperty("password") + systemProperty "OPENSEARCH_JAVA_OPTS", "-Dopensearch.experimental.feature.extensions.enabled=true" + + filter { + includeTestsMatching "org.opensearch.sdk.TestExtensionsRunner" + } + + + // The 'doFirst' delays till execution time. + doFirst { + // Tell the test JVM if the cluster JVM is running under a debugger so that tests can + // use longer timeouts for requests. + def isDebuggingCluster = getDebug() || System.getProperty("test.debug") != null + systemProperty 'cluster.debug', isDebuggingCluster + // Set number of nodes system property to be used in tests + systemProperty 'cluster.number_of_nodes', "1" + // There seems to be an issue when running multi node run or integ tasks with unicast_hosts + // not being written, the waitForAllConditions ensures it's written + getClusters().forEach { cluster -> + cluster.waitForAllConditions() + } + } + + doLast { + exec { + commandLine 'kill', '$(lsof', '-t', '-i:4532)' + } + } + + // The --debug-jvm command-line option makes the cluster debuggable; this makes the tests debuggable + if (System.getProperty("test.debug") != null) { + jvmArgs '-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=*:5005' + } +} + +testClusters.integTest { + extension(new ExtensionsProperties("hello-world", "opensearch-sdk-java-1", "127.0.0.1", "4532", "1.0", "3.0.0", "3.0.0")) + testDistribution = "ARCHIVE" + // Cluster shrink exception thrown if we try to set numberOfNodes to 1, so only apply if > 1 + if (_numNodes > 1) numberOfNodes = _numNodes + // When running integration tests it doesn't forward the --debug-jvm to the cluster anymore + // i.e. we have to use a custom property to flag when we want to debug elasticsearch JVM + // since we also support multi node integration tests we increase debugPort per node + if (System.getProperty("opensearch.debug") != null) { + def debugPort = 5005 + nodes.forEach { node -> + node.jvmArgs("-agentlib:jdwp=transport=dt_socket,server=n,suspend=y,address=*:${debugPort}") + debugPort += 1 + } + } + +} From 52dda1c8b5e23cf2c52284a77a226707f58d8c10 Mon Sep 17 00:00:00 2001 From: Ryan Bogan Date: Wed, 19 Apr 2023 21:31:40 +0000 Subject: [PATCH 02/26] Added closeTestExtension task to kill testExtension process Signed-off-by: Ryan Bogan --- build.gradle | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/build.gradle b/build.gradle index da894ac0..9ceee2f9 100644 --- a/build.gradle +++ b/build.gradle @@ -136,7 +136,7 @@ repositories { dependencies { def opensearchVersion = "3.0.0-SNAPSHOT" - def log4jVersion = "2.20.0" + def log4jVersion = "2.17.1" def nettyVersion = "4.1.91.Final" def jacksonDatabindVersion = "2.14.2" def guavaVersion = "31.1-jre" @@ -265,7 +265,6 @@ integTest { systemProperty "https", System.getProperty("https") systemProperty "user", System.getProperty("user") systemProperty "password", System.getProperty("password") - systemProperty "OPENSEARCH_JAVA_OPTS", "-Dopensearch.experimental.feature.extensions.enabled=true" filter { includeTestsMatching "org.opensearch.sdk.TestExtensionsRunner" @@ -287,18 +286,18 @@ integTest { } } - doLast { - exec { - commandLine 'kill', '$(lsof', '-t', '-i:4532)' - } - } - // The --debug-jvm command-line option makes the cluster debuggable; this makes the tests debuggable if (System.getProperty("test.debug") != null) { jvmArgs '-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=*:5005' } } +task closeTestExtension (type: Exec) { + commandLine 'bash', '-c', "kill \$(lsof -i:4532)" +} + +tasks.named("integTest").configure { finalizedBy(closeTestExtension) } + testClusters.integTest { extension(new ExtensionsProperties("hello-world", "opensearch-sdk-java-1", "127.0.0.1", "4532", "1.0", "3.0.0", "3.0.0")) testDistribution = "ARCHIVE" From 17be5bfd413a10a665044cda5aee7db55bf0bb53 Mon Sep 17 00:00:00 2001 From: Ryan Bogan Date: Wed, 19 Apr 2023 21:59:45 +0000 Subject: [PATCH 03/26] Fix spotless Signed-off-by: Ryan Bogan --- build.gradle | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/build.gradle b/build.gradle index 9ceee2f9..734a73a3 100644 --- a/build.gradle +++ b/build.gradle @@ -255,7 +255,7 @@ task integTest(type: RestIntegTestTask) { tasks.named("check").configure { dependsOn(integTest) } task startTestExtension(type: Exec) { - commandLine 'bash', '-c', "./gradlew helloWorld &" + commandLine 'bash', '-c', "./gradlew helloWorld &" } integTest { @@ -266,11 +266,6 @@ integTest { systemProperty "user", System.getProperty("user") systemProperty "password", System.getProperty("password") - filter { - includeTestsMatching "org.opensearch.sdk.TestExtensionsRunner" - } - - // The 'doFirst' delays till execution time. doFirst { // Tell the test JVM if the cluster JVM is running under a debugger so that tests can @@ -313,5 +308,5 @@ testClusters.integTest { debugPort += 1 } } - + } From 3d96e323268307c54f162848c2a06f9506121ae9 Mon Sep 17 00:00:00 2001 From: Ryan Bogan Date: Mon, 8 May 2023 18:11:27 +0000 Subject: [PATCH 04/26] Implement resolution strategy to fix dependency conflict Signed-off-by: Ryan Bogan --- build.gradle | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index f07f2787..eabc0a65 100644 --- a/build.gradle +++ b/build.gradle @@ -137,10 +137,14 @@ repositories { maven { url "https://d1nvenhzbhpy0q.cloudfront.net/snapshots/lucene/"} } +configurations.all { + resolutionStrategy.force("jakarta.json:jakarta.json-api:2.1.1") +} + dependencies { def opensearchVersion = "3.0.0-SNAPSHOT" - def log4jVersion = "2.20.0" + def log4jVersion = "2.17.1" def nettyVersion = "4.1.92.Final" def jacksonDatabindVersion = "2.15.0" def guavaVersion = "31.1-jre" From af6904eb9935e071fac86db7d1ef4ff45950ab36 Mon Sep 17 00:00:00 2001 From: Ryan Bogan Date: Tue, 9 May 2023 18:06:52 +0000 Subject: [PATCH 05/26] Minor adjustment Signed-off-by: Ryan Bogan --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 48136479..d175388c 100644 --- a/build.gradle +++ b/build.gradle @@ -259,7 +259,7 @@ task integTest(type: RestIntegTestTask) { testClassesDirs = sourceSets.test.output.classesDirs classpath = sourceSets.test.runtimeClasspath } -tasks.named("check").configure { dependsOn(integTest) } +//tasks.named("check").configure { dependsOn(integTest) } task startTestExtension(type: Exec) { commandLine 'bash', '-c', "./gradlew helloWorld &" From d0c9ed888a0956a86dcf950a2a37031217dd4743 Mon Sep 17 00:00:00 2001 From: Ryan Bogan Date: Wed, 10 May 2023 18:41:43 +0000 Subject: [PATCH 06/26] Change jackson version Signed-off-by: Ryan Bogan --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 8ebd5402..6e1a9578 100644 --- a/build.gradle +++ b/build.gradle @@ -146,7 +146,7 @@ dependencies { def opensearchVersion = "3.0.0-SNAPSHOT" def log4jVersion = "2.17.1" def nettyVersion = "4.1.92.Final" - def jacksonDatabindVersion = "2.15.0" + def jacksonDatabindVersion = "2.14.2" def guavaVersion = "31.1-jre" def guiceVersion = "5.1.0" def junit4Version = "4.13.2" From 78a0c3b5d73991e19eee1a34be94d600dfd666a4 Mon Sep 17 00:00:00 2001 From: Ryan Bogan Date: Wed, 10 May 2023 18:44:25 +0000 Subject: [PATCH 07/26] Change jackson dataformat version Signed-off-by: Ryan Bogan --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 6e1a9578..38ee5acc 100644 --- a/build.gradle +++ b/build.gradle @@ -169,7 +169,7 @@ dependencies { } implementation("javax.xml.bind:jaxb-api:${jaxbVersion}") implementation("com.fasterxml.jackson.core:jackson-databind:${jacksonDatabindVersion}") - implementation("com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:${jacksonDatabindVersion}") + implementation("com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.15.0") implementation("com.fasterxml.jackson.datatype:jackson-datatype-jsr310:${jacksonDatabindVersion}") implementation("com.fasterxml.jackson.datatype:jackson-datatype-guava:${jacksonDatabindVersion}") implementation("com.fasterxml.jackson.core:jackson-annotations:${jacksonDatabindVersion}") From 476263e72a42d47a14d57355accedb57b9551ec4 Mon Sep 17 00:00:00 2001 From: Ryan Bogan Date: Wed, 10 May 2023 18:56:25 +0000 Subject: [PATCH 08/26] Revert jackson version change Signed-off-by: Ryan Bogan --- build.gradle | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build.gradle b/build.gradle index 38ee5acc..8ebd5402 100644 --- a/build.gradle +++ b/build.gradle @@ -146,7 +146,7 @@ dependencies { def opensearchVersion = "3.0.0-SNAPSHOT" def log4jVersion = "2.17.1" def nettyVersion = "4.1.92.Final" - def jacksonDatabindVersion = "2.14.2" + def jacksonDatabindVersion = "2.15.0" def guavaVersion = "31.1-jre" def guiceVersion = "5.1.0" def junit4Version = "4.13.2" @@ -169,7 +169,7 @@ dependencies { } implementation("javax.xml.bind:jaxb-api:${jaxbVersion}") implementation("com.fasterxml.jackson.core:jackson-databind:${jacksonDatabindVersion}") - implementation("com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.15.0") + implementation("com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:${jacksonDatabindVersion}") implementation("com.fasterxml.jackson.datatype:jackson-datatype-jsr310:${jacksonDatabindVersion}") implementation("com.fasterxml.jackson.datatype:jackson-datatype-guava:${jacksonDatabindVersion}") implementation("com.fasterxml.jackson.core:jackson-annotations:${jacksonDatabindVersion}") From 1b602a56e0c47aed24bea8f8df72e7ba2fe8fb07 Mon Sep 17 00:00:00 2001 From: Ryan Bogan Date: Fri, 12 May 2023 16:54:20 +0000 Subject: [PATCH 09/26] Add resolution strategy to fix jackson dependency conflict Signed-off-by: Ryan Bogan --- build.gradle | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/build.gradle b/build.gradle index 8ebd5402..e9d803af 100644 --- a/build.gradle +++ b/build.gradle @@ -137,10 +137,6 @@ repositories { maven { url "https://d1nvenhzbhpy0q.cloudfront.net/snapshots/lucene/"} } -configurations.all { - resolutionStrategy.force("jakarta.json:jakarta.json-api:2.1.1") -} - dependencies { def opensearchVersion = "3.0.0-SNAPSHOT" @@ -189,6 +185,12 @@ dependencies { testRuntimeOnly("org.junit.jupiter:junit-jupiter-engine:${junit5Version}") testImplementation("org.opensearch.test:framework:${opensearchVersion}") testRuntimeOnly("org.junit.platform:junit-platform-launcher:${junitPlatform}") + + configurations.all { + resolutionStrategy.force("jakarta.json:jakarta.json-api:2.1.1") + resolutionStrategy.force("com.fasterxml.jackson.core:jackson-databind:${jacksonDatabindVersion}") + resolutionStrategy.force("com.fasterxml.jackson.core:jackson-core:${jacksonDatabindVersion}") + } } // this task runs the helloworld sample extension From e1c389b92cd80c2998bd609c574ca9453665dda9 Mon Sep 17 00:00:00 2001 From: Ryan Bogan Date: Fri, 12 May 2023 18:28:31 +0000 Subject: [PATCH 10/26] Address PR Comments Signed-off-by: Ryan Bogan --- build.gradle | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/build.gradle b/build.gradle index e9d803af..497812c3 100644 --- a/build.gradle +++ b/build.gradle @@ -50,7 +50,7 @@ apply plugin: 'opensearch.testclusters' opensearchplugin { name 'opensearch-sdk-java' description 'OpenSearch Software Development Kit' - classname 'Sample Class' + classname 'org.opensearch.sdk.ExtensionsRunner' } ext { @@ -149,6 +149,7 @@ dependencies { def junit5Version = "5.9.3" def junitPlatform = "1.9.3" def jaxbVersion = "2.3.1" + def jakartaVersion="2.1.1" api("org.opensearch:opensearch:${opensearchVersion}") implementation("org.apache.logging.log4j:log4j-api:${log4jVersion}") @@ -187,7 +188,7 @@ dependencies { testRuntimeOnly("org.junit.platform:junit-platform-launcher:${junitPlatform}") configurations.all { - resolutionStrategy.force("jakarta.json:jakarta.json-api:2.1.1") + resolutionStrategy.force("jakarta.json:jakarta.json-api:${jakartaVersion}") resolutionStrategy.force("com.fasterxml.jackson.core:jackson-databind:${jacksonDatabindVersion}") resolutionStrategy.force("com.fasterxml.jackson.core:jackson-core:${jacksonDatabindVersion}") } From 9f37e544dadff6d86bdc287e65990b1a300f83d9 Mon Sep 17 00:00:00 2001 From: Ryan Bogan Date: Mon, 15 May 2023 21:10:10 +0000 Subject: [PATCH 11/26] Add new implementation to fix error from separate merge Signed-off-by: Ryan Bogan --- build.gradle | 1 + 1 file changed, 1 insertion(+) diff --git a/build.gradle b/build.gradle index 1b9c65a4..662c0f5b 100644 --- a/build.gradle +++ b/build.gradle @@ -160,6 +160,7 @@ dependencies { api("org.opensearch.client:opensearch-java:${opensearchVersion}") implementation("org.opensearch.plugin:transport-netty4-client:${opensearchVersion}") implementation("io.netty:netty-all:${nettyVersion}") + implementation("jakarta.inject:jakarta.inject-api:2.0.0") testCompileOnly("junit:junit:${junit4Version}") { exclude module : 'hamcrest' exclude module : 'hamcrest-core' From e2d16e00de9c3936e91559590674ea88fbe242aa Mon Sep 17 00:00:00 2001 From: Ryan Bogan Date: Tue, 16 May 2023 19:39:40 +0000 Subject: [PATCH 12/26] Address PR Comments Signed-off-by: Ryan Bogan --- build.gradle | 26 ++++++++++++------- .../resources/sample/helloworld-settings.yml | 2 +- 2 files changed, 17 insertions(+), 11 deletions(-) diff --git a/build.gradle b/build.gradle index 662c0f5b..8402bb01 100644 --- a/build.gradle +++ b/build.gradle @@ -60,7 +60,6 @@ ext { } dependencyLicenses.enabled = false -thirdPartyAudit.enabled = false loggerUsageCheck.enabled = false forbiddenApisMain.enabled = false forbiddenApisTest.enabled = false @@ -69,7 +68,8 @@ forbiddenPatterns.enabled = false testingConventions.enabled = false validateNebulaPom.enabled = false -def _numNodes = 1 +def _numNodes = findProperty('numNodes') as Integer ?: 1 +def testPort = 4500 allprojects { project.ext.licenseName = 'The Apache Software License, Version 2.0' @@ -112,7 +112,7 @@ publishing { license.appendNode('url', project.licenseUrl) Node developer = node.appendNode('developers').appendNode('developer') - developer.appendNode('name', 'OpenSearch') + developer.appendNode('name', 'opensearch-sdk-java') developer.appendNode('url', 'https://github.com/opensearch-project/opensearch-sdk-java') } } @@ -149,7 +149,11 @@ dependencies { def junit5Version = "5.9.3" def junitPlatform = "1.9.3" def jaxbVersion = "2.3.1" - def jakartaVersion="2.1.1" + def jakartaVersion = "2.1.1" + def jakartaInjectVersion = "2.0.1" + def javaxVersion = "1" + def guavaFailureAccessVersion = "1.0.1" + def aopallianceVersion = "1.0" api("org.opensearch:opensearch:${opensearchVersion}") implementation("org.apache.logging.log4j:log4j-api:${log4jVersion}") @@ -160,7 +164,7 @@ dependencies { api("org.opensearch.client:opensearch-java:${opensearchVersion}") implementation("org.opensearch.plugin:transport-netty4-client:${opensearchVersion}") implementation("io.netty:netty-all:${nettyVersion}") - implementation("jakarta.inject:jakarta.inject-api:2.0.0") + implementation("jakarta.inject:jakarta.inject-api:${jakartaInjectVersion}") testCompileOnly("junit:junit:${junit4Version}") { exclude module : 'hamcrest' exclude module : 'hamcrest-core' @@ -173,9 +177,9 @@ dependencies { implementation("com.fasterxml.jackson.core:jackson-annotations:${jacksonDatabindVersion}") implementation("com.google.guava:guava:${guavaVersion}") - implementation('javax.inject:javax.inject:1') - implementation('com.google.guava:failureaccess:1.0.1') - implementation('aopalliance:aopalliance:1.0') + implementation("javax.inject:javax.inject:${javaxVersion}") + implementation("com.google.guava:failureaccess:${guavaFailureAccessVersion}") + implementation("aopalliance:aopalliance:${aopallianceVersion}") constraints { implementation("com.google.guava:guava:${guavaVersion}") { @@ -271,6 +275,8 @@ task startTestExtension(type: Exec) { integTest { dependsOn(startTestExtension) + + //TODO: find a way to add permissions for the tests systemProperty 'tests.security.manager', 'false' systemProperty "https", System.getProperty("https") @@ -299,13 +305,13 @@ integTest { } task closeTestExtension (type: Exec) { - commandLine 'bash', '-c', "kill \$(lsof -i:4532)" + commandLine 'bash', '-c', "kill \$(lsof -i:${testPort})" } tasks.named("integTest").configure { finalizedBy(closeTestExtension) } testClusters.integTest { - extension(new ExtensionsProperties("hello-world", "opensearch-sdk-java-1", "127.0.0.1", "4532", "1.0", "3.0.0", "3.0.0")) + extension(new ExtensionsProperties("hello-world", "opensearch-sdk-java-1", "127.0.0.1", "${testPort}", "1.0", "3.0.0", "3.0.0")) testDistribution = "ARCHIVE" // Cluster shrink exception thrown if we try to set numberOfNodes to 1, so only apply if > 1 if (_numNodes > 1) numberOfNodes = _numNodes diff --git a/src/main/resources/sample/helloworld-settings.yml b/src/main/resources/sample/helloworld-settings.yml index 8d46e86b..15cdbdcf 100644 --- a/src/main/resources/sample/helloworld-settings.yml +++ b/src/main/resources/sample/helloworld-settings.yml @@ -1,6 +1,6 @@ extensionName: hello-world hostAddress: 127.0.0.1 -hostPort: 4532 +hostPort: 4500 opensearchAddress: 127.0.0.1 opensearchPort: 9200 #ssl.transport.enabled: true From 660e29ce4fa1d556c9291053d32e377fbfb123b2 Mon Sep 17 00:00:00 2001 From: Ryan Bogan Date: Tue, 16 May 2023 19:41:30 +0000 Subject: [PATCH 13/26] Address PR Comments Signed-off-by: Ryan Bogan --- build.gradle | 9 --------- 1 file changed, 9 deletions(-) diff --git a/build.gradle b/build.gradle index 8402bb01..9f8f1ad9 100644 --- a/build.gradle +++ b/build.gradle @@ -59,15 +59,6 @@ ext { noticeFile = rootProject.file('NOTICE.txt') } -dependencyLicenses.enabled = false -loggerUsageCheck.enabled = false -forbiddenApisMain.enabled = false -forbiddenApisTest.enabled = false -filepermissions.enabled = false -forbiddenPatterns.enabled = false -testingConventions.enabled = false -validateNebulaPom.enabled = false - def _numNodes = findProperty('numNodes') as Integer ?: 1 def testPort = 4500 From 71c92bebb0eaf5f626fb13bd61d01aa692245756 Mon Sep 17 00:00:00 2001 From: Ryan Bogan Date: Tue, 16 May 2023 19:46:46 +0000 Subject: [PATCH 14/26] Revert minor change Signed-off-by: Ryan Bogan --- build.gradle | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/build.gradle b/build.gradle index 9f8f1ad9..277b6cf8 100644 --- a/build.gradle +++ b/build.gradle @@ -59,6 +59,16 @@ ext { noticeFile = rootProject.file('NOTICE.txt') } +dependencyLicenses.enabled = false +thirdPartyAudit.enabled = false +loggerUsageCheck.enabled = false +forbiddenApisMain.enabled = false +forbiddenApisTest.enabled = false +filepermissions.enabled = false +forbiddenPatterns.enabled = false +testingConventions.enabled = false +validateNebulaPom.enabled = false + def _numNodes = findProperty('numNodes') as Integer ?: 1 def testPort = 4500 From e2485fb544b462379966f6817754e6ea25687d6b Mon Sep 17 00:00:00 2001 From: Ryan Bogan Date: Tue, 16 May 2023 20:11:03 +0000 Subject: [PATCH 15/26] Address PR Comments Signed-off-by: Ryan Bogan --- DEVELOPER_GUIDE.md | 10 ++++++++++ .../sdk/{ => integTest}/TransportCommunicationIT.java | 0 2 files changed, 10 insertions(+) rename src/test/java/org/opensearch/sdk/{ => integTest}/TransportCommunicationIT.java (100%) diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index 85fc8065..31da6a89 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -221,6 +221,16 @@ Run tests : ``` ./gradlew clean build test ``` + +Run integration tests : +``` +./gradlew clean build integTest +``` + +## Launching and debugging from an IDE + +See [Launching and debugging from an IDE in OpenSearch](https://github.com/opensearch-project/OpenSearch/blob/main/TESTING.md#launching-and-debugging-from-an-ide) + ## Generate Artifact In opensearch-sdk-java navigate to build/distributions. Look for tar ball in the form `opensearch-sdk-java-1.0.0-SNAPSHOT.tar`. If not found follow the below steps to create one: diff --git a/src/test/java/org/opensearch/sdk/TransportCommunicationIT.java b/src/test/java/org/opensearch/sdk/integTest/TransportCommunicationIT.java similarity index 100% rename from src/test/java/org/opensearch/sdk/TransportCommunicationIT.java rename to src/test/java/org/opensearch/sdk/integTest/TransportCommunicationIT.java From 85d48ba013e43b8a3f5b390e991093a548f7d1e1 Mon Sep 17 00:00:00 2001 From: Ryan Bogan Date: Tue, 16 May 2023 21:15:17 +0000 Subject: [PATCH 16/26] Add reading from yaml for testExtensionCreation Signed-off-by: Ryan Bogan --- build.gradle | 6 ++++-- src/test/resources/hello-world-extension.yml | 7 +++++++ 2 files changed, 11 insertions(+), 2 deletions(-) create mode 100644 src/test/resources/hello-world-extension.yml diff --git a/build.gradle b/build.gradle index 277b6cf8..ef319682 100644 --- a/build.gradle +++ b/build.gradle @@ -305,14 +305,16 @@ integTest { } } +def testExtensionYml = new org.yaml.snakeyaml.Yaml().load(new File("src/test/resources/hello-world-extension.yml").newInputStream()) + task closeTestExtension (type: Exec) { - commandLine 'bash', '-c', "kill \$(lsof -i:${testPort})" + commandLine 'bash', '-c', "kill \$(lsof -i:${testExtensionYml.port})" } tasks.named("integTest").configure { finalizedBy(closeTestExtension) } testClusters.integTest { - extension(new ExtensionsProperties("hello-world", "opensearch-sdk-java-1", "127.0.0.1", "${testPort}", "1.0", "3.0.0", "3.0.0")) + extension(new ExtensionsProperties("${testExtensionYml.name}", "${testExtensionYml.uniqueId}", "${testExtensionYml.hostAddress}", "${testExtensionYml.port}", "${testExtensionYml.version}", "${testExtensionYml.opensearchVersion}", "${testExtensionYml.minimumCompatibleVersion}")) testDistribution = "ARCHIVE" // Cluster shrink exception thrown if we try to set numberOfNodes to 1, so only apply if > 1 if (_numNodes > 1) numberOfNodes = _numNodes diff --git a/src/test/resources/hello-world-extension.yml b/src/test/resources/hello-world-extension.yml new file mode 100644 index 00000000..dd960426 --- /dev/null +++ b/src/test/resources/hello-world-extension.yml @@ -0,0 +1,7 @@ +name: hello-world +uniqueId: opensearch-sdk-java-1 +hostAddress: '127.0.0.1' +port: '4500' +version: '1.0' +opensearchVersion: '3.0.0' +minimumCompatibleVersion: '3.0.0' \ No newline at end of file From 383e7a529700d9b024e7f926867b84f7a01763a5 Mon Sep 17 00:00:00 2001 From: Ryan Bogan Date: Tue, 16 May 2023 21:18:56 +0000 Subject: [PATCH 17/26] Spotless Signed-off-by: Ryan Bogan --- src/test/resources/hello-world-extension.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/resources/hello-world-extension.yml b/src/test/resources/hello-world-extension.yml index dd960426..1ccaec91 100644 --- a/src/test/resources/hello-world-extension.yml +++ b/src/test/resources/hello-world-extension.yml @@ -4,4 +4,4 @@ hostAddress: '127.0.0.1' port: '4500' version: '1.0' opensearchVersion: '3.0.0' -minimumCompatibleVersion: '3.0.0' \ No newline at end of file +minimumCompatibleVersion: '3.0.0' From 71b43c47d4060edb04c0be712a1af95fddc04722 Mon Sep 17 00:00:00 2001 From: Ryan Bogan Date: Tue, 16 May 2023 21:55:24 +0000 Subject: [PATCH 18/26] Minor change Signed-off-by: Ryan Bogan --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index ef319682..1195a5a0 100644 --- a/build.gradle +++ b/build.gradle @@ -50,7 +50,7 @@ apply plugin: 'opensearch.testclusters' opensearchplugin { name 'opensearch-sdk-java' description 'OpenSearch Software Development Kit' - classname 'org.opensearch.sdk.ExtensionsRunner' + classname 'org.opensearch.sdk.sample.helloworld.HelloWorldExtension' } ext { From a921cf182826b608e3d403ab4d2c6be3c13beb32 Mon Sep 17 00:00:00 2001 From: Ryan Bogan Date: Wed, 17 May 2023 17:44:55 +0000 Subject: [PATCH 19/26] Debug test commit Signed-off-by: Ryan Bogan --- build.gradle | 3 ++- src/main/java/org/opensearch/sdk/SDKPlugin.java | 9 +++++++++ .../sdk/sample/helloworld/HelloWorldExtension.java | 1 + 3 files changed, 12 insertions(+), 1 deletion(-) create mode 100644 src/main/java/org/opensearch/sdk/SDKPlugin.java diff --git a/build.gradle b/build.gradle index 1195a5a0..6601d244 100644 --- a/build.gradle +++ b/build.gradle @@ -44,13 +44,14 @@ plugins { apply from: 'gradle/formatting.gradle' apply plugin: 'maven-publish' +apply plugin: 'java' apply plugin: 'opensearch.opensearchplugin' apply plugin: 'opensearch.testclusters' opensearchplugin { name 'opensearch-sdk-java' description 'OpenSearch Software Development Kit' - classname 'org.opensearch.sdk.sample.helloworld.HelloWorldExtension' + classname 'src.main.java.org.opensearch.sdk.SDKPlugin' } ext { diff --git a/src/main/java/org/opensearch/sdk/SDKPlugin.java b/src/main/java/org/opensearch/sdk/SDKPlugin.java new file mode 100644 index 00000000..ba2d7a0e --- /dev/null +++ b/src/main/java/org/opensearch/sdk/SDKPlugin.java @@ -0,0 +1,9 @@ +package org.opensearch.sdk; + +import org.opensearch.plugins.Plugin; + +public class SDKPlugin extends Plugin { + public static void main(String[] args) { + System.out.println("Hi"); + } +} diff --git a/src/main/java/org/opensearch/sdk/sample/helloworld/HelloWorldExtension.java b/src/main/java/org/opensearch/sdk/sample/helloworld/HelloWorldExtension.java index e3459771..c2d05bec 100644 --- a/src/main/java/org/opensearch/sdk/sample/helloworld/HelloWorldExtension.java +++ b/src/main/java/org/opensearch/sdk/sample/helloworld/HelloWorldExtension.java @@ -79,6 +79,7 @@ public List> getSettings() { */ public static void main(String[] args) throws IOException { // Execute this extension by instantiating it and passing to ExtensionsRunner + System.out.println("Hi"); ExtensionsRunner.run(new HelloWorldExtension()); } } From d54c143b60b7f7bd7d22bb19f3f11332f2bbad20 Mon Sep 17 00:00:00 2001 From: Ryan Bogan Date: Tue, 23 May 2023 20:02:07 +0000 Subject: [PATCH 20/26] Minor changes Signed-off-by: Ryan Bogan --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index b34c8bc9..17ee3988 100644 --- a/build.gradle +++ b/build.gradle @@ -51,7 +51,7 @@ apply plugin: 'opensearch.testclusters' opensearchplugin { name 'opensearch-sdk-java' description 'OpenSearch Software Development Kit' - classname 'src.main.java.org.opensearch.sdk.SDKPlugin' + classname 'org.opensearch.sdk.sample.helloworld.HelloWorldExtension' } ext { From 1aa03ec021ebf60842b831cf7ba25bc317a99bdd Mon Sep 17 00:00:00 2001 From: Ryan Bogan Date: Tue, 23 May 2023 20:56:59 +0000 Subject: [PATCH 21/26] Remove extra test class Signed-off-by: Ryan Bogan --- src/main/java/org/opensearch/sdk/SDKPlugin.java | 9 --------- 1 file changed, 9 deletions(-) delete mode 100644 src/main/java/org/opensearch/sdk/SDKPlugin.java diff --git a/src/main/java/org/opensearch/sdk/SDKPlugin.java b/src/main/java/org/opensearch/sdk/SDKPlugin.java deleted file mode 100644 index ba2d7a0e..00000000 --- a/src/main/java/org/opensearch/sdk/SDKPlugin.java +++ /dev/null @@ -1,9 +0,0 @@ -package org.opensearch.sdk; - -import org.opensearch.plugins.Plugin; - -public class SDKPlugin extends Plugin { - public static void main(String[] args) { - System.out.println("Hi"); - } -} From 3d5f1d072671fc724856e47dcd5ddb56c8db2f8f Mon Sep 17 00:00:00 2001 From: Ryan Bogan Date: Wed, 24 May 2023 17:44:07 +0000 Subject: [PATCH 22/26] Add index Signed-off-by: Ryan Bogan --- DEVELOPER_GUIDE.md | 1 + 1 file changed, 1 insertion(+) diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index 31da6a89..034c3a0a 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -13,6 +13,7 @@ - [Publish OpenSearch SDK for Java to Maven Local](#publish-opensearch-sdk-for-java-to-maven-local) - [Perform a REST Request on the Extension](#perform-a-rest-request-on-the-extension) - [Run Tests](#run-tests) + - [Launching and debugging from an IDE](#launching-and-debugging-from-an-ide) - [Submitting Changes](#submitting-changes) ## Introduction From 342caa14240686758eff9494f38863d26a896894 Mon Sep 17 00:00:00 2001 From: Ryan Bogan Date: Wed, 24 May 2023 17:54:21 +0000 Subject: [PATCH 23/26] Fix inadvertent change Signed-off-by: Ryan Bogan --- src/main/resources/sample/helloworld-settings.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/resources/sample/helloworld-settings.yml b/src/main/resources/sample/helloworld-settings.yml index 477ee902..94a2007b 100644 --- a/src/main/resources/sample/helloworld-settings.yml +++ b/src/main/resources/sample/helloworld-settings.yml @@ -1,6 +1,6 @@ # This name is only used as a response string during init and is purely cosmetic extensionName: hello-world -hostAddress: 127.0.0.1 +hostAddress: 0.0.0.0 hostPort: 4500 opensearchAddress: 127.0.0.1 opensearchPort: 9200 From 7f0fd39a33955a8895b27a1eec6ee452a483487f Mon Sep 17 00:00:00 2001 From: Ryan Bogan Date: Wed, 24 May 2023 19:19:53 +0000 Subject: [PATCH 24/26] Minor change Signed-off-by: Ryan Bogan --- src/main/resources/sample/helloworld-settings.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/resources/sample/helloworld-settings.yml b/src/main/resources/sample/helloworld-settings.yml index 94a2007b..15dfa3b9 100644 --- a/src/main/resources/sample/helloworld-settings.yml +++ b/src/main/resources/sample/helloworld-settings.yml @@ -1,5 +1,6 @@ # This name is only used as a response string during init and is purely cosmetic extensionName: hello-world +# The listening port is bound to the interface with this IP. Set to 0.0.0.0 to listen on all interfaces hostAddress: 0.0.0.0 hostPort: 4500 opensearchAddress: 127.0.0.1 From e7540d84a5ba137e54195781db80f730351bb1e3 Mon Sep 17 00:00:00 2001 From: Ryan Bogan Date: Thu, 25 May 2023 16:12:54 +0000 Subject: [PATCH 25/26] Add log4j resolution strategy Signed-off-by: Ryan Bogan --- build.gradle | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 9cf72dc8..072ab29f 100644 --- a/build.gradle +++ b/build.gradle @@ -142,7 +142,7 @@ repositories { dependencies { def opensearchVersion = "3.0.0-SNAPSHOT" - def log4jVersion = "2.17.1" + def log4jVersion = "2.20.0" def nettyVersion = "4.1.92.Final" def jacksonDatabindVersion = "2.15.1" def guavaVersion = "31.1-jre" @@ -198,6 +198,9 @@ dependencies { resolutionStrategy.force("jakarta.json:jakarta.json-api:${jakartaVersion}") resolutionStrategy.force("com.fasterxml.jackson.core:jackson-databind:${jacksonDatabindVersion}") resolutionStrategy.force("com.fasterxml.jackson.core:jackson-core:${jacksonDatabindVersion}") + resolutionStrategy.force("org.apache.logging.log4j:log4j-api:${log4jVersion}") + resolutionStrategy.force("org.apache.logging.log4j:log4j-core:${log4jVersion}") + resolutionStrategy.force("org.apache.logging.log4j:log4j-jul:${log4jVersion}") } } From bdadfa7aed09867d6fab827bb9d57def70dbb247 Mon Sep 17 00:00:00 2001 From: Ryan Bogan Date: Thu, 25 May 2023 18:00:29 +0000 Subject: [PATCH 26/26] Remove testing println Signed-off-by: Ryan Bogan --- .../opensearch/sdk/sample/helloworld/HelloWorldExtension.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/java/org/opensearch/sdk/sample/helloworld/HelloWorldExtension.java b/src/main/java/org/opensearch/sdk/sample/helloworld/HelloWorldExtension.java index c2d05bec..e3459771 100644 --- a/src/main/java/org/opensearch/sdk/sample/helloworld/HelloWorldExtension.java +++ b/src/main/java/org/opensearch/sdk/sample/helloworld/HelloWorldExtension.java @@ -79,7 +79,6 @@ public List> getSettings() { */ public static void main(String[] args) throws IOException { // Execute this extension by instantiating it and passing to ExtensionsRunner - System.out.println("Hi"); ExtensionsRunner.run(new HelloWorldExtension()); } }