diff --git a/qa/build.gradle b/qa/build.gradle index cb812644f..885aef7d3 100644 --- a/qa/build.gradle +++ b/qa/build.gradle @@ -4,9 +4,10 @@ */ import org.apache.tools.ant.taskdefs.condition.Os -import org.w3c.dom.Text import java.nio.file.Files +import java.util.concurrent.Callable +import java.nio.file.Path apply plugin: 'opensearch.testclusters' apply plugin: 'opensearch.build' @@ -60,7 +61,31 @@ if (Os.isFamily(Os.FAMILY_WINDOWS)) { file_ext = "zip" } -task deletetempDirectories { +ext{ + plugins = [provider(new Callable(){ + @Override + RegularFile call() throws Exception { + return new RegularFile() { + @Override + File getAsFile() { + return configurations.zipArchive.asFileTree.matching{include "**/opensearch-ml-plugin-${opensearch_build}.zip"}.getSingleFile() + } + } + } + }), provider(new Callable(){ + @Override + RegularFile call() throws Exception { + return new RegularFile() { + @Override + File getAsFile() { + return configurations.zipArchive.asFileTree.matching{include "**/opensearch-knn-${opensearch_build}.zip"}.getSingleFile() + } + } + } + }), rootProject.tasks.bundlePlugin.archiveFile] +} + +task deleteTempDirectories { doFirst { File[] tempFiles = tmp_dir.listFiles() for (File child : tempFiles) { @@ -73,7 +98,7 @@ task deletetempDirectories { // Task to pull opensearch artifact from archive task pullOpensearchArtifact { - dependsOn "deletetempDirectories" + dependsOn "deleteTempDirectories" doLast{ ext{ @@ -90,9 +115,9 @@ task pullOpensearchArtifact { ) copy { if (Os.isFamily(Os.FAMILY_WINDOWS)) { - from zipTree(java.nio.file.Path.of(tmp_dir.absolutePath, "opensearch-${neural_search_bwc_version_no_qualifier}-${os_platform}-x64.${file_ext}")) + from zipTree(Path.of(tmp_dir.absolutePath, "opensearch-${neural_search_bwc_version_no_qualifier}-${os_platform}-x64.${file_ext}")) } else { - from tarTree(java.nio.file.Path.of(tmp_dir.absolutePath, "opensearch-${neural_search_bwc_version_no_qualifier}-${os_platform}-x64.${file_ext}")) + from tarTree(Path.of(tmp_dir.absolutePath, "opensearch-${neural_search_bwc_version_no_qualifier}-${os_platform}-x64.${file_ext}")) } into tmp_dir.absolutePath } @@ -103,8 +128,8 @@ task pullOpensearchArtifact { task pullMlCommonsBwcPlugin { doLast { copy { - from(java.nio.file.Path.of(tmp_dir.absolutePath, "opensearch-${neural_search_bwc_version_no_qualifier}", "plugins", "opensearch-ml")) - into java.nio.file.Path.of(tmp_dir.absolutePath, "opensearch-ml") + from(Path.of(tmp_dir.absolutePath, "opensearch-${neural_search_bwc_version_no_qualifier}", "plugins", "opensearch-ml")) + into Path.of(tmp_dir.absolutePath, "opensearch-ml") } } } @@ -115,8 +140,8 @@ task pullKnnBwcPlugin { doLast { copy { - from(java.nio.file.Path.of(tmp_dir.absolutePath, "opensearch-${neural_search_bwc_version_no_qualifier}", "plugins", "opensearch-knn")) - into java.nio.file.Path.of(tmp_dir.absolutePath, "opensearch-knn") + from(Path.of(tmp_dir.absolutePath, "opensearch-${neural_search_bwc_version_no_qualifier}", "plugins", "opensearch-knn")) + into Path.of(tmp_dir.absolutePath, "opensearch-knn") } } } @@ -125,21 +150,21 @@ task pullKnnBwcPlugin { task pullBwcPlugin { doLast { copy { - from(java.nio.file.Path.of(tmp_dir.absolutePath, "opensearch-${neural_search_bwc_version_no_qualifier}", "plugins", "opensearch-neural-search")) - into java.nio.file.Path.of(tmp_dir.absolutePath, "opensearch-neural-search") + from(Path.of(tmp_dir.absolutePath, "opensearch-${neural_search_bwc_version_no_qualifier}", "plugins", "opensearch-neural-search")) + into Path.of(tmp_dir.absolutePath, "opensearch-neural-search") } - delete java.nio.file.Path.of(tmp_dir.absolutePath, "opensearch-${neural_search_bwc_version_no_qualifier}"), java.nio.file.Path.of(tmp_dir.absolutePath, "opensearch-${neural_search_bwc_version_no_qualifier}-${os_platform}-x64.${file_ext}") + delete Path.of(tmp_dir.absolutePath, "opensearch-${neural_search_bwc_version_no_qualifier}"), java.nio.file.Path.of(tmp_dir.absolutePath, "opensearch-${neural_search_bwc_version_no_qualifier}-${os_platform}-x64.${file_ext}") } } // Task to zip ml-commons plugin from archive task zipBwcMlCommonsPlugin(type: Zip) { dependsOn "pullMlCommonsBwcPlugin" - from(java.nio.file.Path.of(tmp_dir.absolutePath, "opensearch-ml")) + from(Path.of(tmp_dir.absolutePath, "opensearch-ml")) destinationDirectory = tmp_dir archiveFileName = "opensearch-ml-${neural_search_bwc_version_no_qualifier}.zip" doLast { - delete java.nio.file.Path.of(tmp_dir.absolutePath, "opensearch-ml") + delete Path.of(tmp_dir.absolutePath, "opensearch-ml") } } @@ -147,11 +172,11 @@ task zipBwcMlCommonsPlugin(type: Zip) { task zipBwcKnnPlugin(type: Zip) { dependsOn "pullKnnBwcPlugin" dependsOn "zipBwcMlCommonsPlugin" - from(java.nio.file.Path.of(tmp_dir.absolutePath, "opensearch-knn")) + from(Path.of(tmp_dir.absolutePath, "opensearch-knn")) destinationDirectory = tmp_dir archiveFileName = "opensearch-knn-${neural_search_bwc_version_no_qualifier}.zip" doLast { - delete java.nio.file.Path.of(tmp_dir.absolutePath, "opensearch-knn") + delete Path.of(tmp_dir.absolutePath, "opensearch-knn") } } @@ -159,11 +184,11 @@ task zipBwcKnnPlugin(type: Zip) { task zipBwcPlugin(type: Zip) { dependsOn "zipBwcKnnPlugin" dependsOn "pullBwcPlugin" - from(java.nio.file.Path.of(tmp_dir.absolutePath, "opensearch-neural-search")) + from(Path.of(tmp_dir.absolutePath, "opensearch-neural-search")) destinationDirectory = tmp_dir archiveFileName = "opensearch-neural-search-${neural_search_bwc_version_no_qualifier}.zip" doLast { - delete java.nio.file.Path.of(tmp_dir.absolutePath, "opensearch-neural-search") + delete Path.of(tmp_dir.absolutePath, "opensearch-neural-search") } } diff --git a/qa/restart-upgrade/build.gradle b/qa/restart-upgrade/build.gradle index d585e388d..957c2a45d 100644 --- a/qa/restart-upgrade/build.gradle +++ b/qa/restart-upgrade/build.gradle @@ -4,38 +4,12 @@ */ import org.opensearch.gradle.testclusters.StandaloneRestIntegTestTask -import org.apache.tools.ant.taskdefs.condition.Os -import java.util.concurrent.Callable apply from : "$rootDir/qa/build.gradle" String default_bwc_version = System.getProperty("bwc.version") String neural_search_bwc_version = System.getProperty("tests.bwc.version", default_bwc_version) -boolean isSnapshot = neural_search_bwc_version.contains("-SNAPSHOT") String baseName = "neuralSearchBwcCluster-restart" -String knn_bwc_version_no_qualifier = isSnapshot ? neural_search_bwc_version - "-SNAPSHOT" : neural_search_bwc_version - -List> plugins = [provider(new Callable(){ - @Override - RegularFile call() throws Exception { - return new RegularFile() { - @Override - File getAsFile() { - return configurations.zipArchive.asFileTree.matching{include "**/opensearch-ml-plugin-${opensearch_build}.zip"}.getSingleFile() - } - } - } -}), provider(new Callable(){ - @Override - RegularFile call() throws Exception { - return new RegularFile() { - @Override - File getAsFile() { - return configurations.zipArchive.asFileTree.matching{include "**/opensearch-knn-${opensearch_build}.zip"}.getSingleFile() - } - } - } -}), rootProject.tasks.bundlePlugin.archiveFile] // Creates a test cluster of previous version and loads k-NN plugin of bwcVersion testClusters { @@ -49,8 +23,6 @@ testClusters { plugin(project.tasks.zipBwcPlugin.archiveFile) setting 'path.repo', "${buildDir}/cluster/shared/repo/${baseName}" setting 'http.content_type.required', 'true' - environment "LD_LIBRARY_PATH", "${buildDir}/testclusters/${baseName}-0/distro/${knn_bwc_version_no_qualifier}-ARCHIVE/plugins/opensearch-knn/knnlib;${buildDir}/testclusters/${baseName}-0/distro/${knn_bwc_version_no_qualifier}-ARCHIVE/plugins/opensearch-knn/lib" - systemProperty "java.library.path", "${buildDir}/testclusters/${baseName}-0/distro/${knn_bwc_version_no_qualifier}-ARCHIVE/plugins/opensearch-knn/knnlib:${buildDir}/testclusters/${baseName}-0/distro/${knn_bwc_version_no_qualifier}-ARCHIVE/plugins/opensearch-knn/lib" } } @@ -74,7 +46,7 @@ task testRestartUpgrade(type: StandaloneRestIntegTestTask) { dependsOn rootProject.tasks.assemble useCluster testClusters."${baseName}" doFirst { - testClusters."${baseName}".upgradeAllNodesAndPluginsToNextVersion(plugins) + testClusters."${baseName}".upgradeAllNodesAndPluginsToNextVersion(project.ext.plugins) } systemProperty 'tests.rest.bwcsuite_cluster', 'upgraded_cluster' systemProperty 'tests.skip_delete_model_index', 'true' diff --git a/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/SemanticSearch.java b/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/SemanticSearchIT.java similarity index 97% rename from qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/SemanticSearch.java rename to qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/SemanticSearchIT.java index 48742ce16..fb852b6ff 100644 --- a/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/SemanticSearch.java +++ b/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/SemanticSearchIT.java @@ -15,7 +15,7 @@ import static org.opensearch.neuralsearch.TestUtils.TEXT_EMBEDDING_PROCESSOR; import org.opensearch.neuralsearch.query.NeuralQueryBuilder; -public class SemanticSearch extends AbstractRestartUpgradeRestTestCase { +public class SemanticSearchIT extends AbstractRestartUpgradeRestTestCase { private static final String PIPELINE_NAME = "nlp-pipeline"; private static String DOC_ID = "0"; diff --git a/qa/rolling-upgrade/build.gradle b/qa/rolling-upgrade/build.gradle index 00422f654..db219ad3d 100644 --- a/qa/rolling-upgrade/build.gradle +++ b/qa/rolling-upgrade/build.gradle @@ -4,40 +4,13 @@ */ import org.opensearch.gradle.testclusters.StandaloneRestIntegTestTask -import org.apache.tools.ant.taskdefs.condition.Os - -import java.util.concurrent.Callable apply from : "$rootDir/qa/build.gradle" String default_bwc_version = System.getProperty("bwc.version") String neural_search_bwc_version = System.getProperty("tests.bwc.version", default_bwc_version) -boolean isSnapshot = neural_search_bwc_version.contains("-SNAPSHOT") -String knn_bwc_version_no_qualifier = isSnapshot ? neural_search_bwc_version - "-SNAPSHOT" : neural_search_bwc_version String baseName = "neuralSearchBwcCluster-rolling" -List> plugins = [provider(new Callable(){ - @Override - RegularFile call() throws Exception { - return new RegularFile() { - @Override - File getAsFile() { - return configurations.zipArchive.asFileTree.matching{include "**/opensearch-ml-plugin-${opensearch_build}.zip"}.getSingleFile() - } - } - } -}), provider(new Callable(){ - @Override - RegularFile call() throws Exception { - return new RegularFile() { - @Override - File getAsFile() { - return configurations.zipArchive.asFileTree.matching{include "**/opensearch-knn-${opensearch_build}.zip"}.getSingleFile() - } - } - } -}), rootProject.tasks.bundlePlugin.archiveFile] - // Creates a test cluster of previous version and loads k-NN plugin of bwcVersion testClusters { "${baseName}" { @@ -50,8 +23,6 @@ testClusters { jvmArgs("-Xms1g", "-Xmx4g") setting 'path.repo', "${buildDir}/cluster/shared/repo/${baseName}" setting 'http.content_type.required', 'true' - environment "LD_LIBRARY_PATH", "${buildDir}/testclusters/${baseName}-0/distro/${knn_bwc_version_no_qualifier}-ARCHIVE/plugins/opensearch-knn/knnlib;${buildDir}/testclusters/${baseName}-0/distro/${knn_bwc_version_no_qualifier}-ARCHIVE/plugins/opensearch-knn/lib" - systemProperty "java.library.path", "${buildDir}/testclusters/${baseName}-0/distro/${knn_bwc_version_no_qualifier}-ARCHIVE/plugins/opensearch-knn/knnlib:${buildDir}/testclusters/${baseName}-0/distro/${knn_bwc_version_no_qualifier}-ARCHIVE/plugins/opensearch-knn/lib" } } @@ -75,7 +46,7 @@ task testAgainstOneThirdUpgradedCluster(type: StandaloneRestIntegTestTask) { dependsOn rootProject.tasks.assemble dependsOn "testAgainstOldCluster" doFirst { - testClusters."${baseName}".upgradeNodeAndPluginToNextVersion(plugins) + testClusters."${baseName}".upgradeNodeAndPluginToNextVersion(project.ext.plugins) } systemProperty 'tests.rest.bwcsuite_cluster', 'mixed_cluster' systemProperty 'tests.rest.first_round', 'true' @@ -92,7 +63,7 @@ task testAgainstTwoThirdsUpgradedCluster(type: StandaloneRestIntegTestTask) { dependsOn "testAgainstOneThirdUpgradedCluster" useCluster testClusters."${baseName}" doFirst { - testClusters."${baseName}".upgradeNodeAndPluginToNextVersion(plugins) + testClusters."${baseName}".upgradeNodeAndPluginToNextVersion(project.ext.plugins) } systemProperty 'tests.rest.bwcsuite_cluster', 'mixed_cluster' systemProperty 'tests.rest.first_round', 'false' @@ -109,7 +80,7 @@ task testRollingUpgrade(type: StandaloneRestIntegTestTask) { dependsOn "testAgainstTwoThirdsUpgradedCluster" useCluster testClusters."${baseName}" doFirst { - testClusters."${baseName}".upgradeNodeAndPluginToNextVersion(plugins) + testClusters."${baseName}".upgradeNodeAndPluginToNextVersion(project.ext.plugins) } mustRunAfter "testAgainstOneThirdUpgradedCluster" systemProperty 'tests.rest.bwcsuite_cluster', 'upgraded_cluster' diff --git a/qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/SemanticSearch.java b/qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/SemanticSearchIT.java similarity index 98% rename from qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/SemanticSearch.java rename to qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/SemanticSearchIT.java index 5c8300271..936c7dc11 100644 --- a/qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/SemanticSearch.java +++ b/qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/SemanticSearchIT.java @@ -13,7 +13,7 @@ import static org.opensearch.neuralsearch.TestUtils.TEXT_EMBEDDING_PROCESSOR; import org.opensearch.neuralsearch.query.NeuralQueryBuilder; -public class SemanticSearch extends AbstractRollingUpgradeTestCase { +public class SemanticSearchIT extends AbstractRollingUpgradeTestCase { private static final String PIPELINE_NAME = "nlp-pipeline"; private static final String TEST_FIELD = "test-field"; private static final String TEXT = "Hello world";