From 4a7ea91d5d2d408234b0489f32e684ba026816c0 Mon Sep 17 00:00:00 2001 From: Varun Jain Date: Wed, 27 Dec 2023 12:51:49 -0800 Subject: [PATCH] Addressing comments of naveen Signed-off-by: Varun Jain --- ...backwards_compatibility_tests_workflow.yml | 60 ------------------- gradle.properties | 3 +- qa/build.gradle | 39 +++++++----- qa/restart-upgrade/build.gradle | 16 +---- .../AbstractRestartUpgradeRestTestCase.java | 2 +- qa/rolling-upgrade/build.gradle | 15 ----- .../bwc/AbstractRollingUpgradeTestCase.java | 2 +- 7 files changed, 29 insertions(+), 108 deletions(-) diff --git a/.github/workflows/backwards_compatibility_tests_workflow.yml b/.github/workflows/backwards_compatibility_tests_workflow.yml index 6500011f4..2ef6b2e1c 100644 --- a/.github/workflows/backwards_compatibility_tests_workflow.yml +++ b/.github/workflows/backwards_compatibility_tests_workflow.yml @@ -32,36 +32,6 @@ jobs: with: java-version: ${{ matrix.java }} - - if: startsWith(matrix.os,'ubuntu') - name: Install dependencies on ubuntu - run: | - sudo apt-get install libopenblas-dev gfortran -y - - - if: startsWith(matrix.os,'windows') - name: Install MinGW Using Scoop on Windows - run: | - iex "& {$(irm get.scoop.sh)} -RunAsAdmin" - scoop bucket add main - scoop install mingw - - - if: startsWith(matrix.os,'windows') - name: Add MinGW to PATH on Windows - run: | - echo "C:/Users/runneradmin/scoop/apps/mingw/current/bin" >> $env:GITHUB_PATH - Import-Module "$env:ChocolateyInstall\helpers\chocolateyProfile.psm1" - refreshenv - - - if: startsWith(matrix.os,'windows') - name: Download OpenBLAS on Windows - run: | - curl -L -O https://github.com/xianyi/OpenBLAS/releases/download/v0.3.21/OpenBLAS-0.3.21-x64.zip - mkdir OpenBLAS - Expand-Archive -Path .\OpenBLAS-0.3.21-x64.zip -DestinationPath .\OpenBLAS\ - mkdir ./src/main/resources/windowsDependencies - cp ./OpenBLAS/bin/libopenblas.dll ./src/main/resources/windowsDependencies/ - rm .\OpenBLAS-0.3.21-x64.zip - rm -r .\OpenBLAS\ - - if: startsWith(matrix.os,'windows') name: Run NeuralSearch Restart-Upgrade BWC Tests from BWCVersion-${{ matrix.bwc_version }} to OpenSearch Version-${{ matrix.opensearch_version }} on Windows run: | @@ -97,36 +67,6 @@ jobs: with: java-version: ${{ matrix.java }} - - if: startsWith(matrix.os,'ubuntu') - name: Install dependencies on ubuntu - run: | - sudo apt-get install libopenblas-dev gfortran -y - - - if: startsWith(matrix.os,'windows') - name: Install MinGW Using Scoop on Windows - run: | - iex "& {$(irm get.scoop.sh)} -RunAsAdmin" - scoop bucket add main - scoop install mingw - - - if: startsWith(matrix.os,'windows') - name: Add MinGW to PATH on Windows - run: | - echo "C:/Users/runneradmin/scoop/apps/mingw/current/bin" >> $env:GITHUB_PATH - Import-Module "$env:ChocolateyInstall\helpers\chocolateyProfile.psm1" - refreshenv - - - if: startsWith(matrix.os,'windows') - name: Download OpenBLAS on Windows - run: | - curl -L -O https://github.com/xianyi/OpenBLAS/releases/download/v0.3.21/OpenBLAS-0.3.21-x64.zip - mkdir OpenBLAS - Expand-Archive -Path .\OpenBLAS-0.3.21-x64.zip -DestinationPath .\OpenBLAS\ - mkdir ./src/main/resources/windowsDependencies - cp ./OpenBLAS/bin/libopenblas.dll ./src/main/resources/windowsDependencies/ - rm .\OpenBLAS-0.3.21-x64.zip - rm -r .\OpenBLAS\ - - if: startsWith(matrix.os,'windows') name: Run NeuralSearch Rolling-Upgrade BWC Tests from BWCVersion-${{ matrix.bwc_version }} to OpenSearch Version-${{ matrix.opensearch_version }} on Windows run: | diff --git a/gradle.properties b/gradle.properties index 242771ae4..b4dc3433d 100644 --- a/gradle.properties +++ b/gradle.properties @@ -3,8 +3,7 @@ # SPDX-License-Identifier: Apache-2.0 # -version=2.0.0 -systemProp.bwc.version=2.11.0 +systemProp.bwc.version=2.12.0-SNAPSHOT # For fixing Spotless check with Java 17 org.gradle.jvmargs=--add-exports jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED \ diff --git a/qa/build.gradle b/qa/build.gradle index 86ad41315..ccaba4e24 100644 --- a/qa/build.gradle +++ b/qa/build.gradle @@ -31,7 +31,6 @@ dependencies { zipArchive group: 'org.opensearch.plugin', name:'opensearch-knn', version: "${opensearch_build}" zipArchive group: 'org.opensearch.plugin', name:'opensearch-ml-plugin', version: "${opensearch_build}" compileOnly fileTree(dir: knnJarDirectory, include: '*.jar') - //api group: 'commons-lang', name: 'commons-lang', version: '2.6' api "org.apache.logging.log4j:log4j-api:${versions.log4j}" api "org.apache.logging.log4j:log4j-core:${versions.log4j}" api "junit:junit:${versions.junit}" @@ -72,19 +71,11 @@ task deletetempDirectories { } } -task pullMlCommonsBwcPlugin { - doLast { - copy { - from(java.nio.file.Path.of(tmp_dir.absolutePath, "opensearch-${neural_search_bwc_version_no_qualifier}", "plugins", "opensearch-ml")) - into java.nio.file.Path.of(tmp_dir.absolutePath, "opensearch-ml") - } - } -} - -task pullKnnBwcPlugin { - dependsOn "deletetempDirectories" +// Task to pull opensearch artifact from archive +task pullOpensearchArtifact { + dependsOn "deletetempDirectories" - doLast { + doLast{ ext{ if (isSnapshot) { srcUrl = "https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/${neural_search_bwc_version_no_qualifier}/latest/${os_platform}/x64/${artifact_type}/dist/opensearch/opensearch-${neural_search_bwc_version_no_qualifier}-${os_platform}-x64.${file_ext}" @@ -105,6 +96,24 @@ task pullKnnBwcPlugin { } into tmp_dir.absolutePath } + } +} + +// Task to pull ml plugin from archive +task pullMlCommonsBwcPlugin { + doLast { + copy { + from(java.nio.file.Path.of(tmp_dir.absolutePath, "opensearch-${neural_search_bwc_version_no_qualifier}", "plugins", "opensearch-ml")) + into java.nio.file.Path.of(tmp_dir.absolutePath, "opensearch-ml") + } + } +} + +// Task to pull KNN plugin from archive +task pullKnnBwcPlugin { + dependsOn "pullOpensearchArtifact" + + doLast { copy { from(java.nio.file.Path.of(tmp_dir.absolutePath, "opensearch-${neural_search_bwc_version_no_qualifier}", "plugins", "opensearch-knn")) into java.nio.file.Path.of(tmp_dir.absolutePath, "opensearch-knn") @@ -123,6 +132,7 @@ task pullBwcPlugin { } } +// Task to zip ml-commons plugin from archive task zipBwcMlCommonsPlugin(type: Zip) { dependsOn "pullMlCommonsBwcPlugin" from(java.nio.file.Path.of(tmp_dir.absolutePath, "opensearch-ml")) @@ -133,6 +143,7 @@ task zipBwcMlCommonsPlugin(type: Zip) { } } +// Task to zip knn plugin from archive task zipBwcKnnPlugin(type: Zip) { dependsOn "zipBwcMlCommonsPlugin" dependsOn "pullKnnBwcPlugin" @@ -144,7 +155,7 @@ task zipBwcKnnPlugin(type: Zip) { } } -// Task to zip plugin from archive +// Task to zip neural search plugin from archive task zipBwcPlugin(type: Zip) { dependsOn "zipBwcKnnPlugin" dependsOn "pullBwcPlugin" diff --git a/qa/restart-upgrade/build.gradle b/qa/restart-upgrade/build.gradle index ea8f3b830..c7bd6ae74 100644 --- a/qa/restart-upgrade/build.gradle +++ b/qa/restart-upgrade/build.gradle @@ -13,7 +13,6 @@ String default_bwc_version = System.getProperty("bwc.version") String neural_search_bwc_version = System.getProperty("tests.bwc.version", default_bwc_version) boolean isSnapshot = neural_search_bwc_version.contains("-SNAPSHOT") String baseName = "neuralSearchBwcCluster-restart" -String knn_bwc_version_no_qualifier = isSnapshot ? neural_search_bwc_version - "-SNAPSHOT" : neural_search_bwc_version List> plugins = [provider(new Callable(){ @Override @@ -37,7 +36,7 @@ List> plugins = [provider(new Callable(){ } }), rootProject.tasks.bundlePlugin.archiveFile] -// Creates a test cluster of previous version and loads k-NN plugin of bwcVersion +// Creates a test cluster of previous version and loads neuralsearch plugin of bwcVersion testClusters { "${baseName}" { testDistribution = "ARCHIVE" @@ -49,22 +48,11 @@ testClusters { plugin(project.tasks.zipBwcPlugin.archiveFile) setting 'path.repo', "${buildDir}/cluster/shared/repo/${baseName}" setting 'http.content_type.required', 'true' - environment "LD_LIBRARY_PATH", "${buildDir}/testclusters/${baseName}-0/distro/${knn_bwc_version_no_qualifier}-ARCHIVE/plugins/opensearch-knn/knnlib;${buildDir}/testclusters/${baseName}-0/distro/${knn_bwc_version_no_qualifier}-ARCHIVE/plugins/opensearch-knn/lib" - if (Os.isFamily(Os.FAMILY_WINDOWS)) { - // While running on Windows OS, setting the PATH environment variable to include the paths to dlls of JNI libraries and windows dependencies - environment('PATH', System.getenv('PATH') + ";$rootDir/jni/release" + ";$rootDir/src/main/resources/windowsDependencies") - systemProperty "java.library.path", "${buildDir}/testclusters/${baseName}-0/distro/${knn_bwc_version_no_qualifier}-ARCHIVE/plugins/opensearch-knn/knnlib;${buildDir}/testclusters/${baseName}-0/distro/${knn_bwc_version_no_qualifier}-ARCHIVE/plugins/opensearch-knn/lib" - } else { - systemProperty "java.library.path", "${buildDir}/testclusters/${baseName}-0/distro/${knn_bwc_version_no_qualifier}-ARCHIVE/plugins/opensearch-knn/knnlib:${buildDir}/testclusters/${baseName}-0/distro/${knn_bwc_version_no_qualifier}-ARCHIVE/plugins/opensearch-knn/lib" - } - } } // Task to run BWC tests against the old cluster task testAgainstOldCluster(type: StandaloneRestIntegTestTask) { - dependsOn "zipBwcMlCommonsPlugin" - dependsOn "zipBwcKnnPlugin" dependsOn "zipBwcPlugin" useCluster testClusters."${baseName}" systemProperty 'tests.rest.bwcsuite_cluster', 'old_cluster' @@ -83,8 +71,6 @@ task testRestartUpgrade(type: StandaloneRestIntegTestTask) { dependsOn rootProject.tasks.assemble useCluster testClusters."${baseName}" doFirst { - testClusters."${baseName}".environment("LD_LIBRARY_PATH", "$rootDir/jni/release") - testClusters."${baseName}".systemProperty("java.library.path", "$rootDir/jni/release") testClusters."${baseName}".upgradeAllNodesAndPluginsToNextVersion(plugins) } systemProperty 'tests.rest.bwcsuite_cluster', 'upgraded_cluster' diff --git a/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/AbstractRestartUpgradeRestTestCase.java b/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/AbstractRestartUpgradeRestTestCase.java index aa2fbbe0b..25f591d33 100644 --- a/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/AbstractRestartUpgradeRestTestCase.java +++ b/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/AbstractRestartUpgradeRestTestCase.java @@ -21,7 +21,7 @@ public abstract class AbstractRestartUpgradeRestTestCase extends BaseNeuralSearc @Before protected void setIndex() { - // Creating index name by concatenating "knn-bwc-" prefix with test method name + // Creating index name by concatenating "neural-bwc-" prefix with test method name // for all the tests in this sub-project testIndex = NEURAL_SEARCH_BWC_PREFIX + getTestName().toLowerCase(Locale.ROOT); } diff --git a/qa/rolling-upgrade/build.gradle b/qa/rolling-upgrade/build.gradle index 76560a3cf..d7546f657 100644 --- a/qa/rolling-upgrade/build.gradle +++ b/qa/rolling-upgrade/build.gradle @@ -13,7 +13,6 @@ apply from : "$rootDir/qa/build.gradle" String default_bwc_version = System.getProperty("bwc.version") String neural_search_bwc_version = System.getProperty("tests.bwc.version", default_bwc_version) boolean isSnapshot = neural_search_bwc_version.contains("-SNAPSHOT") -String knn_bwc_version_no_qualifier = isSnapshot ? neural_search_bwc_version - "-SNAPSHOT" : neural_search_bwc_version String baseName = "neuralSearchBwcCluster-rolling" List> plugins = [provider(new Callable(){ @@ -50,22 +49,12 @@ testClusters { jvmArgs("-Xms1g", "-Xmx4g") setting 'path.repo', "${buildDir}/cluster/shared/repo/${baseName}" setting 'http.content_type.required', 'true' - environment "LD_LIBRARY_PATH", "${buildDir}/testclusters/${baseName}-0/distro/${knn_bwc_version_no_qualifier}-ARCHIVE/plugins/opensearch-knn/knnlib;${buildDir}/testclusters/${baseName}-0/distro/${knn_bwc_version_no_qualifier}-ARCHIVE/plugins/opensearch-knn/lib" - if (Os.isFamily(Os.FAMILY_WINDOWS)) { - // While running on Windows OS, setting the PATH environment variable to include the paths to dlls of JNI libraries and windows dependencies - environment('PATH', System.getenv('PATH') + ";$rootDir/jni/release" + ";$rootDir/src/main/resources/windowsDependencies") - systemProperty "java.library.path", "${buildDir}/testclusters/${baseName}-0/distro/${knn_bwc_version_no_qualifier}-ARCHIVE/plugins/opensearch-knn/knnlib;${buildDir}/testclusters/${baseName}-0/distro/${knn_bwc_version_no_qualifier}-ARCHIVE/plugins/opensearch-knn/lib" - } else { - systemProperty "java.library.path", "${buildDir}/testclusters/${baseName}-0/distro/${knn_bwc_version_no_qualifier}-ARCHIVE/plugins/opensearch-knn/knnlib:${buildDir}/testclusters/${baseName}-0/distro/${knn_bwc_version_no_qualifier}-ARCHIVE/plugins/opensearch-knn/lib" - } } } // Task to run BWC tests against the old cluster task testAgainstOldCluster(type: StandaloneRestIntegTestTask) { - dependsOn "zipBwcMlCommonsPlugin" - dependsOn "zipBwcKnnPlugin" dependsOn "zipBwcPlugin" useCluster testClusters."${baseName}" systemProperty 'tests.rest.bwcsuite_cluster', 'old_cluster' @@ -102,8 +91,6 @@ task testAgainstTwoThirdsUpgradedCluster(type: StandaloneRestIntegTestTask) { dependsOn "testAgainstOneThirdUpgradedCluster" useCluster testClusters."${baseName}" doFirst { - testClusters."${baseName}".getNodes().getAt("${baseName}" + "-1").environment("LD_LIBRARY_PATH", "$rootDir/jni/release") - testClusters."${baseName}".getNodes().getAt("${baseName}" + "-1").systemProperty("java.library.path", "$rootDir/jni/release") testClusters."${baseName}".upgradeNodeAndPluginToNextVersion(plugins) } systemProperty 'tests.rest.bwcsuite_cluster', 'mixed_cluster' @@ -121,8 +108,6 @@ task testRollingUpgrade(type: StandaloneRestIntegTestTask) { dependsOn "testAgainstTwoThirdsUpgradedCluster" useCluster testClusters."${baseName}" doFirst { - testClusters."${baseName}".getNodes().getAt("${baseName}" + "-2").environment("LD_LIBRARY_PATH", "$rootDir/jni/release") - testClusters."${baseName}".getNodes().getAt("${baseName}" + "-2").systemProperty("java.library.path", "$rootDir/jni/release") testClusters."${baseName}".upgradeNodeAndPluginToNextVersion(plugins) } mustRunAfter "testAgainstOneThirdUpgradedCluster" diff --git a/qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/AbstractRollingUpgradeTestCase.java b/qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/AbstractRollingUpgradeTestCase.java index 42418f901..49cdf39e8 100644 --- a/qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/AbstractRollingUpgradeTestCase.java +++ b/qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/AbstractRollingUpgradeTestCase.java @@ -25,7 +25,7 @@ public abstract class AbstractRollingUpgradeTestCase extends BaseNeuralSearchIT @Before protected void setIndex() { - // Creating index name by concatenating "knn-bwc-" prefix with test method name + // Creating index name by concatenating "neural-bwc-" prefix with test method name // for all the tests in this sub-project testIndex = NEURAL_SEARCH_BWC_PREFIX + getTestName().toLowerCase(Locale.ROOT); }