Addressing comments of naveen

Signed-off-by: Varun Jain <[email protected]>
opensearch-project · Dec 27, 2023 · 4a7ea91 · 4a7ea91
1 parent 20ec55f
commit 4a7ea91
Show file tree

Hide file tree

Showing 7 changed files with 29 additions and 108 deletions.
diff --git a/.github/workflows/backwards_compatibility_tests_workflow.yml b/.github/workflows/backwards_compatibility_tests_workflow.yml
@@ -32,36 +32,6 @@ jobs:
         with:
           java-version: ${{ matrix.java }}
 
-      - if: startsWith(matrix.os,'ubuntu')
-        name: Install dependencies on ubuntu
-        run: |
-          sudo apt-get install libopenblas-dev gfortran -y
-
-      - if: startsWith(matrix.os,'windows')
-        name: Install MinGW Using Scoop on Windows
-        run: |
-          iex "& {$(irm get.scoop.sh)} -RunAsAdmin"
-          scoop bucket add main
-          scoop install mingw
-
-      - if: startsWith(matrix.os,'windows')
-        name: Add MinGW to PATH on Windows
-        run: |
-          echo "C:/Users/runneradmin/scoop/apps/mingw/current/bin" >> $env:GITHUB_PATH
-          Import-Module "$env:ChocolateyInstall\helpers\chocolateyProfile.psm1"
-          refreshenv
-
-      - if: startsWith(matrix.os,'windows')
-        name: Download OpenBLAS on Windows
-        run: |
-          curl -L -O https://github.com/xianyi/OpenBLAS/releases/download/v0.3.21/OpenBLAS-0.3.21-x64.zip
-          mkdir OpenBLAS
-          Expand-Archive -Path .\OpenBLAS-0.3.21-x64.zip -DestinationPath .\OpenBLAS\
-          mkdir ./src/main/resources/windowsDependencies
-          cp ./OpenBLAS/bin/libopenblas.dll ./src/main/resources/windowsDependencies/
-          rm .\OpenBLAS-0.3.21-x64.zip
-          rm -r .\OpenBLAS\
-
       - if: startsWith(matrix.os,'windows')
         name: Run NeuralSearch Restart-Upgrade BWC Tests from BWCVersion-${{ matrix.bwc_version }} to OpenSearch Version-${{ matrix.opensearch_version }} on Windows
         run: |
@@ -97,36 +67,6 @@ jobs:
         with:
           java-version: ${{ matrix.java }}
 
-      - if: startsWith(matrix.os,'ubuntu')
-        name: Install dependencies on ubuntu
-        run: |
-          sudo apt-get install libopenblas-dev gfortran -y
-
-      - if: startsWith(matrix.os,'windows')
-        name: Install MinGW Using Scoop on Windows
-        run: |
-          iex "& {$(irm get.scoop.sh)} -RunAsAdmin"
-          scoop bucket add main
-          scoop install mingw
-
-      - if: startsWith(matrix.os,'windows')
-        name: Add MinGW to PATH on Windows
-        run: |
-          echo "C:/Users/runneradmin/scoop/apps/mingw/current/bin" >> $env:GITHUB_PATH
-          Import-Module "$env:ChocolateyInstall\helpers\chocolateyProfile.psm1"
-          refreshenv
-
-      - if: startsWith(matrix.os,'windows')
-        name: Download OpenBLAS on Windows
-        run: |
-          curl -L -O https://github.com/xianyi/OpenBLAS/releases/download/v0.3.21/OpenBLAS-0.3.21-x64.zip
-          mkdir OpenBLAS
-          Expand-Archive -Path .\OpenBLAS-0.3.21-x64.zip -DestinationPath .\OpenBLAS\
-          mkdir ./src/main/resources/windowsDependencies
-          cp ./OpenBLAS/bin/libopenblas.dll ./src/main/resources/windowsDependencies/
-          rm .\OpenBLAS-0.3.21-x64.zip
-          rm -r .\OpenBLAS\
-
       - if: startsWith(matrix.os,'windows')
         name: Run NeuralSearch Rolling-Upgrade BWC Tests from BWCVersion-${{ matrix.bwc_version }} to OpenSearch Version-${{ matrix.opensearch_version }} on Windows
         run: |

diff --git a/gradle.properties b/gradle.properties
@@ -3,8 +3,7 @@
 # SPDX-License-Identifier: Apache-2.0
 #
 
-version=2.0.0
-systemProp.bwc.version=2.11.0
+systemProp.bwc.version=2.12.0-SNAPSHOT
 
 # For fixing Spotless check with Java 17
 org.gradle.jvmargs=--add-exports jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED \

diff --git a/qa/build.gradle b/qa/build.gradle
@@ -31,7 +31,6 @@ dependencies {
     zipArchive group: 'org.opensearch.plugin', name:'opensearch-knn', version: "${opensearch_build}"
     zipArchive group: 'org.opensearch.plugin', name:'opensearch-ml-plugin', version: "${opensearch_build}"
     compileOnly fileTree(dir: knnJarDirectory, include: '*.jar')
-    //api group: 'commons-lang', name: 'commons-lang', version: '2.6'
     api "org.apache.logging.log4j:log4j-api:${versions.log4j}"
     api "org.apache.logging.log4j:log4j-core:${versions.log4j}"
     api "junit:junit:${versions.junit}"
@@ -72,19 +71,11 @@ task deletetempDirectories {
     }
 }
 
-task pullMlCommonsBwcPlugin {
-    doLast {
-        copy {
-            from(java.nio.file.Path.of(tmp_dir.absolutePath, "opensearch-${neural_search_bwc_version_no_qualifier}", "plugins", "opensearch-ml"))
-            into java.nio.file.Path.of(tmp_dir.absolutePath, "opensearch-ml")
-        }
-    }
-}
-
-task pullKnnBwcPlugin {
-    dependsOn "deletetempDirectories"
+// Task to pull opensearch artifact from archive
+task pullOpensearchArtifact {
+   dependsOn "deletetempDirectories"
 
-    doLast {
+   doLast{
         ext{
             if (isSnapshot) {
                 srcUrl = "https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/${neural_search_bwc_version_no_qualifier}/latest/${os_platform}/x64/${artifact_type}/dist/opensearch/opensearch-${neural_search_bwc_version_no_qualifier}-${os_platform}-x64.${file_ext}"
@@ -105,6 +96,24 @@ task pullKnnBwcPlugin {
             }
             into tmp_dir.absolutePath
         }
+   }
+}
+
+// Task to pull ml plugin from archive
+task pullMlCommonsBwcPlugin {
+    doLast {
+        copy {
+            from(java.nio.file.Path.of(tmp_dir.absolutePath, "opensearch-${neural_search_bwc_version_no_qualifier}", "plugins", "opensearch-ml"))
+            into java.nio.file.Path.of(tmp_dir.absolutePath, "opensearch-ml")
+        }
+    }
+}
+
+// Task to pull KNN plugin from archive
+task pullKnnBwcPlugin {
+    dependsOn "pullOpensearchArtifact"
+
+    doLast {
         copy {
             from(java.nio.file.Path.of(tmp_dir.absolutePath, "opensearch-${neural_search_bwc_version_no_qualifier}", "plugins", "opensearch-knn"))
             into java.nio.file.Path.of(tmp_dir.absolutePath, "opensearch-knn")
@@ -123,6 +132,7 @@ task pullBwcPlugin {
     }
 }
 
+// Task to zip ml-commons plugin from archive
 task zipBwcMlCommonsPlugin(type: Zip) {
     dependsOn "pullMlCommonsBwcPlugin"
     from(java.nio.file.Path.of(tmp_dir.absolutePath, "opensearch-ml"))
@@ -133,6 +143,7 @@ task zipBwcMlCommonsPlugin(type: Zip) {
     }
 }
 
+// Task to zip knn plugin from archive
 task zipBwcKnnPlugin(type: Zip) {
     dependsOn "zipBwcMlCommonsPlugin"
     dependsOn "pullKnnBwcPlugin"
@@ -144,7 +155,7 @@ task zipBwcKnnPlugin(type: Zip) {
     }
 }
 
-// Task to zip plugin from archive
+// Task to zip neural search plugin from archive
 task zipBwcPlugin(type: Zip) {
     dependsOn "zipBwcKnnPlugin"
     dependsOn "pullBwcPlugin"

diff --git a/qa/restart-upgrade/build.gradle b/qa/restart-upgrade/build.gradle
@@ -13,7 +13,6 @@ String default_bwc_version = System.getProperty("bwc.version")
 String neural_search_bwc_version = System.getProperty("tests.bwc.version", default_bwc_version)
 boolean isSnapshot = neural_search_bwc_version.contains("-SNAPSHOT")
 String baseName = "neuralSearchBwcCluster-restart"
-String knn_bwc_version_no_qualifier = isSnapshot ? neural_search_bwc_version - "-SNAPSHOT" : neural_search_bwc_version
 
 List<Provider<RegularFile>> plugins = [provider(new Callable<RegularFile>(){
     @Override
@@ -37,7 +36,7 @@ List<Provider<RegularFile>> plugins = [provider(new Callable<RegularFile>(){
     }
 }), rootProject.tasks.bundlePlugin.archiveFile]
 
-// Creates a test cluster of previous version and loads k-NN plugin of bwcVersion
+// Creates a test cluster of previous version and loads neuralsearch plugin of bwcVersion
 testClusters {
     "${baseName}" {
         testDistribution = "ARCHIVE"
@@ -49,22 +48,11 @@ testClusters {
         plugin(project.tasks.zipBwcPlugin.archiveFile)
         setting 'path.repo', "${buildDir}/cluster/shared/repo/${baseName}"
         setting 'http.content_type.required', 'true'
-        environment "LD_LIBRARY_PATH", "${buildDir}/testclusters/${baseName}-0/distro/${knn_bwc_version_no_qualifier}-ARCHIVE/plugins/opensearch-knn/knnlib;${buildDir}/testclusters/${baseName}-0/distro/${knn_bwc_version_no_qualifier}-ARCHIVE/plugins/opensearch-knn/lib"
-        if (Os.isFamily(Os.FAMILY_WINDOWS)) {
-            // While running on Windows OS, setting the PATH environment variable to include the paths to dlls of JNI libraries and windows dependencies
-            environment('PATH', System.getenv('PATH') + ";$rootDir/jni/release" + ";$rootDir/src/main/resources/windowsDependencies")
-            systemProperty "java.library.path", "${buildDir}/testclusters/${baseName}-0/distro/${knn_bwc_version_no_qualifier}-ARCHIVE/plugins/opensearch-knn/knnlib;${buildDir}/testclusters/${baseName}-0/distro/${knn_bwc_version_no_qualifier}-ARCHIVE/plugins/opensearch-knn/lib"
-        } else {
-            systemProperty "java.library.path", "${buildDir}/testclusters/${baseName}-0/distro/${knn_bwc_version_no_qualifier}-ARCHIVE/plugins/opensearch-knn/knnlib:${buildDir}/testclusters/${baseName}-0/distro/${knn_bwc_version_no_qualifier}-ARCHIVE/plugins/opensearch-knn/lib"
-        }
-
     }
 }
 
 // Task to run BWC tests against the old cluster
 task testAgainstOldCluster(type: StandaloneRestIntegTestTask) {
-    dependsOn "zipBwcMlCommonsPlugin"
-    dependsOn "zipBwcKnnPlugin"
     dependsOn "zipBwcPlugin"
     useCluster testClusters."${baseName}"
     systemProperty 'tests.rest.bwcsuite_cluster', 'old_cluster'
@@ -83,8 +71,6 @@ task testRestartUpgrade(type: StandaloneRestIntegTestTask) {
     dependsOn rootProject.tasks.assemble
     useCluster testClusters."${baseName}"
     doFirst {
-        testClusters."${baseName}".environment("LD_LIBRARY_PATH", "$rootDir/jni/release")
-        testClusters."${baseName}".systemProperty("java.library.path", "$rootDir/jni/release")
         testClusters."${baseName}".upgradeAllNodesAndPluginsToNextVersion(plugins)
     }
     systemProperty 'tests.rest.bwcsuite_cluster', 'upgraded_cluster'

diff --git a/...ade/src/test/java/org/opensearch/neuralsearch/bwc/AbstractRestartUpgradeRestTestCase.java b/...ade/src/test/java/org/opensearch/neuralsearch/bwc/AbstractRestartUpgradeRestTestCase.java
@@ -21,7 +21,7 @@ public abstract class AbstractRestartUpgradeRestTestCase extends BaseNeuralSearc
 
     @Before
     protected void setIndex() {
-        // Creating index name by concatenating "knn-bwc-" prefix with test method name
+        // Creating index name by concatenating "neural-bwc-" prefix with test method name
         // for all the tests in this sub-project
         testIndex = NEURAL_SEARCH_BWC_PREFIX + getTestName().toLowerCase(Locale.ROOT);
     }

diff --git a/qa/rolling-upgrade/build.gradle b/qa/rolling-upgrade/build.gradle
@@ -13,7 +13,6 @@ apply from : "$rootDir/qa/build.gradle"
 String default_bwc_version = System.getProperty("bwc.version")
 String neural_search_bwc_version = System.getProperty("tests.bwc.version", default_bwc_version)
 boolean isSnapshot = neural_search_bwc_version.contains("-SNAPSHOT")
-String knn_bwc_version_no_qualifier = isSnapshot ? neural_search_bwc_version - "-SNAPSHOT" : neural_search_bwc_version
 String baseName = "neuralSearchBwcCluster-rolling"
 
 List<Provider<RegularFile>> plugins = [provider(new Callable<RegularFile>(){
@@ -50,22 +49,12 @@ testClusters {
         jvmArgs("-Xms1g", "-Xmx4g")
         setting 'path.repo', "${buildDir}/cluster/shared/repo/${baseName}"
         setting 'http.content_type.required', 'true'
-        environment "LD_LIBRARY_PATH", "${buildDir}/testclusters/${baseName}-0/distro/${knn_bwc_version_no_qualifier}-ARCHIVE/plugins/opensearch-knn/knnlib;${buildDir}/testclusters/${baseName}-0/distro/${knn_bwc_version_no_qualifier}-ARCHIVE/plugins/opensearch-knn/lib"
-        if (Os.isFamily(Os.FAMILY_WINDOWS)) {
-            // While running on Windows OS, setting the PATH environment variable to include the paths to dlls of JNI libraries and windows dependencies
-            environment('PATH', System.getenv('PATH') + ";$rootDir/jni/release" + ";$rootDir/src/main/resources/windowsDependencies")
-            systemProperty "java.library.path", "${buildDir}/testclusters/${baseName}-0/distro/${knn_bwc_version_no_qualifier}-ARCHIVE/plugins/opensearch-knn/knnlib;${buildDir}/testclusters/${baseName}-0/distro/${knn_bwc_version_no_qualifier}-ARCHIVE/plugins/opensearch-knn/lib"
-        } else {
-            systemProperty "java.library.path", "${buildDir}/testclusters/${baseName}-0/distro/${knn_bwc_version_no_qualifier}-ARCHIVE/plugins/opensearch-knn/knnlib:${buildDir}/testclusters/${baseName}-0/distro/${knn_bwc_version_no_qualifier}-ARCHIVE/plugins/opensearch-knn/lib"
-        }
     }
 }
 
 
 // Task to run BWC tests against the old cluster
 task testAgainstOldCluster(type: StandaloneRestIntegTestTask) {
-    dependsOn "zipBwcMlCommonsPlugin"
-    dependsOn "zipBwcKnnPlugin"
     dependsOn "zipBwcPlugin"
     useCluster testClusters."${baseName}"
     systemProperty 'tests.rest.bwcsuite_cluster', 'old_cluster'
@@ -102,8 +91,6 @@ task testAgainstTwoThirdsUpgradedCluster(type: StandaloneRestIntegTestTask) {
     dependsOn "testAgainstOneThirdUpgradedCluster"
     useCluster testClusters."${baseName}"
     doFirst {
-        testClusters."${baseName}".getNodes().getAt("${baseName}" + "-1").environment("LD_LIBRARY_PATH", "$rootDir/jni/release")
-        testClusters."${baseName}".getNodes().getAt("${baseName}" + "-1").systemProperty("java.library.path", "$rootDir/jni/release")
         testClusters."${baseName}".upgradeNodeAndPluginToNextVersion(plugins)
     }
     systemProperty 'tests.rest.bwcsuite_cluster', 'mixed_cluster'
@@ -121,8 +108,6 @@ task testRollingUpgrade(type: StandaloneRestIntegTestTask) {
     dependsOn "testAgainstTwoThirdsUpgradedCluster"
     useCluster testClusters."${baseName}"
     doFirst {
-        testClusters."${baseName}".getNodes().getAt("${baseName}" + "-2").environment("LD_LIBRARY_PATH", "$rootDir/jni/release")
-        testClusters."${baseName}".getNodes().getAt("${baseName}" + "-2").systemProperty("java.library.path", "$rootDir/jni/release")
         testClusters."${baseName}".upgradeNodeAndPluginToNextVersion(plugins)
     }
     mustRunAfter "testAgainstOneThirdUpgradedCluster"

diff --git a/...upgrade/src/test/java/org/opensearch/neuralsearch/bwc/AbstractRollingUpgradeTestCase.java b/...upgrade/src/test/java/org/opensearch/neuralsearch/bwc/AbstractRollingUpgradeTestCase.java
@@ -25,7 +25,7 @@ public abstract class AbstractRollingUpgradeTestCase extends BaseNeuralSearchIT
 
     @Before
     protected void setIndex() {
-        // Creating index name by concatenating "knn-bwc-" prefix with test method name
+        // Creating index name by concatenating "neural-bwc-" prefix with test method name
         // for all the tests in this sub-project
         testIndex = NEURAL_SEARCH_BWC_PREFIX + getTestName().toLowerCase(Locale.ROOT);
     }