Bump main to 3.0.0 (opensearch-project#585)

* Bump main to 3.0.0 Signed-off-by: bowenlan-amzn <[email protected]> * Upgrade gradle Signed-off-by: bowenlan-amzn <[email protected]> * Refactor Signed-off-by: bowenlan-amzn <[email protected]> * Refactor for 3.0 Signed-off-by: bowenlan-amzn <[email protected]> * Directly consume plugin zip Signed-off-by: bowenlan-amzn <[email protected]> * Trying to fix bwc tests Signed-off-by: bowenlan-amzn <[email protected]> * Comment out rolling upgrade test Signed-off-by: bowenlan-amzn <[email protected]> --------- Signed-off-by: bowenlan-amzn <[email protected]>
ronnaksaxena · Feb 1, 2023 · e84d9e0 · e84d9e0
1 parent 536eb70
commit e84d9e0
Show file tree

Hide file tree

Showing 45 changed files with 320 additions and 263 deletions.
diff --git a/.github/workflows/bwc-test-workflow.yml b/.github/workflows/bwc-test-workflow.yml
@@ -14,8 +14,9 @@ jobs:
     steps:
       # This step uses the setup-java Github action: https://github.com/actions/setup-java
       - name: Set Up JDK
-        uses: actions/setup-java@v1
+        uses: actions/setup-java@v2
         with:
+          distribution: temurin # Temurin is a distribution of adoptium
           java-version: 17
       # index-management
       - name: Checkout Branch

diff --git a/.github/workflows/docker-security-test-workflow.yml b/.github/workflows/docker-security-test-workflow.yml
@@ -13,8 +13,9 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Set Up JDK
-        uses: actions/setup-java@v1
+        uses: actions/setup-java@v2
         with:
+          distribution: temurin # Temurin is a distribution of adoptium
           java-version: 17
       - name: Checkout Branch
         uses: actions/checkout@v2

diff --git a/.github/workflows/multi-node-test-workflow.yml b/.github/workflows/multi-node-test-workflow.yml
@@ -15,8 +15,9 @@ jobs:
     steps:
       # This step uses the setup-java Github action: https://github.com/actions/setup-java
       - name: Set Up JDK
-        uses: actions/setup-java@v1
+        uses: actions/setup-java@v2
         with:
+          distribution: temurin # Temurin is a distribution of adoptium
           java-version: 17
       # index-management
       - name: Checkout Branch

diff --git a/.github/workflows/release-workflow.yml b/.github/workflows/release-workflow.yml
@@ -20,8 +20,9 @@ jobs:
         uses: actions/checkout@v2
       # This step uses the setup-java Github action: https://github.com/actions/setup-java
       - name: Set Up JDK ${{ matrix.java }}
-        uses: actions/setup-java@v1
+        uses: actions/setup-java@v2
         with:
+          distribution: temurin # Temurin is a distribution of adoptium
           java-version: ${{ matrix.java }}
 
       # Building zip, deb and rpm files

diff --git a/.github/workflows/security-test-workflow.yml b/.github/workflows/security-test-workflow.yml
@@ -15,8 +15,9 @@ jobs:
     steps:
       # This step uses the setup-java Github action: https://github.com/actions/setup-java
       - name: Set Up JDK
-        uses: actions/setup-java@v1
+        uses: actions/setup-java@v2
         with:
+          distribution: temurin # Temurin is a distribution of adoptium
           java-version: 17
       # index-management
       - name: Checkout Branch

diff --git a/.github/workflows/test-and-build-workflow.yml b/.github/workflows/test-and-build-workflow.yml
@@ -30,8 +30,9 @@ jobs:
     steps:
       # This step uses the setup-java Github action: https://github.com/actions/setup-java
       - name: Set Up JDK ${{ matrix.java }}
-        uses: actions/setup-java@v1
+        uses: actions/setup-java@v2
         with:
+          distribution: temurin # Temurin is a distribution of adoptium
           java-version: ${{ matrix.java }}
       # build index management
       - name: Checkout Branch

diff --git a/build.gradle b/build.gradle
diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar
diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties
@@ -1,5 +1,5 @@
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-7.4.2-bin.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-7.5.1-bin.zip
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists
diff --git a/gradlew b/gradlew
@@ -205,6 +205,12 @@ set -- \
         org.gradle.wrapper.GradleWrapperMain \
         "$@"
 
+# Stop when "xargs" is not available.
+if ! command -v xargs >/dev/null 2>&1
+then
+    die "xargs is not available"
+fi
+
 # Use "xargs" to parse quoted args.
 #
 # With -n1 it outputs one arg per line, with the quotes and backslashes removed.

diff --git a/gradlew.bat b/gradlew.bat
@@ -14,7 +14,7 @@
 @rem limitations under the License.
 @rem
 
-@if "%DEBUG%" == "" @echo off
+@if "%DEBUG%"=="" @echo off
 @rem ##########################################################################
 @rem
 @rem  Gradle startup script for Windows
@@ -25,7 +25,7 @@
 if "%OS%"=="Windows_NT" setlocal
 
 set DIRNAME=%~dp0
-if "%DIRNAME%" == "" set DIRNAME=.
+if "%DIRNAME%"=="" set DIRNAME=.
 set APP_BASE_NAME=%~n0
 set APP_HOME=%DIRNAME%
 
@@ -40,7 +40,7 @@ if defined JAVA_HOME goto findJavaFromJavaHome
 
 set JAVA_EXE=java.exe
 %JAVA_EXE% -version >NUL 2>&1
-if "%ERRORLEVEL%" == "0" goto execute
+if %ERRORLEVEL% equ 0 goto execute
 
 echo.
 echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
@@ -75,13 +75,15 @@ set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar
 
 :end
 @rem End local scope for the variables with windows NT shell
-if "%ERRORLEVEL%"=="0" goto mainEnd
+if %ERRORLEVEL% equ 0 goto mainEnd
 
 :fail
 rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of
 rem the _cmd.exe /c_ return code!
-if  not "" == "%GRADLE_EXIT_CONSOLE%" exit 1
-exit /b 1
+set EXIT_CODE=%ERRORLEVEL%
+if %EXIT_CODE% equ 0 set EXIT_CODE=1
+if not ""=="%GRADLE_EXIT_CONSOLE%" exit %EXIT_CODE%
+exit /b %EXIT_CODE%
 
 :mainEnd
 if "%OS%"=="Windows_NT" endlocal

diff --git a/...in/kotlin/org.opensearch.indexmanagement.spi/indexstatemanagement/model/ActionMetaData.kt b/...in/kotlin/org.opensearch.indexmanagement.spi/indexstatemanagement/model/ActionMetaData.kt
@@ -62,7 +62,7 @@ data class ActionMetaData(
     }
 
     fun getMapValueString(): String {
-        return Strings.toString(this, false, false)
+        return Strings.toString(XContentType.JSON, this)
     }
 
     companion object {

diff --git a/.../org.opensearch.indexmanagement.spi/indexstatemanagement/model/PolicyRetryInfoMetaData.kt b/.../org.opensearch.indexmanagement.spi/indexstatemanagement/model/PolicyRetryInfoMetaData.kt
@@ -36,7 +36,7 @@ data class PolicyRetryInfoMetaData(
             .field(CONSUMED_RETRIES, consumedRetries)
     }
 
-    fun getMapValueString(): String = Strings.toString(this, false, false)
+    fun getMapValueString(): String = Strings.toString(XContentType.JSON, this)
 
     companion object {
         const val RETRY_INFO = "retry_info"

diff --git a/...ain/kotlin/org.opensearch.indexmanagement.spi/indexstatemanagement/model/StateMetaData.kt b/...ain/kotlin/org.opensearch.indexmanagement.spi/indexstatemanagement/model/StateMetaData.kt
@@ -38,7 +38,7 @@ data class StateMetaData(
             .field(START_TIME, startTime)
     }
 
-    fun getMapValueString(): String = Strings.toString(this, false, false)
+    fun getMapValueString(): String = Strings.toString(XContentType.JSON, this)
 
     companion object {
         const val STATE = "state"

diff --git a/...main/kotlin/org.opensearch.indexmanagement.spi/indexstatemanagement/model/StepMetaData.kt b/...main/kotlin/org.opensearch.indexmanagement.spi/indexstatemanagement/model/StepMetaData.kt
@@ -46,7 +46,7 @@ data class StepMetaData(
     }
 
     fun getMapValueString(): String {
-        return Strings.toString(this, false, false)
+        return Strings.toString(XContentType.JSON, this)
     }
 
     companion object {

diff --git a/.../kotlin/org.opensearch.indexmanagement.spi/indexstatemanagement/model/ValidationResult.kt b/.../kotlin/org.opensearch.indexmanagement.spi/indexstatemanagement/model/ValidationResult.kt
@@ -40,7 +40,7 @@ data class ValidationResult(
     }
 
     fun getMapValueString(): String {
-        return Strings.toString(this, false, false)
+        return Strings.toString(XContentType.JSON, this)
     }
 
     companion object {

diff --git a/src/main/kotlin/org/opensearch/indexmanagement/rollup/util/QueryShardContextFactory.kt b/src/main/kotlin/org/opensearch/indexmanagement/rollup/util/QueryShardContextFactory.kt
@@ -72,7 +72,7 @@ object QueryShardContextFactory {
         val (index, indexSettings, indexMetadata) = getIndexSettingsAndMetadata(indexName)
         val nodeSettings = Settings.builder()
             .put("node.name", "dummyNodeName")
-            .put(Environment.PATH_HOME_SETTING.key, environment.tmpFile())
+            .put(Environment.PATH_HOME_SETTING.key, environment.tmpDir())
             .build()
         val pluginsService =
             PluginsService(nodeSettings, null, null, null, listOf())

diff --git a/src/test/kotlin/org/opensearch/indexmanagement/IndexManagementIndicesIT.kt b/src/test/kotlin/org/opensearch/indexmanagement/IndexManagementIndicesIT.kt
@@ -5,8 +5,8 @@
 
 package org.opensearch.indexmanagement
 
-import org.apache.http.entity.ContentType
-import org.apache.http.entity.StringEntity
+import org.apache.hc.core5.http.ContentType
+import org.apache.hc.core5.http.io.entity.StringEntity
 import org.opensearch.common.settings.Settings
 import org.opensearch.common.xcontent.ToXContent
 import org.opensearch.common.xcontent.XContentFactory

diff --git a/src/test/kotlin/org/opensearch/indexmanagement/IndexManagementRestTestCase.kt b/src/test/kotlin/org/opensearch/indexmanagement/IndexManagementRestTestCase.kt
@@ -5,8 +5,8 @@
 
 package org.opensearch.indexmanagement
 
-import org.apache.http.entity.ContentType
-import org.apache.http.entity.StringEntity
+import org.apache.hc.core5.http.ContentType
+import org.apache.hc.core5.http.io.entity.StringEntity
 import org.junit.AfterClass
 import org.junit.Before
 import org.junit.rules.DisableOnDebug
@@ -59,7 +59,7 @@ abstract class IndexManagementRestTestCase : ODFERestTestCase() {
     fun initializeManagedIndex() {
         if (!indexExists(IndexManagementPlugin.INDEX_MANAGEMENT_INDEX)) {
             val request = Request("PUT", "/${IndexManagementPlugin.INDEX_MANAGEMENT_INDEX}")
-            var entity = "{\"settings\": " + Strings.toString(Settings.builder().put(INDEX_HIDDEN, true).build())
+            var entity = "{\"settings\": " + Strings.toString(XContentType.JSON, Settings.builder().put(INDEX_HIDDEN, true).build())
             entity += ",\"mappings\" : ${IndexManagementIndices.indexManagementMappings}}"
             request.setJsonEntity(entity)
             client().performRequest(request)
@@ -185,7 +185,7 @@ abstract class IndexManagementRestTestCase : ODFERestTestCase() {
             }
 
             val response = client.performRequest(Request("GET", "/_cat/indices?format=json&expand_wildcards=all"))
-            val xContentType = XContentType.fromMediaType(response.entity.contentType.value)
+            val xContentType = XContentType.fromMediaType(response.entity.contentType)
             xContentType.xContent().createParser(
                 NamedXContentRegistry.EMPTY, DeprecationHandler.THROW_UNSUPPORTED_OPERATION,
                 response.entity.content
@@ -254,7 +254,7 @@ abstract class IndexManagementRestTestCase : ODFERestTestCase() {
         protected fun waitForThreadPools(client: RestClient) {
             val response = client.performRequest(Request("GET", "/_cat/thread_pool?format=json"))
 
-            val xContentType = XContentType.fromMediaType(response.entity.contentType.value)
+            val xContentType = XContentType.fromMediaType(response.entity.contentType)
             xContentType.xContent().createParser(
                 NamedXContentRegistry.EMPTY, DeprecationHandler.THROW_UNSUPPORTED_OPERATION,
                 response.entity.content

diff --git a/src/test/kotlin/org/opensearch/indexmanagement/IndexStateManagementSecurityBehaviorIT.kt b/src/test/kotlin/org/opensearch/indexmanagement/IndexStateManagementSecurityBehaviorIT.kt
@@ -84,7 +84,9 @@ class IndexStateManagementSecurityBehaviorIT : SecurityRestTestCase() {
         assignRoleToUsers(HELPDESK_ROLE, listOf(superIsmUser))
 
         superUserClient =
-            SecureRestClientBuilder(clusterHosts.toTypedArray(), isHttps(), superIsmUser, password).setSocketTimeout(60000)
+            SecureRestClientBuilder(clusterHosts.toTypedArray(), isHttps(), superIsmUser, password).setSocketTimeout(
+                60000
+            ).setConnectionRequestTimeout(180000)
                 .build()
     }
 
@@ -105,6 +107,7 @@ class IndexStateManagementSecurityBehaviorIT : SecurityRestTestCase() {
 
         testClient =
             SecureRestClientBuilder(clusterHosts.toTypedArray(), isHttps(), testUser, password).setSocketTimeout(60000)
+                .setConnectionRequestTimeout(180000)
                 .build()
 
         val indexName = "${AIRLINE_INDEX}_index_basic"
@@ -157,6 +160,7 @@ class IndexStateManagementSecurityBehaviorIT : SecurityRestTestCase() {
 
         testClient =
             SecureRestClientBuilder(clusterHosts.toTypedArray(), isHttps(), testUser, password).setSocketTimeout(60000)
+                .setConnectionRequestTimeout(180000)
                 .build()
         try {
             val testPolicyJson = createReplicaCountTestPolicyRequest(10, "")
@@ -178,6 +182,7 @@ class IndexStateManagementSecurityBehaviorIT : SecurityRestTestCase() {
 
         testClient =
             SecureRestClientBuilder(clusterHosts.toTypedArray(), isHttps(), testUser, password).setSocketTimeout(60000)
+                .setConnectionRequestTimeout(180000)
                 .build()
         try {
             val testPolicyJson = createReplicaCountTestPolicyRequest(10, AIRLINE_INDEX_PATTERN)
@@ -197,10 +202,14 @@ class IndexStateManagementSecurityBehaviorIT : SecurityRestTestCase() {
     }
 
     fun `test delete policy`() {
-        createTestUserWithRole(listOf(EXPLAIN_INDEX, GET_POLICY, EXPLAIN_INDEX), listOf(GET_INDEX_MAPPING, SEARCH_INDEX))
+        createTestUserWithRole(
+            listOf(EXPLAIN_INDEX, GET_POLICY, EXPLAIN_INDEX),
+            listOf(GET_INDEX_MAPPING, SEARCH_INDEX)
+        )
 
         testClient =
             SecureRestClientBuilder(clusterHosts.toTypedArray(), isHttps(), testUser, password).setSocketTimeout(60000)
+                .setConnectionRequestTimeout(180000)
                 .build()
 
         try {

diff --git a/src/test/kotlin/org/opensearch/indexmanagement/ODFERestTestCase.kt b/src/test/kotlin/org/opensearch/indexmanagement/ODFERestTestCase.kt
@@ -5,7 +5,7 @@
 
 package org.opensearch.indexmanagement
 
-import org.apache.http.HttpHost
+import org.apache.hc.core5.http.HttpHost
 import org.opensearch.client.RestClient
 import org.opensearch.common.io.PathUtils
 import org.opensearch.common.settings.Settings
@@ -50,13 +50,16 @@ abstract class ODFERestTestCase : OpenSearchRestTestCase() {
                     // create adminDN (super-admin) client
                     val uri = javaClass.classLoader.getResource("security/sample.pem")?.toURI()
                     val configPath = PathUtils.get(uri).parent.toAbsolutePath()
-                    SecureRestClientBuilder(settings, configPath, hosts).setSocketTimeout(5000).build()
+                    SecureRestClientBuilder(settings, configPath, hosts).setSocketTimeout(60000)
+                        .setConnectionRequestTimeout(180000).build()
                 }
+
                 false -> {
                     // create client with passed user
                     val userName = System.getProperty("user")
                     val password = System.getProperty("password")
-                    SecureRestClientBuilder(hosts, isHttps(), userName, password).setSocketTimeout(5000).build()
+                    SecureRestClientBuilder(hosts, isHttps(), userName, password).setSocketTimeout(60000)
+                        .setConnectionRequestTimeout(180000).build()
                 }
             }
         } else {

diff --git a/src/test/kotlin/org/opensearch/indexmanagement/RollupSecurityBehaviorIT.kt b/src/test/kotlin/org/opensearch/indexmanagement/RollupSecurityBehaviorIT.kt
@@ -72,7 +72,9 @@ class RollupSecurityBehaviorIT : SecurityRestTestCase() {
         assignRoleToUsers(HELPDESK_ROLE, listOf(superRollupUser))
 
         superUserClient =
-            SecureRestClientBuilder(clusterHosts.toTypedArray(), isHttps(), superRollupUser, password).setSocketTimeout(60000)
+            SecureRestClientBuilder(clusterHosts.toTypedArray(), isHttps(), superRollupUser, password).setSocketTimeout(
+                60000
+            ).setConnectionRequestTimeout(180000)
                 .build()
     }
 
@@ -120,6 +122,7 @@ class RollupSecurityBehaviorIT : SecurityRestTestCase() {
 
         testUserClient =
             SecureRestClientBuilder(clusterHosts.toTypedArray(), isHttps(), testUser, password).setSocketTimeout(60000)
+                .setConnectionRequestTimeout(180000)
                 .build()
 
         val rollup = randomRollup()
@@ -149,6 +152,7 @@ class RollupSecurityBehaviorIT : SecurityRestTestCase() {
 
         testUserClient =
             SecureRestClientBuilder(clusterHosts.toTypedArray(), isHttps(), testUser, password).setSocketTimeout(60000)
+                .setConnectionRequestTimeout(180000)
                 .build()
 
         val rollup = randomRollup()
@@ -194,6 +198,7 @@ class RollupSecurityBehaviorIT : SecurityRestTestCase() {
 
         testUserClient =
             SecureRestClientBuilder(clusterHosts.toTypedArray(), isHttps(), testUser, password).setSocketTimeout(60000)
+                .setConnectionRequestTimeout(180000)
                 .build()
         val rollup = randomRollup()
 

diff --git a/src/test/kotlin/org/opensearch/indexmanagement/SecurityBehaviorIT.kt b/src/test/kotlin/org/opensearch/indexmanagement/SecurityBehaviorIT.kt
@@ -53,6 +53,7 @@ class SecurityBehaviorIT : SecurityRestTestCase() {
 
         johnClient =
             SecureRestClientBuilder(clusterHosts.toTypedArray(), isHttps(), john, password).setSocketTimeout(60000)
+                .setConnectionRequestTimeout(180000)
                 .build()
     }
 
@@ -97,9 +98,11 @@ class SecurityBehaviorIT : SecurityRestTestCase() {
 
         val jillClient =
             SecureRestClientBuilder(clusterHosts.toTypedArray(), isHttps(), jill, password).setSocketTimeout(60000)
+                .setConnectionRequestTimeout(180000)
                 .build()
         val janeClient =
             SecureRestClientBuilder(clusterHosts.toTypedArray(), isHttps(), jane, password).setSocketTimeout(60000)
+                .setConnectionRequestTimeout(180000)
                 .build()
 
         setFilterByBackendRole(false)

diff --git a/src/test/kotlin/org/opensearch/indexmanagement/SecurityRestTestCase.kt b/src/test/kotlin/org/opensearch/indexmanagement/SecurityRestTestCase.kt
@@ -11,16 +11,17 @@
 
 package org.opensearch.indexmanagement
 
-import org.apache.http.HttpHeaders
-import org.apache.http.entity.ContentType
-import org.apache.http.entity.StringEntity
-import org.apache.http.message.BasicHeader
+import org.apache.hc.core5.http.HttpHeaders
+import org.apache.hc.core5.http.ContentType
+import org.apache.hc.core5.http.io.entity.StringEntity
+import org.apache.hc.core5.http.message.BasicHeader
 import org.opensearch.client.Request
 import org.opensearch.client.Response
 import org.opensearch.client.ResponseException
 import org.opensearch.client.RestClient
 import org.opensearch.common.Strings
 import org.opensearch.common.settings.Settings
+import org.opensearch.common.xcontent.XContentType
 import org.opensearch.indexmanagement.indexstatemanagement.IndexStateManagementRestTestCase
 import org.opensearch.indexmanagement.indexstatemanagement.model.ManagedIndexConfig
 import org.opensearch.indexmanagement.indexstatemanagement.model.Policy
@@ -254,7 +255,7 @@ abstract class SecurityRestTestCase : IndexManagementRestTestCase() {
         }.build()
 
         val request = Request("PUT", "/$indexName")
-        var entity = "{\"settings\": " + Strings.toString(builtSettings)
+        var entity = "{\"settings\": " + Strings.toString(XContentType.JSON, builtSettings)
         if (sourceIndexMappingString != null) {
             entity = "$entity,\"mappings\" : {$sourceIndexMappingString}"
         }