Merge branch 'main' into listShardsClosedIndexFix

Signed-off-by: Harsh Garg <[email protected]>

CHANGELOG.md

@@ -21,11 +21,12 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Support prefix list for remote repository attributes([#16271](https://github.com/opensearch-project/OpenSearch/pull/16271))
 - Add new configuration setting `synonym_analyzer`, to the `synonym` and `synonym_graph` filters, enabling the specification of a custom analyzer for reading the synonym file ([#16488](https://github.com/opensearch-project/OpenSearch/pull/16488)).
 - Add stats for remote publication failure and move download failure stats to remote methods([#16682](https://github.com/opensearch-project/OpenSearch/pull/16682/))
+- Added a precaution to handle extreme date values during sorting to prevent `arithmetic_exception: long overflow` ([#16812](https://github.com/opensearch-project/OpenSearch/pull/16812)).
 
 ### Dependencies
 - Bump `com.google.cloud:google-cloud-core-http` from 2.23.0 to 2.47.0 ([#16504](https://github.com/opensearch-project/OpenSearch/pull/16504))
 - Bump `google-auth-library-oauth2-http` from 1.7.0 to 1.29.0 in /plugins/repository-gcs ([#16520](https://github.com/opensearch-project/OpenSearch/pull/16520))
-- Bump `com.azure:azure-storage-common` from 12.25.1 to 12.27.1 ([#16521](https://github.com/opensearch-project/OpenSearch/pull/16521))
+- Bump `com.azure:azure-storage-common` from 12.25.1 to 12.28.0 ([#16521](https://github.com/opensearch-project/OpenSearch/pull/16521), [#16808](https://github.com/opensearch-project/OpenSearch/pull/16808))
 - Bump `com.google.apis:google-api-services-compute` from v1-rev20240407-2.0.0 to v1-rev20241105-2.0.0 ([#16502](https://github.com/opensearch-project/OpenSearch/pull/16502), [#16548](https://github.com/opensearch-project/OpenSearch/pull/16548), [#16613](https://github.com/opensearch-project/OpenSearch/pull/16613))
 - Bump `com.azure:azure-storage-blob` from 12.23.0 to 12.28.1 ([#16501](https://github.com/opensearch-project/OpenSearch/pull/16501))
 - Bump `org.apache.hadoop:hadoop-minicluster` from 3.4.0 to 3.4.1 ([#16550](https://github.com/opensearch-project/OpenSearch/pull/16550))
@@ -40,9 +41,11 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Bump `org.apache.logging.log4j:log4j-core` from 2.24.1 to 2.24.2 ([#16718](https://github.com/opensearch-project/OpenSearch/pull/16718))
 - Bump `jackson` from 2.17.2 to 2.18.2 ([#16733](https://github.com/opensearch-project/OpenSearch/pull/16733))
 - Bump `ch.qos.logback:logback-classic` from 1.2.13 to 1.5.12 ([#16716](https://github.com/opensearch-project/OpenSearch/pull/16716))
+- Bump `com.azure:azure-identity` from 1.13.2 to 1.14.2 ([#16778](https://github.com/opensearch-project/OpenSearch/pull/16778))
 
 ### Changed
 - Indexed IP field supports `terms_query` with more than 1025 IP masks [#16391](https://github.com/opensearch-project/OpenSearch/pull/16391)
+- Make entries for dependencies from server/build.gradle to gradle version catalog ([#16707](https://github.com/opensearch-project/OpenSearch/pull/16707))
 
 ### Deprecated
 - Performing update operation with default pipeline or final pipeline is deprecated ([#16712](https://github.com/opensearch-project/OpenSearch/pull/16712))
@@ -61,7 +64,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Fix stale cluster state custom file deletion ([#16670](https://github.com/opensearch-project/OpenSearch/pull/16670))
 - [Tiered Caching] Fix bug in cache stats API ([#16560](https://github.com/opensearch-project/OpenSearch/pull/16560))
 - Bound the size of cache in deprecation logger ([16702](https://github.com/opensearch-project/OpenSearch/issues/16702))
-- Fix _list/shards API failing when closed indices are present in a cluster ([#16606](https://github.com/opensearch-project/OpenSearch/pull/16606))
+- Ensure consistency of system flag on IndexMetadata after diff is applied ([#16644](https://github.com/opensearch-project/OpenSearch/pull/16644))
+- Skip remote-repositories validations for node-joins when RepositoriesService is not in sync with cluster-state ([#16763](https://github.com/opensearch-project/OpenSearch/pull/16763))
+- Fix _list/shards API failing when closed indices are present ([#16606](https://github.com/opensearch-project/OpenSearch/pull/16606))
 
 ### Security
 

buildSrc/src/main/groovy/org/opensearch/gradle/plugin/OptionalDependenciesPlugin.groovy

@@ -79,7 +79,7 @@ class OptionalDependenciesPlugin implements Plugin<Project> {
 
                                 if (foundDep) {
                                     if (foundDep.optional) {
-                                        foundDep.optional.value = 'true'
+                                        foundDep.optional*.value = 'true'
                                     } else {
                                         foundDep.appendNode(OPTIONAL_IDENTIFIER, 'true')
                                     }

buildSrc/src/main/java/org/opensearch/gradle/docker/DockerSupportService.java

@@ -105,16 +105,15 @@ public DockerAvailability getDockerAvailability() {
             Result lastResult = null;
             Version version = null;
             boolean isVersionHighEnough = false;
-            boolean isComposeAvailable = false;
-            boolean isComposeV2Available = false;
+            DockerComposeAvailability dockerComposeAvailability = null;
 
             // Check if the Docker binary exists
             final Optional<String> dockerBinary = getDockerPath();
             if (isExcludedOs() == false && dockerBinary.isPresent()) {
                 dockerPath = dockerBinary.get();
 
                 // Since we use a multi-stage Docker build, check the Docker version meets minimum requirement
-                lastResult = runCommand(dockerPath, "version", "--format", "{{.Server.Version}}");
+                lastResult = runCommand(execOperations, dockerPath, "version", "--format", "{{.Server.Version}}");
 
                 if (lastResult.isSuccess()) {
                     version = Version.fromString(lastResult.stdout.trim(), Version.Mode.RELAXED);
@@ -123,15 +122,11 @@ public DockerAvailability getDockerAvailability() {
 
                     if (isVersionHighEnough) {
                         // Check that we can execute a privileged command
-                        lastResult = runCommand(dockerPath, "images");
-
+                        lastResult = runCommand(execOperations, dockerPath, "images");
                         // If docker all checks out, see if docker-compose is available and working
-                        Optional<String> composePath = getDockerComposePath();
-                        if (lastResult.isSuccess() && composePath.isPresent()) {
-                            isComposeAvailable = runCommand(composePath.get(), "version").isSuccess();
+                        if (lastResult.isSuccess()) {
+                            dockerComposeAvailability = DockerComposeAvailability.detect(execOperations, dockerPath).orElse(null);
                         }
-
-                        isComposeV2Available = runCommand(dockerPath, "compose", "version").isSuccess();
                     }
                 }
             }
@@ -140,8 +135,7 @@ public DockerAvailability getDockerAvailability() {
 
             this.dockerAvailability = new DockerAvailability(
                 isAvailable,
-                isComposeAvailable,
-                isComposeV2Available,
+                dockerComposeAvailability,
                 isVersionHighEnough,
                 dockerPath,
                 version,
@@ -291,17 +285,6 @@ private Optional<String> getDockerPath() {
         return Arrays.asList(DOCKER_BINARIES).stream().filter(path -> new File(path).exists()).findFirst();
     }
 
-    /**
-     * Searches the entries in {@link #DOCKER_COMPOSE_BINARIES} for the Docker Compose CLI. This method does
-     * not check whether the installation appears usable, see {@link #getDockerAvailability()} instead.
-     *
-     * @return the path to a CLI, if available.
-     */
-    private Optional<String> getDockerComposePath() {
-        // Check if the Docker binary exists
-        return Arrays.asList(DOCKER_COMPOSE_BINARIES).stream().filter(path -> new File(path).exists()).findFirst();
-    }
-
     private void throwDockerRequiredException(final String message) {
         throwDockerRequiredException(message, null);
     }
@@ -321,7 +304,7 @@ private void throwDockerRequiredException(final String message, Exception e) {
      * while running the command, or the process was killed after reaching the 10s timeout,
      * then the exit code will be -1.
      */
-    private Result runCommand(String... args) {
+    private static Result runCommand(ExecOperations execOperations, String... args) {
         if (args.length == 0) {
             throw new IllegalArgumentException("Cannot execute with no command");
         }
@@ -356,14 +339,9 @@ public static class DockerAvailability {
         public final boolean isAvailable;
 
         /**
-         * True if docker-compose is available.
+         * Non-null if docker-compose v1 or v2 is available.
          */
-        public final boolean isComposeAvailable;
-
-        /**
-         * True if docker compose is available.
-         */
-        public final boolean isComposeV2Available;
+        public final DockerComposeAvailability dockerComposeAvailability;
 
         /**
          * True if the installed Docker version is &gt;= 17.05
@@ -387,23 +365,70 @@ public static class DockerAvailability {
 
         DockerAvailability(
             boolean isAvailable,
-            boolean isComposeAvailable,
-            boolean isComposeV2Available,
+            DockerComposeAvailability dockerComposeAvailability,
             boolean isVersionHighEnough,
             String path,
             Version version,
             Result lastCommand
         ) {
             this.isAvailable = isAvailable;
-            this.isComposeAvailable = isComposeAvailable;
-            this.isComposeV2Available = isComposeV2Available;
+            this.dockerComposeAvailability = dockerComposeAvailability;
             this.isVersionHighEnough = isVersionHighEnough;
             this.path = path;
             this.version = version;
             this.lastCommand = lastCommand;
         }
+
+        public boolean isDockerComposeAvailable() {
+            return dockerComposeAvailability != null;
+        }
+    }
+
+    /**
+     * Marker interface for Docker Compose availability
+     */
+    private interface DockerComposeAvailability {
+        /**
+         * Detects Docker Compose V1/V2 availability
+         */
+        private static Optional<DockerComposeAvailability> detect(ExecOperations execOperations, String dockerPath) {
+            Optional<String> composePath = getDockerComposePath();
+            if (composePath.isPresent()) {
+                if (runCommand(execOperations, composePath.get(), "version").isSuccess()) {
+                    return Optional.of(new DockerComposeV1Availability());
+                }
+            }
+
+            if (runCommand(execOperations, dockerPath, "compose", "version").isSuccess()) {
+                return Optional.of(new DockerComposeV2Availability());
+            }
+
+            return Optional.empty();
+        }
+
+        /**
+         * Searches the entries in {@link #DOCKER_COMPOSE_BINARIES} for the Docker Compose CLI. This method does
+         * not check whether the installation appears usable, see {@link #getDockerAvailability()} instead.
+         *
+         * @return the path to a CLI, if available.
+         */
+        private static Optional<String> getDockerComposePath() {
+            // Check if the Docker binary exists
+            return Arrays.asList(DOCKER_COMPOSE_BINARIES).stream().filter(path -> new File(path).exists()).findFirst();
+        }
+
     }
 
+    /**
+     * Docker Compose V1 availability
+     */
+    public static class DockerComposeV1Availability implements DockerComposeAvailability {}
+
+    /**
+     * Docker Compose V2 availability
+     */
+    public static class DockerComposeV2Availability implements DockerComposeAvailability {}
+
     /**
      * This class models the result of running a command. It captures the exit code, standard output and standard error.
      */

buildSrc/src/main/java/org/opensearch/gradle/testfixtures/TestFixturesPlugin.java

@@ -43,6 +43,7 @@
 import org.opensearch.gradle.SystemPropertyCommandLineArgumentProvider;
 import org.opensearch.gradle.docker.DockerSupportPlugin;
 import org.opensearch.gradle.docker.DockerSupportService;
+import org.opensearch.gradle.docker.DockerSupportService.DockerComposeV2Availability;
 import org.opensearch.gradle.info.BuildParams;
 import org.opensearch.gradle.precommit.TestingConventionsTasks;
 import org.opensearch.gradle.util.GradleUtils;
@@ -171,11 +172,8 @@ public void execute(Task task) {
                 .findFirst();
 
             composeExtension.getExecutable().set(dockerCompose.isPresent() ? dockerCompose.get() : "/usr/bin/docker");
-            if (dockerSupport.get().getDockerAvailability().isComposeV2Available) {
-                composeExtension.getUseDockerComposeV2().set(true);
-            } else if (dockerSupport.get().getDockerAvailability().isComposeAvailable) {
-                composeExtension.getUseDockerComposeV2().set(false);
-            }
+            composeExtension.getUseDockerComposeV2()
+                .set(dockerSupport.get().getDockerAvailability().dockerComposeAvailability instanceof DockerComposeV2Availability);
 
             tasks.named("composeUp").configure(t -> {
                 // Avoid running docker-compose tasks in parallel in CI due to some issues on certain Linux distributions
@@ -232,8 +230,7 @@ private void maybeSkipTask(Provider<DockerSupportService> dockerSupport, TaskPro
 
     private void maybeSkipTask(Provider<DockerSupportService> dockerSupport, Task task) {
         task.onlyIf(spec -> {
-            boolean isComposeAvailable = dockerSupport.get().getDockerAvailability().isComposeV2Available
-                || dockerSupport.get().getDockerAvailability().isComposeAvailable;
+            boolean isComposeAvailable = dockerSupport.get().getDockerAvailability().isDockerComposeAvailable();
             if (isComposeAvailable == false) {
                 LOGGER.info("Task {} requires docker-compose but it is unavailable. Task will be skipped.", task.getPath());
             }

gradle/libs.versions.toml

@@ -34,6 +34,7 @@ jna               = "5.13.0"
 
 netty             = "4.1.115.Final"
 joda              = "2.12.7"
+roaringbitmap     = "1.3.0"
 
 # project reactor
 reactor_netty     = "1.1.23"
@@ -83,3 +84,52 @@ opentelemetrysemconv  = "1.27.0-alpha"
 # arrow dependencies
 arrow                 = "17.0.0"
 flatbuffers           = "2.0.0"
+
+[libraries]
+hdrhistogram = { group = "org.hdrhistogram", name = "HdrHistogram", version.ref = "hdrhistogram" }
+jakartaannotation = { group = "jakarta.annotation", name = "jakarta.annotation-api", version.ref = "jakarta_annotation" }
+jodatime = { group = "joda-time", name = "joda-time", version.ref = "joda" }
+jna = { group = "net.java.dev.jna", name = "jna", version.ref = "jna" }
+jtscore = { group = "org.locationtech.jts", name = "jts-core", version.ref = "jts" }
+jzlib = { group = "com.jcraft", name = "jzlib", version.ref = "jzlib" }
+log4japi = { group = "org.apache.logging.log4j", name = "log4j-api", version.ref = "log4j" }
+log4jjul = { group = "org.apache.logging.log4j", name = "log4j-jul", version.ref = "log4j" }
+log4jcore = { group = "org.apache.logging.log4j", name = "log4j-core", version.ref = "log4j" }
+lucene-core = { group = "org.apache.lucene", name = "lucene-core", version.ref = "lucene" }
+lucene-analysis-common = { group = "org.apache.lucene", name = "lucene-analysis-common", version.ref = "lucene" }
+lucene-backward-codecs = { group = "org.apache.lucene", name = "lucene-backward-codecs", version.ref = "lucene" }
+lucene-grouping = { group = "org.apache.lucene", name = "lucene-grouping", version.ref = "lucene" }
+lucene-highlighter = { group = "org.apache.lucene", name = "lucene-highlighter", version.ref = "lucene" }
+lucene-join = { group = "org.apache.lucene", name = "lucene-join", version.ref = "lucene" }
+lucene-memory = { group = "org.apache.lucene", name = "lucene-memory", version.ref = "lucene" }
+lucene-misc = { group = "org.apache.lucene", name = "lucene-misc", version.ref = "lucene" }
+lucene-queries = { group = "org.apache.lucene", name = "lucene-queries", version.ref = "lucene" }
+lucene-queryparser = { group = "org.apache.lucene", name = "lucene-queryparser", version.ref = "lucene" }
+lucene-sandbox = { group = "org.apache.lucene", name = "lucene-sandbox", version.ref = "lucene" }
+lucene-spatial-extras = { group = "org.apache.lucene", name = "lucene-spatial-extras", version.ref = "lucene" }
+lucene-spatial3d = { group = "org.apache.lucene", name = "lucene-spatial3d", version.ref = "lucene" }
+lucene-suggest = { group = "org.apache.lucene", name = "lucene-suggest", version.ref = "lucene" }
+protobuf = { group = "com.google.protobuf", name = "protobuf-java", version.ref = "protobuf" }
+reactivestreams = { group = "io.projectreactor", name = "reactor-core", version.ref = "reactor" }
+reactorcore = { group = "org.reactivestreams", name = "reactive-streams", version.ref = "reactivestreams" }
+roaringbitmap = { group = "org.roaringbitmap", name = "RoaringBitmap", version.ref = "roaringbitmap" }
+spatial4j = { group = "org.locationtech.spatial4j", name = "spatial4j", version.ref = "spatial4j" }
+tdigest = { group = "com.tdunning", name = "t-digest", version.ref = "tdigest" }
+
+[bundles]
+lucene = [
+    "lucene-core",
+    "lucene-analysis-common",
+    "lucene-backward-codecs",
+    "lucene-grouping",
+    "lucene-highlighter",
+    "lucene-join",
+    "lucene-memory",
+    "lucene-misc",
+    "lucene-queries",
+    "lucene-queryparser",
+    "lucene-sandbox",
+    "lucene-spatial-extras",
+    "lucene-spatial3d",
+    "lucene-suggest"
+]

plugins/repository-azure/build.gradle

@@ -47,7 +47,7 @@ dependencies {
   api 'com.azure:azure-core:1.51.0'
   api 'com.azure:azure-json:1.3.0'
   api 'com.azure:azure-xml:1.1.0'
-  api 'com.azure:azure-storage-common:12.27.1'
+  api 'com.azure:azure-storage-common:12.28.0'
   api 'com.azure:azure-core-http-netty:1.15.5'
   api "io.netty:netty-codec-dns:${versions.netty}"
   api "io.netty:netty-codec-socks:${versions.netty}"
@@ -57,7 +57,7 @@ dependencies {
   api "io.netty:netty-transport-native-unix-common:${versions.netty}"
   implementation project(':modules:transport-netty4')
   api 'com.azure:azure-storage-blob:12.28.1'
-  api 'com.azure:azure-identity:1.13.2'
+  api 'com.azure:azure-identity:1.14.2'
   // Start of transitive dependencies for azure-identity
   api 'com.microsoft.azure:msal4j-persistence-extension:1.3.0'
   api "net.java.dev.jna:jna-platform:${versions.jna}"
@@ -108,6 +108,7 @@ thirdPartyAudit {
     // Optional and not enabled by Elasticsearch
     'com.google.common.util.concurrent.internal.InternalFutureFailureAccess',
     'com.google.common.util.concurrent.internal.InternalFutures',
+    'com.azure.core.credential.ProofOfPossessionOptions',
     'com.azure.storage.internal.avro.implementation.AvroObject',
     'com.azure.storage.internal.avro.implementation.AvroReader',
     'com.azure.storage.internal.avro.implementation.AvroReaderFactory',

plugins/repository-azure/licenses/azure-identity-1.14.2.jar.sha1

@@ -0,0 +1 @@
+85c45e2add38742009a9c5070d2a9d8f192cf8db

plugins/repository-azure/licenses/azure-storage-common-12.28.0.jar.sha1

@@ -0,0 +1 @@
+3c5b7de96c68947ab74cc7925b27ca2b9f6b91d0

plugins/repository-azure/src/test/java/org/opensearch/repositories/azure/AzureBlobContainerRetriesTests.java

@@ -88,6 +88,7 @@
 
 import fixture.azure.AzureHttpHandler;
 import reactor.core.scheduler.Schedulers;
+import reactor.netty.http.HttpResources;
 
 import static java.nio.charset.StandardCharsets.UTF_8;
 import static org.opensearch.repositories.azure.AzureRepository.Repository.CONTAINER_SETTING;
@@ -142,6 +143,7 @@ public void tearDown() throws Exception {
 
     @AfterClass
     public static void shutdownSchedulers() {
+        HttpResources.disposeLoopsAndConnections();
         Schedulers.shutdownNow();
     }