Merge branch 'opensearch-project:main' into main

.github/workflows/assemble.yml

@@ -7,13 +7,14 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
+        java: [ 11, 17, 21 ]
         os: [ubuntu-latest, windows-latest, macos-latest]
     steps:
       - uses: actions/checkout@v4
-      - name: Set up JDK 11
+      - name: Set up JDK ${{ matrix.java }}
         uses: actions/setup-java@v3
         with:
-          java-version: 11
+          java-version: ${{ matrix.java }}
           distribution: temurin
       - name: Setup docker (missing on MacOS)
         if: runner.os == 'macos'

.github/workflows/precommit.yml

@@ -7,13 +7,14 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
+        java: [ 11, 17, 21 ]
         os: [ubuntu-latest, windows-latest, macos-latest]
     steps:
       - uses: actions/checkout@v4
-      - name: Set up JDK 11
+      - name: Set up JDK ${{ matrix.java }}
         uses: actions/setup-java@v3
         with:
-          java-version: 11
+          java-version: ${{ matrix.java }}
           distribution: temurin
           cache: gradle
       - name: Run Gradle (precommit)

CHANGELOG.md

@@ -64,7 +64,6 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Return 409 Conflict HTTP status instead of 503 on failure to concurrently execute snapshots ([#8986](https://github.com/opensearch-project/OpenSearch/pull/5855))
 - Add task completion count in search backpressure stats API ([#10028](https://github.com/opensearch-project/OpenSearch/pull/10028/))
 - Performance improvement for Datetime field caching ([#4558](https://github.com/opensearch-project/OpenSearch/issues/4558))
-- Performance improvement for MultiTerm Queries on Keyword fields ([#7057](https://github.com/opensearch-project/OpenSearch/issues/7057))
 
 
 ### Deprecated
@@ -100,8 +99,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Introduce ConcurrentQueryProfiler to profile query using concurrent segment search path and support concurrency during rewrite and create weight ([10352](https://github.com/opensearch-project/OpenSearch/pull/10352))
 - Update the indexRandom function to create more segments for concurrent search tests ([10247](https://github.com/opensearch-project/OpenSearch/pull/10247))
 - [Remote cluster state] Make index and global metadata upload timeout dynamic cluster settings ([#10814](https://github.com/opensearch-project/OpenSearch/pull/10814))
-- Added cluster setting cluster.restrict.index.replication_type to restrict setting of index setting replication type ([#10866](https://github.com/opensearch-project/OpenSearch/pull/10866))
 - Add cluster state stats ([#10670](https://github.com/opensearch-project/OpenSearch/pull/10670))
+- Adding slf4j license header to LoggerMessageFormat.java ([#11069](https://github.com/opensearch-project/OpenSearch/pull/11069))
+- [Streaming Indexing] Introduce new experimental server HTTP transport based on Netty 4 and Project Reactor (Reactor Netty) ([#9672](https://github.com/opensearch-project/OpenSearch/pull/9672))
 
 ### Dependencies
 - Bump `com.google.api.grpc:proto-google-common-protos` from 2.10.0 to 2.25.1 ([#10208](https://github.com/opensearch-project/OpenSearch/pull/10208), [#10298](https://github.com/opensearch-project/OpenSearch/pull/10298))
@@ -126,6 +126,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Add telemetry tracer/metric enable flag and integ test. ([#10395](https://github.com/opensearch-project/OpenSearch/pull/10395))
 - Add instrumentation for indexing in transport bulk action and transport shard bulk action. ([#10273](https://github.com/opensearch-project/OpenSearch/pull/10273))
 - [BUG] Disable sort optimization for HALF_FLOAT ([#10999](https://github.com/opensearch-project/OpenSearch/pull/10999))
+- Performance improvement for MultiTerm Queries on Keyword fields ([#7057](https://github.com/opensearch-project/OpenSearch/issues/7057))
+- Disable concurrent aggs for Diversified Sampler and Sampler aggs ([#11087](https://github.com/opensearch-project/OpenSearch/issues/11087))
 
 ### Deprecated
 
@@ -138,6 +140,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Fix Segment Replication ShardLockObtainFailedException bug during index corruption ([10370](https://github.com/opensearch-project/OpenSearch/pull/10370))
 - Fix some test methods in SimulatePipelineRequestParsingTests never run and fix test failure ([#10496](https://github.com/opensearch-project/OpenSearch/pull/10496))
 - Fix passing wrong parameter when calling newConfigurationException() in DotExpanderProcessor ([#10737](https://github.com/opensearch-project/OpenSearch/pull/10737))
+- Adding version condition while adding geoshape doc values to the index, to ensure backward compatibility.([#11095](https://github.com/opensearch-project/OpenSearch/pull/11095))
 
 ### Security
 

buildSrc/version.properties

@@ -29,6 +29,10 @@ jna               = 5.13.0
 netty             = 4.1.100.Final
 joda              = 2.12.2
 
+# project reactor
+reactor_netty     = 1.1.12
+reactor           = 3.5.11
+
 # client dependencies
 httpclient5       = 5.2.1
 httpcore5         = 5.2.2

client/rest-high-level/src/test/java/org/opensearch/client/PitIT.java

@@ -24,7 +24,6 @@
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
-import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
 import java.util.stream.Collectors;
 
@@ -72,7 +71,7 @@ public void testCreateAndDeletePit() throws IOException {
         assertTrue(deletePitResponse.getDeletePitResults().get(0).getPitId().equals(createPitResponse.getId()));
     }
 
-    public void testDeleteAllAndListAllPits() throws IOException, InterruptedException {
+    public void testDeleteAllAndListAllPits() throws Exception {
         CreatePitRequest pitRequest = new CreatePitRequest(new TimeValue(1, TimeUnit.DAYS), true, "index");
         CreatePitResponse pitResponse = execute(pitRequest, highLevelClient()::createPit, highLevelClient()::createPitAsync);
         CreatePitResponse pitResponse1 = execute(pitRequest, highLevelClient()::createPit, highLevelClient()::createPitAsync);
@@ -91,31 +90,30 @@ public void testDeleteAllAndListAllPits() throws IOException, InterruptedExcepti
         List<String> pits = getAllPitResponse.getPitInfos().stream().map(r -> r.getPitId()).collect(Collectors.toList());
         assertTrue(pits.contains(pitResponse.getId()));
         assertTrue(pits.contains(pitResponse1.getId()));
-        CountDownLatch countDownLatch = new CountDownLatch(1);
         ActionListener<DeletePitResponse> deletePitListener = new ActionListener<>() {
             @Override
             public void onResponse(DeletePitResponse response) {
-                countDownLatch.countDown();
                 for (DeletePitInfo deletePitInfo : response.getDeletePitResults()) {
                     assertTrue(deletePitInfo.isSuccessful());
                 }
             }
 
             @Override
             public void onFailure(Exception e) {
-                countDownLatch.countDown();
                 if (!(e instanceof OpenSearchStatusException)) {
                     throw new AssertionError("Delete all failed");
                 }
             }
         };
         final CreatePitResponse pitResponse3 = execute(pitRequest, highLevelClient()::createPit, highLevelClient()::createPitAsync);
-
+        assertTrue(pitResponse3.getId() != null);
         ActionListener<GetAllPitNodesResponse> getPitsListener = new ActionListener<GetAllPitNodesResponse>() {
             @Override
             public void onResponse(GetAllPitNodesResponse response) {
                 List<String> pits = response.getPitInfos().stream().map(r -> r.getPitId()).collect(Collectors.toList());
                 assertTrue(pits.contains(pitResponse3.getId()));
+                // delete all pits
+                highLevelClient().deleteAllPitsAsync(RequestOptions.DEFAULT, deletePitListener);
             }
 
             @Override
@@ -126,11 +124,12 @@ public void onFailure(Exception e) {
             }
         };
         highLevelClient().getAllPitsAsync(RequestOptions.DEFAULT, getPitsListener);
-        highLevelClient().deleteAllPitsAsync(RequestOptions.DEFAULT, deletePitListener);
-        assertTrue(countDownLatch.await(10, TimeUnit.SECONDS));
+
         // validate no pits case
-        getAllPitResponse = highLevelClient().getAllPits(RequestOptions.DEFAULT);
-        assertTrue(getAllPitResponse.getPitInfos().size() == 0);
-        highLevelClient().deleteAllPitsAsync(RequestOptions.DEFAULT, deletePitListener);
+        assertBusy(() -> {
+            GetAllPitNodesResponse getAllPitResponse1 = highLevelClient().getAllPits(RequestOptions.DEFAULT);
+            assertTrue(getAllPitResponse1.getPitInfos().size() == 0);
+            highLevelClient().deleteAllPitsAsync(RequestOptions.DEFAULT, deletePitListener);
+        });
     }
 }

libs/core/src/main/java/org/opensearch/core/common/logging/LoggerMessageFormat.java

@@ -30,6 +30,13 @@
  * GitHub history for details.
  */
 
+/*
+ * This code is based on code from SFL4J 1.5.11
+ * Copyright (c) 2004-2007 QOS.ch
+ * All rights reserved.
+ * SPDX-License-Identifier: MIT
+ */
+
 package org.opensearch.core.common.logging;
 
 import java.util.HashSet;

modules/geo/src/internalClusterTest/java/org/opensearch/geo/search/aggregations/bucket/AbstractGeoBucketAggregationIntegTest.java

@@ -87,7 +87,7 @@ protected boolean forbidPrivateIndexSettings() {
      */
     protected void prepareGeoShapeIndexForAggregations(final Random random) throws Exception {
         expectedDocsCountForGeoShapes = new HashMap<>();
-        final Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, version).build();
+        final Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT).build();
         final List<IndexRequestBuilder> geoshapes = new ArrayList<>();
         assertAcked(prepareCreate(GEO_SHAPE_INDEX_NAME).setSettings(settings).setMapping(GEO_SHAPE_FIELD_NAME, "type" + "=geo_shape"));
         boolean isShapeIntersectingBB = false;
@@ -136,7 +136,7 @@ protected void prepareSingleValueGeoPointIndex(final Random random) throws Excep
         expectedDocCountsForSingleGeoPoint = new HashMap<>();
         createIndex("idx_unmapped");
         final Settings settings = Settings.builder()
-            .put(IndexMetadata.SETTING_VERSION_CREATED, version)
+            .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT)
             .put("index.number_of_shards", 4)
             .put("index.number_of_replicas", 0)
             .build();
@@ -160,7 +160,7 @@ protected void prepareSingleValueGeoPointIndex(final Random random) throws Excep
 
     protected void prepareMultiValuedGeoPointIndex(final Random random) throws Exception {
         multiValuedExpectedDocCountsGeoPoint = new HashMap<>();
-        final Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, version).build();
+        final Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT).build();
         final List<IndexRequestBuilder> cities = new ArrayList<>();
         assertAcked(
             prepareCreate("multi_valued_idx").setSettings(settings)

modules/transport-netty4/src/main/java/org/opensearch/http/netty4/Netty4HttpServerTransport.java

@@ -116,6 +116,9 @@
 import static org.opensearch.http.HttpTransportSettings.SETTING_HTTP_TCP_SEND_BUFFER_SIZE;
 import static org.opensearch.http.HttpTransportSettings.SETTING_PIPELINING_MAX_EVENTS;
 
+/**
+ * The HTTP transport implementations based on Netty 4.
+ */
 public class Netty4HttpServerTransport extends AbstractHttpServerTransport {
     private static final Logger logger = LogManager.getLogger(Netty4HttpServerTransport.class);
 
@@ -184,6 +187,17 @@ public class Netty4HttpServerTransport extends AbstractHttpServerTransport {
     private volatile ServerBootstrap serverBootstrap;
     private volatile SharedGroupFactory.SharedGroup sharedGroup;
 
+    /**
+     * Creates new HTTP transport implementations based on Netty 4
+     * @param settings seetings
+     * @param networkService network service
+     * @param bigArrays big array allocator
+     * @param threadPool thread pool instance
+     * @param xContentRegistry XContent registry instance
+     * @param dispatcher dispatcher instance
+     * @param clusterSettings cluster settings
+     * @param sharedGroupFactory shared group factory
+     */
     public Netty4HttpServerTransport(
         Settings settings,
         NetworkService networkService,

plugins/repository-azure/build.gradle

@@ -56,11 +56,8 @@ dependencies {
   api "io.netty:netty-transport-native-unix-common:${versions.netty}"
   implementation project(':modules:transport-netty4')
   api 'com.azure:azure-storage-blob:12.23.0'
-  api 'org.reactivestreams:reactive-streams:1.0.4'
-  api 'io.projectreactor:reactor-core:3.5.6'
-  api 'io.projectreactor.netty:reactor-netty:1.1.8'
-  api 'io.projectreactor.netty:reactor-netty-core:1.1.8'
-  api 'io.projectreactor.netty:reactor-netty-http:1.1.9'
+  api "io.projectreactor.netty:reactor-netty-core:${versions.reactor_netty}"
+  api "io.projectreactor.netty:reactor-netty-http:${versions.reactor_netty}"
   api "org.slf4j:slf4j-api:${versions.slf4j}"
   api "com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}"
   api "com.fasterxml.jackson.core:jackson-databind:${versions.jackson_databind}"
@@ -101,10 +98,6 @@ thirdPartyAudit {
     'com.azure.storage.internal.avro.implementation.schema.AvroSchema',
     'com.ctc.wstx.shaded.msv_core.driver.textui.Driver',
     'io.micrometer.context.ContextAccessor',
-    'io.micrometer.context.ContextRegistry',
-    'io.micrometer.context.ContextSnapshot',
-    'io.micrometer.context.ContextSnapshot$Scope',
-    'io.micrometer.core.instrument.Clock',
     'io.micrometer.core.instrument.Counter',
     'io.micrometer.core.instrument.Counter$Builder',
     'io.micrometer.core.instrument.DistributionSummary',
@@ -114,14 +107,10 @@ thirdPartyAudit {
     'io.micrometer.core.instrument.Meter',
     'io.micrometer.core.instrument.MeterRegistry',
     'io.micrometer.core.instrument.Metrics',
-    'io.micrometer.core.instrument.Tag',
-    'io.micrometer.core.instrument.Tags',
     'io.micrometer.core.instrument.Timer',
     'io.micrometer.core.instrument.Timer$Builder',
     'io.micrometer.core.instrument.Timer$Sample',
-    'io.micrometer.core.instrument.binder.jvm.ExecutorServiceMetrics',
     'io.micrometer.core.instrument.composite.CompositeMeterRegistry',
-    'io.micrometer.core.instrument.search.Search',
     'io.netty.channel.epoll.Epoll',
     'io.netty.channel.epoll.EpollDatagramChannel',
     'io.netty.channel.epoll.EpollServerSocketChannel',
@@ -168,9 +157,6 @@ thirdPartyAudit {
     'org.slf4j.impl.StaticLoggerBinder',
     'org.slf4j.impl.StaticMDCBinder',
     'org.slf4j.impl.StaticMarkerBinder',
-    'reactor.blockhound.BlockHound$Builder',
-    'reactor.blockhound.integration.BlockHoundIntegration',
-    'io.micrometer.context.ThreadLocalAccessor',
     'io.micrometer.common.KeyValue',
     'io.micrometer.common.KeyValues',
     'io.micrometer.common.docs.KeyName',
@@ -190,6 +176,7 @@ thirdPartyAudit {
     'io.micrometer.tracing.handler.PropagatingSenderTracingObservationHandler',
     'io.micrometer.tracing.propagation.Propagator',
     'io.micrometer.core.instrument.observation.MeterObservationHandler',
+    'io.micrometer.core.instrument.Tags',
     'io.micrometer.observation.ObservationHandler',
     'io.micrometer.observation.ObservationRegistry',
     'io.micrometer.observation.ObservationRegistry$ObservationConfig',
@@ -210,8 +197,7 @@ thirdPartyAudit {
     'com.google.common.util.concurrent.AbstractFuture$UnsafeAtomicHelper$1',
     'com.google.common.hash.LittleEndianByteArray$UnsafeByteArray',
     'com.google.common.primitives.UnsignedBytes$LexicographicalComparatorHolder$UnsafeComparator',
-    'com.google.common.primitives.UnsignedBytes$LexicographicalComparatorHolder$UnsafeComparator$1',
-    'reactor.core.publisher.Traces$SharedSecretsCallSiteSupplierFactory$TracingException'
+    'com.google.common.primitives.UnsignedBytes$LexicographicalComparatorHolder$UnsafeComparator$1'
   )
 }
 

plugins/repository-azure/licenses/reactor-netty-core-1.1.12.jar.sha1

@@ -0,0 +1 @@
+378dc5a375e6440099e837b22cf4b01341cbe4ea

plugins/repository-azure/licenses/reactor-netty-http-1.1.12.jar.sha1

@@ -0,0 +1 @@
+e839fadb8f45d8a7a2783466faedd03373366c23

plugins/repository-s3/build.gradle

@@ -70,7 +70,6 @@ dependencies {
   api "software.amazon.awssdk:sts:${versions.aws}"
   api "software.amazon.awssdk:netty-nio-client:${versions.aws}"
 
-  api "org.reactivestreams:reactive-streams:${versions.reactivestreams}"
   api "org.apache.httpcomponents:httpclient:${versions.httpclient}"
   api "org.apache.httpcomponents:httpcore:${versions.httpcore}"
   api "commons-logging:commons-logging:${versions.commonslogging}"