Merge branch 'main' into snapshot-pinned-timestamp

Signed-off-by: Anshu Agarwal <[email protected]>

.github/workflows/gradle-check.yml

@@ -20,7 +20,7 @@ jobs:
     - uses: actions/checkout@v4
     - name: Get changed files
       id: changed-files-specific
-      uses: tj-actions/changed-files@v44
+      uses: tj-actions/changed-files@v45
       with:
         files_ignore: |
           release-notes/*.md

CHANGELOG.md

@@ -27,12 +27,13 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Adding access to noSubMatches and noOverlappingMatches in Hyphenation ([#13895](https://github.com/opensearch-project/OpenSearch/pull/13895))
 - Add support for index level max slice count setting for concurrent segment search ([#15336](https://github.com/opensearch-project/OpenSearch/pull/15336))
 - Add support for centralize snapshot creation with pinned timestamp ([#15124](https://github.com/opensearch-project/OpenSearch/pull/15124))
+- Add concurrent search support for Derived Fields ([#15326](https://github.com/opensearch-project/OpenSearch/pull/15326))
 
 ### Dependencies
 - Bump `netty` from 4.1.111.Final to 4.1.112.Final ([#15081](https://github.com/opensearch-project/OpenSearch/pull/15081))
 - Bump `org.apache.commons:commons-lang3` from 3.14.0 to 3.16.0 ([#14861](https://github.com/opensearch-project/OpenSearch/pull/14861), [#15205](https://github.com/opensearch-project/OpenSearch/pull/15205))
 - OpenJDK Update (July 2024 Patch releases) ([#14998](https://github.com/opensearch-project/OpenSearch/pull/14998))
-- Bump `com.microsoft.azure:msal4j` from 1.16.1 to 1.16.2 ([#14995](https://github.com/opensearch-project/OpenSearch/pull/14995))
+- Bump `com.microsoft.azure:msal4j` from 1.16.1 to 1.17.0 ([#14995](https://github.com/opensearch-project/OpenSearch/pull/14995), [#15420](https://github.com/opensearch-project/OpenSearch/pull/15420))
 - Bump `actions/github-script` from 6 to 7 ([#14997](https://github.com/opensearch-project/OpenSearch/pull/14997))
 - Bump `org.tukaani:xz` from 1.9 to 1.10 ([#15110](https://github.com/opensearch-project/OpenSearch/pull/15110))
 - Bump `actions/setup-java` from 1 to 4 ([#15104](https://github.com/opensearch-project/OpenSearch/pull/15104))
@@ -47,6 +48,12 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Bump `com.azure:azure-core-http-netty` from 1.15.1 to 1.15.3 ([#15300](https://github.com/opensearch-project/OpenSearch/pull/15300))
 - Bump `com.gradle.develocity` from 3.17.6 to 3.18 ([#15297](https://github.com/opensearch-project/OpenSearch/pull/15297))
 - Bump `commons-cli:commons-cli` from 1.8.0 to 1.9.0 ([#15298](https://github.com/opensearch-project/OpenSearch/pull/15298))
+- Bump `opentelemetry` from 1.40.0 to 1.41.0 ([#15361](https://github.com/opensearch-project/OpenSearch/pull/15361))
+- Bump `opentelemetry-semconv` from 1.26.0-alpha to 1.27.0-alpha ([#15361](https://github.com/opensearch-project/OpenSearch/pull/15361))
+- Bump `tj-actions/changed-files` from 44 to 45 ([#15422](https://github.com/opensearch-project/OpenSearch/pull/15422))
+- Bump `dnsjava:dnsjava` from 3.6.0 to 3.6.1 ([#15418](https://github.com/opensearch-project/OpenSearch/pull/15418))
+- Bump `com.netflix.nebula.ospackage-base` from 11.9.1 to 11.10.0 ([#15419](https://github.com/opensearch-project/OpenSearch/pull/15419))
+- Bump `org.roaringbitmap:RoaringBitmap` from 1.1.0 to 1.2.1 ([#15423](https://github.com/opensearch-project/OpenSearch/pull/15423))
 
 ### Changed
 - Add lower limit for primary and replica batch allocators timeout ([#14979](https://github.com/opensearch-project/OpenSearch/pull/14979))
@@ -66,6 +73,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Fixed array field name omission in flat_object function for nested JSON ([#13620](https://github.com/opensearch-project/OpenSearch/pull/13620))
 - Fix range aggregation optimization ignoring top level queries ([#15194](https://github.com/opensearch-project/OpenSearch/pull/15194))
 - Fix incorrect parameter names in MinHash token filter configuration handling ([#15233](https://github.com/opensearch-project/OpenSearch/pull/15233))
+- Fix indexing error when flat_object field is explicitly null ([#15375](https://github.com/opensearch-project/OpenSearch/pull/15375))
+- Fix split response processor not included in allowlist ([#15393](https://github.com/opensearch-project/OpenSearch/pull/15393))
+- Fix unchecked cast in dynamic action map getter ([#15394](https://github.com/opensearch-project/OpenSearch/pull/15394))
 
 ### Security
 

buildSrc/version.properties

@@ -74,5 +74,5 @@ jzlib             = 1.1.3
 resteasy          = 6.2.4.Final
 
 # opentelemetry dependencies
-opentelemetry         = 1.40.0
-opentelemetrysemconv  = 1.26.0-alpha
+opentelemetry         = 1.41.0
+opentelemetrysemconv  = 1.27.0-alpha

distribution/packages/build.gradle

@@ -63,7 +63,7 @@ import java.util.regex.Pattern
  */
 
 plugins {
-  id "com.netflix.nebula.ospackage-base" version "11.9.1"
+  id "com.netflix.nebula.ospackage-base" version "11.10.0"
 }
 
 void addProcessFilesTask(String type, boolean jdk) {

modules/lang-painless/src/internalClusterTest/java/org/opensearch/painless/SimplePainlessIT.java

@@ -188,10 +188,6 @@ public void testTermsValuesSource() throws Exception {
     }
 
     public void testSimpleDerivedFieldsQuery() {
-        assumeFalse(
-            "Derived fields do not support concurrent search https://github.com/opensearch-project/OpenSearch/issues/15007",
-            internalCluster().clusterService().getClusterSettings().get(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING)
-        );
         SearchRequest searchRequest = new SearchRequest("test-df").source(
             SearchSourceBuilder.searchSource()
                 .derivedField("result", "keyword", new Script("emit(params._source[\"field\"])"))
@@ -204,10 +200,6 @@ public void testSimpleDerivedFieldsQuery() {
     }
 
     public void testSimpleDerivedFieldsAgg() {
-        assumeFalse(
-            "Derived fields do not support concurrent search https://github.com/opensearch-project/OpenSearch/issues/15007",
-            internalCluster().clusterService().getClusterSettings().get(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING)
-        );
         SearchRequest searchRequest = new SearchRequest("test-df").source(
             SearchSourceBuilder.searchSource()
                 .derivedField("result", "keyword", new Script("emit(params._source[\"field\"])"))

modules/search-pipeline-common/src/main/java/org/opensearch/search/pipeline/common/SearchPipelineCommonModulePlugin.java

@@ -97,6 +97,8 @@ public Map<String, Processor.Factory<SearchResponseProcessor>> getResponseProces
                 new TruncateHitsResponseProcessor.Factory(),
                 CollapseResponseProcessor.TYPE,
                 new CollapseResponseProcessor.Factory(),
+                SplitResponseProcessor.TYPE,
+                new SplitResponseProcessor.Factory(),
                 SortResponseProcessor.TYPE,
                 new SortResponseProcessor.Factory()
             )

modules/search-pipeline-common/src/test/java/org/opensearch/search/pipeline/common/SearchPipelineCommonModulePluginTests.java

@@ -47,6 +47,7 @@ public void testResponseProcessorAllowlist() throws IOException {
             List.of("rename_field", "truncate_hits", "collapse"),
             SearchPipelineCommonModulePlugin::getResponseProcessors
         );
+        runAllowlistTest(key, List.of("split", "sort"), SearchPipelineCommonModulePlugin::getResponseProcessors);
 
         final IllegalArgumentException e = expectThrows(
             IllegalArgumentException.class,
@@ -82,7 +83,7 @@ public void testAllowlistNotSpecified() throws IOException {
         try (SearchPipelineCommonModulePlugin plugin = new SearchPipelineCommonModulePlugin()) {
             assertEquals(Set.of("oversample", "filter_query", "script"), plugin.getRequestProcessors(createParameters(settings)).keySet());
             assertEquals(
-                Set.of("rename_field", "truncate_hits", "collapse", "sort"),
+                Set.of("rename_field", "truncate_hits", "collapse", "split", "sort"),
                 plugin.getResponseProcessors(createParameters(settings)).keySet()
             );
             assertEquals(Set.of(), plugin.getSearchPhaseResultsProcessors(createParameters(settings)).keySet());

plugins/repository-azure/build.gradle

@@ -61,7 +61,7 @@ dependencies {
   // Start of transitive dependencies for azure-identity
   api 'com.microsoft.azure:msal4j-persistence-extension:1.3.0'
   api "net.java.dev.jna:jna-platform:${versions.jna}"
-  api 'com.microsoft.azure:msal4j:1.16.2'
+  api 'com.microsoft.azure:msal4j:1.17.0'
   api 'com.nimbusds:oauth2-oidc-sdk:11.9.1'
   api 'com.nimbusds:nimbus-jose-jwt:9.40'
   api 'com.nimbusds:content-type:2.3'

plugins/repository-azure/licenses/msal4j-1.17.0.jar.sha1

@@ -0,0 +1 @@
+7d37157da92b719f250b0023234ac9dda922a2a5

plugins/telemetry-otel/build.gradle

@@ -86,7 +86,9 @@ thirdPartyAudit {
     'io.opentelemetry.sdk.autoconfigure.spi.traces.ConfigurableSpanExporterProvider',
     'kotlin.io.path.PathsKt',
     'io.opentelemetry.sdk.autoconfigure.spi.traces.ConfigurableSpanExporterProvider',
-    'io.opentelemetry.sdk.autoconfigure.spi.internal.AutoConfigureListener'
+    'io.opentelemetry.sdk.autoconfigure.spi.internal.AutoConfigureListener',
+    'io.opentelemetry.sdk.autoconfigure.spi.internal.ComponentProvider',
+    'io.opentelemetry.sdk.autoconfigure.spi.internal.StructuredConfigProperties'
   )
 }
 

plugins/telemetry-otel/licenses/opentelemetry-api-1.41.0.jar.sha1

@@ -0,0 +1 @@
+ec5ad3b420c9fba4b340e85a3199fd0f2accd023

plugins/telemetry-otel/licenses/opentelemetry-api-incubator-1.41.0-alpha.jar.sha1

@@ -0,0 +1 @@
+fd387313cc37a6e93062e9a80a2526634d22cb19

plugins/telemetry-otel/licenses/opentelemetry-context-1.41.0.jar.sha1

@@ -0,0 +1 @@
+3d7cf15ef425053e24e825160ca7b4ac08d721aa

plugins/telemetry-otel/licenses/opentelemetry-exporter-common-1.41.0.jar.sha1

@@ -0,0 +1 @@
+cf92f4c1b60c2359c12f6f323f6a2a623c333910

plugins/telemetry-otel/licenses/opentelemetry-exporter-logging-1.41.0.jar.sha1

@@ -0,0 +1 @@
+8dee21440b811004ecc1c36c1cd44f9d3494546c

plugins/telemetry-otel/licenses/opentelemetry-exporter-otlp-1.41.0.jar.sha1

@@ -0,0 +1 @@
+d86e60b6d49e389ebe5797d42a7288a20d30c162

plugins/telemetry-otel/licenses/opentelemetry-exporter-otlp-common-1.41.0.jar.sha1

@@ -0,0 +1 @@
+aeba3075b8dfd97779edadc0a3711d999bb0e396

plugins/telemetry-otel/licenses/opentelemetry-exporter-sender-okhttp-1.41.0.jar.sha1

@@ -0,0 +1 @@
+368d7905d6a0a313c63e3a91f895a3a08500519e

plugins/telemetry-otel/licenses/opentelemetry-sdk-1.41.0.jar.sha1

@@ -0,0 +1 @@
+c740e8f7d0d914d6acd310ac53901bb8753c6e8d

plugins/telemetry-otel/licenses/opentelemetry-sdk-common-1.41.0.jar.sha1

@@ -0,0 +1 @@
+b820861f85ba83db0ad896c47f723208d7473d5a

plugins/telemetry-otel/licenses/opentelemetry-sdk-logs-1.41.0.jar.sha1

@@ -0,0 +1 @@
+f88ee292f5605c87dfe85c8d90131bce9f0b3b8e

plugins/telemetry-otel/licenses/opentelemetry-sdk-metrics-1.41.0.jar.sha1

@@ -0,0 +1 @@
+9d1200befb28e3e9f61073ac3de23cc55e509dc7

plugins/telemetry-otel/licenses/opentelemetry-sdk-trace-1.41.0.jar.sha1

@@ -0,0 +1 @@
+d9bbc2e2e800317d72fbf3141ae8391e95fa6229

plugins/telemetry-otel/licenses/opentelemetry-semconv-1.27.0-alpha.jar.sha1

@@ -0,0 +1 @@
+906d916bee46f60260c09314284b5948c54a0662

release-notes/opensearch.release-notes-1.3.19.md

@@ -0,0 +1,5 @@
+## 2024-08-22 Version 1.3.19 Release Notes
+
+### Upgrades
+- OpenJDK Update (July 2024 Patch releases) ([#15002](https://github.com/opensearch-project/OpenSearch/pull/15002))
+- Bump `netty` from 4.1.111.Final to 4.1.112.Final ([#15081](https://github.com/opensearch-project/OpenSearch/pull/15081))

rest-api-spec/src/main/resources/rest-api-spec/test/index/100_partial_flat_object.yml

@@ -88,7 +88,16 @@ setup:
             } ]
           }
         }
-
+  - do:
+      index:
+        index: test_partial_flat_object
+        id: 4
+        body: {
+          "issue": {
+            "number": 999,
+            "labels": null
+          }
+        }
   - do:
       indices.refresh:
         index: test_partial_flat_object
@@ -135,7 +144,7 @@ teardown:
           }
         }
 
-  - length: { hits.hits: 3 }
+  - length: { hits.hits: 4 }
 
   # Match Query with exact dot path.
   - do:

server/build.gradle

@@ -127,7 +127,7 @@ dependencies {
   api "jakarta.annotation:jakarta.annotation-api:${versions.jakarta_annotation}"
 
   // https://mvnrepository.com/artifact/org.roaringbitmap/RoaringBitmap
-  implementation 'org.roaringbitmap:RoaringBitmap:1.1.0'
+  implementation 'org.roaringbitmap:RoaringBitmap:1.2.1'
 
   testImplementation(project(":test:framework")) {
     // tests use the locally compiled version of server

server/licenses/RoaringBitmap-1.2.1.jar.sha1

@@ -0,0 +1 @@
+828eb489b5e8c8762f2471010e9c7f20c7de596d

server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteStatePublicationIT.java

@@ -50,15 +50,22 @@
 public class RemoteStatePublicationIT extends RemoteStoreBaseIntegTestCase {
 
     private static String INDEX_NAME = "test-index";
+    private boolean isRemoteStateEnabled = true;
+    private String isRemotePublicationEnabled = "true";
 
     @Before
     public void setup() {
         asyncUploadMockFsRepo = false;
+        isRemoteStateEnabled = true;
+        isRemotePublicationEnabled = "true";
     }
 
     @Override
     protected Settings featureFlagSettings() {
-        return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.REMOTE_PUBLICATION_EXPERIMENTAL, "true").build();
+        return Settings.builder()
+            .put(super.featureFlagSettings())
+            .put(FeatureFlags.REMOTE_PUBLICATION_EXPERIMENTAL, isRemotePublicationEnabled)
+            .build();
     }
 
     @Override
@@ -76,7 +83,7 @@ protected Settings nodeSettings(int nodeOrdinal) {
         );
         return Settings.builder()
             .put(super.nodeSettings(nodeOrdinal))
-            .put(REMOTE_CLUSTER_STATE_ENABLED_SETTING.getKey(), true)
+            .put(REMOTE_CLUSTER_STATE_ENABLED_SETTING.getKey(), isRemoteStateEnabled)
             .put("node.attr." + REMOTE_STORE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY, routingTableRepoName)
             .put(routingTableRepoTypeAttributeKey, ReloadableFsRepository.TYPE)
             .put(routingTableRepoSettingsAttributeKeyPrefix + "location", segmentRepoPath)
@@ -136,6 +143,18 @@ public void testPublication() throws Exception {
         }
     }
 
+    public void testRemotePublicationDisableIfRemoteStateDisabled() {
+        // only disable remote state
+        isRemoteStateEnabled = false;
+        // create cluster with multi node with in-consistent settings
+        prepareCluster(3, 2, INDEX_NAME, 1, 2);
+        // assert cluster is stable, ensuring publication falls back to legacy transport with inconsistent settings
+        ensureStableCluster(5);
+        ensureGreen(INDEX_NAME);
+
+        assertNull(internalCluster().getCurrentClusterManagerNodeInstance(RemoteClusterStateService.class));
+    }
+
     private Map<String, Integer> getMetadataFiles(BlobStoreRepository repository, String subDirectory) throws IOException {
         BlobPath metadataPath = repository.basePath()
             .add(
@@ -151,5 +170,4 @@ private Map<String, Integer> getMetadataFiles(BlobStoreRepository repository, St
             return fileName.split(DELIMITER)[0];
         }).collect(Collectors.toMap(Function.identity(), key -> 1, Integer::sum));
     }
-
 }

server/src/internalClusterTest/java/org/opensearch/index/mapper/StarTreeMapperIT.java

@@ -264,15 +264,16 @@ public void testValidCompositeIndex() {
                     );
                     assertEquals(expectedTimeUnits, dateDim.getIntervals());
                     assertEquals("numeric_dv", starTreeFieldType.getDimensions().get(1).getField());
+                    assertEquals(2, starTreeFieldType.getMetrics().size());
                     assertEquals("numeric_dv", starTreeFieldType.getMetrics().get(0).getField());
-                    List<MetricStat> expectedMetrics = Arrays.asList(
-                        MetricStat.AVG,
-                        MetricStat.VALUE_COUNT,
-                        MetricStat.SUM,
-                        MetricStat.MAX,
-                        MetricStat.MIN
-                    );
+
+                    // Assert default metrics
+                    List<MetricStat> expectedMetrics = Arrays.asList(MetricStat.VALUE_COUNT, MetricStat.SUM, MetricStat.AVG);
                     assertEquals(expectedMetrics, starTreeFieldType.getMetrics().get(0).getMetrics());
+
+                    assertEquals("_doc_count", starTreeFieldType.getMetrics().get(1).getField());
+                    assertEquals(List.of(MetricStat.DOC_COUNT), starTreeFieldType.getMetrics().get(1).getMetrics());
+
                     assertEquals(10000, starTreeFieldType.getStarTreeConfig().maxLeafDocs());
                     assertEquals(
                         StarTreeFieldConfiguration.StarTreeBuildMode.OFF_HEAP,
@@ -349,13 +350,9 @@ public void testUpdateIndexWhenMappingIsSame() {
                     assertEquals(expectedTimeUnits, dateDim.getIntervals());
                     assertEquals("numeric_dv", starTreeFieldType.getDimensions().get(1).getField());
                     assertEquals("numeric_dv", starTreeFieldType.getMetrics().get(0).getField());
-                    List<MetricStat> expectedMetrics = Arrays.asList(
-                        MetricStat.AVG,
-                        MetricStat.VALUE_COUNT,
-                        MetricStat.SUM,
-                        MetricStat.MAX,
-                        MetricStat.MIN
-                    );
+
+                    // Assert default metrics
+                    List<MetricStat> expectedMetrics = Arrays.asList(MetricStat.VALUE_COUNT, MetricStat.SUM, MetricStat.AVG);
                     assertEquals(expectedMetrics, starTreeFieldType.getMetrics().get(0).getMetrics());
                     assertEquals(10000, starTreeFieldType.getStarTreeConfig().maxLeafDocs());
                     assertEquals(