Fix CHANGELOG.md conflict

Signed-off-by: Prudhvi Godithi <[email protected]>

CHANGELOG.md

@@ -14,13 +14,18 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Increase segrep pressure checkpoint default limit to 30 ([#16577](https://github.com/opensearch-project/OpenSearch/pull/16577/files))
 - Add dynamic setting allowing size > 0 requests to be cached in the request cache ([#16483](https://github.com/opensearch-project/OpenSearch/pull/16483))
 - Make IndexStoreListener a pluggable interface ([#16583](https://github.com/opensearch-project/OpenSearch/pull/16583))
+- Support for keyword fields in star-tree index ([#16233](https://github.com/opensearch-project/OpenSearch/pull/16233))
+- Add a flag in QueryShardContext to differentiate inner hit query ([#16600](https://github.com/opensearch-project/OpenSearch/pull/16600))
+- Add vertical scaling and SoftReference for snapshot repository data cache ([#16489](https://github.com/opensearch-project/OpenSearch/pull/16489))
 - Add new configuration setting `synonym_analyzer`, to the `synonym` and `synonym_graph` filters, enabling the specification of a custom analyzer for reading the synonym file ([#16488](https://github.com/opensearch-project/OpenSearch/pull/16488)).
 
 ### Dependencies
 - Bump `com.azure:azure-storage-common` from 12.25.1 to 12.27.1 ([#16521](https://github.com/opensearch-project/OpenSearch/pull/16521))
 - Bump `com.google.apis:google-api-services-compute` from v1-rev20240407-2.0.0 to v1-rev20241021-2.0.0 ([#16502](https://github.com/opensearch-project/OpenSearch/pull/16502), [#16548](https://github.com/opensearch-project/OpenSearch/pull/16548))
 - Bump `com.azure:azure-storage-blob` from 12.23.0 to 12.28.1 ([#16501](https://github.com/opensearch-project/OpenSearch/pull/16501))
 - Bump `org.apache.hadoop:hadoop-minicluster` from 3.4.0 to 3.4.1 ([#16550](https://github.com/opensearch-project/OpenSearch/pull/16550))
+- Bump `org.apache.xmlbeans:xmlbeans` from 5.2.1 to 5.2.2 ([#16612](https://github.com/opensearch-project/OpenSearch/pull/16612))
+- Bump `com.nimbusds:nimbus-jose-jwt` from 9.41.1 to 9.46 ([#16611](https://github.com/opensearch-project/OpenSearch/pull/16611))
 - Bump `lycheeverse/lychee-action` from 2.0.2 to 2.1.0 ([#16610](https://github.com/opensearch-project/OpenSearch/pull/16610))
 - Bump `me.champeau.gradle.japicmp` from 0.4.4 to 0.4.5 ([#16614](https://github.com/opensearch-project/OpenSearch/pull/16614))
 

modules/search-pipeline-common/src/internalClusterTest/java/org/opensearch/search/pipeline/common/SearchPipelineCommonIT.java

@@ -8,41 +8,187 @@
 
 package org.opensearch.search.pipeline.common;
 
-import org.opensearch.action.admin.indices.delete.DeleteIndexRequest;
 import org.opensearch.action.admin.indices.refresh.RefreshRequest;
 import org.opensearch.action.admin.indices.refresh.RefreshResponse;
+import org.opensearch.action.admin.indices.settings.put.UpdateSettingsRequest;
 import org.opensearch.action.index.IndexRequest;
 import org.opensearch.action.index.IndexResponse;
 import org.opensearch.action.search.DeleteSearchPipelineRequest;
+import org.opensearch.action.search.GetSearchPipelineRequest;
+import org.opensearch.action.search.GetSearchPipelineResponse;
 import org.opensearch.action.search.PutSearchPipelineRequest;
 import org.opensearch.action.search.SearchRequest;
 import org.opensearch.action.search.SearchResponse;
 import org.opensearch.action.support.master.AcknowledgedResponse;
+import org.opensearch.common.settings.Settings;
 import org.opensearch.core.common.bytes.BytesArray;
+import org.opensearch.core.common.bytes.BytesReference;
 import org.opensearch.core.rest.RestStatus;
 import org.opensearch.core.xcontent.MediaTypeRegistry;
 import org.opensearch.index.query.MatchAllQueryBuilder;
+import org.opensearch.ingest.PipelineConfiguration;
 import org.opensearch.plugins.Plugin;
 import org.opensearch.search.builder.SearchSourceBuilder;
 import org.opensearch.test.OpenSearchIntegTestCase;
+import org.junit.After;
+import org.junit.Before;
 
 import java.util.Collection;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
 @OpenSearchIntegTestCase.SuiteScopeTestCase
 public class SearchPipelineCommonIT extends OpenSearchIntegTestCase {
 
+    private static final String TEST_INDEX = "myindex";
+    private static final String PIPELINE_NAME = "test_pipeline";
+
     @Override
     protected Collection<Class<? extends Plugin>> nodePlugins() {
         return List.of(SearchPipelineCommonModulePlugin.class);
     }
 
+    @Before
+    public void setup() throws Exception {
+        createIndex(TEST_INDEX);
+
+        IndexRequest doc1 = new IndexRequest(TEST_INDEX).id("doc1").source(Map.of("field", "value"));
+        IndexRequest doc2 = new IndexRequest(TEST_INDEX).id("doc2").source(Map.of("field", "something else"));
+
+        IndexResponse ir = client().index(doc1).actionGet();
+        assertSame(RestStatus.CREATED, ir.status());
+        ir = client().index(doc2).actionGet();
+        assertSame(RestStatus.CREATED, ir.status());
+
+        RefreshResponse refRsp = client().admin().indices().refresh(new RefreshRequest(TEST_INDEX)).actionGet();
+        assertSame(RestStatus.OK, refRsp.getStatus());
+    }
+
+    @After
+    public void cleanup() throws Exception {
+        internalCluster().wipeIndices(TEST_INDEX);
+    }
+
     public void testFilterQuery() {
         // Create a pipeline with a filter_query processor.
-        String pipelineName = "foo";
+        createPipeline();
+
+        // Search without the pipeline. Should see both documents.
+        SearchRequest req = new SearchRequest(TEST_INDEX).source(new SearchSourceBuilder().query(new MatchAllQueryBuilder()));
+        SearchResponse rsp = client().search(req).actionGet();
+        assertEquals(2, rsp.getHits().getTotalHits().value);
+
+        // Search with the pipeline. Should only see document with "field":"value".
+        req.pipeline(PIPELINE_NAME);
+        rsp = client().search(req).actionGet();
+        assertEquals(1, rsp.getHits().getTotalHits().value);
+
+        // Clean up.
+        deletePipeline();
+    }
+
+    public void testSearchWithTemporaryPipeline() throws Exception {
+
+        // Search without the pipeline. Should see both documents.
+        SearchRequest req = new SearchRequest(TEST_INDEX).source(new SearchSourceBuilder().query(new MatchAllQueryBuilder()));
+        SearchResponse rsp = client().search(req).actionGet();
+        assertEquals(2, rsp.getHits().getTotalHits().value);
+
+        // Search with temporary pipeline
+        Map<String, Object> pipelineSourceMap = new HashMap<>();
+        Map<String, Object> requestProcessorConfig = new HashMap<>();
+
+        Map<String, Object> filterQuery = new HashMap<>();
+        filterQuery.put("query", Map.of("term", Map.of("field", "value")));
+        requestProcessorConfig.put("filter_query", filterQuery);
+        pipelineSourceMap.put("request_processors", List.of(requestProcessorConfig));
+
+        req = new SearchRequest(TEST_INDEX).source(
+            new SearchSourceBuilder().query(new MatchAllQueryBuilder()).searchPipelineSource(pipelineSourceMap)
+        );
+
+        SearchResponse rspWithTempPipeline = client().search(req).actionGet();
+        assertEquals(1, rspWithTempPipeline.getHits().getTotalHits().value);
+    }
+
+    public void testSearchWithDefaultPipeline() throws Exception {
+        // Create pipeline
+        createPipeline();
+
+        // Search without the pipeline. Should see both documents.
+        SearchRequest req = new SearchRequest(TEST_INDEX).source(new SearchSourceBuilder().query(new MatchAllQueryBuilder()));
+        SearchResponse rsp = client().search(req).actionGet();
+        assertEquals(2, rsp.getHits().getTotalHits().value);
+
+        // Set pipeline as default for the index
+        UpdateSettingsRequest updateSettingsRequest = new UpdateSettingsRequest(TEST_INDEX);
+        updateSettingsRequest.settings(Settings.builder().put("index.search.default_pipeline", PIPELINE_NAME));
+        AcknowledgedResponse updateSettingsResponse = client().admin().indices().updateSettings(updateSettingsRequest).actionGet();
+        assertTrue(updateSettingsResponse.isAcknowledged());
+
+        // Search with the default pipeline. Should only see document with "field":"value".
+        rsp = client().search(req).actionGet();
+        assertEquals(1, rsp.getHits().getTotalHits().value);
+
+        // Clean up: Remove default pipeline setting
+        updateSettingsRequest = new UpdateSettingsRequest(TEST_INDEX);
+        updateSettingsRequest.settings(Settings.builder().putNull("index.search.default_pipeline"));
+        updateSettingsResponse = client().admin().indices().updateSettings(updateSettingsRequest).actionGet();
+        assertTrue(updateSettingsResponse.isAcknowledged());
+
+        // Clean up.
+        deletePipeline();
+    }
+
+    public void testUpdateSearchPipeline() throws Exception {
+        // Create initial pipeline
+        createPipeline();
+
+        // Verify initial pipeline
+        SearchRequest req = new SearchRequest(TEST_INDEX).source(new SearchSourceBuilder().query(new MatchAllQueryBuilder()));
+        req.pipeline(PIPELINE_NAME);
+        SearchResponse initialRsp = client().search(req).actionGet();
+        assertEquals(1, initialRsp.getHits().getTotalHits().value);
+
+        BytesReference pipelineConfig = new BytesArray(
+            "{"
+                + "\"description\": \"Updated pipeline\","
+                + "\"request_processors\": ["
+                + "{"
+                + "\"filter_query\" : {"
+                + "\"query\": {"
+                + "\"term\" : {"
+                + "\"field\" : \"something else\""
+                + "}"
+                + "}"
+                + "}"
+                + "}"
+                + "]"
+                + "}"
+        );
+
+        PipelineConfiguration pipeline = new PipelineConfiguration(PIPELINE_NAME, pipelineConfig, MediaTypeRegistry.JSON);
+
+        // Update pipeline
+        PutSearchPipelineRequest updateRequest = new PutSearchPipelineRequest(pipeline.getId(), pipelineConfig, MediaTypeRegistry.JSON);
+        AcknowledgedResponse ackRsp = client().admin().cluster().putSearchPipeline(updateRequest).actionGet();
+        assertTrue(ackRsp.isAcknowledged());
+
+        // Verify pipeline description
+        GetSearchPipelineResponse getPipelineResponse = client().admin()
+            .cluster()
+            .getSearchPipeline(new GetSearchPipelineRequest(PIPELINE_NAME))
+            .actionGet();
+        assertEquals(PIPELINE_NAME, getPipelineResponse.pipelines().get(0).getId());
+        assertEquals(pipeline.getConfigAsMap(), getPipelineResponse.pipelines().get(0).getConfigAsMap());
+        // Clean up.
+        deletePipeline();
+    }
+
+    private void createPipeline() {
         PutSearchPipelineRequest putSearchPipelineRequest = new PutSearchPipelineRequest(
-            pipelineName,
+            PIPELINE_NAME,
             new BytesArray(
                 "{"
                     + "\"request_processors\": ["
@@ -62,35 +208,13 @@ public void testFilterQuery() {
         );
         AcknowledgedResponse ackRsp = client().admin().cluster().putSearchPipeline(putSearchPipelineRequest).actionGet();
         assertTrue(ackRsp.isAcknowledged());
+    }
 
-        // Index some documents.
-        String indexName = "myindex";
-        IndexRequest doc1 = new IndexRequest(indexName).id("doc1").source(Map.of("field", "value"));
-        IndexRequest doc2 = new IndexRequest(indexName).id("doc2").source(Map.of("field", "something else"));
-
-        IndexResponse ir = client().index(doc1).actionGet();
-        assertSame(RestStatus.CREATED, ir.status());
-        ir = client().index(doc2).actionGet();
-        assertSame(RestStatus.CREATED, ir.status());
-
-        // Refresh so the documents are visible to search.
-        RefreshResponse refRsp = client().admin().indices().refresh(new RefreshRequest(indexName)).actionGet();
-        assertSame(RestStatus.OK, refRsp.getStatus());
-
-        // Search without the pipeline. Should see both documents.
-        SearchRequest req = new SearchRequest(indexName).source(new SearchSourceBuilder().query(new MatchAllQueryBuilder()));
-        SearchResponse rsp = client().search(req).actionGet();
-        assertEquals(2, rsp.getHits().getTotalHits().value);
-
-        // Search with the pipeline. Should only see document with "field":"value".
-        req.pipeline(pipelineName);
-        rsp = client().search(req).actionGet();
-        assertEquals(1, rsp.getHits().getTotalHits().value);
-
-        // Clean up.
-        ackRsp = client().admin().cluster().deleteSearchPipeline(new DeleteSearchPipelineRequest(pipelineName)).actionGet();
-        assertTrue(ackRsp.isAcknowledged());
-        ackRsp = client().admin().indices().delete(new DeleteIndexRequest(indexName)).actionGet();
+    private void deletePipeline() {
+        AcknowledgedResponse ackRsp = client().admin()
+            .cluster()
+            .deleteSearchPipeline(new DeleteSearchPipelineRequest(PIPELINE_NAME))
+            .actionGet();
         assertTrue(ackRsp.isAcknowledged());
     }
 }

plugins/ingest-attachment/build.gradle

@@ -89,7 +89,7 @@ dependencies {
   api "org.apache.poi:poi:${versions.poi}"
   api "org.apache.poi:poi-ooxml-lite:${versions.poi}"
   api "commons-codec:commons-codec:${versions.commonscodec}"
-  api 'org.apache.xmlbeans:xmlbeans:5.2.1'
+  api 'org.apache.xmlbeans:xmlbeans:5.2.2'
   api 'org.apache.commons:commons-collections4:4.4'
   // MS Office
   api "org.apache.poi:poi-scratchpad:${versions.poi}"

plugins/ingest-attachment/licenses/xmlbeans-5.2.2.jar.sha1

@@ -0,0 +1 @@
+586ffe10ae9864e19e85c24bd060790a70586f72

server/src/internalClusterTest/java/org/opensearch/index/mapper/StarTreeMapperIT.java

@@ -56,7 +56,7 @@ public class StarTreeMapperIT extends OpenSearchIntegTestCase {
         .put(IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey(), new ByteSizeValue(512, ByteSizeUnit.MB))
         .build();
 
-    private static XContentBuilder createMinimalTestMapping(boolean invalidDim, boolean invalidMetric, boolean keywordDim) {
+    private static XContentBuilder createMinimalTestMapping(boolean invalidDim, boolean invalidMetric, boolean ipdim) {
         try {
             return jsonBuilder().startObject()
                 .startObject("composite")
@@ -68,12 +68,15 @@ private static XContentBuilder createMinimalTestMapping(boolean invalidDim, bool
                 .endObject()
                 .startArray("ordered_dimensions")
                 .startObject()
-                .field("name", getDim(invalidDim, keywordDim))
+                .field("name", getDim(invalidDim, ipdim))
+                .endObject()
+                .startObject()
+                .field("name", "keyword_dv")
                 .endObject()
                 .endArray()
                 .startArray("metrics")
                 .startObject()
-                .field("name", getDim(invalidMetric, false))
+                .field("name", getMetric(invalidMetric, false))
                 .endObject()
                 .endArray()
                 .endObject()
@@ -99,6 +102,10 @@ private static XContentBuilder createMinimalTestMapping(boolean invalidDim, bool
                 .field("type", "keyword")
                 .field("doc_values", false)
                 .endObject()
+                .startObject("ip")
+                .field("type", "ip")
+                .field("doc_values", false)
+                .endObject()
                 .endObject()
                 .endObject();
         } catch (IOException e) {
@@ -356,10 +363,19 @@ private XContentBuilder getMappingWithDuplicateFields(boolean isDuplicateDim, bo
     }
 
     private static String getDim(boolean hasDocValues, boolean isKeyword) {
+        if (hasDocValues) {
+            return random().nextBoolean() ? "numeric" : "keyword";
+        } else if (isKeyword) {
+            return "ip";
+        }
+        return "numeric_dv";
+    }
+
+    private static String getMetric(boolean hasDocValues, boolean isKeyword) {
         if (hasDocValues) {
             return "numeric";
         } else if (isKeyword) {
-            return "keyword";
+            return "ip";
         }
         return "numeric_dv";
     }
@@ -398,6 +414,7 @@ public void testValidCompositeIndex() {
                         assertEquals(expectedTimeUnits.get(i).shortName(), dateDim.getSortedCalendarIntervals().get(i).shortName());
                     }
                     assertEquals("numeric_dv", starTreeFieldType.getDimensions().get(1).getField());
+                    assertEquals("keyword_dv", starTreeFieldType.getDimensions().get(2).getField());
                     assertEquals("numeric_dv", starTreeFieldType.getMetrics().get(0).getField());
                     List<MetricStat> expectedMetrics = Arrays.asList(MetricStat.VALUE_COUNT, MetricStat.SUM, MetricStat.AVG);
                     assertEquals(expectedMetrics, starTreeFieldType.getMetrics().get(0).getMetrics());
@@ -665,10 +682,7 @@ public void testInvalidDimCompositeIndex() {
             IllegalArgumentException.class,
             () -> prepareCreate(TEST_INDEX).setSettings(settings).setMapping(createMinimalTestMapping(true, false, false)).get()
         );
-        assertEquals(
-            "Aggregations not supported for the dimension field [numeric] with field type [integer] as part of star tree field",
-            ex.getMessage()
-        );
+        assertTrue(ex.getMessage().startsWith("Aggregations not supported for the dimension field "));
     }
 
     public void testMaxDimsCompositeIndex() {
@@ -734,7 +748,7 @@ public void testUnsupportedDim() {
             () -> prepareCreate(TEST_INDEX).setSettings(settings).setMapping(createMinimalTestMapping(false, false, true)).get()
         );
         assertEquals(
-            "Failed to parse mapping [_doc]: unsupported field type associated with dimension [keyword] as part of star tree field [startree-1]",
+            "Failed to parse mapping [_doc]: unsupported field type associated with dimension [ip] as part of star tree field [startree-1]",
             ex.getMessage()
         );
     }

server/src/main/java/org/apache/lucene/index/DocValuesWriterWrapper.java

@@ -0,0 +1,18 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.apache.lucene.index;
+
+import org.apache.lucene.search.DocIdSetIterator;
+
+/**
+ * Base wrapper class for DocValuesWriter.
+ */
+public interface DocValuesWriterWrapper<T extends DocIdSetIterator> {
+    T getDocValues();
+}