Merge branch 'refs/heads/2.x' into backport/backport-12190-to-2.x

.github/workflows/assemble.yml

@@ -19,6 +19,8 @@ jobs:
       - name: Setup docker (missing on MacOS)
         if: runner.os == 'macos'
         uses: douglascamata/setup-docker-macos-action@main
+        with:
+          upgrade-qemu: true
       - name: Run Gradle (assemble)
         run: |
           ./gradlew assemble --parallel --no-build-cache -PDISABLE_BUILD_CACHE

.github/workflows/gradle-check.yml

@@ -12,13 +12,28 @@ permissions:
   contents: read # to fetch code (actions/checkout)
 
 jobs:
+  check-files:
+    runs-on: ubuntu-latest
+    outputs: 
+      RUN_GRADLE_CHECK: ${{ steps.changed-files-specific.outputs.any_changed }}
+    steps:
+    - uses: actions/checkout@v3
+    - name: Get changed files
+      id: changed-files-specific
+      uses: tj-actions/changed-files@v44
+      with:
+        files_ignore: |
+          release-notes/*.md
+          .github/**
+          *.md
+    
   gradle-check:
-    if: github.repository == 'opensearch-project/OpenSearch'
+    needs: check-files
+    if: github.repository == 'opensearch-project/OpenSearch' && needs.check-files.outputs.RUN_GRADLE_CHECK == 'true'
     permissions:
       contents: read # to fetch code (actions/checkout)
       pull-requests: write # to create or update comment (peter-evans/create-or-update-comment)
       issues: write # To create an issue if check fails on push.
-
     runs-on: ubuntu-latest
     timeout-minutes: 130
     steps:
@@ -151,3 +166,12 @@ jobs:
         with:
           assignees: ${{ github.event.head_commit.author.username }}, ${{ github.triggering_actor }}
           filename: .github/ISSUE_TEMPLATE/failed_check.md
+
+  check-result:
+    needs: [check-files, gradle-check]
+    if: always()
+    runs-on: ubuntu-latest
+    steps:
+      - name: Fail if gradle-check fails
+        if: ${{ needs.check-files.outputs.RUN_GRADLE_CHECK && needs.gradle-check.result == 'failure' }}
+        run: exit 1

CHANGELOG.md

@@ -10,6 +10,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Make outbound side of transport protocol dependent ([#13293](https://github.com/opensearch-project/OpenSearch/pull/13293))
 - [Remote Store] Upload translog checkpoint as object metadata to translog.tlog([#13637](https://github.com/opensearch-project/OpenSearch/pull/13637))
 - [Remote Store] Add dynamic cluster settings to set timeout for segments upload to Remote Store ([#13679](https://github.com/opensearch-project/OpenSearch/pull/13679))
+- Add getMetadataFields to MapperService ([#13819](https://github.com/opensearch-project/OpenSearch/pull/13819))
 
 ### Dependencies
 - Bump `com.github.spullara.mustache.java:compiler` from 0.9.10 to 0.9.13 ([#13329](https://github.com/opensearch-project/OpenSearch/pull/13329), [#13559](https://github.com/opensearch-project/OpenSearch/pull/13559))
@@ -24,6 +25,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Bump `jackson` from 2.17.0 to 2.17.1 ([#13817](https://github.com/opensearch-project/OpenSearch/pull/13817))
 - Bump `reactor` from 3.5.15 to 3.5.17 ([#13825](https://github.com/opensearch-project/OpenSearch/pull/13825))
 - Bump `reactor-netty` from 1.1.17 to 1.1.19 ([#13825](https://github.com/opensearch-project/OpenSearch/pull/13825))
+- Bump `commons-cli:commons-cli` from 1.7.0 to 1.8.0 ([#13840](https://github.com/opensearch-project/OpenSearch/pull/13840))
+- Bump `org.apache.xmlbeans:xmlbeans` from 5.2.0 to 5.2.1 ([#13839](https://github.com/opensearch-project/OpenSearch/pull/13839))
+- Bump `com.netflix.nebula.ospackage-base` from 11.9.0 to 11.9.1 ([#13933](https://github.com/opensearch-project/OpenSearch/pull/13933))
 
 ### Changed
 - Add ability for Boolean and date field queries to run when only doc_values are enabled ([#11650](https://github.com/opensearch-project/OpenSearch/pull/11650))
@@ -36,9 +40,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Remove handling of index.mapper.dynamic in AutoCreateIndex([#13067](https://github.com/opensearch-project/OpenSearch/pull/13067))
 
 ### Fixed
-- Fix negative RequestStats metric issue ([#13553](https://github.com/opensearch-project/OpenSearch/pull/13553))
 - Fix get field mapping API returns 404 error in mixed cluster with multiple versions ([#13624](https://github.com/opensearch-project/OpenSearch/pull/13624))
 - Allow clearing `remote_store.compatibility_mode` setting ([#13646](https://github.com/opensearch-project/OpenSearch/pull/13646))
+- Painless: ensure type "UnmodifiableMap" for params ([#13885](https://github.com/opensearch-project/OpenSearch/pull/13885))
+- Pass parent filter to inner hit query ([#13903](https://github.com/opensearch-project/OpenSearch/pull/13903))
 
 ### Security
 

distribution/packages/build.gradle

@@ -63,7 +63,7 @@ import java.util.regex.Pattern
  */
 
 plugins {
-  id "com.netflix.nebula.ospackage-base" version "11.9.0"
+  id "com.netflix.nebula.ospackage-base" version "11.9.1"
 }
 
 void addProcessFilesTask(String type, boolean jdk) {

libs/core/src/main/java/org/opensearch/core/xcontent/XContentBuilder.java

@@ -157,6 +157,9 @@ public static XContentBuilder builder(XContent xContent, Set<String> includes, S
 
     /**
      * Returns a string representation of the builder (only applicable for text based xcontent).
+     * Note: explicitly or implicitly (from debugger) calling toString() could cause XContentBuilder
+     * to close which is a side effect done by @see BytesReference#bytes().
+     * Trying to write more contents after toString() will cause NPE. Use it with caution.
      */
     @Override
     public String toString() {

modules/lang-painless/src/test/java/org/opensearch/painless/WhenThingsGoWrongTests.java

@@ -354,6 +354,9 @@ public void testInvalidAssignment() {
         assertEquals(iae.getMessage(), "invalid assignment: cannot assign a value to addition operation [+]");
         iae = expectScriptThrows(IllegalArgumentException.class, () -> exec("Double.x() = 1;"));
         assertEquals(iae.getMessage(), "invalid assignment: cannot assign a value to method call [x/0]");
+
+        expectScriptThrows(UnsupportedOperationException.class, () -> exec("params['modifyingParamsMap'] = 2;"));
+        expectScriptThrows(UnsupportedOperationException.class, () -> exec("params.modifyingParamsMap = 2;"));
     }
 
     public void testCannotResolveSymbol() {

modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/17_update_error.yml

@@ -13,3 +13,50 @@
   - match: { error.root_cause.0.position.offset: 13 }
   - match: { error.root_cause.0.position.start: 0 }
   - match: { error.root_cause.0.position.end: 38 }
+
+---
+"Test modifying params map from script leads to exception":
+  - skip:
+      features: "node_selector"
+
+  - do:
+      put_script:
+        id: "except"
+        body: {"script": {"lang": "painless", "source": "params.that = 3"}}
+
+  - do:
+      indices.create:
+        index: "test"
+        body:
+          settings:
+            index:
+              number_of_shards: 1
+              number_of_replicas: 0
+          mappings:
+            properties:
+              this:
+                type: "integer"
+              that:
+                type: "integer"
+
+  - do:
+      index:
+        index: "test"
+        id: 1
+        body: {"this": 1, "that": 2}
+
+  - do:
+      catch: /unsupported_operation_exception/
+      node_selector:
+        version: "2.15.0 - "
+      update:
+        index: "test"
+        id: 1
+        body:
+          script:
+            id: "except"
+            params: {"this": 2}
+
+  - match: { error.caused_by.position.offset: 6 }
+  - match: { error.caused_by.position.start: 0 }
+  - match: { error.caused_by.position.end: 15 }

plugins/ingest-attachment/build.gradle

@@ -89,7 +89,7 @@ dependencies {
   api "org.apache.poi:poi:${versions.poi}"
   api "org.apache.poi:poi-ooxml-lite:${versions.poi}"
   api "commons-codec:commons-codec:${versions.commonscodec}"
-  api 'org.apache.xmlbeans:xmlbeans:5.2.0'
+  api 'org.apache.xmlbeans:xmlbeans:5.2.1'
   api 'org.apache.commons:commons-collections4:4.4'
   // MS Office
   api "org.apache.poi:poi-scratchpad:${versions.poi}"

plugins/ingest-attachment/licenses/xmlbeans-5.2.1.jar.sha1

@@ -0,0 +1 @@
+e16ddf17fe181c202b097e0dcc0ee2fed91cb7da

plugins/repository-hdfs/build.gradle

@@ -70,7 +70,7 @@ dependencies {
   api 'com.google.code.gson:gson:2.11.0'
   runtimeOnly "com.google.guava:guava:${versions.guava}"
   api "commons-logging:commons-logging:${versions.commonslogging}"
-  api 'commons-cli:commons-cli:1.7.0'
+  api 'commons-cli:commons-cli:1.8.0'
   api "commons-codec:commons-codec:${versions.commonscodec}"
   api 'commons-collections:commons-collections:3.2.2'
   api "org.apache.commons:commons-compress:${versions.commonscompress}"

plugins/repository-hdfs/licenses/commons-cli-1.8.0.jar.sha1

@@ -0,0 +1 @@
+41a4bff12057eecb6daaf9c7f36c237815be3da1

plugins/repository-s3/src/internalClusterTest/java/org/opensearch/repositories/s3/S3BlobStoreRepositoryTests.java

@@ -82,7 +82,6 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.Objects;
 import java.util.stream.StreamSupport;
 
 import fixture.s3.S3HttpHandler;
@@ -228,7 +227,6 @@ public void testEnforcedCooldownPeriod() throws IOException {
         assertThat(repository.threadPool().relativeTimeInNanos() - beforeFastDelete, lessThan(TEST_COOLDOWN_PERIOD.getNanos()));
     }
 
-    @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/10735")
     @Override
     public void testRequestStats() throws Exception {
         final String repository = createRepository(randomName());
@@ -270,7 +268,12 @@ public void testRequestStats() throws Exception {
             } catch (RepositoryMissingException e) {
                 return null;
             }
-        }).filter(Objects::nonNull).map(Repository::stats).reduce(RepositoryStats::merge).get();
+        }).filter(b -> {
+            if (b instanceof BlobStoreRepository) {
+                return ((BlobStoreRepository) b).blobStore() != null;
+            }
+            return false;
+        }).map(Repository::stats).reduce(RepositoryStats::merge).get();
 
         Map<BlobStore.Metric, Map<String, Long>> extendedStats = repositoryStats.extendedStats;
         Map<String, Long> aggregatedStats = new HashMap<>();
@@ -312,6 +315,8 @@ protected S3Repository createRepository(
             ClusterService clusterService,
             RecoverySettings recoverySettings
         ) {
+            GenericStatsMetricPublisher genericStatsMetricPublisher = new GenericStatsMetricPublisher(10000L, 10, 10000L, 10);
+
             return new S3Repository(
                 metadata,
                 registry,
@@ -326,7 +331,7 @@ protected S3Repository createRepository(
                 false,
                 null,
                 null,
-                null
+                genericStatsMetricPublisher
             ) {
 
                 @Override

release-notes/opensearch.release-notes-2.14.0.md

@@ -84,4 +84,5 @@
 - Improve the error messages for _stats with closed indices ([#13012](https://github.com/opensearch-project/OpenSearch/pull/13012))
 - Ignore BaseRestHandler unconsumed content check as it's always consumed. ([#13290](https://github.com/opensearch-project/OpenSearch/pull/13290))
 - Fix mapper_parsing_exception when using flat_object fields with names longer than 11 characters ([#13259](https://github.com/opensearch-project/OpenSearch/pull/13259))
-- DATETIME_FORMATTER_CACHING_SETTING experimental feature should not default to 'true' ([#13532](https://github.com/opensearch-project/OpenSearch/pull/13532))
+- DATETIME_FORMATTER_CACHING_SETTING experimental feature should not default to 'true' ([#13532](https://github.com/opensearch-project/OpenSearch/pull/13532))
+- Fix negative RequestStats metric issue ([#13553](https://github.com/opensearch-project/OpenSearch/pull/13553))

server/build.gradle

@@ -36,7 +36,7 @@ plugins {
   id('opensearch.publish')
   id('opensearch.internal-cluster-test')
   id('opensearch.optional-dependencies')
-  id('me.champeau.gradle.japicmp') version '0.4.2'
+  id('me.champeau.gradle.japicmp') version '0.4.3'
 }
 
 publishing {

server/src/main/java/org/opensearch/index/mapper/MapperService.java

@@ -710,6 +710,13 @@ public boolean isMetadataField(String field) {
         return mapperRegistry.isMetadataField(indexVersionCreated, field);
     }
 
+    /**
+     * Returns a set containing the registered metadata fields
+     */
+    public Set<String> getMetadataFields() {
+        return Collections.unmodifiableSet(mapperRegistry.getMetadataMapperParsers(indexVersionCreated).keySet());
+    }
+
     /**
      * An analyzer wrapper that can lookup fields within the index mappings
      */

server/src/main/java/org/opensearch/index/query/NestedQueryBuilder.java

@@ -401,16 +401,32 @@ protected void doBuild(SearchContext parentSearchContext, InnerHitsContext inner
                 }
             }
             String name = innerHitBuilder.getName() != null ? innerHitBuilder.getName() : nestedObjectMapper.fullPath();
-            ObjectMapper parentObjectMapper = queryShardContext.nestedScope().nextLevel(nestedObjectMapper);
-            NestedInnerHitSubContext nestedInnerHits = new NestedInnerHitSubContext(
-                name,
-                parentSearchContext,
-                parentObjectMapper,
-                nestedObjectMapper
-            );
-            setupInnerHitsContext(queryShardContext, nestedInnerHits);
-            queryShardContext.nestedScope().previousLevel();
-            innerHitsContext.addInnerHitDefinition(nestedInnerHits);
+            ObjectMapper parentObjectMapper = queryShardContext.nestedScope().getObjectMapper();
+            BitSetProducer parentFilter;
+            if (parentObjectMapper == null) {
+                parentFilter = queryShardContext.bitsetFilter(Queries.newNonNestedFilter());
+            } else {
+                parentFilter = queryShardContext.bitsetFilter(parentObjectMapper.nestedTypeFilter());
+            }
+            BitSetProducer previousParentFilter = queryShardContext.getParentFilter();
+            try {
+                queryShardContext.setParentFilter(parentFilter);
+                queryShardContext.nestedScope().nextLevel(nestedObjectMapper);
+                try {
+                    NestedInnerHitSubContext nestedInnerHits = new NestedInnerHitSubContext(
+                        name,
+                        parentSearchContext,
+                        parentObjectMapper,
+                        nestedObjectMapper
+                    );
+                    setupInnerHitsContext(queryShardContext, nestedInnerHits);
+                    innerHitsContext.addInnerHitDefinition(nestedInnerHits);
+                } finally {
+                    queryShardContext.nestedScope().previousLevel();
+                }
+            } finally {
+                queryShardContext.setParentFilter(previousParentFilter);
+            }
         }
     }
 

server/src/main/java/org/opensearch/script/Script.java

@@ -589,7 +589,7 @@ public Script(StreamInput in) throws IOException {
         @SuppressWarnings("unchecked")
         Map<String, String> options = (Map<String, String>) (Map) in.readMap();
         this.options = options;
-        this.params = in.readMap();
+        this.params = Collections.unmodifiableMap(in.readMap());
     }
 
     @Override

server/src/main/java/org/opensearch/script/UpdateScript.java

@@ -32,6 +32,7 @@
 
 package org.opensearch.script;
 
+import java.util.Collections;
 import java.util.Map;
 
 /**
@@ -53,7 +54,7 @@ public abstract class UpdateScript {
     private final Map<String, Object> ctx;
 
     public UpdateScript(Map<String, Object> params, Map<String, Object> ctx) {
-        this.params = params;
+        this.params = Collections.unmodifiableMap(params);
         this.ctx = ctx;
     }
 

server/src/test/java/org/opensearch/index/engine/InternalEngineTests.java

@@ -1873,7 +1873,7 @@ public void testForceMergeWithSoftDeletesRetentionAndRecoverySource() throws Exc
         try (
             Store store = createStore();
             InternalEngine engine = createEngine(
-                config(indexSettings, store, createTempDir(), newMergePolicy(), null, null, globalCheckpoint::get)
+                config(indexSettings, store, createTempDir(), newMergePolicy(random(), false), null, null, globalCheckpoint::get)
             )
         ) {
             int numDocs = scaledRandomIntBetween(10, 100);

server/src/test/java/org/opensearch/index/mapper/MapperServiceTests.java

@@ -52,6 +52,7 @@
 import org.opensearch.index.mapper.KeywordFieldMapper.KeywordFieldType;
 import org.opensearch.index.mapper.MapperService.MergeReason;
 import org.opensearch.index.mapper.NumberFieldMapper.NumberFieldType;
+import org.opensearch.indices.IndicesModule;
 import org.opensearch.indices.InvalidTypeNameException;
 import org.opensearch.indices.analysis.AnalysisModule.AnalysisProvider;
 import org.opensearch.plugins.AnalysisPlugin;
@@ -87,6 +88,11 @@ public void testTypeValidation() {
         MapperService.validateTypeName("_doc"); // no exception
     }
 
+    public void testGetMetadataFieldsReturnsExpectedSet() throws Throwable {
+        final MapperService mapperService = createIndex("test1").mapperService();
+        assertEquals(mapperService.getMetadataFields(), IndicesModule.getBuiltInMetadataFields());
+    }
+
     public void testPreflightUpdateDoesNotChangeMapping() throws Throwable {
         final MapperService mapperService = createIndex("test1").mapperService();
         final CompressedXContent mapping = createMappingSpecifyingNumberOfFields(1);