Merge branch 'main' into plugin-aware-thread-context

.github/CODEOWNERS

@@ -13,15 +13,25 @@
 # Default ownership for all repo files
 * @anasalkouz @andrross @ashking94 @Bukhtawar @CEHENKLE @dblock @dbwiddis @gbbafna @kotwanikunal @mch2 @msfroh @nknize @owaiskazi19 @reta @Rishikesh1159 @sachinpkale @saratvemulapalli @shwetathareja @sohami @VachaShah
 
+/modules/lang-painless/ @anasalkouz @andrross @ashking94 @Bukhtawar @CEHENKLE @dblock @dbwiddis @gbbafna @jed326 @kotwanikunal @mch2 @msfroh @nknize @owaiskazi19 @reta @Rishikesh1159 @sachinpkale @saratvemulapalli @shwetathareja @sohami @VachaShah
+/modules/parent-join/ @anasalkouz @andrross @ashking94 @Bukhtawar @CEHENKLE @dblock @dbwiddis @gbbafna @jed326 @kotwanikunal @mch2 @msfroh @nknize @owaiskazi19 @reta @Rishikesh1159 @sachinpkale @saratvemulapalli @shwetathareja @sohami @VachaShah
 /modules/transport-netty4/ @peternied
 
 /plugins/identity-shiro/ @peternied
 
+/server/src/internalClusterTest/java/org/opensearch/index/ @anasalkouz @andrross @ashking94 @Bukhtawar @CEHENKLE @dblock @dbwiddis @gbbafna @jed326 @kotwanikunal @mch2 @msfroh @nknize @owaiskazi19 @reta @Rishikesh1159 @sachinpkale @saratvemulapalli @shwetathareja @sohami @VachaShah
+/server/src/internalClusterTest/java/org/opensearch/search/ @anasalkouz @andrross @ashking94 @Bukhtawar @CEHENKLE @dblock @dbwiddis @gbbafna @jed326 @kotwanikunal @mch2 @msfroh @nknize @owaiskazi19 @reta @Rishikesh1159 @sachinpkale @saratvemulapalli @shwetathareja @sohami @VachaShah
+
 /server/src/main/java/org/opensearch/extensions/ @peternied
 /server/src/main/java/org/opensearch/identity/ @peternied
-/server/src/main/java/org/opensearch/threadpool/ @peternied
+/server/src/main/java/org/opensearch/index/ @anasalkouz @andrross @ashking94 @Bukhtawar @CEHENKLE @dblock @dbwiddis @gbbafna @jed326 @kotwanikunal @mch2 @msfroh @nknize @owaiskazi19 @reta @Rishikesh1159 @sachinpkale @saratvemulapalli @shwetathareja @sohami @VachaShah
+/server/src/main/java/org/opensearch/search/ @anasalkouz @andrross @ashking94 @Bukhtawar @CEHENKLE @dblock @dbwiddis @gbbafna @jed326 @kotwanikunal @mch2 @msfroh @nknize @owaiskazi19 @reta @Rishikesh1159 @sachinpkale @saratvemulapalli @shwetathareja @sohami @VachaShah
+/server/src/main/java/org/opensearch/threadpool/ @jed326 @peternied
 /server/src/main/java/org/opensearch/transport/ @peternied
 
-/.github/ @peternied
+/server/src/test/java/org/opensearch/index/ @anasalkouz @andrross @ashking94 @Bukhtawar @CEHENKLE @dblock @dbwiddis @gbbafna @jed326 @kotwanikunal @mch2 @msfroh @nknize @owaiskazi19 @reta @Rishikesh1159 @sachinpkale @saratvemulapalli @shwetathareja @sohami @VachaShah
+/server/src/test/java/org/opensearch/search/ @anasalkouz @andrross @ashking94 @Bukhtawar @CEHENKLE @dblock @dbwiddis @gbbafna @jed326 @kotwanikunal @mch2 @msfroh @nknize @owaiskazi19 @reta @Rishikesh1159 @sachinpkale @saratvemulapalli @shwetathareja @sohami @VachaShah
+
+/.github/ @jed326 @peternied
 
 /MAINTAINERS.md @anasalkouz @andrross @ashking94 @Bukhtawar @CEHENKLE @dblock @dbwiddis @gaobinlong @gbbafna @jed326 @kotwanikunal @mch2 @msfroh @nknize @owaiskazi19 @peternied @reta @Rishikesh1159 @sachinpkale @saratvemulapalli @shwetathareja @sohami @VachaShah

.github/benchmark-configs.json

@@ -14,7 +14,8 @@
     "cluster_configuration": {
       "size": "Single-Node",
       "data_instance_config": "4vCPU, 32G Mem, 16G Heap"
-    }
+    },
+    "baseline_cluster_config": "x64-r5.xlarge-single-node-1-shard-0-replica-baseline"
   },
   "id_2": {
     "description": "Indexing only configuration for HTTP_LOGS workload",
@@ -30,7 +31,8 @@
     "cluster_configuration": {
       "size": "Single-Node",
       "data_instance_config": "4vCPU, 32G Mem, 16G Heap"
-    }
+    },
+    "baseline_cluster_config": "x64-r5.xlarge-single-node-1-shard-0-replica-baseline"
   },
   "id_3": {
     "description": "Search only test-procedure for NYC_TAXIS, uses snapshot to restore the data for OS-3.0.0",
@@ -46,7 +48,8 @@
     "cluster_configuration": {
       "size": "Single-Node",
       "data_instance_config": "4vCPU, 32G Mem, 16G Heap"
-    }
+    },
+    "baseline_cluster_config": "x64-r5.xlarge-1-shard-0-replica-snapshot-baseline"
   },
   "id_4": {
     "description": "Search only test-procedure for HTTP_LOGS, uses snapshot to restore the data for OS-3.0.0",
@@ -62,10 +65,11 @@
     "cluster_configuration": {
       "size": "Single-Node",
       "data_instance_config": "4vCPU, 32G Mem, 16G Heap"
-    }
+    },
+    "baseline_cluster_config": "x64-r5.xlarge-1-shard-0-replica-snapshot-baseline"
   },
   "id_5": {
-    "description": "Search only test-procedure for HTTP_LOGS, uses snapshot to restore the data for OS-3.0.0",
+    "description": "Search only test-procedure for big5, uses snapshot to restore the data for OS-3.0.0",
     "supported_major_versions": ["3"],
     "cluster-benchmark-configs": {
       "SINGLE_NODE_CLUSTER": "true",
@@ -78,7 +82,8 @@
     "cluster_configuration": {
       "size": "Single-Node",
       "data_instance_config": "4vCPU, 32G Mem, 16G Heap"
-    }
+    },
+    "baseline_cluster_config": "x64-r5.xlarge-1-shard-0-replica-snapshot-baseline"
   },
   "id_6": {
     "description": "Search only test-procedure for NYC_TAXIS, uses snapshot to restore the data for OS-2.x",
@@ -94,7 +99,8 @@
     "cluster_configuration": {
       "size": "Single-Node",
       "data_instance_config": "4vCPU, 32G Mem, 16G Heap"
-    }
+    },
+    "baseline_cluster_config": "x64-r5.xlarge-1-shard-0-replica-snapshot-baseline"
   },
   "id_7": {
     "description": "Search only test-procedure for HTTP_LOGS, uses snapshot to restore the data for OS-2.x",
@@ -110,10 +116,11 @@
     "cluster_configuration": {
       "size": "Single-Node",
       "data_instance_config": "4vCPU, 32G Mem, 16G Heap"
-    }
+    },
+    "baseline_cluster_config": "x64-r5.xlarge-1-shard-0-replica-snapshot-baseline"
   },
   "id_8": {
-    "description": "Search only test-procedure for HTTP_LOGS, uses snapshot to restore the data for OS-2.x",
+    "description": "Search only test-procedure for big5, uses snapshot to restore the data for OS-2.x",
     "supported_major_versions": ["2"],
     "cluster-benchmark-configs": {
       "SINGLE_NODE_CLUSTER": "true",
@@ -126,7 +133,8 @@
     "cluster_configuration": {
       "size": "Single-Node",
       "data_instance_config": "4vCPU, 32G Mem, 16G Heap"
-    }
+    },
+    "baseline_cluster_config": "x64-r5.xlarge-1-shard-0-replica-snapshot-baseline"
   },
   "id_9": {
     "description": "Indexing and search configuration for pmc workload",
@@ -141,7 +149,8 @@
     "cluster_configuration": {
       "size": "Single-Node",
       "data_instance_config": "4vCPU, 32G Mem, 16G Heap"
-    }
+    },
+    "baseline_cluster_config": "x64-r5.xlarge-single-node-1-shard-0-replica-baseline"
   },
   "id_10": {
     "description": "Indexing only configuration for stack-overflow workload",
@@ -156,6 +165,7 @@
     "cluster_configuration": {
       "size": "Single-Node",
       "data_instance_config": "4vCPU, 32G Mem, 16G Heap"
-    }
+    },
+    "baseline_cluster_config": "x64-r5.xlarge-single-node-1-shard-0-replica-baseline"
   }
 }

.github/workflows/benchmark-pull-request.yml

@@ -60,6 +60,8 @@ jobs:
             for (const [key, value] of Object.entries(clusterBenchmarkConfigs)) {
               core.exportVariable(key, value);
             }
+            if (benchmarkConfigs[configId].hasOwnProperty('baseline_cluster_config')) {
+              core.exportVariable('BASELINE_CLUSTER_CONFIG', benchmarkConfigs[configId]['baseline_cluster_config']);
       - name: Post invalid format comment
         if: steps.check_comment.outputs.invalid == 'true'
         uses: actions/github-script@v7

CHANGELOG.md

@@ -14,6 +14,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Add ThreadContextPermission for stashAndMergeHeaders and stashWithOrigin ([#15039](https://github.com/opensearch-project/OpenSearch/pull/15039))
 - [Concurrent Segment Search] Support composite aggregations with scripting ([#15072](https://github.com/opensearch-project/OpenSearch/pull/15072))
 - Add `rangeQuery` and `regexpQuery` for `constant_keyword` field type ([#14711](https://github.com/opensearch-project/OpenSearch/pull/14711))
+- Add took time to request nodes stats ([#15054](https://github.com/opensearch-project/OpenSearch/pull/15054))
+- [Workload Management] QueryGroup resource tracking framework changes ([#13897](https://github.com/opensearch-project/OpenSearch/pull/13897))
 
 ### Dependencies
 - Bump `netty` from 4.1.111.Final to 4.1.112.Final ([#15081](https://github.com/opensearch-project/OpenSearch/pull/15081))
@@ -24,6 +26,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Bump `org.tukaani:xz` from 1.9 to 1.10 ([#15110](https://github.com/opensearch-project/OpenSearch/pull/15110))
 - Bump `actions/setup-java` from 1 to 4 ([#15104](https://github.com/opensearch-project/OpenSearch/pull/15104))
 - Bump `org.apache.avro:avro` from 1.11.3 to 1.12.0 in /plugins/repository-hdfs ([#15119](https://github.com/opensearch-project/OpenSearch/pull/15119))
+- Bump `org.bouncycastle:bcpg-fips` from 1.0.7.1 to 2.0.8 and `org.bouncycastle:bc-fips` from 1.0.2.5 to 2.0.0 in /distribution/tools/plugin-cli ([#15103](https://github.com/opensearch-project/OpenSearch/pull/15103))
+- Bump `com.azure:azure-core` from 1.49.1 to 1.51.0 ([#15111](https://github.com/opensearch-project/OpenSearch/pull/15111))
 
 ### Changed
 - Add lower limit for primary and replica batch allocators timeout ([#14979](https://github.com/opensearch-project/OpenSearch/pull/14979))
@@ -34,6 +38,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 
 ### Fixed
 - Fix constraint bug which allows more primary shards than average primary shards per index ([#14908](https://github.com/opensearch-project/OpenSearch/pull/14908))
+- Fix NPE when bulk ingest with empty pipeline ([#15033](https://github.com/opensearch-project/OpenSearch/pull/15033))
 - Fix missing value of FieldSort for unsigned_long ([#14963](https://github.com/opensearch-project/OpenSearch/pull/14963))
 - Fix delete index template failed when the index template matches a data stream but is unused ([#15080](https://github.com/opensearch-project/OpenSearch/pull/15080))
 

distribution/tools/plugin-cli/build.gradle

@@ -37,8 +37,8 @@ base {
 dependencies {
   compileOnly project(":server")
   compileOnly project(":libs:opensearch-cli")
-  api "org.bouncycastle:bcpg-fips:1.0.7.1"
-  api "org.bouncycastle:bc-fips:1.0.2.5"
+  api "org.bouncycastle:bcpg-fips:2.0.8"
+  api "org.bouncycastle:bc-fips:2.0.0"
   testImplementation project(":test:framework")
   testImplementation 'com.google.jimfs:jimfs:1.3.0'
   testRuntimeOnly("com.google.guava:guava:${versions.guava}") {
@@ -58,33 +58,6 @@ test {
   jvmArgs += [ "-Djava.security.egd=file:/dev/urandom" ]
 }
 
-/*
- * these two classes intentionally use the following JDK internal APIs in order to offer the necessary
- * functionality
- *
- * sun.security.internal.spec.TlsKeyMaterialParameterSpec
- * sun.security.internal.spec.TlsKeyMaterialSpec
- * sun.security.internal.spec.TlsMasterSecretParameterSpec
- * sun.security.internal.spec.TlsPrfParameterSpec
- * sun.security.internal.spec.TlsRsaPremasterSecretParameterSpec
- * sun.security.provider.SecureRandom
- *
- */
-thirdPartyAudit.ignoreViolations(
-  'org.bouncycastle.jcajce.provider.BouncyCastleFipsProvider$CoreSecureRandom',
-  'org.bouncycastle.jcajce.provider.ProvSunTLSKDF',
-  'org.bouncycastle.jcajce.provider.ProvSunTLSKDF$BaseTLSKeyGeneratorSpi',
-  'org.bouncycastle.jcajce.provider.ProvSunTLSKDF$TLSKeyMaterialGenerator',
-  'org.bouncycastle.jcajce.provider.ProvSunTLSKDF$TLSKeyMaterialGenerator$2',
-  'org.bouncycastle.jcajce.provider.ProvSunTLSKDF$TLSMasterSecretGenerator',
-  'org.bouncycastle.jcajce.provider.ProvSunTLSKDF$TLSMasterSecretGenerator$2',
-  'org.bouncycastle.jcajce.provider.ProvSunTLSKDF$TLSPRFKeyGenerator',
-  'org.bouncycastle.jcajce.provider.ProvSunTLSKDF$TLSRsaPreMasterSecretGenerator',
-  'org.bouncycastle.jcajce.provider.ProvSunTLSKDF$TLSRsaPreMasterSecretGenerator$2',
-  'org.bouncycastle.jcajce.provider.ProvSunTLSKDF$TLSExtendedMasterSecretGenerator',
-  'org.bouncycastle.jcajce.provider.ProvSunTLSKDF$TLSExtendedMasterSecretGenerator$2'
-)
-
 thirdPartyAudit.ignoreMissingClasses(
   'org.brotli.dec.BrotliInputStream',
   'org.objectweb.asm.AnnotationVisitor',

distribution/tools/plugin-cli/licenses/bc-fips-2.0.0.jar.sha1

@@ -0,0 +1 @@
+ee9ac432cf08f9a9ebee35d7cf8a45f94959a7ab

distribution/tools/plugin-cli/licenses/bcpg-fips-2.0.8.jar.sha1

@@ -0,0 +1 @@
+51c2f633e0c32d10de1ebab4c86f93310ff820f8

plugins/repository-azure/build.gradle

@@ -44,7 +44,7 @@ opensearchplugin {
 }
 
 dependencies {
-  api 'com.azure:azure-core:1.49.1'
+  api 'com.azure:azure-core:1.51.0'
   api 'com.azure:azure-json:1.1.0'
   api 'com.azure:azure-xml:1.0.0'
   api 'com.azure:azure-storage-common:12.25.1'

plugins/repository-azure/licenses/azure-core-1.51.0.jar.sha1

@@ -0,0 +1 @@
+ff5d0aedf75ca45ec0ace24673f790d2f7a57096

rest-api-spec/src/main/resources/rest-api-spec/test/index/110_constant_keyword.yml → rest-api-spec/src/main/resources/rest-api-spec/test/index/115_constant_keyword.yml

@@ -70,8 +70,8 @@
 ---
 "Queries":
   - skip:
-      version: " - 2.99.99"
-      reason: "rangeQuery and regexpQuery are supported in 3.0.0 in main branch"
+      version: " - 2.16.99"
+      reason: "rangeQuery and regexpQuery are introduced in 2.17.0"
 
   - do:
       indices.create:

server/src/internalClusterTest/java/org/opensearch/index/mapper/StarTreeMapperIT.java

@@ -275,7 +275,7 @@ public void testValidCompositeIndex() {
                     assertEquals(expectedMetrics, starTreeFieldType.getMetrics().get(0).getMetrics());
                     assertEquals(10000, starTreeFieldType.getStarTreeConfig().maxLeafDocs());
                     assertEquals(
-                        StarTreeFieldConfiguration.StarTreeBuildMode.ON_HEAP,
+                        StarTreeFieldConfiguration.StarTreeBuildMode.OFF_HEAP,
                         starTreeFieldType.getStarTreeConfig().getBuildMode()
                     );
                     assertEquals(Collections.emptySet(), starTreeFieldType.getStarTreeConfig().getSkipStarNodeCreationInDims());
@@ -359,7 +359,7 @@ public void testUpdateIndexWhenMappingIsSame() {
                     assertEquals(expectedMetrics, starTreeFieldType.getMetrics().get(0).getMetrics());
                     assertEquals(10000, starTreeFieldType.getStarTreeConfig().maxLeafDocs());
                     assertEquals(
-                        StarTreeFieldConfiguration.StarTreeBuildMode.ON_HEAP,
+                        StarTreeFieldConfiguration.StarTreeBuildMode.OFF_HEAP,
                         starTreeFieldType.getStarTreeConfig().getBuildMode()
                     );
                     assertEquals(Collections.emptySet(), starTreeFieldType.getStarTreeConfig().getSkipStarNodeCreationInDims());

server/src/main/java/org/opensearch/action/search/SearchRequestStats.java

@@ -27,6 +27,7 @@
 @PublicApi(since = "2.11.0")
 public final class SearchRequestStats extends SearchRequestOperationsListener {
     Map<SearchPhaseName, StatsHolder> phaseStatsMap = new EnumMap<>(SearchPhaseName.class);
+    StatsHolder tookStatsHolder;
 
     public static final String SEARCH_REQUEST_STATS_ENABLED_KEY = "search.request_stats_enabled";
     public static final Setting<Boolean> SEARCH_REQUEST_STATS_ENABLED = Setting.boolSetting(
@@ -40,6 +41,7 @@ public final class SearchRequestStats extends SearchRequestOperationsListener {
     public SearchRequestStats(ClusterSettings clusterSettings) {
         this.setEnabled(clusterSettings.get(SEARCH_REQUEST_STATS_ENABLED));
         clusterSettings.addSettingsUpdateConsumer(SEARCH_REQUEST_STATS_ENABLED, this::setEnabled);
+        tookStatsHolder = new StatsHolder();
         for (SearchPhaseName searchPhaseName : SearchPhaseName.values()) {
             phaseStatsMap.put(searchPhaseName, new StatsHolder());
         }
@@ -57,6 +59,18 @@ public long getPhaseMetric(SearchPhaseName searchPhaseName) {
         return phaseStatsMap.get(searchPhaseName).timing.sum();
     }
 
+    public long getTookCurrent() {
+        return tookStatsHolder.current.count();
+    }
+
+    public long getTookTotal() {
+        return tookStatsHolder.total.count();
+    }
+
+    public long getTookMetric() {
+        return tookStatsHolder.timing.sum();
+    }
+
     @Override
     protected void onPhaseStart(SearchPhaseContext context) {
         phaseStatsMap.get(context.getCurrentPhase().getSearchPhaseName()).current.inc();
@@ -75,6 +89,23 @@ protected void onPhaseFailure(SearchPhaseContext context, Throwable cause) {
         phaseStatsMap.get(context.getCurrentPhase().getSearchPhaseName()).current.dec();
     }
 
+    @Override
+    protected void onRequestStart(SearchRequestContext searchRequestContext) {
+        tookStatsHolder.current.inc();
+    }
+
+    @Override
+    protected void onRequestEnd(SearchPhaseContext context, SearchRequestContext searchRequestContext) {
+        tookStatsHolder.current.dec();
+        tookStatsHolder.total.inc();
+        tookStatsHolder.timing.inc(TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - searchRequestContext.getAbsoluteStartNanos()));
+    }
+
+    @Override
+    protected void onRequestFailure(SearchPhaseContext context, SearchRequestContext searchRequestContext) {
+        tookStatsHolder.current.dec();
+    }
+
     /**
      * Holder of statistics values
      *

server/src/main/java/org/opensearch/cluster/metadata/Metadata.java

@@ -1391,7 +1391,7 @@ public Builder put(final QueryGroup queryGroup) {
             return queryGroups(existing);
         }
 
-        private Map<String, QueryGroup> getQueryGroups() {
+        public Map<String, QueryGroup> getQueryGroups() {
             return Optional.ofNullable(this.customs.get(QueryGroupMetadata.TYPE))
                 .map(o -> (QueryGroupMetadata) o)
                 .map(QueryGroupMetadata::queryGroups)