Merge branch 'main' into 11654-segment-matchall

CHANGELOG.md

@@ -188,9 +188,11 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Bump `com.diffplug.spotless` from 6.23.2 to 6.25.0 ([#11962](https://github.com/opensearch-project/OpenSearch/pull/11962), [#12055](https://github.com/opensearch-project/OpenSearch/pull/12055))
 - Bump `com.google.cloud:google-cloud-core` from 2.5.10 to 2.30.0 ([#11961](https://github.com/opensearch-project/OpenSearch/pull/11961))
 - Bump `reactor-core` from 3.5.11 to 3.5.14 ([#12042](https://github.com/opensearch-project/OpenSearch/pull/12042))
+- Bump `org.apache.shiro:shiro-core` from 1.11.0 to 1.13.0 ([#12200](https://github.com/opensearch-project/OpenSearch/pull/12200))
 - Bump `com.google.http-client:google-http-client-jackson2` from 1.43.3 to 1.44.1 ([#12059](https://github.com/opensearch-project/OpenSearch/pull/12059))
 - Bump `peter-evans/create-issue-from-file` from 4 to 5 ([#12057](https://github.com/opensearch-project/OpenSearch/pull/12057))
 - Bump `org.gradle.test-retry` from 1.5.4 to 1.5.8 ([#12168](https://github.com/opensearch-project/OpenSearch/pull/12168))
+- Bump `org.apache.kerby:kerb-admin` from 1.0.1 to 2.0.3 ([#12194](https://github.com/opensearch-project/OpenSearch/pull/12194))
 
 ### Changed
 - Mute the query profile IT with concurrent execution ([#9840](https://github.com/opensearch-project/OpenSearch/pull/9840))

plugins/identity-shiro/build.gradle

@@ -17,7 +17,7 @@ opensearchplugin {
 }
 
 dependencies {
-  implementation 'org.apache.shiro:shiro-core:1.11.0'
+  implementation 'org.apache.shiro:shiro-core:1.13.0'
 
   // Needed for shiro
   implementation "org.slf4j:slf4j-api:${versions.slf4j}"

plugins/identity-shiro/licenses/shiro-core-1.13.0.jar.sha1

@@ -0,0 +1 @@
+7e542e3d614b197bf10005e98e19f9f19cb943e7

server/src/main/java/org/opensearch/action/admin/cluster/node/stats/NodeStats.java

@@ -229,8 +229,7 @@ public NodeStats(StreamInput in) throws IOException {
         } else {
             repositoriesStats = null;
         }
-        // TODO: change to V_2_12_0 on main after backport to 2.x
-        if (in.getVersion().onOrAfter(Version.V_3_0_0)) {
+        if (in.getVersion().onOrAfter(Version.V_2_12_0)) {
             admissionControlStats = in.readOptionalWriteable(AdmissionControlStats::new);
         } else {
             admissionControlStats = null;
@@ -504,8 +503,7 @@ public void writeTo(StreamOutput out) throws IOException {
         if (out.getVersion().onOrAfter(Version.V_2_12_0)) {
             out.writeOptionalWriteable(repositoriesStats);
         }
-        // TODO: change to V_2_12_0 on main after backport to 2.x
-        if (out.getVersion().onOrAfter(Version.V_3_0_0)) {
+        if (out.getVersion().onOrAfter(Version.V_2_12_0)) {
             out.writeOptionalWriteable(admissionControlStats);
         }
     }

server/src/main/java/org/opensearch/common/lucene/search/function/ScriptScoreQuery.java

@@ -45,6 +45,7 @@
 import org.apache.lucene.search.Scorable;
 import org.apache.lucene.search.ScoreMode;
 import org.apache.lucene.search.Scorer;
+import org.apache.lucene.search.TwoPhaseIterator;
 import org.apache.lucene.search.Weight;
 import org.apache.lucene.util.Bits;
 import org.opensearch.Version;
@@ -302,6 +303,11 @@ public DocIdSetIterator iterator() {
             return subQueryScorer.iterator();
         }
 
+        @Override
+        public TwoPhaseIterator twoPhaseIterator() {
+            return subQueryScorer.twoPhaseIterator();
+        }
+
         @Override
         public float getMaxScore(int upTo) {
             return Float.MAX_VALUE; // TODO: what would be a good upper bound?

server/src/main/java/org/opensearch/threadpool/ThreadPool.java

@@ -281,7 +281,13 @@ public ThreadPool(
         );
         builders.put(
             Names.INDEX_SEARCHER,
-            new ResizableExecutorBuilder(settings, Names.INDEX_SEARCHER, allocatedProcessors, 1000, runnableTaskListener)
+            new ResizableExecutorBuilder(
+                settings,
+                Names.INDEX_SEARCHER,
+                twiceAllocatedProcessors(allocatedProcessors),
+                1000,
+                runnableTaskListener
+            )
         );
 
         for (final ExecutorBuilder<?> builder : customBuilders) {

server/src/test/java/org/opensearch/search/query/ScriptScoreQueryTests.java

@@ -39,23 +39,31 @@
 import org.apache.lucene.index.DirectoryReader;
 import org.apache.lucene.index.IndexWriter;
 import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.search.DocIdSetIterator;
 import org.apache.lucene.search.Explanation;
 import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.MatchAllDocsQuery;
+import org.apache.lucene.search.Query;
 import org.apache.lucene.search.ScoreMode;
+import org.apache.lucene.search.Scorer;
+import org.apache.lucene.search.TwoPhaseIterator;
 import org.apache.lucene.search.Weight;
 import org.apache.lucene.store.Directory;
 import org.opensearch.Version;
 import org.opensearch.common.lucene.search.Queries;
 import org.opensearch.common.lucene.search.function.ScriptScoreQuery;
 import org.opensearch.script.ScoreScript;
 import org.opensearch.script.Script;
+import org.opensearch.script.ScriptType;
 import org.opensearch.search.lookup.LeafSearchLookup;
 import org.opensearch.search.lookup.SearchLookup;
 import org.opensearch.test.OpenSearchTestCase;
 import org.junit.After;
 import org.junit.Before;
 
 import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
 import java.util.function.Function;
 
 import static org.hamcrest.CoreMatchers.containsString;
@@ -177,6 +185,37 @@ public void testScriptScoreErrorOnNegativeScore() {
         assertTrue(e.getMessage().contains("Must be a non-negative score!"));
     }
 
+    public void testTwoPhaseIteratorDelegation() throws IOException {
+        Map<String, Object> params = new HashMap<>();
+        String scriptSource = "doc['field'].value != null ? 2.0 : 0.0"; // Adjust based on actual field and logic
+        Script script = new Script(ScriptType.INLINE, "painless", scriptSource, params);
+        float minScore = 1.0f; // This should be below the score produced by the script for all docs
+        ScoreScript.LeafFactory factory = newFactory(script, false, explanation -> 2.0);
+
+        Query subQuery = new MatchAllDocsQuery();
+        ScriptScoreQuery scriptScoreQuery = new ScriptScoreQuery(subQuery, script, factory, minScore, "index", 0, Version.CURRENT);
+
+        Weight weight = searcher.createWeight(searcher.rewrite(scriptScoreQuery), ScoreMode.COMPLETE, 1f);
+
+        boolean foundMatchingDoc = false;
+        for (LeafReaderContext leafContext : searcher.getIndexReader().leaves()) {
+            Scorer scorer = weight.scorer(leafContext);
+            if (scorer != null) {
+                TwoPhaseIterator twoPhaseIterator = scorer.twoPhaseIterator();
+                assertNotNull("TwoPhaseIterator should not be null", twoPhaseIterator);
+                DocIdSetIterator docIdSetIterator = twoPhaseIterator.approximation();
+                int docId;
+                while ((docId = docIdSetIterator.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {
+                    if (twoPhaseIterator.matches()) {
+                        foundMatchingDoc = true;
+                        break;
+                    }
+                }
+            }
+        }
+        assertTrue("Expected to find at least one matching document", foundMatchingDoc);
+    }
+
     private ScoreScript.LeafFactory newFactory(
         Script script,
         boolean needsScore,
@@ -203,5 +242,4 @@ public double execute(ExplanationHolder explanation) {
             }
         };
     }
-
 }

test/fixtures/hdfs-fixture/build.gradle

@@ -51,6 +51,7 @@ dependencies {
     exclude module: "logback-core"
     exclude module: "logback-classic"
     exclude module: "avro"
+    exclude group: 'org.apache.kerby'
   }
   api "org.codehaus.jettison:jettison:${versions.jettison}"
   api "org.apache.commons:commons-compress:${versions.commonscompress}"
@@ -72,6 +73,7 @@ dependencies {
   api "commons-net:commons-net:3.10.0"
   api "ch.qos.logback:logback-core:1.2.13"
   api "ch.qos.logback:logback-classic:1.2.13"
+  api 'org.apache.kerby:kerb-admin:2.0.3'
   runtimeOnly "com.google.guava:guava:${versions.guava}"
   runtimeOnly("com.squareup.okhttp3:okhttp:4.12.0") {
     exclude group: "com.squareup.okio"