[Backport 2.x] Bump to Lucene99

CHANGELOG.md

@@ -76,6 +76,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Bump `commons-cli:commons-cli` from 1.5.0 to 1.6.0 ([#10996](https://github.com/opensearch-project/OpenSearch/pull/10996))
 - Bump `org.apache.commons:commons-lang3` from 3.13.0 to 3.14.0 ([#11691](https://github.com/opensearch-project/OpenSearch/pull/11691))
 - Bump `net.java.dev.jna:jna` from 5.13.0 to 5.14.0 ([#11798](https://github.com/opensearch-project/OpenSearch/pull/11798))
+- Bump `Lucene` from 9.8.0 to 9.9.1 ([#11421](https://github.com/opensearch-project/OpenSearch/pull/11421))
 
 ### Changed
 - Force merge with `only_expunge_deletes` honors max segment size ([#10036](https://github.com/opensearch-project/OpenSearch/pull/10036))

buildSrc/version.properties

@@ -1,5 +1,5 @@
 opensearch        = 2.12.0
-lucene            = 9.8.0
+lucene            = 9.9.1
 
 bundled_jdk_vendor = adoptium
 bundled_jdk = 21.0.1+12

libs/core/licenses/lucene-core-9.9.1.jar.sha1

@@ -0,0 +1 @@
+55249fa9a0ed321adcf8283c6f3b649a6812b0a9

libs/core/src/main/java/org/opensearch/Version.java

@@ -123,7 +123,7 @@ public class Version implements Comparable<Version>, ToXContentFragment {
     public static final Version V_2_11_0 = new Version(2110099, org.apache.lucene.util.Version.LUCENE_9_7_0);
     public static final Version V_2_11_1 = new Version(2110199, org.apache.lucene.util.Version.LUCENE_9_7_0);
     public static final Version V_2_11_2 = new Version(2110299, org.apache.lucene.util.Version.LUCENE_9_7_0);
-    public static final Version V_2_12_0 = new Version(2120099, org.apache.lucene.util.Version.LUCENE_9_8_0);
+    public static final Version V_2_12_0 = new Version(2120099, org.apache.lucene.util.Version.LUCENE_9_9_1);
     public static final Version CURRENT = V_2_12_0;
 
     public static Version fromId(int id) {

modules/lang-expression/licenses/lucene-expressions-9.9.1.jar.sha1

@@ -0,0 +1 @@
+1782a69d0e83af9cc3c65db0dcd2e7e7c1e5f90e

modules/mapper-extras/src/test/java/org/opensearch/index/mapper/ScaledFloatFieldTypeTests.java

@@ -135,35 +135,40 @@ public void testRangeQuery() throws IOException {
     public void testRoundsUpperBoundCorrectly() {
         ScaledFloatFieldMapper.ScaledFloatFieldType ft = new ScaledFloatFieldMapper.ScaledFloatFieldType("scaled_float", 100);
         Query scaledFloatQ = ft.rangeQuery(null, 0.1, true, false, MOCK_QSC);
-        assertEquals("scaled_float:[-9223372036854775808 TO 9]", scaledFloatQ.toString());
+        assertEquals("scaled_float:[-9223372036854775808 TO 9]", getQueryString(scaledFloatQ));
         scaledFloatQ = ft.rangeQuery(null, 0.1, true, true, MOCK_QSC);
-        assertEquals("scaled_float:[-9223372036854775808 TO 10]", scaledFloatQ.toString());
+        assertEquals("scaled_float:[-9223372036854775808 TO 10]", getQueryString(scaledFloatQ));
         scaledFloatQ = ft.rangeQuery(null, 0.095, true, false, MOCK_QSC);
-        assertEquals("scaled_float:[-9223372036854775808 TO 9]", scaledFloatQ.toString());
+        assertEquals("scaled_float:[-9223372036854775808 TO 9]", getQueryString(scaledFloatQ));
         scaledFloatQ = ft.rangeQuery(null, 0.095, true, true, MOCK_QSC);
-        assertEquals("scaled_float:[-9223372036854775808 TO 9]", scaledFloatQ.toString());
+        assertEquals("scaled_float:[-9223372036854775808 TO 9]", getQueryString(scaledFloatQ));
         scaledFloatQ = ft.rangeQuery(null, 0.105, true, false, MOCK_QSC);
-        assertEquals("scaled_float:[-9223372036854775808 TO 10]", scaledFloatQ.toString());
+        assertEquals("scaled_float:[-9223372036854775808 TO 10]", getQueryString(scaledFloatQ));
         scaledFloatQ = ft.rangeQuery(null, 0.105, true, true, MOCK_QSC);
-        assertEquals("scaled_float:[-9223372036854775808 TO 10]", scaledFloatQ.toString());
+        assertEquals("scaled_float:[-9223372036854775808 TO 10]", getQueryString(scaledFloatQ));
         scaledFloatQ = ft.rangeQuery(null, 79.99, true, true, MOCK_QSC);
-        assertEquals("scaled_float:[-9223372036854775808 TO 7999]", scaledFloatQ.toString());
+        assertEquals("scaled_float:[-9223372036854775808 TO 7999]", getQueryString(scaledFloatQ));
     }
 
     public void testRoundsLowerBoundCorrectly() {
         ScaledFloatFieldMapper.ScaledFloatFieldType ft = new ScaledFloatFieldMapper.ScaledFloatFieldType("scaled_float", 100);
         Query scaledFloatQ = ft.rangeQuery(-0.1, null, false, true, MOCK_QSC);
-        assertEquals("scaled_float:[-9 TO 9223372036854775807]", scaledFloatQ.toString());
+        assertEquals("scaled_float:[-9 TO 9223372036854775807]", getQueryString(scaledFloatQ));
         scaledFloatQ = ft.rangeQuery(-0.1, null, true, true, MOCK_QSC);
-        assertEquals("scaled_float:[-10 TO 9223372036854775807]", scaledFloatQ.toString());
+        assertEquals("scaled_float:[-10 TO 9223372036854775807]", getQueryString(scaledFloatQ));
         scaledFloatQ = ft.rangeQuery(-0.095, null, false, true, MOCK_QSC);
-        assertEquals("scaled_float:[-9 TO 9223372036854775807]", scaledFloatQ.toString());
+        assertEquals("scaled_float:[-9 TO 9223372036854775807]", getQueryString(scaledFloatQ));
         scaledFloatQ = ft.rangeQuery(-0.095, null, true, true, MOCK_QSC);
-        assertEquals("scaled_float:[-9 TO 9223372036854775807]", scaledFloatQ.toString());
+        assertEquals("scaled_float:[-9 TO 9223372036854775807]", getQueryString(scaledFloatQ));
         scaledFloatQ = ft.rangeQuery(-0.105, null, false, true, MOCK_QSC);
-        assertEquals("scaled_float:[-10 TO 9223372036854775807]", scaledFloatQ.toString());
+        assertEquals("scaled_float:[-10 TO 9223372036854775807]", getQueryString(scaledFloatQ));
         scaledFloatQ = ft.rangeQuery(-0.105, null, true, true, MOCK_QSC);
-        assertEquals("scaled_float:[-10 TO 9223372036854775807]", scaledFloatQ.toString());
+        assertEquals("scaled_float:[-10 TO 9223372036854775807]", getQueryString(scaledFloatQ));
+    }
+
+    private String getQueryString(Query query) {
+        assertTrue(query instanceof IndexOrDocValuesQuery);
+        return ((IndexOrDocValuesQuery) query).getIndexQuery().toString();
     }
 
     public void testValueForSearch() {

modules/transport-netty4/src/test/java/org/opensearch/http/netty4/Netty4HttpPipeliningHandlerTests.java

@@ -79,7 +79,7 @@ public class Netty4HttpPipeliningHandlerTests extends OpenSearchTestCase {
     @After
     public void tearDown() throws Exception {
         waitingRequests.keySet().forEach(this::finishRequest);
-        shutdownExecutorService();
+        shutdownExecutorServices();
         super.tearDown();
     }
 
@@ -88,7 +88,7 @@ private CountDownLatch finishRequest(String url) {
         return finishingRequests.get(url);
     }
 
-    private void shutdownExecutorService() throws InterruptedException {
+    private void shutdownExecutorServices() throws InterruptedException {
         if (!handlerService.isShutdown()) {
             handlerService.shutdown();
             handlerService.awaitTermination(10, TimeUnit.SECONDS);

plugins/analysis-icu/licenses/lucene-analysis-icu-9.9.1.jar.sha1

@@ -0,0 +1 @@
+147cb42a90a29501d9ca6094ea0db1d213f3076a

plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.9.1.jar.sha1

@@ -0,0 +1 @@
+b034dd3a975763e083c7e11b5d0f7d516ab72590

plugins/analysis-kuromoji/src/main/java/org/opensearch/index/analysis/KuromojiTokenizerFactory.java

@@ -38,7 +38,7 @@
 import org.apache.lucene.analysis.ja.JapaneseTokenizer;
 import org.apache.lucene.analysis.ja.JapaneseTokenizer.Mode;
 import org.apache.lucene.analysis.ja.dict.UserDictionary;
-import org.apache.lucene.analysis.ja.util.CSVUtil;
+import org.apache.lucene.analysis.util.CSVUtil;
 import org.opensearch.OpenSearchException;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.env.Environment;

plugins/analysis-nori/licenses/lucene-analysis-nori-9.9.1.jar.sha1

@@ -0,0 +1 @@
+c405f2f7d0fc127d88dfbadd753469b2028fdf52

plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.9.1.jar.sha1

@@ -0,0 +1 @@
+970e5775876c2d7e1b9af7421a4b17d96f63faf4

plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.9.1.jar.sha1

@@ -0,0 +1 @@
+2421e5238e9b8484929291744d709dd743c01da1

plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.9.1.jar.sha1

@@ -0,0 +1 @@
+a23e7de4cd9ae7af285c89dc1c55e0ac3f157fd3

plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.9.1.jar.sha1

@@ -0,0 +1 @@
+8d9bce1ea51db279878c51091dd9aefc7b335da4

plugins/transport-nio/src/test/java/org/opensearch/http/nio/NioHttpPipeliningHandlerTests.java

@@ -80,15 +80,15 @@ public class NioHttpPipeliningHandlerTests extends OpenSearchTestCase {
     @After
     public void cleanup() throws Exception {
         waitingRequests.keySet().forEach(this::finishRequest);
-        shutdownExecutorService();
+        shutdownExecutorServices();
     }
 
     private CountDownLatch finishRequest(String url) {
         waitingRequests.get(url).countDown();
         return finishingRequests.get(url);
     }
 
-    private void shutdownExecutorService() throws InterruptedException {
+    private void shutdownExecutorServices() throws InterruptedException {
         if (!handlerService.isShutdown()) {
             handlerService.shutdown();
             handlerService.awaitTermination(10, TimeUnit.SECONDS);

server/licenses/lucene-analysis-common-9.9.1.jar.sha1

@@ -0,0 +1 @@
+24c8401b530308f9568eb7b408c2029c63f564c6

server/licenses/lucene-backward-codecs-9.9.1.jar.sha1

@@ -0,0 +1 @@
+11c46007366bb037be7d271ab0a5849b1d544662

server/licenses/lucene-core-9.9.1.jar.sha1

@@ -0,0 +1 @@
+55249fa9a0ed321adcf8283c6f3b649a6812b0a9

server/licenses/lucene-grouping-9.9.1.jar.sha1

@@ -0,0 +1 @@
+2f2785e17c5c823cc8f41a7ddb4647aaca8ee773

server/licenses/lucene-highlighter-9.9.1.jar.sha1

@@ -0,0 +1 @@
+30928513461bf79a5cb057e84da7d34a1e53227d

server/licenses/lucene-join-9.9.1.jar.sha1

@@ -0,0 +1 @@
+b9c8cc99632280148f92b4c0a64111c482d5d0ac

server/licenses/lucene-memory-9.9.1.jar.sha1

@@ -0,0 +1 @@
+49f820b1b321860fa42a4f7583e8ed8f77b9c1c2

server/licenses/lucene-misc-9.9.1.jar.sha1

@@ -0,0 +1 @@
+db7c30217602dfcda394a4d0f0a9e68140d385a6

server/licenses/lucene-queries-9.9.1.jar.sha1

@@ -0,0 +1 @@
+d157547bd24edc8e9d9d59c273107dc3ac5fde5e

server/licenses/lucene-queryparser-9.9.1.jar.sha1

@@ -0,0 +1 @@
+12d844fe224f6f97c510ac20d68903ed7f626f6c

server/licenses/lucene-sandbox-9.9.1.jar.sha1

@@ -0,0 +1 @@
+272e588fd3d8c0a401b28a1ac715f27044bf62ec

server/licenses/lucene-spatial-extras-9.9.1.jar.sha1

@@ -0,0 +1 @@
+e066432e7ab02b2a4914f989bcd8c44adbf340ad

server/licenses/lucene-spatial3d-9.9.1.jar.sha1

@@ -0,0 +1 @@
+fa54c9b962778e28ebc0efb9f75297781350361a

server/licenses/lucene-suggest-9.9.1.jar.sha1

@@ -0,0 +1 @@
+9554de5b22ae7483b344b94a9a956960b7a5d49c

server/src/internalClusterTest/java/org/opensearch/index/store/CorruptedFileIT.java

@@ -710,6 +710,7 @@ public void testPrimaryCorruptionDuringReplicationDoesNotFailReplicaShard() thro
 
         final NodeStats primaryNode = dataNodeStats.get(0);
         final NodeStats replicaNode = dataNodeStats.get(1);
+
         assertAcked(
             prepareCreate("test").setSettings(
                 Settings.builder()
@@ -795,6 +796,17 @@ public void testPrimaryCorruptionDuringReplicationDoesNotFailReplicaShard() thro
 
         // Assert the cluster returns to green status because the replica will be promoted to primary
         ensureGreen();
+
+        // After Lucene 9.9 check index will flag corruption with old (not the latest) commit points.
+        // For this test our previous corrupt commit "segments_2" will remain on the primary.
+        // This asserts this is the case, and then resets check index status.
+        assertEquals("Check index has a single failure", 1, checkIndexFailures.size());
+        assertTrue(
+            checkIndexFailures.get(0)
+                .getMessage()
+                .contains("could not read old (not latest) commit point segments file \"segments_2\" in directory")
+        );
+        resetCheckIndexStatus();
     }
 
     private int numShards(String... index) {

server/src/internalClusterTest/java/org/opensearch/validate/SimpleValidateQueryIT.java

@@ -270,7 +270,10 @@ public void testExplainDateRangeInQueryString() {
 
         long twoMonthsAgo = now.minus(2, ChronoUnit.MONTHS).truncatedTo(ChronoUnit.DAYS).toEpochSecond() * 1000;
         long rangeEnd = (now.plus(1, ChronoUnit.DAYS).truncatedTo(ChronoUnit.DAYS).toEpochSecond() * 1000) - 1;
-        assertThat(response.getQueryExplanation().get(0).getExplanation(), equalTo("past:[" + twoMonthsAgo + " TO " + rangeEnd + "]"));
+        assertThat(
+            response.getQueryExplanation().get(0).getExplanation(),
+            containsString("past:[" + twoMonthsAgo + " TO " + rangeEnd + "]")
+        );
         assertThat(response.isValid(), equalTo(true));
     }
 

server/src/main/java/org/apache/lucene/queries/BlendedTermQuery.java

@@ -100,11 +100,10 @@ public Query rewrite(IndexSearcher searcher) throws IOException {
             return rewritten;
         }
         IndexReader reader = searcher.getIndexReader();
-        IndexReaderContext context = reader.getContext();
         TermStates[] ctx = new TermStates[terms.length];
         int[] docFreqs = new int[ctx.length];
         for (int i = 0; i < terms.length; i++) {
-            ctx[i] = TermStates.build(context, terms[i], true);
+            ctx[i] = TermStates.build(searcher, terms[i], true);
             docFreqs[i] = ctx[i].docFreq();
         }
 

server/src/main/java/org/apache/lucene/search/grouping/CollapseTopFieldDocs.java

@@ -33,6 +33,7 @@
 
 import org.apache.lucene.search.FieldComparator;
 import org.apache.lucene.search.FieldDoc;
+import org.apache.lucene.search.Pruning;
 import org.apache.lucene.search.ScoreDoc;
 import org.apache.lucene.search.Sort;
 import org.apache.lucene.search.SortField;
@@ -144,7 +145,7 @@ private static class MergeSortQueue extends PriorityQueue<ShardRef> {
             reverseMul = new int[sortFields.length];
             for (int compIDX = 0; compIDX < sortFields.length; compIDX++) {
                 final SortField sortField = sortFields[compIDX];
-                comparators[compIDX] = sortField.getComparator(1, false);
+                comparators[compIDX] = sortField.getComparator(1, Pruning.NONE);
                 reverseMul[compIDX] = sortField.getReverse() ? -1 : 1;
             }
         }

server/src/main/java/org/opensearch/action/search/BottomSortValuesCollector.java

@@ -34,6 +34,7 @@
 
 import org.apache.lucene.search.FieldComparator;
 import org.apache.lucene.search.FieldDoc;
+import org.apache.lucene.search.Pruning;
 import org.apache.lucene.search.SortField;
 import org.apache.lucene.search.TopFieldDocs;
 import org.opensearch.search.DocValueFormat;
@@ -59,7 +60,7 @@ class BottomSortValuesCollector {
         this.reverseMuls = new int[sortFields.length];
         this.sortFields = sortFields;
         for (int i = 0; i < sortFields.length; i++) {
-            comparators[i] = sortFields[i].getComparator(1, false);
+            comparators[i] = sortFields[i].getComparator(1, Pruning.NONE);
             reverseMuls[i] = sortFields[i].getReverse() ? -1 : 1;
         }
     }