[Backport 2.x] [Upgrade] Lucene 9.7.0 release (#8272) (#8277)

Upgrades to the official 9.7.0 release of lucene Signed-off-by: Nicholas Walter Knize <[email protected]>
opensearch-project · Jun 27, 2023 · 5f80c24 · 5f80c24
1 parent 2c01342
commit 5f80c24
Show file tree

Hide file tree

Showing 53 changed files with 55 additions and 44 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -42,6 +42,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Bump `mockito` from 5.2.0 to 5.4.0 ([#8181](https://github.com/opensearch-project/OpenSearch/pull/8181))
 - Bump `netty` from 4.1.93.Final to 4.1.94.Final ([#8191](https://github.com/opensearch-project/OpenSearch/pull/8191))
 - Bump `io.projectreactor.netty:reactor-netty-http` from 1.1.7 to 1.1.8 (#8256)
+- [Upgrade] Lucene 9.7.0 release (#8272)
 
 ### Changed
 - Replace jboss-annotations-api_1.2_spec with jakarta.annotation-api ([#7836](https://github.com/opensearch-project/OpenSearch/pull/7836))

diff --git a/buildSrc/version.properties b/buildSrc/version.properties
@@ -1,5 +1,5 @@
 opensearch        = 2.9.0
-lucene            = 9.6.0
+lucene            = 9.7.0
 
 bundled_jdk_vendor = adoptium
 bundled_jdk = 17.0.7+7

diff --git a/libs/core/licenses/lucene-core-9.6.0.jar.sha1 b/libs/core/licenses/lucene-core-9.6.0.jar.sha1
diff --git a/libs/core/licenses/lucene-core-9.7.0.jar.sha1 b/libs/core/licenses/lucene-core-9.7.0.jar.sha1
@@ -0,0 +1 @@
+ad391210ffd806931334be9670a35af00c56f959
diff --git a/modules/lang-expression/licenses/lucene-expressions-9.6.0.jar.sha1 b/modules/lang-expression/licenses/lucene-expressions-9.6.0.jar.sha1
diff --git a/modules/lang-expression/licenses/lucene-expressions-9.7.0.jar.sha1 b/modules/lang-expression/licenses/lucene-expressions-9.7.0.jar.sha1
@@ -0,0 +1 @@
+297e1cfade4ef71466cc9d4f361d81807c8dc4c8
diff --git a/modules/parent-join/src/test/java/org/opensearch/join/query/HasChildQueryBuilderTests.java b/modules/parent-join/src/test/java/org/opensearch/join/query/HasChildQueryBuilderTests.java
@@ -303,10 +303,9 @@ static void assertLateParsingQuery(Query query, String type, String id) throws I
         assertThat(booleanQuery.clauses().get(0).getOccur(), equalTo(BooleanClause.Occur.MUST));
         assertThat(booleanQuery.clauses().get(0).getQuery(), instanceOf(TermInSetQuery.class));
         TermInSetQuery termsQuery = (TermInSetQuery) booleanQuery.clauses().get(0).getQuery();
-        Query rewrittenTermsQuery = termsQuery.rewrite(null);
         // The query is of type MultiTermQueryConstantScoreBlendedWrapper and is sealed inside Apache Lucene,
         // no access to inner queries without using the reflection, falling back to stringified query comparison
-        assertThat(rewrittenTermsQuery.toString(), equalTo("_id:([ff 69 64])"));
+        assertThat(termsQuery.toString(), equalTo("_id:([ff 69 64])"));
         // check the type filter
         assertThat(booleanQuery.clauses().get(1).getOccur(), equalTo(BooleanClause.Occur.FILTER));
         assertEquals(new TermQuery(new Term("join_field", type)), booleanQuery.clauses().get(1).getQuery());

diff --git a/plugins/analysis-icu/licenses/lucene-analysis-icu-9.6.0.jar.sha1 b/plugins/analysis-icu/licenses/lucene-analysis-icu-9.6.0.jar.sha1
diff --git a/plugins/analysis-icu/licenses/lucene-analysis-icu-9.7.0.jar.sha1 b/plugins/analysis-icu/licenses/lucene-analysis-icu-9.7.0.jar.sha1
@@ -0,0 +1 @@
+94293b169fb8572f440a5a4a523320ecf9778ffe
diff --git a/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.6.0.jar.sha1 b/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.6.0.jar.sha1
diff --git a/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.7.0.jar.sha1 b/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.7.0.jar.sha1
@@ -0,0 +1 @@
+2df800a38b64867b8dcd61fc2cd986114e4a80cb
diff --git a/plugins/analysis-nori/licenses/lucene-analysis-nori-9.6.0.jar.sha1 b/plugins/analysis-nori/licenses/lucene-analysis-nori-9.6.0.jar.sha1
diff --git a/plugins/analysis-nori/licenses/lucene-analysis-nori-9.7.0.jar.sha1 b/plugins/analysis-nori/licenses/lucene-analysis-nori-9.7.0.jar.sha1
@@ -0,0 +1 @@
+a01e8153f34d72e8c8c0180c1dea5b10f677dd3a
diff --git a/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.6.0.jar.sha1 b/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.6.0.jar.sha1
diff --git a/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.7.0.jar.sha1 b/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.7.0.jar.sha1
@@ -0,0 +1 @@
+b7d47d54683b0b1e09b271c32d1b7d3eb1990f49
diff --git a/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.6.0.jar.sha1 b/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.6.0.jar.sha1
diff --git a/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.7.0.jar.sha1 b/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.7.0.jar.sha1
@@ -0,0 +1 @@
+5e68b9816e6cff8ee15f5b350cf2ffa54f9828b7
diff --git a/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.6.0.jar.sha1 b/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.6.0.jar.sha1
diff --git a/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.7.0.jar.sha1 b/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.7.0.jar.sha1
@@ -0,0 +1 @@
+d23b1f05b471e05d0d6068b3ece7c8c65672eae7
diff --git a/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.6.0.jar.sha1 b/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.6.0.jar.sha1
diff --git a/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.7.0.jar.sha1 b/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.7.0.jar.sha1
@@ -0,0 +1 @@
+dfb4313f3c68d337310522840d7144c1605d084a
diff --git a/server/licenses/lucene-analysis-common-9.6.0.jar.sha1 b/server/licenses/lucene-analysis-common-9.6.0.jar.sha1
diff --git a/server/licenses/lucene-analysis-common-9.7.0.jar.sha1 b/server/licenses/lucene-analysis-common-9.7.0.jar.sha1
@@ -0,0 +1 @@
+27ba6caaa4587a982cd451f7217b5a982bcfc44a
diff --git a/server/licenses/lucene-backward-codecs-9.6.0.jar.sha1 b/server/licenses/lucene-backward-codecs-9.6.0.jar.sha1
diff --git a/server/licenses/lucene-backward-codecs-9.7.0.jar.sha1 b/server/licenses/lucene-backward-codecs-9.7.0.jar.sha1
@@ -0,0 +1 @@
+6389463bfbfcf902c8d31d12e9513a6818ac9d5e
diff --git a/server/licenses/lucene-core-9.6.0.jar.sha1 b/server/licenses/lucene-core-9.6.0.jar.sha1
diff --git a/server/licenses/lucene-core-9.7.0.jar.sha1 b/server/licenses/lucene-core-9.7.0.jar.sha1
@@ -0,0 +1 @@
+ad391210ffd806931334be9670a35af00c56f959
diff --git a/server/licenses/lucene-grouping-9.6.0.jar.sha1 b/server/licenses/lucene-grouping-9.6.0.jar.sha1
diff --git a/server/licenses/lucene-grouping-9.7.0.jar.sha1 b/server/licenses/lucene-grouping-9.7.0.jar.sha1
@@ -0,0 +1 @@
+8e6f0c229f4861be641047c33b05067176e4279c
diff --git a/server/licenses/lucene-highlighter-9.6.0.jar.sha1 b/server/licenses/lucene-highlighter-9.6.0.jar.sha1
diff --git a/server/licenses/lucene-highlighter-9.7.0.jar.sha1 b/server/licenses/lucene-highlighter-9.7.0.jar.sha1
@@ -0,0 +1 @@
+facb7c7ee0f75ed457a2d98f10d6430e25a53691
diff --git a/server/licenses/lucene-join-9.6.0.jar.sha1 b/server/licenses/lucene-join-9.6.0.jar.sha1
diff --git a/server/licenses/lucene-join-9.7.0.jar.sha1 b/server/licenses/lucene-join-9.7.0.jar.sha1
@@ -0,0 +1 @@
+d041bdc0947a14223cf68357407ee18b21027587
diff --git a/server/licenses/lucene-memory-9.6.0.jar.sha1 b/server/licenses/lucene-memory-9.6.0.jar.sha1
diff --git a/server/licenses/lucene-memory-9.7.0.jar.sha1 b/server/licenses/lucene-memory-9.7.0.jar.sha1
@@ -0,0 +1 @@
+0fade51ee353e15ddbbc45262aafe6f99ed020f1
diff --git a/server/licenses/lucene-misc-9.6.0.jar.sha1 b/server/licenses/lucene-misc-9.6.0.jar.sha1
diff --git a/server/licenses/lucene-misc-9.7.0.jar.sha1 b/server/licenses/lucene-misc-9.7.0.jar.sha1
@@ -0,0 +1 @@
+7fcf451e2376526c3a027958812866cc5b0ff13f
diff --git a/server/licenses/lucene-queries-9.6.0.jar.sha1 b/server/licenses/lucene-queries-9.6.0.jar.sha1
diff --git a/server/licenses/lucene-queries-9.7.0.jar.sha1 b/server/licenses/lucene-queries-9.7.0.jar.sha1
@@ -0,0 +1 @@
+126989d4622419aa06fcbf3a342e859cab8c8799
diff --git a/server/licenses/lucene-queryparser-9.6.0.jar.sha1 b/server/licenses/lucene-queryparser-9.6.0.jar.sha1
diff --git a/server/licenses/lucene-queryparser-9.7.0.jar.sha1 b/server/licenses/lucene-queryparser-9.7.0.jar.sha1
@@ -0,0 +1 @@
+6e77bde908ff698354e4a2149e6dd4658b56d7b0
diff --git a/server/licenses/lucene-sandbox-9.6.0.jar.sha1 b/server/licenses/lucene-sandbox-9.6.0.jar.sha1
diff --git a/server/licenses/lucene-sandbox-9.7.0.jar.sha1 b/server/licenses/lucene-sandbox-9.7.0.jar.sha1
@@ -0,0 +1 @@
+9f3e8e1947f2f1c5784132444af51a060ff0b4bf
diff --git a/server/licenses/lucene-spatial-extras-9.6.0.jar.sha1 b/server/licenses/lucene-spatial-extras-9.6.0.jar.sha1
diff --git a/server/licenses/lucene-spatial-extras-9.7.0.jar.sha1 b/server/licenses/lucene-spatial-extras-9.7.0.jar.sha1
@@ -0,0 +1 @@
+01b0bc7a407d8c35a70a1adf7966bb3e7caae928
diff --git a/server/licenses/lucene-spatial3d-9.6.0.jar.sha1 b/server/licenses/lucene-spatial3d-9.6.0.jar.sha1
diff --git a/server/licenses/lucene-spatial3d-9.7.0.jar.sha1 b/server/licenses/lucene-spatial3d-9.7.0.jar.sha1
@@ -0,0 +1 @@
+7c6b1b6e0a70c9cd177371e648648c2f896742a2
diff --git a/server/licenses/lucene-suggest-9.6.0.jar.sha1 b/server/licenses/lucene-suggest-9.6.0.jar.sha1
diff --git a/server/licenses/lucene-suggest-9.7.0.jar.sha1 b/server/licenses/lucene-suggest-9.7.0.jar.sha1
@@ -0,0 +1 @@
+5c37fd9a5d71dc87fe1cd4c18ff295ec8cfac170
diff --git a/server/src/main/java/org/apache/lucene/search/vectorhighlight/CustomFieldQuery.java b/server/src/main/java/org/apache/lucene/search/vectorhighlight/CustomFieldQuery.java
@@ -38,6 +38,7 @@
 import org.apache.lucene.queries.spans.SpanTermQuery;
 import org.apache.lucene.search.BoostQuery;
 import org.apache.lucene.search.ConstantScoreQuery;
+import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.MultiPhraseQuery;
 import org.apache.lucene.search.PhraseQuery;
 import org.apache.lucene.search.Query;
@@ -66,45 +67,45 @@ public CustomFieldQuery(Query query, IndexReader reader, boolean phraseHighlight
     }
 
     @Override
-    protected void flatten(Query sourceQuery, IndexReader reader, Collection<Query> flatQueries, float boost) throws IOException {
+    protected void flatten(Query sourceQuery, IndexSearcher searcher, Collection<Query> flatQueries, float boost) throws IOException {
         if (sourceQuery instanceof BoostQuery) {
             BoostQuery bq = (BoostQuery) sourceQuery;
             sourceQuery = bq.getQuery();
             boost *= bq.getBoost();
-            flatten(sourceQuery, reader, flatQueries, boost);
+            flatten(sourceQuery, searcher, flatQueries, boost);
         } else if (sourceQuery instanceof SpanTermQuery) {
-            super.flatten(new TermQuery(((SpanTermQuery) sourceQuery).getTerm()), reader, flatQueries, boost);
+            super.flatten(new TermQuery(((SpanTermQuery) sourceQuery).getTerm()), searcher, flatQueries, boost);
         } else if (sourceQuery instanceof ConstantScoreQuery) {
-            flatten(((ConstantScoreQuery) sourceQuery).getQuery(), reader, flatQueries, boost);
+            flatten(((ConstantScoreQuery) sourceQuery).getQuery(), searcher, flatQueries, boost);
         } else if (sourceQuery instanceof FunctionScoreQuery) {
-            flatten(((FunctionScoreQuery) sourceQuery).getSubQuery(), reader, flatQueries, boost);
+            flatten(((FunctionScoreQuery) sourceQuery).getSubQuery(), searcher, flatQueries, boost);
         } else if (sourceQuery instanceof MultiPhrasePrefixQuery) {
-            flatten(sourceQuery.rewrite(reader), reader, flatQueries, boost);
+            flatten(sourceQuery.rewrite(searcher), searcher, flatQueries, boost);
         } else if (sourceQuery instanceof MultiPhraseQuery) {
             MultiPhraseQuery q = ((MultiPhraseQuery) sourceQuery);
-            convertMultiPhraseQuery(0, new int[q.getTermArrays().length], q, q.getTermArrays(), q.getPositions(), reader, flatQueries);
+            convertMultiPhraseQuery(0, new int[q.getTermArrays().length], q, q.getTermArrays(), q.getPositions(), searcher, flatQueries);
         } else if (sourceQuery instanceof BlendedTermQuery) {
             final BlendedTermQuery blendedTermQuery = (BlendedTermQuery) sourceQuery;
-            flatten(blendedTermQuery.rewrite(reader), reader, flatQueries, boost);
+            flatten(blendedTermQuery.rewrite(searcher), searcher, flatQueries, boost);
         } else if (sourceQuery instanceof org.apache.lucene.queries.function.FunctionScoreQuery) {
             org.apache.lucene.queries.function.FunctionScoreQuery funcScoreQuery =
                 (org.apache.lucene.queries.function.FunctionScoreQuery) sourceQuery;
             // flatten query with query boost
-            flatten(funcScoreQuery.getWrappedQuery(), reader, flatQueries, boost);
+            flatten(funcScoreQuery.getWrappedQuery(), searcher, flatQueries, boost);
         } else if (sourceQuery instanceof SynonymQuery) {
             // SynonymQuery should be handled by the parent class directly.
             // This statement should be removed when https://issues.apache.org/jira/browse/LUCENE-7484 is merged.
             SynonymQuery synQuery = (SynonymQuery) sourceQuery;
             for (Term term : synQuery.getTerms()) {
-                flatten(new TermQuery(term), reader, flatQueries, boost);
+                flatten(new TermQuery(term), searcher, flatQueries, boost);
             }
         } else if (sourceQuery instanceof OpenSearchToParentBlockJoinQuery) {
             Query childQuery = ((OpenSearchToParentBlockJoinQuery) sourceQuery).getChildQuery();
             if (childQuery != null) {
-                flatten(childQuery, reader, flatQueries, boost);
+                flatten(childQuery, searcher, flatQueries, boost);
             }
         } else {
-            super.flatten(sourceQuery, reader, flatQueries, boost);
+            super.flatten(sourceQuery, searcher, flatQueries, boost);
         }
     }
 
@@ -114,7 +115,7 @@ private void convertMultiPhraseQuery(
         MultiPhraseQuery orig,
         Term[][] terms,
         int[] pos,
-        IndexReader reader,
+        IndexSearcher searcher,
         Collection<Query> flatQueries
     ) throws IOException {
         if (currentPos == 0) {
@@ -126,7 +127,7 @@ private void convertMultiPhraseQuery(
             if (numTerms > 16) {
                 for (Term[] currentPosTerm : terms) {
                     for (Term term : currentPosTerm) {
-                        super.flatten(new TermQuery(term), reader, flatQueries, 1F);
+                        super.flatten(new TermQuery(term), searcher, flatQueries, 1F);
                     }
                 }
                 return;
@@ -143,12 +144,12 @@ private void convertMultiPhraseQuery(
                 queryBuilder.add(terms[i][termsIdx[i]], pos[i]);
             }
             Query query = queryBuilder.build();
-            this.flatten(query, reader, flatQueries, 1F);
+            this.flatten(query, searcher, flatQueries, 1F);
         } else {
             Term[] t = terms[currentPos];
             for (int i = 0; i < t.length; i++) {
                 termsIdx[currentPos] = i;
-                convertMultiPhraseQuery(currentPos + 1, termsIdx, orig, terms, pos, reader, flatQueries);
+                convertMultiPhraseQuery(currentPos + 1, termsIdx, orig, terms, pos, searcher, flatQueries);
             }
         }
     }

diff --git a/server/src/main/java/org/opensearch/Version.java b/server/src/main/java/org/opensearch/Version.java
@@ -116,7 +116,7 @@ public class Version implements Comparable<Version>, ToXContentFragment {
     public static final Version V_2_7_1 = new Version(2070199, org.apache.lucene.util.Version.LUCENE_9_5_0);
     public static final Version V_2_8_0 = new Version(2080099, org.apache.lucene.util.Version.LUCENE_9_6_0);
     public static final Version V_2_8_1 = new Version(2080199, org.apache.lucene.util.Version.LUCENE_9_6_0);
-    public static final Version V_2_9_0 = new Version(2090099, org.apache.lucene.util.Version.LUCENE_9_6_0);
+    public static final Version V_2_9_0 = new Version(2090099, org.apache.lucene.util.Version.LUCENE_9_7_0);
     public static final Version CURRENT = V_2_9_0;
 
     public static Version readVersion(StreamInput in) throws IOException {

diff --git a/server/src/test/java/org/opensearch/lucene/queries/SpanMatchNoDocsQueryTests.java b/server/src/test/java/org/opensearch/lucene/queries/SpanMatchNoDocsQueryTests.java
@@ -56,11 +56,19 @@
 
 public class SpanMatchNoDocsQueryTests extends OpenSearchTestCase {
     public void testSimple() throws Exception {
+        Directory dir = newDirectory();
+        IndexWriter iw = new IndexWriter(dir, newIndexWriterConfig());
+        IndexReader ir = DirectoryReader.open(iw);
+
         SpanMatchNoDocsQuery query = new SpanMatchNoDocsQuery("field", "a good reason");
         assertEquals(query.toString(), "SpanMatchNoDocsQuery(\"a good reason\")");
-        Query rewrite = query.rewrite(null);
+        Query rewrite = query.rewrite(ir);
         assertTrue(rewrite instanceof SpanMatchNoDocsQuery);
         assertEquals(rewrite.toString(), "SpanMatchNoDocsQuery(\"a good reason\")");
+
+        iw.close();
+        ir.close();
+        dir.close();
     }
 
     public void testQuery() throws Exception {

diff --git a/test/framework/src/main/java/org/opensearch/indices/analysis/AnalysisFactoryTestCase.java b/test/framework/src/main/java/org/opensearch/indices/analysis/AnalysisFactoryTestCase.java
@@ -222,6 +222,8 @@ public abstract class AnalysisFactoryTestCase extends OpenSearchTestCase {
         // LUCENE-10352
         .put("daitchmokotoffsoundex", Void.class)
         .put("persianstem", Void.class)
+        // https://github.com/apache/lucene/pull/12169
+        .put("word2vecsynonym", Void.class)
         .immutableMap();
 
     static final Map<String, Class<?>> KNOWN_CHARFILTERS = new MapBuilder<String, Class<?>>()