From 35f7b8053c1ff5283f70759b6610992e256f798c Mon Sep 17 00:00:00 2001 From: Marc Handalian Date: Thu, 30 Nov 2023 14:38:32 -0800 Subject: [PATCH 01/26] Bump version to 9.9.0-snapshot Signed-off-by: Marc Handalian --- buildSrc/version.properties | 2 +- libs/core/src/main/java/org/opensearch/Version.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/buildSrc/version.properties b/buildSrc/version.properties index 74d655cfb1045..4d8413d335434 100644 --- a/buildSrc/version.properties +++ b/buildSrc/version.properties @@ -1,5 +1,5 @@ opensearch = 3.0.0 -lucene = 9.8.0 +lucene = 9.9.0-snapshot-eb697aa bundled_jdk_vendor = adoptium bundled_jdk = 21.0.1+12 diff --git a/libs/core/src/main/java/org/opensearch/Version.java b/libs/core/src/main/java/org/opensearch/Version.java index d94be3f25b53d..7d6dba1b06890 100644 --- a/libs/core/src/main/java/org/opensearch/Version.java +++ b/libs/core/src/main/java/org/opensearch/Version.java @@ -99,7 +99,7 @@ public class Version implements Comparable, ToXContentFragment { public static final Version V_2_11_1 = new Version(2110199, org.apache.lucene.util.Version.LUCENE_9_7_0); public static final Version V_2_11_2 = new Version(2110299, org.apache.lucene.util.Version.LUCENE_9_7_0); public static final Version V_2_12_0 = new Version(2120099, org.apache.lucene.util.Version.LUCENE_9_8_0); - public static final Version V_3_0_0 = new Version(3000099, org.apache.lucene.util.Version.LUCENE_9_8_0); + public static final Version V_3_0_0 = new Version(3000099, org.apache.lucene.util.Version.LUCENE_9_9_0); public static final Version CURRENT = V_3_0_0; public static Version fromId(int id) { From 08a9ae7ddd24079549340859ffcc2376b85eb01f Mon Sep 17 00:00:00 2001 From: Marc Handalian Date: Thu, 30 Nov 2023 14:41:43 -0800 Subject: [PATCH 02/26] Support change https://github.com/apache/lucene/pull/12405 Updates to support interface change to FieldComparatorSource to use Pruning enum instead of enableSkipping boolean. Signed-off-by: Marc Handalian --- .../search/grouping/CollapseTopFieldDocs.java | 10 ++----- .../search/BottomSortValuesCollector.java | 7 ++--- .../index/fielddata/IndexFieldData.java | 16 +++--------- .../BytesRefFieldComparatorSource.java | 9 +++---- .../DoubleValuesComparatorSource.java | 10 +++---- .../FloatValuesComparatorSource.java | 10 +++---- .../HalfFloatValuesComparatorSource.java | 5 ++-- .../IntValuesComparatorSource.java | 9 +++---- .../LongValuesComparatorSource.java | 9 +++---- .../UnsignedLongValuesComparatorSource.java | 9 +++---- .../comparators/HalfFloatComparator.java | 5 ++-- .../comparators/UnsignedLongComparator.java | 5 ++-- .../queries/SearchAfterSortedDocQuery.java | 17 ++---------- .../bucket/composite/CompositeAggregator.java | 21 +++------------ .../search/sort/GeoDistanceSortBuilder.java | 9 +++---- .../sort/SortedWiderNumericSortField.java | 7 ++--- .../BottomSortValuesCollectorTests.java | 8 ++---- .../metrics/InternalTopHitsTests.java | 10 ++----- .../search/query/QueryPhaseTests.java | 26 ++----------------- .../search/query/QueryProfilePhaseTests.java | 19 ++------------ .../searchafter/SearchAfterBuilderTests.java | 7 ++--- 21 files changed, 57 insertions(+), 171 deletions(-) diff --git a/server/src/main/java/org/apache/lucene/search/grouping/CollapseTopFieldDocs.java b/server/src/main/java/org/apache/lucene/search/grouping/CollapseTopFieldDocs.java index e93e5cdcc3f7b..aa9696468424d 100644 --- a/server/src/main/java/org/apache/lucene/search/grouping/CollapseTopFieldDocs.java +++ b/server/src/main/java/org/apache/lucene/search/grouping/CollapseTopFieldDocs.java @@ -31,13 +31,7 @@ package org.apache.lucene.search.grouping; -import org.apache.lucene.search.FieldComparator; -import org.apache.lucene.search.FieldDoc; -import org.apache.lucene.search.ScoreDoc; -import org.apache.lucene.search.Sort; -import org.apache.lucene.search.SortField; -import org.apache.lucene.search.TopFieldDocs; -import org.apache.lucene.search.TotalHits; +import org.apache.lucene.search.*; import org.apache.lucene.util.PriorityQueue; import org.opensearch.core.common.util.CollectionUtils; @@ -144,7 +138,7 @@ private static class MergeSortQueue extends PriorityQueue { reverseMul = new int[sortFields.length]; for (int compIDX = 0; compIDX < sortFields.length; compIDX++) { final SortField sortField = sortFields[compIDX]; - comparators[compIDX] = sortField.getComparator(1, false); + comparators[compIDX] = sortField.getComparator(1, Pruning.NONE); reverseMul[compIDX] = sortField.getReverse() ? -1 : 1; } } diff --git a/server/src/main/java/org/opensearch/action/search/BottomSortValuesCollector.java b/server/src/main/java/org/opensearch/action/search/BottomSortValuesCollector.java index c831c80b6455c..64f177f6ac597 100644 --- a/server/src/main/java/org/opensearch/action/search/BottomSortValuesCollector.java +++ b/server/src/main/java/org/opensearch/action/search/BottomSortValuesCollector.java @@ -32,10 +32,7 @@ package org.opensearch.action.search; -import org.apache.lucene.search.FieldComparator; -import org.apache.lucene.search.FieldDoc; -import org.apache.lucene.search.SortField; -import org.apache.lucene.search.TopFieldDocs; +import org.apache.lucene.search.*; import org.opensearch.search.DocValueFormat; import org.opensearch.search.SearchSortValuesAndFormats; @@ -59,7 +56,7 @@ class BottomSortValuesCollector { this.reverseMuls = new int[sortFields.length]; this.sortFields = sortFields; for (int i = 0; i < sortFields.length; i++) { - comparators[i] = sortFields[i].getComparator(1, false); + comparators[i] = sortFields[i].getComparator(1, Pruning.NONE); reverseMuls[i] = sortFields[i].getReverse() ? -1 : 1; } } diff --git a/server/src/main/java/org/opensearch/index/fielddata/IndexFieldData.java b/server/src/main/java/org/opensearch/index/fielddata/IndexFieldData.java index 6a79679cc2f09..7aaf47691cbf4 100644 --- a/server/src/main/java/org/opensearch/index/fielddata/IndexFieldData.java +++ b/server/src/main/java/org/opensearch/index/fielddata/IndexFieldData.java @@ -34,15 +34,7 @@ import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.LeafReaderContext; -import org.apache.lucene.search.DocIdSet; -import org.apache.lucene.search.DocIdSetIterator; -import org.apache.lucene.search.FieldComparatorSource; -import org.apache.lucene.search.IndexSearcher; -import org.apache.lucene.search.Query; -import org.apache.lucene.search.ScoreMode; -import org.apache.lucene.search.Scorer; -import org.apache.lucene.search.SortField; -import org.apache.lucene.search.Weight; +import org.apache.lucene.search.*; import org.apache.lucene.search.join.BitSetProducer; import org.apache.lucene.util.BitDocIdSet; import org.apache.lucene.util.BitSet; @@ -129,13 +121,13 @@ abstract class XFieldComparatorSource extends FieldComparatorSource { protected final MultiValueMode sortMode; protected final Object missingValue; protected final Nested nested; - protected boolean enableSkipping; + protected Pruning pruning; public XFieldComparatorSource(Object missingValue, MultiValueMode sortMode, Nested nested) { this.sortMode = sortMode; this.missingValue = missingValue; this.nested = nested; - this.enableSkipping = true; // true by default + this.pruning = Pruning.GREATER_THAN; // true by default } public MultiValueMode sortMode() { @@ -147,7 +139,7 @@ public Nested nested() { } public void disableSkipping() { - this.enableSkipping = false; + this.pruning = Pruning.NONE; } /** diff --git a/server/src/main/java/org/opensearch/index/fielddata/fieldcomparator/BytesRefFieldComparatorSource.java b/server/src/main/java/org/opensearch/index/fielddata/fieldcomparator/BytesRefFieldComparatorSource.java index 3aa4d9cb782ca..8e049e4fa0b19 100644 --- a/server/src/main/java/org/opensearch/index/fielddata/fieldcomparator/BytesRefFieldComparatorSource.java +++ b/server/src/main/java/org/opensearch/index/fielddata/fieldcomparator/BytesRefFieldComparatorSource.java @@ -36,10 +36,7 @@ import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.SortedDocValues; import org.apache.lucene.index.SortedSetDocValues; -import org.apache.lucene.search.DocIdSetIterator; -import org.apache.lucene.search.FieldComparator; -import org.apache.lucene.search.Scorable; -import org.apache.lucene.search.SortField; +import org.apache.lucene.search.*; import org.apache.lucene.search.comparators.TermOrdValComparator; import org.apache.lucene.util.BitSet; import org.apache.lucene.util.BytesRef; @@ -94,7 +91,7 @@ protected SortedBinaryDocValues getValues(LeafReaderContext context) throws IOEx protected void setScorer(Scorable scorer, LeafReaderContext context) {} @Override - public FieldComparator newComparator(String fieldname, int numHits, boolean enableSkipping, boolean reversed) { + public FieldComparator newComparator(String fieldname, int numHits, Pruning pruning, boolean reversed) { assert indexFieldData == null || fieldname.equals(indexFieldData.getFieldName()); final boolean sortMissingLast = sortMissingLast(missingValue) ^ reversed; @@ -105,7 +102,7 @@ public FieldComparator newComparator(String fieldname, int numHits, boolean e indexFieldData.getFieldName(), sortMissingLast, reversed, - enableSkipping + pruning ) { @Override diff --git a/server/src/main/java/org/opensearch/index/fielddata/fieldcomparator/DoubleValuesComparatorSource.java b/server/src/main/java/org/opensearch/index/fielddata/fieldcomparator/DoubleValuesComparatorSource.java index 4f9f05addc8c7..f8773b391faa7 100644 --- a/server/src/main/java/org/opensearch/index/fielddata/fieldcomparator/DoubleValuesComparatorSource.java +++ b/server/src/main/java/org/opensearch/index/fielddata/fieldcomparator/DoubleValuesComparatorSource.java @@ -34,11 +34,7 @@ import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.NumericDocValues; -import org.apache.lucene.search.DocIdSetIterator; -import org.apache.lucene.search.FieldComparator; -import org.apache.lucene.search.LeafFieldComparator; -import org.apache.lucene.search.Scorable; -import org.apache.lucene.search.SortField; +import org.apache.lucene.search.*; import org.apache.lucene.search.comparators.DoubleComparator; import org.apache.lucene.util.BitSet; import org.opensearch.common.Nullable; @@ -98,11 +94,11 @@ private NumericDoubleValues getNumericDocValues(LeafReaderContext context, doubl protected void setScorer(Scorable scorer, LeafReaderContext context) {} @Override - public FieldComparator newComparator(String fieldname, int numHits, boolean enableSkipping, boolean reversed) { + public FieldComparator newComparator(String fieldname, int numHits, Pruning pruning, boolean reversed) { assert indexFieldData == null || fieldname.equals(indexFieldData.getFieldName()); final double dMissingValue = (Double) missingObject(missingValue, reversed); - return new DoubleComparator(numHits, fieldname, dMissingValue, reversed, enableSkipping && this.enableSkipping) { + return new DoubleComparator(numHits, fieldname, dMissingValue, reversed, pruning) { @Override public LeafFieldComparator getLeafComparator(LeafReaderContext context) throws IOException { return new DoubleLeafComparator(context) { diff --git a/server/src/main/java/org/opensearch/index/fielddata/fieldcomparator/FloatValuesComparatorSource.java b/server/src/main/java/org/opensearch/index/fielddata/fieldcomparator/FloatValuesComparatorSource.java index 3fb8b5c6ce393..27badbe9c2a00 100644 --- a/server/src/main/java/org/opensearch/index/fielddata/fieldcomparator/FloatValuesComparatorSource.java +++ b/server/src/main/java/org/opensearch/index/fielddata/fieldcomparator/FloatValuesComparatorSource.java @@ -33,11 +33,7 @@ import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.NumericDocValues; -import org.apache.lucene.search.DocIdSetIterator; -import org.apache.lucene.search.FieldComparator; -import org.apache.lucene.search.LeafFieldComparator; -import org.apache.lucene.search.Scorable; -import org.apache.lucene.search.SortField; +import org.apache.lucene.search.*; import org.apache.lucene.search.comparators.FloatComparator; import org.apache.lucene.util.BitSet; import org.opensearch.common.Nullable; @@ -91,11 +87,11 @@ private NumericDoubleValues getNumericDocValues(LeafReaderContext context, float } @Override - public FieldComparator newComparator(String fieldname, int numHits, boolean enableSkipping, boolean reversed) { + public FieldComparator newComparator(String fieldname, int numHits, Pruning pruning, boolean reversed) { assert indexFieldData == null || fieldname.equals(indexFieldData.getFieldName()); final float fMissingValue = (Float) missingObject(missingValue, reversed); - return new FloatComparator(numHits, fieldname, fMissingValue, reversed, enableSkipping && this.enableSkipping) { + return new FloatComparator(numHits, fieldname, fMissingValue, reversed, pruning) { @Override public LeafFieldComparator getLeafComparator(LeafReaderContext context) throws IOException { return new FloatLeafComparator(context) { diff --git a/server/src/main/java/org/opensearch/index/fielddata/fieldcomparator/HalfFloatValuesComparatorSource.java b/server/src/main/java/org/opensearch/index/fielddata/fieldcomparator/HalfFloatValuesComparatorSource.java index e0227083f4181..99edec225099e 100644 --- a/server/src/main/java/org/opensearch/index/fielddata/fieldcomparator/HalfFloatValuesComparatorSource.java +++ b/server/src/main/java/org/opensearch/index/fielddata/fieldcomparator/HalfFloatValuesComparatorSource.java @@ -13,6 +13,7 @@ import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.FieldComparator; import org.apache.lucene.search.LeafFieldComparator; +import org.apache.lucene.search.Pruning; import org.apache.lucene.util.BitSet; import org.opensearch.index.fielddata.FieldData; import org.opensearch.index.fielddata.IndexNumericFieldData; @@ -42,11 +43,11 @@ public HalfFloatValuesComparatorSource( } @Override - public FieldComparator newComparator(String fieldname, int numHits, boolean enableSkipping, boolean reversed) { + public FieldComparator newComparator(String fieldname, int numHits, Pruning pruning, boolean reversed) { assert indexFieldData == null || fieldname.equals(indexFieldData.getFieldName()); final float fMissingValue = (Float) missingObject(missingValue, reversed); - return new HalfFloatComparator(numHits, fieldname, fMissingValue, reversed, enableSkipping && this.enableSkipping) { + return new HalfFloatComparator(numHits, fieldname, fMissingValue, reversed, pruning) { @Override public LeafFieldComparator getLeafComparator(LeafReaderContext context) throws IOException { return new HalfFloatLeafComparator(context) { diff --git a/server/src/main/java/org/opensearch/index/fielddata/fieldcomparator/IntValuesComparatorSource.java b/server/src/main/java/org/opensearch/index/fielddata/fieldcomparator/IntValuesComparatorSource.java index 3a3fdb85c3415..732999f7dd752 100644 --- a/server/src/main/java/org/opensearch/index/fielddata/fieldcomparator/IntValuesComparatorSource.java +++ b/server/src/main/java/org/opensearch/index/fielddata/fieldcomparator/IntValuesComparatorSource.java @@ -16,10 +16,7 @@ import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.NumericDocValues; import org.apache.lucene.index.SortedNumericDocValues; -import org.apache.lucene.search.DocIdSetIterator; -import org.apache.lucene.search.FieldComparator; -import org.apache.lucene.search.LeafFieldComparator; -import org.apache.lucene.search.SortField; +import org.apache.lucene.search.*; import org.apache.lucene.search.comparators.IntComparator; import org.apache.lucene.util.BitSet; import org.opensearch.common.Nullable; @@ -70,11 +67,11 @@ private NumericDocValues getNumericDocValues(LeafReaderContext context, int miss } @Override - public FieldComparator newComparator(String fieldname, int numHits, boolean enableSkipping, boolean reversed) { + public FieldComparator newComparator(String fieldname, int numHits, Pruning pruning, boolean reversed) { assert indexFieldData == null || fieldname.equals(indexFieldData.getFieldName()); final int iMissingValue = (Integer) missingObject(missingValue, reversed); - return new IntComparator(numHits, fieldname, iMissingValue, reversed, enableSkipping && this.enableSkipping) { + return new IntComparator(numHits, fieldname, iMissingValue, reversed, pruning) { @Override public LeafFieldComparator getLeafComparator(LeafReaderContext context) throws IOException { return new IntLeafComparator(context) { diff --git a/server/src/main/java/org/opensearch/index/fielddata/fieldcomparator/LongValuesComparatorSource.java b/server/src/main/java/org/opensearch/index/fielddata/fieldcomparator/LongValuesComparatorSource.java index 57568aa84fa5c..331e06baf0b41 100644 --- a/server/src/main/java/org/opensearch/index/fielddata/fieldcomparator/LongValuesComparatorSource.java +++ b/server/src/main/java/org/opensearch/index/fielddata/fieldcomparator/LongValuesComparatorSource.java @@ -34,10 +34,7 @@ import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.NumericDocValues; import org.apache.lucene.index.SortedNumericDocValues; -import org.apache.lucene.search.DocIdSetIterator; -import org.apache.lucene.search.FieldComparator; -import org.apache.lucene.search.LeafFieldComparator; -import org.apache.lucene.search.SortField; +import org.apache.lucene.search.*; import org.apache.lucene.search.comparators.LongComparator; import org.apache.lucene.util.BitSet; import org.opensearch.common.Nullable; @@ -114,11 +111,11 @@ private NumericDocValues getNumericDocValues(LeafReaderContext context, long mis } @Override - public FieldComparator newComparator(String fieldname, int numHits, boolean enableSkipping, boolean reversed) { + public FieldComparator newComparator(String fieldname, int numHits, Pruning pruning, boolean reversed) { assert indexFieldData == null || fieldname.equals(indexFieldData.getFieldName()); final long lMissingValue = (Long) missingObject(missingValue, reversed); - return new LongComparator(numHits, fieldname, lMissingValue, reversed, enableSkipping && this.enableSkipping) { + return new LongComparator(numHits, fieldname, lMissingValue, reversed, pruning) { @Override public LeafFieldComparator getLeafComparator(LeafReaderContext context) throws IOException { return new LongLeafComparator(context) { diff --git a/server/src/main/java/org/opensearch/index/fielddata/fieldcomparator/UnsignedLongValuesComparatorSource.java b/server/src/main/java/org/opensearch/index/fielddata/fieldcomparator/UnsignedLongValuesComparatorSource.java index 9d1edde1fe08f..b598608c4cd6c 100644 --- a/server/src/main/java/org/opensearch/index/fielddata/fieldcomparator/UnsignedLongValuesComparatorSource.java +++ b/server/src/main/java/org/opensearch/index/fielddata/fieldcomparator/UnsignedLongValuesComparatorSource.java @@ -11,10 +11,7 @@ import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.NumericDocValues; import org.apache.lucene.index.SortedNumericDocValues; -import org.apache.lucene.search.DocIdSetIterator; -import org.apache.lucene.search.FieldComparator; -import org.apache.lucene.search.LeafFieldComparator; -import org.apache.lucene.search.SortField; +import org.apache.lucene.search.*; import org.apache.lucene.util.BitSet; import org.opensearch.common.Nullable; import org.opensearch.common.Numbers; @@ -88,11 +85,11 @@ public Object missingObject(Object missingValue, boolean reversed) { } @Override - public FieldComparator newComparator(String fieldname, int numHits, boolean enableSkipping, boolean reversed) { + public FieldComparator newComparator(String fieldname, int numHits, Pruning pruning, boolean reversed) { assert indexFieldData == null || fieldname.equals(indexFieldData.getFieldName()); final BigInteger ulMissingValue = (BigInteger) missingObject(missingValue, reversed); - return new UnsignedLongComparator(numHits, fieldname, ulMissingValue, reversed, enableSkipping && this.enableSkipping) { + return new UnsignedLongComparator(numHits, fieldname, ulMissingValue, reversed, pruning) { @Override public LeafFieldComparator getLeafComparator(LeafReaderContext context) throws IOException { return new UnsignedLongLeafComparator(context) { diff --git a/server/src/main/java/org/opensearch/index/search/comparators/HalfFloatComparator.java b/server/src/main/java/org/opensearch/index/search/comparators/HalfFloatComparator.java index 6244fa647b042..b2e2ba8001b88 100644 --- a/server/src/main/java/org/opensearch/index/search/comparators/HalfFloatComparator.java +++ b/server/src/main/java/org/opensearch/index/search/comparators/HalfFloatComparator.java @@ -11,6 +11,7 @@ import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.sandbox.document.HalfFloatPoint; import org.apache.lucene.search.LeafFieldComparator; +import org.apache.lucene.search.Pruning; import org.apache.lucene.search.comparators.NumericComparator; import java.io.IOException; @@ -25,8 +26,8 @@ public class HalfFloatComparator extends NumericComparator { protected float topValue; protected float bottom; - public HalfFloatComparator(int numHits, String field, Float missingValue, boolean reverse, boolean enableSkipping) { - super(field, missingValue != null ? missingValue : 0.0f, reverse, enableSkipping, HalfFloatPoint.BYTES); + public HalfFloatComparator(int numHits, String field, Float missingValue, boolean reverse, Pruning pruning) { + super(field, missingValue != null ? missingValue : 0.0f, reverse, pruning, HalfFloatPoint.BYTES); values = new float[numHits]; } diff --git a/server/src/main/java/org/opensearch/index/search/comparators/UnsignedLongComparator.java b/server/src/main/java/org/opensearch/index/search/comparators/UnsignedLongComparator.java index d46b34fe97356..2b6bd9933e553 100644 --- a/server/src/main/java/org/opensearch/index/search/comparators/UnsignedLongComparator.java +++ b/server/src/main/java/org/opensearch/index/search/comparators/UnsignedLongComparator.java @@ -11,6 +11,7 @@ import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.sandbox.document.BigIntegerPoint; import org.apache.lucene.search.LeafFieldComparator; +import org.apache.lucene.search.Pruning; import org.apache.lucene.search.comparators.NumericComparator; import org.opensearch.common.Numbers; @@ -23,8 +24,8 @@ public class UnsignedLongComparator extends NumericComparator { protected BigInteger topValue; protected BigInteger bottom; - public UnsignedLongComparator(int numHits, String field, BigInteger missingValue, boolean reverse, boolean enableSkipping) { - super(field, missingValue != null ? missingValue : Numbers.MIN_UNSIGNED_LONG_VALUE, reverse, enableSkipping, BigIntegerPoint.BYTES); + public UnsignedLongComparator(int numHits, String field, BigInteger missingValue, boolean reverse, Pruning pruning) { + super(field, missingValue != null ? missingValue : Numbers.MIN_UNSIGNED_LONG_VALUE, reverse, pruning, BigIntegerPoint.BYTES); values = new BigInteger[numHits]; } diff --git a/server/src/main/java/org/opensearch/lucene/queries/SearchAfterSortedDocQuery.java b/server/src/main/java/org/opensearch/lucene/queries/SearchAfterSortedDocQuery.java index e417a2eaa7cf4..179d112699b35 100644 --- a/server/src/main/java/org/opensearch/lucene/queries/SearchAfterSortedDocQuery.java +++ b/server/src/main/java/org/opensearch/lucene/queries/SearchAfterSortedDocQuery.java @@ -33,20 +33,7 @@ package org.opensearch.lucene.queries; import org.apache.lucene.index.LeafReaderContext; -import org.apache.lucene.search.ConstantScoreScorer; -import org.apache.lucene.search.ConstantScoreWeight; -import org.apache.lucene.search.DocIdSetIterator; -import org.apache.lucene.search.FieldComparator; -import org.apache.lucene.search.FieldDoc; -import org.apache.lucene.search.IndexSearcher; -import org.apache.lucene.search.LeafFieldComparator; -import org.apache.lucene.search.Query; -import org.apache.lucene.search.QueryVisitor; -import org.apache.lucene.search.ScoreMode; -import org.apache.lucene.search.Scorer; -import org.apache.lucene.search.Sort; -import org.apache.lucene.search.SortField; -import org.apache.lucene.search.Weight; +import org.apache.lucene.search.*; import org.opensearch.common.lucene.Lucene; import java.io.IOException; @@ -77,7 +64,7 @@ public SearchAfterSortedDocQuery(Sort sort, FieldDoc after) { this.reverseMuls = new int[numFields]; for (int i = 0; i < numFields; i++) { SortField sortField = sort.getSort()[i]; - FieldComparator fieldComparator = sortField.getComparator(1, false); + FieldComparator fieldComparator = sortField.getComparator(1, Pruning.NONE); @SuppressWarnings("unchecked") FieldComparator comparator = (FieldComparator) fieldComparator; comparator.setTopValue(after.fields[i]); diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/composite/CompositeAggregator.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/composite/CompositeAggregator.java index f2a4d5cd46127..2b566c51e4c22 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/composite/CompositeAggregator.java +++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/composite/CompositeAggregator.java @@ -37,22 +37,7 @@ import org.apache.lucene.index.NumericDocValues; import org.apache.lucene.index.SortedNumericDocValues; import org.apache.lucene.index.SortedSetDocValues; -import org.apache.lucene.search.BooleanClause; -import org.apache.lucene.search.BooleanQuery; -import org.apache.lucene.search.CollectionTerminatedException; -import org.apache.lucene.search.DocIdSet; -import org.apache.lucene.search.DocIdSetIterator; -import org.apache.lucene.search.FieldComparator; -import org.apache.lucene.search.FieldDoc; -import org.apache.lucene.search.LeafFieldComparator; -import org.apache.lucene.search.Query; -import org.apache.lucene.search.ScoreMode; -import org.apache.lucene.search.Scorer; -import org.apache.lucene.search.Sort; -import org.apache.lucene.search.SortField; -import org.apache.lucene.search.SortedNumericSelector; -import org.apache.lucene.search.SortedNumericSortField; -import org.apache.lucene.search.Weight; +import org.apache.lucene.search.*; import org.apache.lucene.search.comparators.LongComparator; import org.apache.lucene.util.Bits; import org.apache.lucene.util.RoaringDocIdSet; @@ -354,8 +339,8 @@ public int hashCode() { } @Override - public FieldComparator getComparator(int numHits, boolean enableSkipping) { - return new LongComparator(1, delegate.getField(), (Long) missingValue, delegate.getReverse(), false) { + public FieldComparator getComparator(int numHits, Pruning pruning) { + return new LongComparator(1, delegate.getField(), (Long) missingValue, delegate.getReverse(), pruning) { @Override public LeafFieldComparator getLeafComparator(LeafReaderContext context) throws IOException { return new LongLeafComparator(context) { diff --git a/server/src/main/java/org/opensearch/search/sort/GeoDistanceSortBuilder.java b/server/src/main/java/org/opensearch/search/sort/GeoDistanceSortBuilder.java index c4bb54de2eedd..852ae19edb6dc 100644 --- a/server/src/main/java/org/opensearch/search/sort/GeoDistanceSortBuilder.java +++ b/server/src/main/java/org/opensearch/search/sort/GeoDistanceSortBuilder.java @@ -35,10 +35,7 @@ import org.apache.lucene.document.LatLonDocValuesField; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.NumericDocValues; -import org.apache.lucene.search.DocIdSetIterator; -import org.apache.lucene.search.FieldComparator; -import org.apache.lucene.search.LeafFieldComparator; -import org.apache.lucene.search.SortField; +import org.apache.lucene.search.*; import org.apache.lucene.search.comparators.DoubleComparator; import org.apache.lucene.util.BitSet; import org.opensearch.OpenSearchParseException; @@ -734,8 +731,8 @@ private NumericDoubleValues getNumericDoubleValues(LeafReaderContext context) th } @Override - public FieldComparator newComparator(String fieldname, int numHits, boolean enableSkipping, boolean reversed) { - return new DoubleComparator(numHits, null, null, reversed, enableSkipping) { + public FieldComparator newComparator(String fieldname, int numHits, Pruning pruning, boolean reversed) { + return new DoubleComparator(numHits, null, null, reversed, pruning) { @Override public LeafFieldComparator getLeafComparator(LeafReaderContext context) throws IOException { return new DoubleLeafComparator(context) { diff --git a/server/src/main/java/org/opensearch/search/sort/SortedWiderNumericSortField.java b/server/src/main/java/org/opensearch/search/sort/SortedWiderNumericSortField.java index bded2417ba6c1..cfd1fd712d162 100644 --- a/server/src/main/java/org/opensearch/search/sort/SortedWiderNumericSortField.java +++ b/server/src/main/java/org/opensearch/search/sort/SortedWiderNumericSortField.java @@ -16,6 +16,7 @@ import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.search.FieldComparator; import org.apache.lucene.search.LeafFieldComparator; +import org.apache.lucene.search.Pruning; import org.apache.lucene.search.SortedNumericSortField; import org.apache.lucene.search.comparators.NumericComparator; @@ -44,13 +45,13 @@ public SortedWiderNumericSortField(String field, Type type, boolean reverse) { * Creates and return a comparator, which always converts Numeric to double * and compare to support multi type comparison between numeric values * @param numHits number of top hits the queue will store - * @param enableSkipping true if the comparator can skip documents via {@link + * @param pruning true if the comparator can skip documents via {@link * LeafFieldComparator#competitiveIterator()} * @return NumericComparator */ @Override - public FieldComparator getComparator(int numHits, boolean enableSkipping) { - return new NumericComparator(getField(), (Number) getMissingValue(), getReverse(), enableSkipping, Double.BYTES) { + public FieldComparator getComparator(int numHits, Pruning pruning) { + return new NumericComparator(getField(), (Number) getMissingValue(), getReverse(), pruning, Double.BYTES) { @Override public int compare(int slot1, int slot2) { throw new UnsupportedOperationException(); diff --git a/server/src/test/java/org/opensearch/action/search/BottomSortValuesCollectorTests.java b/server/src/test/java/org/opensearch/action/search/BottomSortValuesCollectorTests.java index 8042a7e296869..2bc48edd3376f 100644 --- a/server/src/test/java/org/opensearch/action/search/BottomSortValuesCollectorTests.java +++ b/server/src/test/java/org/opensearch/action/search/BottomSortValuesCollectorTests.java @@ -32,11 +32,7 @@ package org.opensearch.action.search; -import org.apache.lucene.search.FieldComparator; -import org.apache.lucene.search.FieldDoc; -import org.apache.lucene.search.SortField; -import org.apache.lucene.search.TopFieldDocs; -import org.apache.lucene.search.TotalHits; +import org.apache.lucene.search.*; import org.apache.lucene.util.BytesRef; import org.opensearch.common.time.DateUtils; import org.opensearch.index.mapper.DateFieldMapper; @@ -264,7 +260,7 @@ private Object[] newDateNanoArray(String... values) { private TopFieldDocs createTopDocs(SortField sortField, int totalHits, Object[] values) { FieldDoc[] fieldDocs = new FieldDoc[values.length]; - FieldComparator cmp = sortField.getComparator(1, false); + FieldComparator cmp = sortField.getComparator(1, Pruning.NONE); for (int i = 0; i < values.length; i++) { fieldDocs[i] = new FieldDoc(i, Float.NaN, new Object[] { values[i] }); } diff --git a/server/src/test/java/org/opensearch/search/aggregations/metrics/InternalTopHitsTests.java b/server/src/test/java/org/opensearch/search/aggregations/metrics/InternalTopHitsTests.java index 9358fa568db6a..8a6c6f1a81cb2 100644 --- a/server/src/test/java/org/opensearch/search/aggregations/metrics/InternalTopHitsTests.java +++ b/server/src/test/java/org/opensearch/search/aggregations/metrics/InternalTopHitsTests.java @@ -33,13 +33,7 @@ package org.opensearch.search.aggregations.metrics; import org.apache.lucene.index.IndexWriter; -import org.apache.lucene.search.FieldComparator; -import org.apache.lucene.search.FieldDoc; -import org.apache.lucene.search.ScoreDoc; -import org.apache.lucene.search.SortField; -import org.apache.lucene.search.TopDocs; -import org.apache.lucene.search.TopFieldDocs; -import org.apache.lucene.search.TotalHits; +import org.apache.lucene.search.*; import org.apache.lucene.util.BytesRef; import org.opensearch.common.collect.Tuple; import org.opensearch.common.document.DocumentField; @@ -356,7 +350,7 @@ private Comparator sortFieldsComparator(SortField[] sortFields) { FieldComparator[] comparators = new FieldComparator[sortFields.length]; for (int i = 0; i < sortFields.length; i++) { // Values passed to getComparator shouldn't matter - comparators[i] = sortFields[i].getComparator(0, false); + comparators[i] = sortFields[i].getComparator(0, Pruning.NONE); } return (lhs, rhs) -> { FieldDoc l = (FieldDoc) lhs; diff --git a/server/src/test/java/org/opensearch/search/query/QueryPhaseTests.java b/server/src/test/java/org/opensearch/search/query/QueryPhaseTests.java index 39126a607f968..c24bdcf888a5f 100644 --- a/server/src/test/java/org/opensearch/search/query/QueryPhaseTests.java +++ b/server/src/test/java/org/opensearch/search/query/QueryPhaseTests.java @@ -54,30 +54,8 @@ import org.apache.lucene.index.Term; import org.apache.lucene.queries.spans.SpanNearQuery; import org.apache.lucene.queries.spans.SpanTermQuery; +import org.apache.lucene.search.*; import org.apache.lucene.search.BooleanClause.Occur; -import org.apache.lucene.search.BooleanQuery; -import org.apache.lucene.search.Collector; -import org.apache.lucene.search.ConstantScoreQuery; -import org.apache.lucene.search.DocValuesFieldExistsQuery; -import org.apache.lucene.search.FieldComparator; -import org.apache.lucene.search.FieldDoc; -import org.apache.lucene.search.FilterCollector; -import org.apache.lucene.search.FilterLeafCollector; -import org.apache.lucene.search.IndexSearcher; -import org.apache.lucene.search.LeafCollector; -import org.apache.lucene.search.MatchAllDocsQuery; -import org.apache.lucene.search.MatchNoDocsQuery; -import org.apache.lucene.search.MultiTermQuery; -import org.apache.lucene.search.PrefixQuery; -import org.apache.lucene.search.Query; -import org.apache.lucene.search.ScoreDoc; -import org.apache.lucene.search.Sort; -import org.apache.lucene.search.SortField; -import org.apache.lucene.search.TermQuery; -import org.apache.lucene.search.TopDocs; -import org.apache.lucene.search.TotalHitCountCollector; -import org.apache.lucene.search.TotalHits; -import org.apache.lucene.search.Weight; import org.apache.lucene.search.grouping.CollapseTopFieldDocs; import org.apache.lucene.search.join.BitSetProducer; import org.apache.lucene.search.join.ScoreMode; @@ -659,7 +637,7 @@ public void testIndexSortScrollOptimization() throws Exception { @SuppressWarnings("unchecked") FieldComparator comparator = (FieldComparator) searchSortAndFormat.sort.getSort()[i].getComparator( 1, - false + Pruning.NONE ); int cmp = comparator.compareValues(firstDoc.fields[i], lastDoc.fields[i]); if (cmp == 0) { diff --git a/server/src/test/java/org/opensearch/search/query/QueryProfilePhaseTests.java b/server/src/test/java/org/opensearch/search/query/QueryProfilePhaseTests.java index 62dcf54e25578..67c2ef1011966 100644 --- a/server/src/test/java/org/opensearch/search/query/QueryProfilePhaseTests.java +++ b/server/src/test/java/org/opensearch/search/query/QueryProfilePhaseTests.java @@ -24,23 +24,8 @@ import org.apache.lucene.index.Term; import org.apache.lucene.queries.spans.SpanNearQuery; import org.apache.lucene.queries.spans.SpanTermQuery; +import org.apache.lucene.search.*; import org.apache.lucene.search.BooleanClause.Occur; -import org.apache.lucene.search.BooleanQuery; -import org.apache.lucene.search.Collector; -import org.apache.lucene.search.FieldComparator; -import org.apache.lucene.search.FieldDoc; -import org.apache.lucene.search.FilterCollector; -import org.apache.lucene.search.FilterLeafCollector; -import org.apache.lucene.search.IndexSearcher; -import org.apache.lucene.search.LeafCollector; -import org.apache.lucene.search.MatchAllDocsQuery; -import org.apache.lucene.search.MatchNoDocsQuery; -import org.apache.lucene.search.Query; -import org.apache.lucene.search.Sort; -import org.apache.lucene.search.SortField; -import org.apache.lucene.search.TermQuery; -import org.apache.lucene.search.TotalHits; -import org.apache.lucene.search.Weight; import org.apache.lucene.search.grouping.CollapseTopFieldDocs; import org.apache.lucene.store.Directory; import org.apache.lucene.tests.index.RandomIndexWriter; @@ -1056,7 +1041,7 @@ public void testIndexSortScrollOptimization() throws Exception { @SuppressWarnings("unchecked") FieldComparator comparator = (FieldComparator) searchSortAndFormat.sort.getSort()[i].getComparator( i, - randomBoolean() + randomFrom(Pruning.values()) ); int cmp = comparator.compareValues(firstDoc.fields[i], lastDoc.fields[i]); if (cmp == 0) { diff --git a/server/src/test/java/org/opensearch/search/searchafter/SearchAfterBuilderTests.java b/server/src/test/java/org/opensearch/search/searchafter/SearchAfterBuilderTests.java index cc5c205f7f9fe..48c93809cfcd3 100644 --- a/server/src/test/java/org/opensearch/search/searchafter/SearchAfterBuilderTests.java +++ b/server/src/test/java/org/opensearch/search/searchafter/SearchAfterBuilderTests.java @@ -33,10 +33,7 @@ package org.opensearch.search.searchafter; import org.apache.lucene.document.LatLonDocValuesField; -import org.apache.lucene.search.FieldComparator; -import org.apache.lucene.search.SortField; -import org.apache.lucene.search.SortedNumericSortField; -import org.apache.lucene.search.SortedSetSortField; +import org.apache.lucene.search.*; import org.opensearch.common.geo.GeoPoint; import org.opensearch.common.util.BigArrays; import org.opensearch.common.xcontent.XContentFactory; @@ -279,7 +276,7 @@ public SortField.Type reducedType() { } @Override - public FieldComparator newComparator(String fieldname, int numHits, boolean enableSkipping, boolean reversed) { + public FieldComparator newComparator(String fieldname, int numHits, Pruning pruning, boolean reversed) { return null; } From 455f99993af01e626e08561393c2bdefc23731f3 Mon Sep 17 00:00:00 2001 From: Marc Handalian Date: Thu, 30 Nov 2023 14:45:32 -0800 Subject: [PATCH 03/26] Codec bump Signed-off-by: Marc Handalian --- .../lucene/queries/BlendedTermQuery.java | 3 +-- .../org/opensearch/common/lucene/Lucene.java | 2 +- .../opensearch/index/codec/CodecService.java | 12 +++++------ .../PerFieldMappingPostingFormatCodec.java | 4 ++-- .../opensearch/index/codec/CodecTests.java | 20 +++++++++---------- .../engine/CompletionStatsCacheTests.java | 4 ++-- 6 files changed, 22 insertions(+), 23 deletions(-) diff --git a/server/src/main/java/org/apache/lucene/queries/BlendedTermQuery.java b/server/src/main/java/org/apache/lucene/queries/BlendedTermQuery.java index 345be330f048c..b47b974b96fed 100644 --- a/server/src/main/java/org/apache/lucene/queries/BlendedTermQuery.java +++ b/server/src/main/java/org/apache/lucene/queries/BlendedTermQuery.java @@ -100,11 +100,10 @@ public Query rewrite(IndexSearcher searcher) throws IOException { return rewritten; } IndexReader reader = searcher.getIndexReader(); - IndexReaderContext context = reader.getContext(); TermStates[] ctx = new TermStates[terms.length]; int[] docFreqs = new int[ctx.length]; for (int i = 0; i < terms.length; i++) { - ctx[i] = TermStates.build(context, terms[i], true); + ctx[i] = TermStates.build(searcher, terms[i], true); docFreqs[i] = ctx[i].docFreq(); } diff --git a/server/src/main/java/org/opensearch/common/lucene/Lucene.java b/server/src/main/java/org/opensearch/common/lucene/Lucene.java index 5b521081ac63b..2c7b6b552b43f 100644 --- a/server/src/main/java/org/opensearch/common/lucene/Lucene.java +++ b/server/src/main/java/org/opensearch/common/lucene/Lucene.java @@ -110,7 +110,7 @@ * @opensearch.internal */ public class Lucene { - public static final String LATEST_CODEC = "Lucene95"; + public static final String LATEST_CODEC = "Lucene99"; public static final String SOFT_DELETES_FIELD = "__soft_deletes"; diff --git a/server/src/main/java/org/opensearch/index/codec/CodecService.java b/server/src/main/java/org/opensearch/index/codec/CodecService.java index 775fc88b504f5..67f38536a0d11 100644 --- a/server/src/main/java/org/opensearch/index/codec/CodecService.java +++ b/server/src/main/java/org/opensearch/index/codec/CodecService.java @@ -34,8 +34,8 @@ import org.apache.logging.log4j.Logger; import org.apache.lucene.codecs.Codec; -import org.apache.lucene.codecs.lucene95.Lucene95Codec; -import org.apache.lucene.codecs.lucene95.Lucene95Codec.Mode; +import org.apache.lucene.codecs.lucene99.Lucene99Codec; +import org.apache.lucene.codecs.lucene99.Lucene99Codec.Mode; import org.opensearch.common.Nullable; import org.opensearch.common.collect.MapBuilder; import org.opensearch.index.IndexSettings; @@ -68,10 +68,10 @@ public CodecService(@Nullable MapperService mapperService, IndexSettings indexSe final MapBuilder codecs = MapBuilder.newMapBuilder(); assert null != indexSettings; if (mapperService == null) { - codecs.put(DEFAULT_CODEC, new Lucene95Codec()); - codecs.put(LZ4, new Lucene95Codec()); - codecs.put(BEST_COMPRESSION_CODEC, new Lucene95Codec(Mode.BEST_COMPRESSION)); - codecs.put(ZLIB, new Lucene95Codec(Mode.BEST_COMPRESSION)); + codecs.put(DEFAULT_CODEC, new Lucene99Codec()); + codecs.put(LZ4, new Lucene99Codec()); + codecs.put(BEST_COMPRESSION_CODEC, new Lucene99Codec(Mode.BEST_COMPRESSION)); + codecs.put(ZLIB, new Lucene99Codec(Mode.BEST_COMPRESSION)); } else { codecs.put(DEFAULT_CODEC, new PerFieldMappingPostingFormatCodec(Mode.BEST_SPEED, mapperService, logger)); codecs.put(LZ4, new PerFieldMappingPostingFormatCodec(Mode.BEST_SPEED, mapperService, logger)); diff --git a/server/src/main/java/org/opensearch/index/codec/PerFieldMappingPostingFormatCodec.java b/server/src/main/java/org/opensearch/index/codec/PerFieldMappingPostingFormatCodec.java index d3207557273a5..dc28ad2d6dc07 100644 --- a/server/src/main/java/org/opensearch/index/codec/PerFieldMappingPostingFormatCodec.java +++ b/server/src/main/java/org/opensearch/index/codec/PerFieldMappingPostingFormatCodec.java @@ -37,7 +37,7 @@ import org.apache.lucene.codecs.DocValuesFormat; import org.apache.lucene.codecs.PostingsFormat; import org.apache.lucene.codecs.lucene90.Lucene90DocValuesFormat; -import org.apache.lucene.codecs.lucene95.Lucene95Codec; +import org.apache.lucene.codecs.lucene99.Lucene99Codec; import org.opensearch.common.lucene.Lucene; import org.opensearch.index.mapper.CompletionFieldMapper; import org.opensearch.index.mapper.MappedFieldType; @@ -53,7 +53,7 @@ * * @opensearch.internal */ -public class PerFieldMappingPostingFormatCodec extends Lucene95Codec { +public class PerFieldMappingPostingFormatCodec extends Lucene99Codec { private final Logger logger; private final MapperService mapperService; private final DocValuesFormat dvFormat = new Lucene90DocValuesFormat(); diff --git a/server/src/test/java/org/opensearch/index/codec/CodecTests.java b/server/src/test/java/org/opensearch/index/codec/CodecTests.java index 6f9ff78aa0143..b31edd79411d0 100644 --- a/server/src/test/java/org/opensearch/index/codec/CodecTests.java +++ b/server/src/test/java/org/opensearch/index/codec/CodecTests.java @@ -35,7 +35,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.lucene.codecs.Codec; import org.apache.lucene.codecs.lucene90.Lucene90StoredFieldsFormat; -import org.apache.lucene.codecs.lucene95.Lucene95Codec; +import org.apache.lucene.codecs.lucene99.Lucene99Codec; import org.apache.lucene.document.Document; import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexWriter; @@ -68,28 +68,28 @@ public class CodecTests extends OpenSearchTestCase { public void testResolveDefaultCodecs() throws Exception { CodecService codecService = createCodecService(false); assertThat(codecService.codec("default"), instanceOf(PerFieldMappingPostingFormatCodec.class)); - assertThat(codecService.codec("default"), instanceOf(Lucene95Codec.class)); + assertThat(codecService.codec("default"), instanceOf(Lucene99Codec.class)); } public void testDefault() throws Exception { Codec codec = createCodecService(false).codec("default"); - assertStoredFieldsCompressionEquals(Lucene95Codec.Mode.BEST_SPEED, codec); + assertStoredFieldsCompressionEquals(Lucene99Codec.Mode.BEST_SPEED, codec); } public void testBestCompression() throws Exception { Codec codec = createCodecService(false).codec("best_compression"); - assertStoredFieldsCompressionEquals(Lucene95Codec.Mode.BEST_COMPRESSION, codec); + assertStoredFieldsCompressionEquals(Lucene99Codec.Mode.BEST_COMPRESSION, codec); } public void testLZ4() throws Exception { Codec codec = createCodecService(false).codec("lz4"); - assertStoredFieldsCompressionEquals(Lucene95Codec.Mode.BEST_SPEED, codec); + assertStoredFieldsCompressionEquals(Lucene99Codec.Mode.BEST_SPEED, codec); assert codec instanceof PerFieldMappingPostingFormatCodec; } public void testZlib() throws Exception { Codec codec = createCodecService(false).codec("zlib"); - assertStoredFieldsCompressionEquals(Lucene95Codec.Mode.BEST_COMPRESSION, codec); + assertStoredFieldsCompressionEquals(Lucene99Codec.Mode.BEST_COMPRESSION, codec); assert codec instanceof PerFieldMappingPostingFormatCodec; } @@ -125,12 +125,12 @@ public void testLuceneCodecsWithCompressionLevel() { public void testDefaultMapperServiceNull() throws Exception { Codec codec = createCodecService(true).codec("default"); - assertStoredFieldsCompressionEquals(Lucene95Codec.Mode.BEST_SPEED, codec); + assertStoredFieldsCompressionEquals(Lucene99Codec.Mode.BEST_SPEED, codec); } public void testBestCompressionMapperServiceNull() throws Exception { Codec codec = createCodecService(true).codec("best_compression"); - assertStoredFieldsCompressionEquals(Lucene95Codec.Mode.BEST_COMPRESSION, codec); + assertStoredFieldsCompressionEquals(Lucene99Codec.Mode.BEST_COMPRESSION, codec); } public void testExceptionCodecNull() { @@ -142,11 +142,11 @@ public void testExceptionIndexSettingsNull() { } // write some docs with it, inspect .si to see this was the used compression - private void assertStoredFieldsCompressionEquals(Lucene95Codec.Mode expected, Codec actual) throws Exception { + private void assertStoredFieldsCompressionEquals(Lucene99Codec.Mode expected, Codec actual) throws Exception { SegmentReader sr = getSegmentReader(actual); String v = sr.getSegmentInfo().info.getAttribute(Lucene90StoredFieldsFormat.MODE_KEY); assertNotNull(v); - assertEquals(expected, Lucene95Codec.Mode.valueOf(v)); + assertEquals(expected, Lucene99Codec.Mode.valueOf(v)); } private CodecService createCodecService(boolean isMapperServiceNull) throws IOException { diff --git a/server/src/test/java/org/opensearch/index/engine/CompletionStatsCacheTests.java b/server/src/test/java/org/opensearch/index/engine/CompletionStatsCacheTests.java index b746d0ba8a56d..cf5d0f8f97f1f 100644 --- a/server/src/test/java/org/opensearch/index/engine/CompletionStatsCacheTests.java +++ b/server/src/test/java/org/opensearch/index/engine/CompletionStatsCacheTests.java @@ -32,7 +32,7 @@ package org.opensearch.index.engine; import org.apache.lucene.codecs.PostingsFormat; -import org.apache.lucene.codecs.lucene95.Lucene95Codec; +import org.apache.lucene.codecs.lucene99.Lucene99Codec; import org.apache.lucene.document.Document; import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexWriter; @@ -70,7 +70,7 @@ public void testExceptionsAreNotCached() { public void testCompletionStatsCache() throws IOException, InterruptedException { final IndexWriterConfig indexWriterConfig = newIndexWriterConfig(); final PostingsFormat postingsFormat = new Completion90PostingsFormat(); - indexWriterConfig.setCodec(new Lucene95Codec() { + indexWriterConfig.setCodec(new Lucene99Codec() { @Override public PostingsFormat getPostingsFormatForField(String field) { return postingsFormat; // all fields are suggest fields From 0e355fc859abfbba7b15f1ae71d58c35e67c32cc Mon Sep 17 00:00:00 2001 From: Marc Handalian Date: Mon, 4 Dec 2023 11:25:25 -0800 Subject: [PATCH 04/26] Update to honor enableSkipping in XFieldComparatorSource Signed-off-by: Marc Handalian --- .../index/fielddata/IndexFieldData.java | 13 +++++++++--- .../BytesRefFieldComparatorSource.java | 2 +- .../DoubleValuesComparatorSource.java | 2 +- .../FloatValuesComparatorSource.java | 2 +- .../HalfFloatValuesComparatorSource.java | 2 +- .../IntValuesComparatorSource.java | 2 +- .../LongValuesComparatorSource.java | 2 +- .../UnsignedLongValuesComparatorSource.java | 2 +- .../bucket/composite/CompositeAggregator.java | 20 +++++++++++++++++-- .../search/sort/GeoDistanceSortBuilder.java | 2 +- 10 files changed, 36 insertions(+), 13 deletions(-) diff --git a/server/src/main/java/org/opensearch/index/fielddata/IndexFieldData.java b/server/src/main/java/org/opensearch/index/fielddata/IndexFieldData.java index 7aaf47691cbf4..09d87989c45bb 100644 --- a/server/src/main/java/org/opensearch/index/fielddata/IndexFieldData.java +++ b/server/src/main/java/org/opensearch/index/fielddata/IndexFieldData.java @@ -121,13 +121,13 @@ abstract class XFieldComparatorSource extends FieldComparatorSource { protected final MultiValueMode sortMode; protected final Object missingValue; protected final Nested nested; - protected Pruning pruning; + protected boolean enableSkipping; public XFieldComparatorSource(Object missingValue, MultiValueMode sortMode, Nested nested) { this.sortMode = sortMode; this.missingValue = missingValue; this.nested = nested; - this.pruning = Pruning.GREATER_THAN; // true by default + this.enableSkipping = true; // true by default } public MultiValueMode sortMode() { @@ -139,7 +139,14 @@ public Nested nested() { } public void disableSkipping() { - this.pruning = Pruning.NONE; + this.enableSkipping = false; + } + + protected Pruning filterPruning(Pruning pruning) { + if (this.enableSkipping) { + return pruning; + } + return Pruning.NONE; } /** diff --git a/server/src/main/java/org/opensearch/index/fielddata/fieldcomparator/BytesRefFieldComparatorSource.java b/server/src/main/java/org/opensearch/index/fielddata/fieldcomparator/BytesRefFieldComparatorSource.java index 8e049e4fa0b19..976b4e64d1611 100644 --- a/server/src/main/java/org/opensearch/index/fielddata/fieldcomparator/BytesRefFieldComparatorSource.java +++ b/server/src/main/java/org/opensearch/index/fielddata/fieldcomparator/BytesRefFieldComparatorSource.java @@ -102,7 +102,7 @@ public FieldComparator newComparator(String fieldname, int numHits, Pruning p indexFieldData.getFieldName(), sortMissingLast, reversed, - pruning + filterPruning(pruning) ) { @Override diff --git a/server/src/main/java/org/opensearch/index/fielddata/fieldcomparator/DoubleValuesComparatorSource.java b/server/src/main/java/org/opensearch/index/fielddata/fieldcomparator/DoubleValuesComparatorSource.java index f8773b391faa7..c46c740b35b82 100644 --- a/server/src/main/java/org/opensearch/index/fielddata/fieldcomparator/DoubleValuesComparatorSource.java +++ b/server/src/main/java/org/opensearch/index/fielddata/fieldcomparator/DoubleValuesComparatorSource.java @@ -98,7 +98,7 @@ public FieldComparator newComparator(String fieldname, int numHits, Pruning p assert indexFieldData == null || fieldname.equals(indexFieldData.getFieldName()); final double dMissingValue = (Double) missingObject(missingValue, reversed); - return new DoubleComparator(numHits, fieldname, dMissingValue, reversed, pruning) { + return new DoubleComparator(numHits, fieldname, dMissingValue, reversed, filterPruning(pruning)) { @Override public LeafFieldComparator getLeafComparator(LeafReaderContext context) throws IOException { return new DoubleLeafComparator(context) { diff --git a/server/src/main/java/org/opensearch/index/fielddata/fieldcomparator/FloatValuesComparatorSource.java b/server/src/main/java/org/opensearch/index/fielddata/fieldcomparator/FloatValuesComparatorSource.java index 27badbe9c2a00..4911a58e7a1f2 100644 --- a/server/src/main/java/org/opensearch/index/fielddata/fieldcomparator/FloatValuesComparatorSource.java +++ b/server/src/main/java/org/opensearch/index/fielddata/fieldcomparator/FloatValuesComparatorSource.java @@ -91,7 +91,7 @@ public FieldComparator newComparator(String fieldname, int numHits, Pruning p assert indexFieldData == null || fieldname.equals(indexFieldData.getFieldName()); final float fMissingValue = (Float) missingObject(missingValue, reversed); - return new FloatComparator(numHits, fieldname, fMissingValue, reversed, pruning) { + return new FloatComparator(numHits, fieldname, fMissingValue, reversed, filterPruning(pruning)) { @Override public LeafFieldComparator getLeafComparator(LeafReaderContext context) throws IOException { return new FloatLeafComparator(context) { diff --git a/server/src/main/java/org/opensearch/index/fielddata/fieldcomparator/HalfFloatValuesComparatorSource.java b/server/src/main/java/org/opensearch/index/fielddata/fieldcomparator/HalfFloatValuesComparatorSource.java index 99edec225099e..e2e56dcb14fdf 100644 --- a/server/src/main/java/org/opensearch/index/fielddata/fieldcomparator/HalfFloatValuesComparatorSource.java +++ b/server/src/main/java/org/opensearch/index/fielddata/fieldcomparator/HalfFloatValuesComparatorSource.java @@ -47,7 +47,7 @@ public FieldComparator newComparator(String fieldname, int numHits, Pruning p assert indexFieldData == null || fieldname.equals(indexFieldData.getFieldName()); final float fMissingValue = (Float) missingObject(missingValue, reversed); - return new HalfFloatComparator(numHits, fieldname, fMissingValue, reversed, pruning) { + return new HalfFloatComparator(numHits, fieldname, fMissingValue, reversed, filterPruning(pruning)) { @Override public LeafFieldComparator getLeafComparator(LeafReaderContext context) throws IOException { return new HalfFloatLeafComparator(context) { diff --git a/server/src/main/java/org/opensearch/index/fielddata/fieldcomparator/IntValuesComparatorSource.java b/server/src/main/java/org/opensearch/index/fielddata/fieldcomparator/IntValuesComparatorSource.java index 732999f7dd752..8741e5d8bbdb2 100644 --- a/server/src/main/java/org/opensearch/index/fielddata/fieldcomparator/IntValuesComparatorSource.java +++ b/server/src/main/java/org/opensearch/index/fielddata/fieldcomparator/IntValuesComparatorSource.java @@ -71,7 +71,7 @@ public FieldComparator newComparator(String fieldname, int numHits, Pruning p assert indexFieldData == null || fieldname.equals(indexFieldData.getFieldName()); final int iMissingValue = (Integer) missingObject(missingValue, reversed); - return new IntComparator(numHits, fieldname, iMissingValue, reversed, pruning) { + return new IntComparator(numHits, fieldname, iMissingValue, reversed, filterPruning(pruning)) { @Override public LeafFieldComparator getLeafComparator(LeafReaderContext context) throws IOException { return new IntLeafComparator(context) { diff --git a/server/src/main/java/org/opensearch/index/fielddata/fieldcomparator/LongValuesComparatorSource.java b/server/src/main/java/org/opensearch/index/fielddata/fieldcomparator/LongValuesComparatorSource.java index 331e06baf0b41..a22ff94b1221f 100644 --- a/server/src/main/java/org/opensearch/index/fielddata/fieldcomparator/LongValuesComparatorSource.java +++ b/server/src/main/java/org/opensearch/index/fielddata/fieldcomparator/LongValuesComparatorSource.java @@ -115,7 +115,7 @@ public FieldComparator newComparator(String fieldname, int numHits, Pruning p assert indexFieldData == null || fieldname.equals(indexFieldData.getFieldName()); final long lMissingValue = (Long) missingObject(missingValue, reversed); - return new LongComparator(numHits, fieldname, lMissingValue, reversed, pruning) { + return new LongComparator(numHits, fieldname, lMissingValue, reversed, filterPruning(pruning)) { @Override public LeafFieldComparator getLeafComparator(LeafReaderContext context) throws IOException { return new LongLeafComparator(context) { diff --git a/server/src/main/java/org/opensearch/index/fielddata/fieldcomparator/UnsignedLongValuesComparatorSource.java b/server/src/main/java/org/opensearch/index/fielddata/fieldcomparator/UnsignedLongValuesComparatorSource.java index b598608c4cd6c..31f8d84301a10 100644 --- a/server/src/main/java/org/opensearch/index/fielddata/fieldcomparator/UnsignedLongValuesComparatorSource.java +++ b/server/src/main/java/org/opensearch/index/fielddata/fieldcomparator/UnsignedLongValuesComparatorSource.java @@ -89,7 +89,7 @@ public FieldComparator newComparator(String fieldname, int numHits, Pruning p assert indexFieldData == null || fieldname.equals(indexFieldData.getFieldName()); final BigInteger ulMissingValue = (BigInteger) missingObject(missingValue, reversed); - return new UnsignedLongComparator(numHits, fieldname, ulMissingValue, reversed, pruning) { + return new UnsignedLongComparator(numHits, fieldname, ulMissingValue, reversed, filterPruning(pruning)) { @Override public LeafFieldComparator getLeafComparator(LeafReaderContext context) throws IOException { return new UnsignedLongLeafComparator(context) { diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/composite/CompositeAggregator.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/composite/CompositeAggregator.java index 2b566c51e4c22..317c2a357bac5 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/composite/CompositeAggregator.java +++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/composite/CompositeAggregator.java @@ -37,7 +37,23 @@ import org.apache.lucene.index.NumericDocValues; import org.apache.lucene.index.SortedNumericDocValues; import org.apache.lucene.index.SortedSetDocValues; -import org.apache.lucene.search.*; +import org.apache.lucene.search.BooleanClause; +import org.apache.lucene.search.BooleanQuery; +import org.apache.lucene.search.CollectionTerminatedException; +import org.apache.lucene.search.DocIdSet; +import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.search.FieldComparator; +import org.apache.lucene.search.FieldDoc; +import org.apache.lucene.search.LeafFieldComparator; +import org.apache.lucene.search.Pruning; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.ScoreMode; +import org.apache.lucene.search.Scorer; +import org.apache.lucene.search.Sort; +import org.apache.lucene.search.SortField; +import org.apache.lucene.search.SortedNumericSelector; +import org.apache.lucene.search.SortedNumericSortField; +import org.apache.lucene.search.Weight; import org.apache.lucene.search.comparators.LongComparator; import org.apache.lucene.util.Bits; import org.apache.lucene.util.RoaringDocIdSet; @@ -340,7 +356,7 @@ public int hashCode() { @Override public FieldComparator getComparator(int numHits, Pruning pruning) { - return new LongComparator(1, delegate.getField(), (Long) missingValue, delegate.getReverse(), pruning) { + return new LongComparator(1, delegate.getField(), (Long) missingValue, delegate.getReverse(), Pruning.NONE) { @Override public LeafFieldComparator getLeafComparator(LeafReaderContext context) throws IOException { return new LongLeafComparator(context) { diff --git a/server/src/main/java/org/opensearch/search/sort/GeoDistanceSortBuilder.java b/server/src/main/java/org/opensearch/search/sort/GeoDistanceSortBuilder.java index 852ae19edb6dc..3845d3d72a61f 100644 --- a/server/src/main/java/org/opensearch/search/sort/GeoDistanceSortBuilder.java +++ b/server/src/main/java/org/opensearch/search/sort/GeoDistanceSortBuilder.java @@ -732,7 +732,7 @@ private NumericDoubleValues getNumericDoubleValues(LeafReaderContext context) th @Override public FieldComparator newComparator(String fieldname, int numHits, Pruning pruning, boolean reversed) { - return new DoubleComparator(numHits, null, null, reversed, pruning) { + return new DoubleComparator(numHits, null, null, reversed, filterPruning(pruning)) { @Override public LeafFieldComparator getLeafComparator(LeafReaderContext context) throws IOException { return new DoubleLeafComparator(context) { From 13f5b3b28ed6bb745e2595489e77390811ba19d2 Mon Sep 17 00:00:00 2001 From: Marc Handalian Date: Mon, 4 Dec 2023 11:25:51 -0800 Subject: [PATCH 05/26] Use released 9.9 version Signed-off-by: Marc Handalian --- buildSrc/version.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildSrc/version.properties b/buildSrc/version.properties index 4d8413d335434..223bf5983fb8f 100644 --- a/buildSrc/version.properties +++ b/buildSrc/version.properties @@ -1,5 +1,5 @@ opensearch = 3.0.0 -lucene = 9.9.0-snapshot-eb697aa +lucene = 9.9.0 bundled_jdk_vendor = adoptium bundled_jdk = 21.0.1+12 From 4cc1ce61bd6fac421520fb5d4a1a94265bea7352 Mon Sep 17 00:00:00 2001 From: Marc Handalian Date: Mon, 4 Dec 2023 12:52:25 -0800 Subject: [PATCH 06/26] spotless Signed-off-by: Marc Handalian --- .../search/grouping/CollapseTopFieldDocs.java | 9 ++++++- .../search/BottomSortValuesCollector.java | 6 ++++- .../index/fielddata/IndexFieldData.java | 11 +++++++- .../BytesRefFieldComparatorSource.java | 6 ++++- .../DoubleValuesComparatorSource.java | 7 +++++- .../FloatValuesComparatorSource.java | 7 +++++- .../IntValuesComparatorSource.java | 6 ++++- .../LongValuesComparatorSource.java | 6 ++++- .../UnsignedLongValuesComparatorSource.java | 6 ++++- .../queries/SearchAfterSortedDocQuery.java | 16 +++++++++++- .../search/sort/GeoDistanceSortBuilder.java | 6 ++++- .../BottomSortValuesCollectorTests.java | 7 +++++- .../engine/CompletionStatsCacheTests.java | 4 +-- .../index/mapper/RangeFieldTypeTests.java | 4 +-- .../query/GeoDistanceQueryBuilderTests.java | 5 +++- .../metrics/InternalTopHitsTests.java | 9 ++++++- .../search/query/QueryPhaseTests.java | 25 ++++++++++++++++++- .../search/query/QueryProfilePhaseTests.java | 18 ++++++++++++- .../searchafter/SearchAfterBuilderTests.java | 6 ++++- 19 files changed, 143 insertions(+), 21 deletions(-) diff --git a/server/src/main/java/org/apache/lucene/search/grouping/CollapseTopFieldDocs.java b/server/src/main/java/org/apache/lucene/search/grouping/CollapseTopFieldDocs.java index aa9696468424d..961587113173d 100644 --- a/server/src/main/java/org/apache/lucene/search/grouping/CollapseTopFieldDocs.java +++ b/server/src/main/java/org/apache/lucene/search/grouping/CollapseTopFieldDocs.java @@ -31,7 +31,14 @@ package org.apache.lucene.search.grouping; -import org.apache.lucene.search.*; +import org.apache.lucene.search.FieldComparator; +import org.apache.lucene.search.FieldDoc; +import org.apache.lucene.search.Pruning; +import org.apache.lucene.search.ScoreDoc; +import org.apache.lucene.search.Sort; +import org.apache.lucene.search.SortField; +import org.apache.lucene.search.TopFieldDocs; +import org.apache.lucene.search.TotalHits; import org.apache.lucene.util.PriorityQueue; import org.opensearch.core.common.util.CollectionUtils; diff --git a/server/src/main/java/org/opensearch/action/search/BottomSortValuesCollector.java b/server/src/main/java/org/opensearch/action/search/BottomSortValuesCollector.java index 64f177f6ac597..bce8d9fb2b1ca 100644 --- a/server/src/main/java/org/opensearch/action/search/BottomSortValuesCollector.java +++ b/server/src/main/java/org/opensearch/action/search/BottomSortValuesCollector.java @@ -32,7 +32,11 @@ package org.opensearch.action.search; -import org.apache.lucene.search.*; +import org.apache.lucene.search.FieldComparator; +import org.apache.lucene.search.FieldDoc; +import org.apache.lucene.search.Pruning; +import org.apache.lucene.search.SortField; +import org.apache.lucene.search.TopFieldDocs; import org.opensearch.search.DocValueFormat; import org.opensearch.search.SearchSortValuesAndFormats; diff --git a/server/src/main/java/org/opensearch/index/fielddata/IndexFieldData.java b/server/src/main/java/org/opensearch/index/fielddata/IndexFieldData.java index 09d87989c45bb..a63e1d418e3ba 100644 --- a/server/src/main/java/org/opensearch/index/fielddata/IndexFieldData.java +++ b/server/src/main/java/org/opensearch/index/fielddata/IndexFieldData.java @@ -34,7 +34,16 @@ import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.LeafReaderContext; -import org.apache.lucene.search.*; +import org.apache.lucene.search.DocIdSet; +import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.search.FieldComparatorSource; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Pruning; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.ScoreMode; +import org.apache.lucene.search.Scorer; +import org.apache.lucene.search.SortField; +import org.apache.lucene.search.Weight; import org.apache.lucene.search.join.BitSetProducer; import org.apache.lucene.util.BitDocIdSet; import org.apache.lucene.util.BitSet; diff --git a/server/src/main/java/org/opensearch/index/fielddata/fieldcomparator/BytesRefFieldComparatorSource.java b/server/src/main/java/org/opensearch/index/fielddata/fieldcomparator/BytesRefFieldComparatorSource.java index 976b4e64d1611..4c6eba2f0c7ec 100644 --- a/server/src/main/java/org/opensearch/index/fielddata/fieldcomparator/BytesRefFieldComparatorSource.java +++ b/server/src/main/java/org/opensearch/index/fielddata/fieldcomparator/BytesRefFieldComparatorSource.java @@ -36,7 +36,11 @@ import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.SortedDocValues; import org.apache.lucene.index.SortedSetDocValues; -import org.apache.lucene.search.*; +import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.search.FieldComparator; +import org.apache.lucene.search.Pruning; +import org.apache.lucene.search.Scorable; +import org.apache.lucene.search.SortField; import org.apache.lucene.search.comparators.TermOrdValComparator; import org.apache.lucene.util.BitSet; import org.apache.lucene.util.BytesRef; diff --git a/server/src/main/java/org/opensearch/index/fielddata/fieldcomparator/DoubleValuesComparatorSource.java b/server/src/main/java/org/opensearch/index/fielddata/fieldcomparator/DoubleValuesComparatorSource.java index c46c740b35b82..fd4fac3877f2f 100644 --- a/server/src/main/java/org/opensearch/index/fielddata/fieldcomparator/DoubleValuesComparatorSource.java +++ b/server/src/main/java/org/opensearch/index/fielddata/fieldcomparator/DoubleValuesComparatorSource.java @@ -34,7 +34,12 @@ import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.NumericDocValues; -import org.apache.lucene.search.*; +import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.search.FieldComparator; +import org.apache.lucene.search.LeafFieldComparator; +import org.apache.lucene.search.Pruning; +import org.apache.lucene.search.Scorable; +import org.apache.lucene.search.SortField; import org.apache.lucene.search.comparators.DoubleComparator; import org.apache.lucene.util.BitSet; import org.opensearch.common.Nullable; diff --git a/server/src/main/java/org/opensearch/index/fielddata/fieldcomparator/FloatValuesComparatorSource.java b/server/src/main/java/org/opensearch/index/fielddata/fieldcomparator/FloatValuesComparatorSource.java index 4911a58e7a1f2..b106ba268bcb9 100644 --- a/server/src/main/java/org/opensearch/index/fielddata/fieldcomparator/FloatValuesComparatorSource.java +++ b/server/src/main/java/org/opensearch/index/fielddata/fieldcomparator/FloatValuesComparatorSource.java @@ -33,7 +33,12 @@ import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.NumericDocValues; -import org.apache.lucene.search.*; +import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.search.FieldComparator; +import org.apache.lucene.search.LeafFieldComparator; +import org.apache.lucene.search.Pruning; +import org.apache.lucene.search.Scorable; +import org.apache.lucene.search.SortField; import org.apache.lucene.search.comparators.FloatComparator; import org.apache.lucene.util.BitSet; import org.opensearch.common.Nullable; diff --git a/server/src/main/java/org/opensearch/index/fielddata/fieldcomparator/IntValuesComparatorSource.java b/server/src/main/java/org/opensearch/index/fielddata/fieldcomparator/IntValuesComparatorSource.java index 8741e5d8bbdb2..8f540cc6ae9d9 100644 --- a/server/src/main/java/org/opensearch/index/fielddata/fieldcomparator/IntValuesComparatorSource.java +++ b/server/src/main/java/org/opensearch/index/fielddata/fieldcomparator/IntValuesComparatorSource.java @@ -16,7 +16,11 @@ import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.NumericDocValues; import org.apache.lucene.index.SortedNumericDocValues; -import org.apache.lucene.search.*; +import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.search.FieldComparator; +import org.apache.lucene.search.LeafFieldComparator; +import org.apache.lucene.search.Pruning; +import org.apache.lucene.search.SortField; import org.apache.lucene.search.comparators.IntComparator; import org.apache.lucene.util.BitSet; import org.opensearch.common.Nullable; diff --git a/server/src/main/java/org/opensearch/index/fielddata/fieldcomparator/LongValuesComparatorSource.java b/server/src/main/java/org/opensearch/index/fielddata/fieldcomparator/LongValuesComparatorSource.java index a22ff94b1221f..3666cd8d6dfea 100644 --- a/server/src/main/java/org/opensearch/index/fielddata/fieldcomparator/LongValuesComparatorSource.java +++ b/server/src/main/java/org/opensearch/index/fielddata/fieldcomparator/LongValuesComparatorSource.java @@ -34,7 +34,11 @@ import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.NumericDocValues; import org.apache.lucene.index.SortedNumericDocValues; -import org.apache.lucene.search.*; +import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.search.FieldComparator; +import org.apache.lucene.search.LeafFieldComparator; +import org.apache.lucene.search.Pruning; +import org.apache.lucene.search.SortField; import org.apache.lucene.search.comparators.LongComparator; import org.apache.lucene.util.BitSet; import org.opensearch.common.Nullable; diff --git a/server/src/main/java/org/opensearch/index/fielddata/fieldcomparator/UnsignedLongValuesComparatorSource.java b/server/src/main/java/org/opensearch/index/fielddata/fieldcomparator/UnsignedLongValuesComparatorSource.java index 31f8d84301a10..3714561b63e44 100644 --- a/server/src/main/java/org/opensearch/index/fielddata/fieldcomparator/UnsignedLongValuesComparatorSource.java +++ b/server/src/main/java/org/opensearch/index/fielddata/fieldcomparator/UnsignedLongValuesComparatorSource.java @@ -11,7 +11,11 @@ import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.NumericDocValues; import org.apache.lucene.index.SortedNumericDocValues; -import org.apache.lucene.search.*; +import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.search.FieldComparator; +import org.apache.lucene.search.LeafFieldComparator; +import org.apache.lucene.search.Pruning; +import org.apache.lucene.search.SortField; import org.apache.lucene.util.BitSet; import org.opensearch.common.Nullable; import org.opensearch.common.Numbers; diff --git a/server/src/main/java/org/opensearch/lucene/queries/SearchAfterSortedDocQuery.java b/server/src/main/java/org/opensearch/lucene/queries/SearchAfterSortedDocQuery.java index 179d112699b35..600ba5b5a92d8 100644 --- a/server/src/main/java/org/opensearch/lucene/queries/SearchAfterSortedDocQuery.java +++ b/server/src/main/java/org/opensearch/lucene/queries/SearchAfterSortedDocQuery.java @@ -33,7 +33,21 @@ package org.opensearch.lucene.queries; import org.apache.lucene.index.LeafReaderContext; -import org.apache.lucene.search.*; +import org.apache.lucene.search.ConstantScoreScorer; +import org.apache.lucene.search.ConstantScoreWeight; +import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.search.FieldComparator; +import org.apache.lucene.search.FieldDoc; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.LeafFieldComparator; +import org.apache.lucene.search.Pruning; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.QueryVisitor; +import org.apache.lucene.search.ScoreMode; +import org.apache.lucene.search.Scorer; +import org.apache.lucene.search.Sort; +import org.apache.lucene.search.SortField; +import org.apache.lucene.search.Weight; import org.opensearch.common.lucene.Lucene; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/search/sort/GeoDistanceSortBuilder.java b/server/src/main/java/org/opensearch/search/sort/GeoDistanceSortBuilder.java index 3845d3d72a61f..0499bba3245c6 100644 --- a/server/src/main/java/org/opensearch/search/sort/GeoDistanceSortBuilder.java +++ b/server/src/main/java/org/opensearch/search/sort/GeoDistanceSortBuilder.java @@ -35,7 +35,11 @@ import org.apache.lucene.document.LatLonDocValuesField; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.NumericDocValues; -import org.apache.lucene.search.*; +import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.search.FieldComparator; +import org.apache.lucene.search.LeafFieldComparator; +import org.apache.lucene.search.Pruning; +import org.apache.lucene.search.SortField; import org.apache.lucene.search.comparators.DoubleComparator; import org.apache.lucene.util.BitSet; import org.opensearch.OpenSearchParseException; diff --git a/server/src/test/java/org/opensearch/action/search/BottomSortValuesCollectorTests.java b/server/src/test/java/org/opensearch/action/search/BottomSortValuesCollectorTests.java index 2bc48edd3376f..4f929a71429a6 100644 --- a/server/src/test/java/org/opensearch/action/search/BottomSortValuesCollectorTests.java +++ b/server/src/test/java/org/opensearch/action/search/BottomSortValuesCollectorTests.java @@ -32,7 +32,12 @@ package org.opensearch.action.search; -import org.apache.lucene.search.*; +import org.apache.lucene.search.FieldComparator; +import org.apache.lucene.search.FieldDoc; +import org.apache.lucene.search.Pruning; +import org.apache.lucene.search.SortField; +import org.apache.lucene.search.TopFieldDocs; +import org.apache.lucene.search.TotalHits; import org.apache.lucene.util.BytesRef; import org.opensearch.common.time.DateUtils; import org.opensearch.index.mapper.DateFieldMapper; diff --git a/server/src/test/java/org/opensearch/index/engine/CompletionStatsCacheTests.java b/server/src/test/java/org/opensearch/index/engine/CompletionStatsCacheTests.java index cf5d0f8f97f1f..0c87c384e0749 100644 --- a/server/src/test/java/org/opensearch/index/engine/CompletionStatsCacheTests.java +++ b/server/src/test/java/org/opensearch/index/engine/CompletionStatsCacheTests.java @@ -39,7 +39,7 @@ import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.search.Query; import org.apache.lucene.search.QueryCachingPolicy; -import org.apache.lucene.search.suggest.document.Completion90PostingsFormat; +import org.apache.lucene.search.suggest.document.Completion99PostingsFormat; import org.apache.lucene.search.suggest.document.SuggestField; import org.apache.lucene.store.Directory; import org.opensearch.OpenSearchException; @@ -69,7 +69,7 @@ public void testExceptionsAreNotCached() { public void testCompletionStatsCache() throws IOException, InterruptedException { final IndexWriterConfig indexWriterConfig = newIndexWriterConfig(); - final PostingsFormat postingsFormat = new Completion90PostingsFormat(); + final PostingsFormat postingsFormat = new Completion99PostingsFormat(); indexWriterConfig.setCodec(new Lucene99Codec() { @Override public PostingsFormat getPostingsFormatForField(String field) { diff --git a/server/src/test/java/org/opensearch/index/mapper/RangeFieldTypeTests.java b/server/src/test/java/org/opensearch/index/mapper/RangeFieldTypeTests.java index 755d77c6ae392..00b48240d0567 100644 --- a/server/src/test/java/org/opensearch/index/mapper/RangeFieldTypeTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/RangeFieldTypeTests.java @@ -277,12 +277,12 @@ public void testDateRangeQueryUsingMappingFormat() { RangeFieldType fieldType = new RangeFieldType("field", formatter); final Query query = fieldType.rangeQuery(from, to, true, true, relation, null, fieldType.dateMathParser(), context); - assertEquals("field:", query.toString()); + assertEquals("field:", ((IndexOrDocValuesQuery) query).getIndexQuery().toString()); // compare lower and upper bounds with what we would get on a `date` field DateFieldType dateFieldType = new DateFieldType("field", DateFieldMapper.Resolution.MILLISECONDS, formatter); final Query queryOnDateField = dateFieldType.rangeQuery(from, to, true, true, relation, null, fieldType.dateMathParser(), context); - assertEquals("field:[1465975790000 TO 1466062190999]", queryOnDateField.toString()); + assertEquals("field:[1465975790000 TO 1466062190999]", ((IndexOrDocValuesQuery) queryOnDateField).getIndexQuery().toString()); } /** diff --git a/server/src/test/java/org/opensearch/index/query/GeoDistanceQueryBuilderTests.java b/server/src/test/java/org/opensearch/index/query/GeoDistanceQueryBuilderTests.java index 1b62fef30d255..0dfe47e83fedc 100644 --- a/server/src/test/java/org/opensearch/index/query/GeoDistanceQueryBuilderTests.java +++ b/server/src/test/java/org/opensearch/index/query/GeoDistanceQueryBuilderTests.java @@ -358,7 +358,10 @@ private void assertGeoDistanceRangeQuery(String query, double lat, double lon, d Query parsedQuery = parseQuery(query).toQuery(createShardContext()); // The parsedQuery contains IndexOrDocValuesQuery, which wraps LatLonPointDistanceQuery which in turn has default visibility, // so we cannot access its fields directly to check and have to use toString() here instead. - assertEquals(parsedQuery.toString(), "mapped_geo_point:" + lat + "," + lon + " +/- " + distanceUnit.toMeters(distance) + " meters"); + assertEquals( + ((IndexOrDocValuesQuery) parsedQuery).getIndexQuery().toString(), + "mapped_geo_point:" + lat + "," + lon + " +/- " + distanceUnit.toMeters(distance) + " meters" + ); } public void testFromJson() throws IOException { diff --git a/server/src/test/java/org/opensearch/search/aggregations/metrics/InternalTopHitsTests.java b/server/src/test/java/org/opensearch/search/aggregations/metrics/InternalTopHitsTests.java index 8a6c6f1a81cb2..01e259f84660d 100644 --- a/server/src/test/java/org/opensearch/search/aggregations/metrics/InternalTopHitsTests.java +++ b/server/src/test/java/org/opensearch/search/aggregations/metrics/InternalTopHitsTests.java @@ -33,7 +33,14 @@ package org.opensearch.search.aggregations.metrics; import org.apache.lucene.index.IndexWriter; -import org.apache.lucene.search.*; +import org.apache.lucene.search.FieldComparator; +import org.apache.lucene.search.FieldDoc; +import org.apache.lucene.search.Pruning; +import org.apache.lucene.search.ScoreDoc; +import org.apache.lucene.search.SortField; +import org.apache.lucene.search.TopDocs; +import org.apache.lucene.search.TopFieldDocs; +import org.apache.lucene.search.TotalHits; import org.apache.lucene.util.BytesRef; import org.opensearch.common.collect.Tuple; import org.opensearch.common.document.DocumentField; diff --git a/server/src/test/java/org/opensearch/search/query/QueryPhaseTests.java b/server/src/test/java/org/opensearch/search/query/QueryPhaseTests.java index c24bdcf888a5f..d0e01c5461c79 100644 --- a/server/src/test/java/org/opensearch/search/query/QueryPhaseTests.java +++ b/server/src/test/java/org/opensearch/search/query/QueryPhaseTests.java @@ -54,8 +54,31 @@ import org.apache.lucene.index.Term; import org.apache.lucene.queries.spans.SpanNearQuery; import org.apache.lucene.queries.spans.SpanTermQuery; -import org.apache.lucene.search.*; import org.apache.lucene.search.BooleanClause.Occur; +import org.apache.lucene.search.BooleanQuery; +import org.apache.lucene.search.Collector; +import org.apache.lucene.search.ConstantScoreQuery; +import org.apache.lucene.search.DocValuesFieldExistsQuery; +import org.apache.lucene.search.FieldComparator; +import org.apache.lucene.search.FieldDoc; +import org.apache.lucene.search.FilterCollector; +import org.apache.lucene.search.FilterLeafCollector; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.LeafCollector; +import org.apache.lucene.search.MatchAllDocsQuery; +import org.apache.lucene.search.MatchNoDocsQuery; +import org.apache.lucene.search.MultiTermQuery; +import org.apache.lucene.search.PrefixQuery; +import org.apache.lucene.search.Pruning; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.ScoreDoc; +import org.apache.lucene.search.Sort; +import org.apache.lucene.search.SortField; +import org.apache.lucene.search.TermQuery; +import org.apache.lucene.search.TopDocs; +import org.apache.lucene.search.TotalHitCountCollector; +import org.apache.lucene.search.TotalHits; +import org.apache.lucene.search.Weight; import org.apache.lucene.search.grouping.CollapseTopFieldDocs; import org.apache.lucene.search.join.BitSetProducer; import org.apache.lucene.search.join.ScoreMode; diff --git a/server/src/test/java/org/opensearch/search/query/QueryProfilePhaseTests.java b/server/src/test/java/org/opensearch/search/query/QueryProfilePhaseTests.java index 67c2ef1011966..9031ca837d984 100644 --- a/server/src/test/java/org/opensearch/search/query/QueryProfilePhaseTests.java +++ b/server/src/test/java/org/opensearch/search/query/QueryProfilePhaseTests.java @@ -24,8 +24,24 @@ import org.apache.lucene.index.Term; import org.apache.lucene.queries.spans.SpanNearQuery; import org.apache.lucene.queries.spans.SpanTermQuery; -import org.apache.lucene.search.*; import org.apache.lucene.search.BooleanClause.Occur; +import org.apache.lucene.search.BooleanQuery; +import org.apache.lucene.search.Collector; +import org.apache.lucene.search.FieldComparator; +import org.apache.lucene.search.FieldDoc; +import org.apache.lucene.search.FilterCollector; +import org.apache.lucene.search.FilterLeafCollector; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.LeafCollector; +import org.apache.lucene.search.MatchAllDocsQuery; +import org.apache.lucene.search.MatchNoDocsQuery; +import org.apache.lucene.search.Pruning; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.Sort; +import org.apache.lucene.search.SortField; +import org.apache.lucene.search.TermQuery; +import org.apache.lucene.search.TotalHits; +import org.apache.lucene.search.Weight; import org.apache.lucene.search.grouping.CollapseTopFieldDocs; import org.apache.lucene.store.Directory; import org.apache.lucene.tests.index.RandomIndexWriter; diff --git a/server/src/test/java/org/opensearch/search/searchafter/SearchAfterBuilderTests.java b/server/src/test/java/org/opensearch/search/searchafter/SearchAfterBuilderTests.java index 48c93809cfcd3..53c07674f5bd7 100644 --- a/server/src/test/java/org/opensearch/search/searchafter/SearchAfterBuilderTests.java +++ b/server/src/test/java/org/opensearch/search/searchafter/SearchAfterBuilderTests.java @@ -33,7 +33,11 @@ package org.opensearch.search.searchafter; import org.apache.lucene.document.LatLonDocValuesField; -import org.apache.lucene.search.*; +import org.apache.lucene.search.FieldComparator; +import org.apache.lucene.search.Pruning; +import org.apache.lucene.search.SortField; +import org.apache.lucene.search.SortedNumericSortField; +import org.apache.lucene.search.SortedSetSortField; import org.opensearch.common.geo.GeoPoint; import org.opensearch.common.util.BigArrays; import org.opensearch.common.xcontent.XContentFactory; From ed62ad166d007eaebbbf61b104b355baa41439c3 Mon Sep 17 00:00:00 2001 From: Marc Handalian Date: Mon, 4 Dec 2023 13:45:45 -0800 Subject: [PATCH 07/26] Fix javadoc in SortedWiderNumericSortField Signed-off-by: Marc Handalian --- .../org/opensearch/search/sort/SortedWiderNumericSortField.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/org/opensearch/search/sort/SortedWiderNumericSortField.java b/server/src/main/java/org/opensearch/search/sort/SortedWiderNumericSortField.java index cfd1fd712d162..10cc832fdb684 100644 --- a/server/src/main/java/org/opensearch/search/sort/SortedWiderNumericSortField.java +++ b/server/src/main/java/org/opensearch/search/sort/SortedWiderNumericSortField.java @@ -45,7 +45,7 @@ public SortedWiderNumericSortField(String field, Type type, boolean reverse) { * Creates and return a comparator, which always converts Numeric to double * and compare to support multi type comparison between numeric values * @param numHits number of top hits the queue will store - * @param pruning true if the comparator can skip documents via {@link + * @param pruning controls how the comparator skips documents via {@link * LeafFieldComparator#competitiveIterator()} * @return NumericComparator */ From 771f40c3ad4fa5de400e939e96294492e0b48fc4 Mon Sep 17 00:00:00 2001 From: Marc Handalian Date: Tue, 5 Dec 2023 16:36:12 -0800 Subject: [PATCH 08/26] Fix tests mocking now final IndexSearcher#getSlices method Signed-off-by: Marc Handalian --- .../search/internal/ContextIndexSearcher.java | 1 + .../AggregationProcessorTests.java | 38 ++++++++++++++++--- .../internal/ContextIndexSearcherTests.java | 4 +- 3 files changed, 36 insertions(+), 7 deletions(-) diff --git a/server/src/main/java/org/opensearch/search/internal/ContextIndexSearcher.java b/server/src/main/java/org/opensearch/search/internal/ContextIndexSearcher.java index b042f3cf41d61..403b0b545c113 100644 --- a/server/src/main/java/org/opensearch/search/internal/ContextIndexSearcher.java +++ b/server/src/main/java/org/opensearch/search/internal/ContextIndexSearcher.java @@ -102,6 +102,7 @@ public class ContextIndexSearcher extends IndexSearcher implements Releasable { * The interval at which we check for search cancellation when we cannot use * a {@link CancellableBulkScorer}. See {@link #intersectScorerAndBitSet}. */ + private static final int CHECK_CANCELLED_SCORER_INTERVAL = 1 << 11; private AggregatedDfs aggregatedDfs; diff --git a/server/src/test/java/org/opensearch/search/aggregations/AggregationProcessorTests.java b/server/src/test/java/org/opensearch/search/aggregations/AggregationProcessorTests.java index 1ae201666bc54..d68b0911d3d01 100644 --- a/server/src/test/java/org/opensearch/search/aggregations/AggregationProcessorTests.java +++ b/server/src/test/java/org/opensearch/search/aggregations/AggregationProcessorTests.java @@ -8,10 +8,13 @@ package org.opensearch.search.aggregations; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.search.Collector; import org.apache.lucene.search.CollectorManager; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Query; +import org.opensearch.index.engine.Engine; import org.opensearch.search.aggregations.bucket.global.GlobalAggregator; import org.opensearch.search.internal.ContextIndexSearcher; import org.opensearch.search.profile.query.CollectorResult; @@ -20,14 +23,17 @@ import java.util.ArrayList; import java.util.Collection; +import java.util.List; +import java.util.concurrent.ExecutorService; import org.mockito.ArgumentMatchers; import static org.mockito.ArgumentMatchers.nullable; +import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; public class AggregationProcessorTests extends AggregationSetupTests { private final AggregationProcessor testAggregationProcessor = new ConcurrentAggregationProcessor(); @@ -152,9 +158,31 @@ private void testPostProcessCommon( globalCollectors.add(context.queryCollectorManagers().get(GlobalAggCollectorManager.class).newCollector()); } } - final ContextIndexSearcher testSearcher = mock(ContextIndexSearcher.class); final IndexSearcher.LeafSlice[] slicesToReturn = new IndexSearcher.LeafSlice[numSlices]; - when(testSearcher.getSlices()).thenReturn(slicesToReturn); + + // Build a ContextIndexSearcher that stubs slices to return slicesToReturn. Slices is protected in IndexReader + // so this builds a real object. The DirectoryReader fetched to build the object is not used for any searches. + final DirectoryReader reader; + try (Engine.Searcher searcher = context.indexShard().acquireSearcher("test")) { + reader = searcher.getDirectoryReader(); + } + ContextIndexSearcher testSearcher = spy( + new ContextIndexSearcher( + reader, + IndexSearcher.getDefaultSimilarity(), + IndexSearcher.getDefaultQueryCache(), + IndexSearcher.getDefaultQueryCachingPolicy(), + randomBoolean(), + mock(ExecutorService.class), + context + ) { + @Override + protected LeafSlice[] slices(List leaves) { + return slicesToReturn; + } + } + ); + ((TestSearchContext) context).setSearcher(testSearcher); AggregationCollectorManager collectorManager; if (expectedNonGlobalAggsPerSlice > 0) { @@ -164,8 +192,8 @@ private void testPostProcessCommon( if (expectedGlobalAggs > 0) { collectorManager = (AggregationCollectorManager) context.queryCollectorManagers().get(GlobalAggCollectorManager.class); ReduceableSearchResult result = collectorManager.reduce(globalCollectors); - when(testSearcher.search(nullable(Query.class), ArgumentMatchers.>any())) - .thenReturn(result); + doReturn(result).when(testSearcher) + .search(nullable(Query.class), ArgumentMatchers.>any()); } assertTrue(context.queryResult().hasAggs()); if (withProfilers) { diff --git a/server/src/test/java/org/opensearch/search/internal/ContextIndexSearcherTests.java b/server/src/test/java/org/opensearch/search/internal/ContextIndexSearcherTests.java index b1f70dfce176c..b8966b76d0193 100644 --- a/server/src/test/java/org/opensearch/search/internal/ContextIndexSearcherTests.java +++ b/server/src/test/java/org/opensearch/search/internal/ContextIndexSearcherTests.java @@ -393,8 +393,8 @@ public void testGetSlicesWithNonNullExecutorButCSDisabled() throws Exception { null, searchContext ); - // Case 1: Verify getSlices return null when concurrent segment search is disabled - assertNull(searcher.getSlices()); + // Case 1: Verify getSlices returns not null when concurrent segment search is disabled + assertNotNull(searcher.getSlices()); // Case 2: Verify the slice count when custom max slice computation is used searcher = new ContextIndexSearcher( From 5a773ce7727be0271e74b459a02cc92e1ebe754a Mon Sep 17 00:00:00 2001 From: Marc Handalian Date: Tue, 5 Dec 2023 16:41:10 -0800 Subject: [PATCH 09/26] Bump CorrelationCodec to V_9_9_0 Signed-off-by: Marc Handalian --- .../core/index/codec/CorrelationCodecVersion.java | 8 ++++---- .../index/codec/correlation950/CorrelationCodec.java | 2 +- .../PerFieldCorrelationVectorsFormat.java | 10 +++++----- .../codec/correlation950/CorrelationCodecTests.java | 4 ++-- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/core/index/codec/CorrelationCodecVersion.java b/plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/core/index/codec/CorrelationCodecVersion.java index 5e2cb8bfbc03a..1ef6bd1d8f747 100644 --- a/plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/core/index/codec/CorrelationCodecVersion.java +++ b/plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/core/index/codec/CorrelationCodecVersion.java @@ -9,7 +9,7 @@ package org.opensearch.plugin.correlation.core.index.codec; import org.apache.lucene.codecs.Codec; -import org.apache.lucene.codecs.lucene95.Lucene95Codec; +import org.apache.lucene.codecs.lucene99.Lucene99Codec; import org.opensearch.index.mapper.MapperService; import org.opensearch.plugin.correlation.core.index.codec.correlation950.CorrelationCodec; import org.opensearch.plugin.correlation.core.index.codec.correlation950.PerFieldCorrelationVectorsFormat; @@ -24,15 +24,15 @@ * @opensearch.internal */ public enum CorrelationCodecVersion { - V_9_5_0( + V_9_9_0( "CorrelationCodec", - new Lucene95Codec(), + new Lucene99Codec(), new PerFieldCorrelationVectorsFormat(Optional.empty()), (userCodec, mapperService) -> new CorrelationCodec(userCodec, new PerFieldCorrelationVectorsFormat(Optional.of(mapperService))), CorrelationCodec::new ); - private static final CorrelationCodecVersion CURRENT = V_9_5_0; + private static final CorrelationCodecVersion CURRENT = V_9_9_0; private final String codecName; private final Codec defaultCodecDelegate; private final PerFieldCorrelationVectorsFormat perFieldCorrelationVectorsFormat; diff --git a/plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/core/index/codec/correlation950/CorrelationCodec.java b/plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/core/index/codec/correlation950/CorrelationCodec.java index f91ba429fbea9..6da71bda9ae5c 100644 --- a/plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/core/index/codec/correlation950/CorrelationCodec.java +++ b/plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/core/index/codec/correlation950/CorrelationCodec.java @@ -19,7 +19,7 @@ * @opensearch.internal */ public class CorrelationCodec extends FilterCodec { - private static final CorrelationCodecVersion VERSION = CorrelationCodecVersion.V_9_5_0; + private static final CorrelationCodecVersion VERSION = CorrelationCodecVersion.V_9_9_0; private final PerFieldCorrelationVectorsFormat perFieldCorrelationVectorsFormat; /** diff --git a/plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/core/index/codec/correlation950/PerFieldCorrelationVectorsFormat.java b/plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/core/index/codec/correlation950/PerFieldCorrelationVectorsFormat.java index f6862ecc17736..185267c53a231 100644 --- a/plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/core/index/codec/correlation950/PerFieldCorrelationVectorsFormat.java +++ b/plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/core/index/codec/correlation950/PerFieldCorrelationVectorsFormat.java @@ -8,7 +8,7 @@ package org.opensearch.plugin.correlation.core.index.codec.correlation950; -import org.apache.lucene.codecs.lucene95.Lucene95HnswVectorsFormat; +import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat; import org.opensearch.index.mapper.MapperService; import org.opensearch.plugin.correlation.core.index.codec.BasePerFieldCorrelationVectorsFormat; @@ -26,10 +26,10 @@ public class PerFieldCorrelationVectorsFormat extends BasePerFieldCorrelationVec public PerFieldCorrelationVectorsFormat(final Optional mapperService) { super( mapperService, - Lucene95HnswVectorsFormat.DEFAULT_MAX_CONN, - Lucene95HnswVectorsFormat.DEFAULT_BEAM_WIDTH, - Lucene95HnswVectorsFormat::new, - Lucene95HnswVectorsFormat::new + Lucene99HnswVectorsFormat.DEFAULT_MAX_CONN, + Lucene99HnswVectorsFormat.DEFAULT_BEAM_WIDTH, + Lucene99HnswVectorsFormat::new, + Lucene99HnswVectorsFormat::new ); } } diff --git a/plugins/events-correlation-engine/src/test/java/org/opensearch/plugin/correlation/core/index/codec/correlation950/CorrelationCodecTests.java b/plugins/events-correlation-engine/src/test/java/org/opensearch/plugin/correlation/core/index/codec/correlation950/CorrelationCodecTests.java index b93172537d419..df679fd7ca678 100644 --- a/plugins/events-correlation-engine/src/test/java/org/opensearch/plugin/correlation/core/index/codec/correlation950/CorrelationCodecTests.java +++ b/plugins/events-correlation-engine/src/test/java/org/opensearch/plugin/correlation/core/index/codec/correlation950/CorrelationCodecTests.java @@ -32,7 +32,7 @@ import static org.opensearch.plugin.correlation.core.index.codec.BasePerFieldCorrelationVectorsFormat.METHOD_PARAMETER_EF_CONSTRUCTION; import static org.opensearch.plugin.correlation.core.index.codec.BasePerFieldCorrelationVectorsFormat.METHOD_PARAMETER_M; -import static org.opensearch.plugin.correlation.core.index.codec.CorrelationCodecVersion.V_9_5_0; +import static org.opensearch.plugin.correlation.core.index.codec.CorrelationCodecVersion.V_9_9_0; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.spy; @@ -56,7 +56,7 @@ public void testCorrelationVectorIndex() throws Exception { Function perFieldCorrelationVectorsProvider = mapperService -> new PerFieldCorrelationVectorsFormat(Optional.of(mapperService)); Function correlationCodecProvider = (correlationVectorsFormat -> new CorrelationCodec( - V_9_5_0.getDefaultCodecDelegate(), + V_9_9_0.getDefaultCodecDelegate(), correlationVectorsFormat )); testCorrelationVectorIndex(correlationCodecProvider, perFieldCorrelationVectorsProvider); From b2546b249791a8246bc3e1c373227eb5c79b93bd Mon Sep 17 00:00:00 2001 From: Marc Handalian Date: Tue, 5 Dec 2023 16:42:58 -0800 Subject: [PATCH 10/26] update javadoc linking to old codec Signed-off-by: Marc Handalian --- server/src/main/java/org/opensearch/index/IndexModule.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/org/opensearch/index/IndexModule.java b/server/src/main/java/org/opensearch/index/IndexModule.java index 545623287bae8..6ac10a221d49e 100644 --- a/server/src/main/java/org/opensearch/index/IndexModule.java +++ b/server/src/main/java/org/opensearch/index/IndexModule.java @@ -161,7 +161,7 @@ public final class IndexModule { /** Which lucene file extensions to load with the mmap directory when using hybridfs store. This settings is ignored if {@link #INDEX_STORE_HYBRID_NIO_EXTENSIONS} is set. * This is an expert setting. - * @see Lucene File Extensions. + * @see Lucene File Extensions. * * @deprecated This setting will be removed in OpenSearch 3.x. Use {@link #INDEX_STORE_HYBRID_NIO_EXTENSIONS} instead. */ @@ -206,7 +206,7 @@ public Iterator> settings() { /** Which lucene file extensions to load with nio. All others will default to mmap. Takes precedence over {@link #INDEX_STORE_HYBRID_MMAP_EXTENSIONS}. * This is an expert setting. - * @see Lucene File Extensions. + * @see Lucene File Extensions. */ public static final Setting> INDEX_STORE_HYBRID_NIO_EXTENSIONS = Setting.listSetting( "index.store.hybrid.nio.extensions", From 1e04b1b9c45421ab457aacbda8dc38e5bcec632e Mon Sep 17 00:00:00 2001 From: Marc Handalian Date: Tue, 5 Dec 2023 16:48:31 -0800 Subject: [PATCH 11/26] update SHAs Signed-off-by: Marc Handalian --- libs/core/licenses/lucene-core-9.8.0.jar.sha1 | 1 - libs/core/licenses/lucene-core-9.9.0.jar.sha1 | 1 + .../lang-expression/licenses/lucene-expressions-9.8.0.jar.sha1 | 1 - .../lang-expression/licenses/lucene-expressions-9.9.0.jar.sha1 | 1 + plugins/analysis-icu/licenses/lucene-analysis-icu-9.8.0.jar.sha1 | 1 - plugins/analysis-icu/licenses/lucene-analysis-icu-9.9.0.jar.sha1 | 1 + .../licenses/lucene-analysis-kuromoji-9.8.0.jar.sha1 | 1 - .../licenses/lucene-analysis-kuromoji-9.9.0.jar.sha1 | 1 + .../analysis-nori/licenses/lucene-analysis-nori-9.8.0.jar.sha1 | 1 - .../analysis-nori/licenses/lucene-analysis-nori-9.9.0.jar.sha1 | 1 + .../licenses/lucene-analysis-phonetic-9.8.0.jar.sha1 | 1 - .../licenses/lucene-analysis-phonetic-9.9.0.jar.sha1 | 1 + .../licenses/lucene-analysis-smartcn-9.8.0.jar.sha1 | 1 - .../licenses/lucene-analysis-smartcn-9.9.0.jar.sha1 | 1 + .../licenses/lucene-analysis-stempel-9.8.0.jar.sha1 | 1 - .../licenses/lucene-analysis-stempel-9.9.0.jar.sha1 | 1 + .../licenses/lucene-analysis-morfologik-9.8.0.jar.sha1 | 1 - .../licenses/lucene-analysis-morfologik-9.9.0.jar.sha1 | 1 + server/licenses/lucene-analysis-common-9.8.0.jar.sha1 | 1 - server/licenses/lucene-analysis-common-9.9.0.jar.sha1 | 1 + server/licenses/lucene-backward-codecs-9.8.0.jar.sha1 | 1 - server/licenses/lucene-backward-codecs-9.9.0.jar.sha1 | 1 + server/licenses/lucene-core-9.8.0.jar.sha1 | 1 - server/licenses/lucene-core-9.9.0.jar.sha1 | 1 + server/licenses/lucene-grouping-9.8.0.jar.sha1 | 1 - server/licenses/lucene-grouping-9.9.0.jar.sha1 | 1 + server/licenses/lucene-highlighter-9.8.0.jar.sha1 | 1 - server/licenses/lucene-highlighter-9.9.0.jar.sha1 | 1 + server/licenses/lucene-join-9.8.0.jar.sha1 | 1 - server/licenses/lucene-join-9.9.0.jar.sha1 | 1 + server/licenses/lucene-memory-9.8.0.jar.sha1 | 1 - server/licenses/lucene-memory-9.9.0.jar.sha1 | 1 + server/licenses/lucene-misc-9.8.0.jar.sha1 | 1 - server/licenses/lucene-misc-9.9.0.jar.sha1 | 1 + server/licenses/lucene-queries-9.8.0.jar.sha1 | 1 - server/licenses/lucene-queries-9.9.0.jar.sha1 | 1 + server/licenses/lucene-queryparser-9.8.0.jar.sha1 | 1 - server/licenses/lucene-queryparser-9.9.0.jar.sha1 | 1 + server/licenses/lucene-sandbox-9.8.0.jar.sha1 | 1 - server/licenses/lucene-sandbox-9.9.0.jar.sha1 | 1 + server/licenses/lucene-spatial-extras-9.8.0.jar.sha1 | 1 - server/licenses/lucene-spatial-extras-9.9.0.jar.sha1 | 1 + server/licenses/lucene-spatial3d-9.8.0.jar.sha1 | 1 - server/licenses/lucene-spatial3d-9.9.0.jar.sha1 | 1 + server/licenses/lucene-suggest-9.8.0.jar.sha1 | 1 - server/licenses/lucene-suggest-9.9.0.jar.sha1 | 1 + 46 files changed, 23 insertions(+), 23 deletions(-) delete mode 100644 libs/core/licenses/lucene-core-9.8.0.jar.sha1 create mode 100644 libs/core/licenses/lucene-core-9.9.0.jar.sha1 delete mode 100644 modules/lang-expression/licenses/lucene-expressions-9.8.0.jar.sha1 create mode 100644 modules/lang-expression/licenses/lucene-expressions-9.9.0.jar.sha1 delete mode 100644 plugins/analysis-icu/licenses/lucene-analysis-icu-9.8.0.jar.sha1 create mode 100644 plugins/analysis-icu/licenses/lucene-analysis-icu-9.9.0.jar.sha1 delete mode 100644 plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.8.0.jar.sha1 create mode 100644 plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.9.0.jar.sha1 delete mode 100644 plugins/analysis-nori/licenses/lucene-analysis-nori-9.8.0.jar.sha1 create mode 100644 plugins/analysis-nori/licenses/lucene-analysis-nori-9.9.0.jar.sha1 delete mode 100644 plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.8.0.jar.sha1 create mode 100644 plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.9.0.jar.sha1 delete mode 100644 plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.8.0.jar.sha1 create mode 100644 plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.9.0.jar.sha1 delete mode 100644 plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.8.0.jar.sha1 create mode 100644 plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.9.0.jar.sha1 delete mode 100644 plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.8.0.jar.sha1 create mode 100644 plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.9.0.jar.sha1 delete mode 100644 server/licenses/lucene-analysis-common-9.8.0.jar.sha1 create mode 100644 server/licenses/lucene-analysis-common-9.9.0.jar.sha1 delete mode 100644 server/licenses/lucene-backward-codecs-9.8.0.jar.sha1 create mode 100644 server/licenses/lucene-backward-codecs-9.9.0.jar.sha1 delete mode 100644 server/licenses/lucene-core-9.8.0.jar.sha1 create mode 100644 server/licenses/lucene-core-9.9.0.jar.sha1 delete mode 100644 server/licenses/lucene-grouping-9.8.0.jar.sha1 create mode 100644 server/licenses/lucene-grouping-9.9.0.jar.sha1 delete mode 100644 server/licenses/lucene-highlighter-9.8.0.jar.sha1 create mode 100644 server/licenses/lucene-highlighter-9.9.0.jar.sha1 delete mode 100644 server/licenses/lucene-join-9.8.0.jar.sha1 create mode 100644 server/licenses/lucene-join-9.9.0.jar.sha1 delete mode 100644 server/licenses/lucene-memory-9.8.0.jar.sha1 create mode 100644 server/licenses/lucene-memory-9.9.0.jar.sha1 delete mode 100644 server/licenses/lucene-misc-9.8.0.jar.sha1 create mode 100644 server/licenses/lucene-misc-9.9.0.jar.sha1 delete mode 100644 server/licenses/lucene-queries-9.8.0.jar.sha1 create mode 100644 server/licenses/lucene-queries-9.9.0.jar.sha1 delete mode 100644 server/licenses/lucene-queryparser-9.8.0.jar.sha1 create mode 100644 server/licenses/lucene-queryparser-9.9.0.jar.sha1 delete mode 100644 server/licenses/lucene-sandbox-9.8.0.jar.sha1 create mode 100644 server/licenses/lucene-sandbox-9.9.0.jar.sha1 delete mode 100644 server/licenses/lucene-spatial-extras-9.8.0.jar.sha1 create mode 100644 server/licenses/lucene-spatial-extras-9.9.0.jar.sha1 delete mode 100644 server/licenses/lucene-spatial3d-9.8.0.jar.sha1 create mode 100644 server/licenses/lucene-spatial3d-9.9.0.jar.sha1 delete mode 100644 server/licenses/lucene-suggest-9.8.0.jar.sha1 create mode 100644 server/licenses/lucene-suggest-9.9.0.jar.sha1 diff --git a/libs/core/licenses/lucene-core-9.8.0.jar.sha1 b/libs/core/licenses/lucene-core-9.8.0.jar.sha1 deleted file mode 100644 index f9a3e2f3cbee6..0000000000000 --- a/libs/core/licenses/lucene-core-9.8.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -5e8421c5f8573bcf22e9265fc7e19469545a775a \ No newline at end of file diff --git a/libs/core/licenses/lucene-core-9.9.0.jar.sha1 b/libs/core/licenses/lucene-core-9.9.0.jar.sha1 new file mode 100644 index 0000000000000..b6df006f2f947 --- /dev/null +++ b/libs/core/licenses/lucene-core-9.9.0.jar.sha1 @@ -0,0 +1 @@ +2f0876192ee21f2b6f85b9435c99b9f0e04f3fcc \ No newline at end of file diff --git a/modules/lang-expression/licenses/lucene-expressions-9.8.0.jar.sha1 b/modules/lang-expression/licenses/lucene-expressions-9.8.0.jar.sha1 deleted file mode 100644 index 892865a017f48..0000000000000 --- a/modules/lang-expression/licenses/lucene-expressions-9.8.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -7725476acfcb9bdfeff1b813ce15c39c6b857dc2 \ No newline at end of file diff --git a/modules/lang-expression/licenses/lucene-expressions-9.9.0.jar.sha1 b/modules/lang-expression/licenses/lucene-expressions-9.9.0.jar.sha1 new file mode 100644 index 0000000000000..5c25ba1e48cfd --- /dev/null +++ b/modules/lang-expression/licenses/lucene-expressions-9.9.0.jar.sha1 @@ -0,0 +1 @@ +a36fc5b5087b704852a5f228c38cb3a47fea8c36 \ No newline at end of file diff --git a/plugins/analysis-icu/licenses/lucene-analysis-icu-9.8.0.jar.sha1 b/plugins/analysis-icu/licenses/lucene-analysis-icu-9.8.0.jar.sha1 deleted file mode 100644 index ef410899981ca..0000000000000 --- a/plugins/analysis-icu/licenses/lucene-analysis-icu-9.8.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -7133d34e92770f59eb28686f4d511b9f3f32e970 \ No newline at end of file diff --git a/plugins/analysis-icu/licenses/lucene-analysis-icu-9.9.0.jar.sha1 b/plugins/analysis-icu/licenses/lucene-analysis-icu-9.9.0.jar.sha1 new file mode 100644 index 0000000000000..85793f76f884b --- /dev/null +++ b/plugins/analysis-icu/licenses/lucene-analysis-icu-9.9.0.jar.sha1 @@ -0,0 +1 @@ +fa8ddcddd8ad086d123dbcf1b2804683a14f252f \ No newline at end of file diff --git a/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.8.0.jar.sha1 b/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.8.0.jar.sha1 deleted file mode 100644 index 46b83c9e40b3a..0000000000000 --- a/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.8.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -be44282e1f6b91a0650fcceb558053d6bdd4863d \ No newline at end of file diff --git a/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.9.0.jar.sha1 b/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.9.0.jar.sha1 new file mode 100644 index 0000000000000..a95fb85beed94 --- /dev/null +++ b/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.9.0.jar.sha1 @@ -0,0 +1 @@ +70903a94800e3f09898f279987a591e1443e7f11 \ No newline at end of file diff --git a/plugins/analysis-nori/licenses/lucene-analysis-nori-9.8.0.jar.sha1 b/plugins/analysis-nori/licenses/lucene-analysis-nori-9.8.0.jar.sha1 deleted file mode 100644 index 36664695a7818..0000000000000 --- a/plugins/analysis-nori/licenses/lucene-analysis-nori-9.8.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -bd1f80d33346f7e588685484ef29a304db5190e4 \ No newline at end of file diff --git a/plugins/analysis-nori/licenses/lucene-analysis-nori-9.9.0.jar.sha1 b/plugins/analysis-nori/licenses/lucene-analysis-nori-9.9.0.jar.sha1 new file mode 100644 index 0000000000000..e9b373a9c0ae3 --- /dev/null +++ b/plugins/analysis-nori/licenses/lucene-analysis-nori-9.9.0.jar.sha1 @@ -0,0 +1 @@ +521d3e833ffb371dd4712b8241239cae031507e5 \ No newline at end of file diff --git a/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.8.0.jar.sha1 b/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.8.0.jar.sha1 deleted file mode 100644 index 003ccdf8b0727..0000000000000 --- a/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.8.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -b9ffdc7a52d2087ecb03318ec06305b480cdfe82 \ No newline at end of file diff --git a/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.9.0.jar.sha1 b/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.9.0.jar.sha1 new file mode 100644 index 0000000000000..1d17b615f88e7 --- /dev/null +++ b/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.9.0.jar.sha1 @@ -0,0 +1 @@ +40d1fb2e36efc071d1b96958f23bfe3d5e3d5758 \ No newline at end of file diff --git a/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.8.0.jar.sha1 b/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.8.0.jar.sha1 deleted file mode 100644 index e22eaa474016f..0000000000000 --- a/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.8.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -f73e2007b133fb699e517ef13b4952844f0150d8 \ No newline at end of file diff --git a/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.9.0.jar.sha1 b/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.9.0.jar.sha1 new file mode 100644 index 0000000000000..b290b2015a7c0 --- /dev/null +++ b/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.9.0.jar.sha1 @@ -0,0 +1 @@ +4a4f7ba40b82f108a2c3d150624cb46f8281d212 \ No newline at end of file diff --git a/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.8.0.jar.sha1 b/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.8.0.jar.sha1 deleted file mode 100644 index 1ebe42a2a2f56..0000000000000 --- a/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.8.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -2c09cbc021a8f81a01600a1d2a999361e70f7aed \ No newline at end of file diff --git a/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.9.0.jar.sha1 b/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.9.0.jar.sha1 new file mode 100644 index 0000000000000..1c5f02aa45451 --- /dev/null +++ b/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.9.0.jar.sha1 @@ -0,0 +1 @@ +067dd0df852c5df001d77505dd048a4e90a4b365 \ No newline at end of file diff --git a/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.8.0.jar.sha1 b/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.8.0.jar.sha1 deleted file mode 100644 index 3c4523d45c0f5..0000000000000 --- a/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.8.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -b054f2c7b11fc7c5601b4c3cdf18aa7508612898 \ No newline at end of file diff --git a/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.9.0.jar.sha1 b/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.9.0.jar.sha1 new file mode 100644 index 0000000000000..9e3d15356d69a --- /dev/null +++ b/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.9.0.jar.sha1 @@ -0,0 +1 @@ +ac5225e8a5fa4e1d9784b46f64942b75b2716b26 \ No newline at end of file diff --git a/server/licenses/lucene-analysis-common-9.8.0.jar.sha1 b/server/licenses/lucene-analysis-common-9.8.0.jar.sha1 deleted file mode 100644 index 6ad304fa52c12..0000000000000 --- a/server/licenses/lucene-analysis-common-9.8.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -36f0363325ca7bf62c180160d1ed5165c7c37795 \ No newline at end of file diff --git a/server/licenses/lucene-analysis-common-9.9.0.jar.sha1 b/server/licenses/lucene-analysis-common-9.9.0.jar.sha1 new file mode 100644 index 0000000000000..52c91a5a7babf --- /dev/null +++ b/server/licenses/lucene-analysis-common-9.9.0.jar.sha1 @@ -0,0 +1 @@ +445448c09a0c8ba40f81df7d86ac97303ce0b19e \ No newline at end of file diff --git a/server/licenses/lucene-backward-codecs-9.8.0.jar.sha1 b/server/licenses/lucene-backward-codecs-9.8.0.jar.sha1 deleted file mode 100644 index f104c4207d390..0000000000000 --- a/server/licenses/lucene-backward-codecs-9.8.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -e98fb408028f40170e6d87c16422bfdc0bb2e392 \ No newline at end of file diff --git a/server/licenses/lucene-backward-codecs-9.9.0.jar.sha1 b/server/licenses/lucene-backward-codecs-9.9.0.jar.sha1 new file mode 100644 index 0000000000000..62cf8eb86ff5f --- /dev/null +++ b/server/licenses/lucene-backward-codecs-9.9.0.jar.sha1 @@ -0,0 +1 @@ +fd7a9d02972955bb85896ded80e9bf5ffa173b2b \ No newline at end of file diff --git a/server/licenses/lucene-core-9.8.0.jar.sha1 b/server/licenses/lucene-core-9.8.0.jar.sha1 deleted file mode 100644 index f9a3e2f3cbee6..0000000000000 --- a/server/licenses/lucene-core-9.8.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -5e8421c5f8573bcf22e9265fc7e19469545a775a \ No newline at end of file diff --git a/server/licenses/lucene-core-9.9.0.jar.sha1 b/server/licenses/lucene-core-9.9.0.jar.sha1 new file mode 100644 index 0000000000000..b6df006f2f947 --- /dev/null +++ b/server/licenses/lucene-core-9.9.0.jar.sha1 @@ -0,0 +1 @@ +2f0876192ee21f2b6f85b9435c99b9f0e04f3fcc \ No newline at end of file diff --git a/server/licenses/lucene-grouping-9.8.0.jar.sha1 b/server/licenses/lucene-grouping-9.8.0.jar.sha1 deleted file mode 100644 index ab132121b2edc..0000000000000 --- a/server/licenses/lucene-grouping-9.8.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -d39184518351178c404ed9669fc6cb6111f2288d \ No newline at end of file diff --git a/server/licenses/lucene-grouping-9.9.0.jar.sha1 b/server/licenses/lucene-grouping-9.9.0.jar.sha1 new file mode 100644 index 0000000000000..4be6aae4c8be1 --- /dev/null +++ b/server/licenses/lucene-grouping-9.9.0.jar.sha1 @@ -0,0 +1 @@ +becfa739de2d3e29c129ed0347890a2dcab24b9b \ No newline at end of file diff --git a/server/licenses/lucene-highlighter-9.8.0.jar.sha1 b/server/licenses/lucene-highlighter-9.8.0.jar.sha1 deleted file mode 100644 index c7cb678fb7b72..0000000000000 --- a/server/licenses/lucene-highlighter-9.8.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -1ac38c8278dbd63dfab30744a41dd955a415a31c \ No newline at end of file diff --git a/server/licenses/lucene-highlighter-9.9.0.jar.sha1 b/server/licenses/lucene-highlighter-9.9.0.jar.sha1 new file mode 100644 index 0000000000000..6addb610c1b28 --- /dev/null +++ b/server/licenses/lucene-highlighter-9.9.0.jar.sha1 @@ -0,0 +1 @@ +fc7392800a3baff95efa592b1dca0df047e862e5 \ No newline at end of file diff --git a/server/licenses/lucene-join-9.8.0.jar.sha1 b/server/licenses/lucene-join-9.8.0.jar.sha1 deleted file mode 100644 index 2b6cb8af4faf6..0000000000000 --- a/server/licenses/lucene-join-9.8.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -3d64fc57bb6e718d906413a9f73c713e6d4d8bb0 \ No newline at end of file diff --git a/server/licenses/lucene-join-9.9.0.jar.sha1 b/server/licenses/lucene-join-9.9.0.jar.sha1 new file mode 100644 index 0000000000000..8194e3e50a065 --- /dev/null +++ b/server/licenses/lucene-join-9.9.0.jar.sha1 @@ -0,0 +1 @@ +58e2ff4e9dbb7bad804f08a1f482f08b72676421 \ No newline at end of file diff --git a/server/licenses/lucene-memory-9.8.0.jar.sha1 b/server/licenses/lucene-memory-9.8.0.jar.sha1 deleted file mode 100644 index 5fdfee401dd0a..0000000000000 --- a/server/licenses/lucene-memory-9.8.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -5283ac71d6ccecb5e00c7b52df2faec012f2625a \ No newline at end of file diff --git a/server/licenses/lucene-memory-9.9.0.jar.sha1 b/server/licenses/lucene-memory-9.9.0.jar.sha1 new file mode 100644 index 0000000000000..5d1c37d721951 --- /dev/null +++ b/server/licenses/lucene-memory-9.9.0.jar.sha1 @@ -0,0 +1 @@ +266c6dc593217f606e06903d15d20aaa502d3ad2 \ No newline at end of file diff --git a/server/licenses/lucene-misc-9.8.0.jar.sha1 b/server/licenses/lucene-misc-9.8.0.jar.sha1 deleted file mode 100644 index cf815cba15862..0000000000000 --- a/server/licenses/lucene-misc-9.8.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -9a57b049cf51a5e9c9c1909c420f645f1b6f9a54 \ No newline at end of file diff --git a/server/licenses/lucene-misc-9.9.0.jar.sha1 b/server/licenses/lucene-misc-9.9.0.jar.sha1 new file mode 100644 index 0000000000000..1d892105059e8 --- /dev/null +++ b/server/licenses/lucene-misc-9.9.0.jar.sha1 @@ -0,0 +1 @@ +873e4beb02f028a41c981a86e04c6d86a4e4cd68 \ No newline at end of file diff --git a/server/licenses/lucene-queries-9.8.0.jar.sha1 b/server/licenses/lucene-queries-9.8.0.jar.sha1 deleted file mode 100644 index 09f369ef18e12..0000000000000 --- a/server/licenses/lucene-queries-9.8.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -628db4ef46f1c6a05145bdac1d1bc4ace6341b13 \ No newline at end of file diff --git a/server/licenses/lucene-queries-9.9.0.jar.sha1 b/server/licenses/lucene-queries-9.9.0.jar.sha1 new file mode 100644 index 0000000000000..1d93435a9af2a --- /dev/null +++ b/server/licenses/lucene-queries-9.9.0.jar.sha1 @@ -0,0 +1 @@ +298606babedbd3a67b7539fd1f84ccca7140c4de \ No newline at end of file diff --git a/server/licenses/lucene-queryparser-9.8.0.jar.sha1 b/server/licenses/lucene-queryparser-9.8.0.jar.sha1 deleted file mode 100644 index 2a42a8956b18b..0000000000000 --- a/server/licenses/lucene-queryparser-9.8.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -982faf2bfa55542bf57fbadef54c19ac00f57cae \ No newline at end of file diff --git a/server/licenses/lucene-queryparser-9.9.0.jar.sha1 b/server/licenses/lucene-queryparser-9.9.0.jar.sha1 new file mode 100644 index 0000000000000..2fe4c83bf4a1a --- /dev/null +++ b/server/licenses/lucene-queryparser-9.9.0.jar.sha1 @@ -0,0 +1 @@ +1968cd1b3ccaff22d30ffa5e5e54ec51b4a4e66e \ No newline at end of file diff --git a/server/licenses/lucene-sandbox-9.8.0.jar.sha1 b/server/licenses/lucene-sandbox-9.8.0.jar.sha1 deleted file mode 100644 index 64a0b07f72d29..0000000000000 --- a/server/licenses/lucene-sandbox-9.8.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -06493dbd14d02537716822254866a94458f4d842 \ No newline at end of file diff --git a/server/licenses/lucene-sandbox-9.9.0.jar.sha1 b/server/licenses/lucene-sandbox-9.9.0.jar.sha1 new file mode 100644 index 0000000000000..349b90f7a259a --- /dev/null +++ b/server/licenses/lucene-sandbox-9.9.0.jar.sha1 @@ -0,0 +1 @@ +25472d0b679ba12ce34dfe925b7541c8b7d74025 \ No newline at end of file diff --git a/server/licenses/lucene-spatial-extras-9.8.0.jar.sha1 b/server/licenses/lucene-spatial-extras-9.8.0.jar.sha1 deleted file mode 100644 index d1bcb0581435c..0000000000000 --- a/server/licenses/lucene-spatial-extras-9.8.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -9d9a731822ad6eefa1ba288a0c158d478522f165 \ No newline at end of file diff --git a/server/licenses/lucene-spatial-extras-9.9.0.jar.sha1 b/server/licenses/lucene-spatial-extras-9.9.0.jar.sha1 new file mode 100644 index 0000000000000..a801a810e10c6 --- /dev/null +++ b/server/licenses/lucene-spatial-extras-9.9.0.jar.sha1 @@ -0,0 +1 @@ +37b89d9250edf4f2b1940395c053c66869b4449d \ No newline at end of file diff --git a/server/licenses/lucene-spatial3d-9.8.0.jar.sha1 b/server/licenses/lucene-spatial3d-9.8.0.jar.sha1 deleted file mode 100644 index d17459cc569a9..0000000000000 --- a/server/licenses/lucene-spatial3d-9.8.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -ce752a52b2d4eac90633c7df7982e29504f99e76 \ No newline at end of file diff --git a/server/licenses/lucene-spatial3d-9.9.0.jar.sha1 b/server/licenses/lucene-spatial3d-9.9.0.jar.sha1 new file mode 100644 index 0000000000000..702b20318890a --- /dev/null +++ b/server/licenses/lucene-spatial3d-9.9.0.jar.sha1 @@ -0,0 +1 @@ +e51e7d3ca30c1789b7779b54afce4ab140830718 \ No newline at end of file diff --git a/server/licenses/lucene-suggest-9.8.0.jar.sha1 b/server/licenses/lucene-suggest-9.8.0.jar.sha1 deleted file mode 100644 index ff47b87672d2c..0000000000000 --- a/server/licenses/lucene-suggest-9.8.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -f977f96f2093b7fddea6b67caa2e1c5b10edebf6 \ No newline at end of file diff --git a/server/licenses/lucene-suggest-9.9.0.jar.sha1 b/server/licenses/lucene-suggest-9.9.0.jar.sha1 new file mode 100644 index 0000000000000..016eb9add1ea3 --- /dev/null +++ b/server/licenses/lucene-suggest-9.9.0.jar.sha1 @@ -0,0 +1 @@ +400c160b7fef8a4d142aa7cfc18f98469a5aec24 \ No newline at end of file From 77945072ac4775795b96b05070dbbc00f907629a Mon Sep 17 00:00:00 2001 From: Marc Handalian Date: Tue, 5 Dec 2023 17:20:40 -0800 Subject: [PATCH 12/26] rename shutdownExecutorService in tests that now conflict with LuceneTestCase method of the same name Signed-off-by: Marc Handalian --- .../http/netty4/Netty4HttpPipeliningHandlerTests.java | 4 ++-- .../opensearch/http/nio/NioHttpPipeliningHandlerTests.java | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/modules/transport-netty4/src/test/java/org/opensearch/http/netty4/Netty4HttpPipeliningHandlerTests.java b/modules/transport-netty4/src/test/java/org/opensearch/http/netty4/Netty4HttpPipeliningHandlerTests.java index 743b51e979fb7..99d576bed01c7 100644 --- a/modules/transport-netty4/src/test/java/org/opensearch/http/netty4/Netty4HttpPipeliningHandlerTests.java +++ b/modules/transport-netty4/src/test/java/org/opensearch/http/netty4/Netty4HttpPipeliningHandlerTests.java @@ -79,7 +79,7 @@ public class Netty4HttpPipeliningHandlerTests extends OpenSearchTestCase { @After public void tearDown() throws Exception { waitingRequests.keySet().forEach(this::finishRequest); - shutdownExecutorService(); + shutdownExecutorServices(); super.tearDown(); } @@ -88,7 +88,7 @@ private CountDownLatch finishRequest(String url) { return finishingRequests.get(url); } - private void shutdownExecutorService() throws InterruptedException { + private void shutdownExecutorServices() throws InterruptedException { if (!handlerService.isShutdown()) { handlerService.shutdown(); handlerService.awaitTermination(10, TimeUnit.SECONDS); diff --git a/plugins/transport-nio/src/test/java/org/opensearch/http/nio/NioHttpPipeliningHandlerTests.java b/plugins/transport-nio/src/test/java/org/opensearch/http/nio/NioHttpPipeliningHandlerTests.java index 46cf6ae708d1c..d0c0406bd7774 100644 --- a/plugins/transport-nio/src/test/java/org/opensearch/http/nio/NioHttpPipeliningHandlerTests.java +++ b/plugins/transport-nio/src/test/java/org/opensearch/http/nio/NioHttpPipeliningHandlerTests.java @@ -80,7 +80,7 @@ public class NioHttpPipeliningHandlerTests extends OpenSearchTestCase { @After public void cleanup() throws Exception { waitingRequests.keySet().forEach(this::finishRequest); - shutdownExecutorService(); + shutdownExecutorServices(); } private CountDownLatch finishRequest(String url) { @@ -88,7 +88,7 @@ private CountDownLatch finishRequest(String url) { return finishingRequests.get(url); } - private void shutdownExecutorService() throws InterruptedException { + private void shutdownExecutorServices() throws InterruptedException { if (!handlerService.isShutdown()) { handlerService.shutdown(); handlerService.awaitTermination(10, TimeUnit.SECONDS); From 410cb69b9666fb2f16c6fd6992912543f130b0d4 Mon Sep 17 00:00:00 2001 From: Marc Handalian Date: Tue, 5 Dec 2023 17:21:20 -0800 Subject: [PATCH 13/26] more toString test fixes Signed-off-by: Marc Handalian --- .../mapper/ScaledFloatFieldTypeTests.java | 32 +++++++++++-------- 1 file changed, 19 insertions(+), 13 deletions(-) diff --git a/modules/mapper-extras/src/test/java/org/opensearch/index/mapper/ScaledFloatFieldTypeTests.java b/modules/mapper-extras/src/test/java/org/opensearch/index/mapper/ScaledFloatFieldTypeTests.java index be12c49321b87..1d46298caa757 100644 --- a/modules/mapper-extras/src/test/java/org/opensearch/index/mapper/ScaledFloatFieldTypeTests.java +++ b/modules/mapper-extras/src/test/java/org/opensearch/index/mapper/ScaledFloatFieldTypeTests.java @@ -39,6 +39,7 @@ import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.search.IndexOrDocValuesQuery; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Query; import org.apache.lucene.store.Directory; @@ -122,35 +123,40 @@ public void testRangeQuery() throws IOException { public void testRoundsUpperBoundCorrectly() { ScaledFloatFieldMapper.ScaledFloatFieldType ft = new ScaledFloatFieldMapper.ScaledFloatFieldType("scaled_float", 100); Query scaledFloatQ = ft.rangeQuery(null, 0.1, true, false, MOCK_QSC); - assertEquals("scaled_float:[-9223372036854775808 TO 9]", scaledFloatQ.toString()); + assertEquals("scaled_float:[-9223372036854775808 TO 9]", getQueryString(scaledFloatQ)); scaledFloatQ = ft.rangeQuery(null, 0.1, true, true, MOCK_QSC); - assertEquals("scaled_float:[-9223372036854775808 TO 10]", scaledFloatQ.toString()); + assertEquals("scaled_float:[-9223372036854775808 TO 10]", getQueryString(scaledFloatQ)); scaledFloatQ = ft.rangeQuery(null, 0.095, true, false, MOCK_QSC); - assertEquals("scaled_float:[-9223372036854775808 TO 9]", scaledFloatQ.toString()); + assertEquals("scaled_float:[-9223372036854775808 TO 9]", getQueryString(scaledFloatQ)); scaledFloatQ = ft.rangeQuery(null, 0.095, true, true, MOCK_QSC); - assertEquals("scaled_float:[-9223372036854775808 TO 9]", scaledFloatQ.toString()); + assertEquals("scaled_float:[-9223372036854775808 TO 9]", getQueryString(scaledFloatQ)); scaledFloatQ = ft.rangeQuery(null, 0.105, true, false, MOCK_QSC); - assertEquals("scaled_float:[-9223372036854775808 TO 10]", scaledFloatQ.toString()); + assertEquals("scaled_float:[-9223372036854775808 TO 10]", getQueryString(scaledFloatQ)); scaledFloatQ = ft.rangeQuery(null, 0.105, true, true, MOCK_QSC); - assertEquals("scaled_float:[-9223372036854775808 TO 10]", scaledFloatQ.toString()); + assertEquals("scaled_float:[-9223372036854775808 TO 10]", getQueryString(scaledFloatQ)); scaledFloatQ = ft.rangeQuery(null, 79.99, true, true, MOCK_QSC); - assertEquals("scaled_float:[-9223372036854775808 TO 7999]", scaledFloatQ.toString()); + assertEquals("scaled_float:[-9223372036854775808 TO 7999]", getQueryString(scaledFloatQ)); } public void testRoundsLowerBoundCorrectly() { ScaledFloatFieldMapper.ScaledFloatFieldType ft = new ScaledFloatFieldMapper.ScaledFloatFieldType("scaled_float", 100); Query scaledFloatQ = ft.rangeQuery(-0.1, null, false, true, MOCK_QSC); - assertEquals("scaled_float:[-9 TO 9223372036854775807]", scaledFloatQ.toString()); + assertEquals("scaled_float:[-9 TO 9223372036854775807]", getQueryString(scaledFloatQ)); scaledFloatQ = ft.rangeQuery(-0.1, null, true, true, MOCK_QSC); - assertEquals("scaled_float:[-10 TO 9223372036854775807]", scaledFloatQ.toString()); + assertEquals("scaled_float:[-10 TO 9223372036854775807]", getQueryString(scaledFloatQ)); scaledFloatQ = ft.rangeQuery(-0.095, null, false, true, MOCK_QSC); - assertEquals("scaled_float:[-9 TO 9223372036854775807]", scaledFloatQ.toString()); + assertEquals("scaled_float:[-9 TO 9223372036854775807]", getQueryString(scaledFloatQ)); scaledFloatQ = ft.rangeQuery(-0.095, null, true, true, MOCK_QSC); - assertEquals("scaled_float:[-9 TO 9223372036854775807]", scaledFloatQ.toString()); + assertEquals("scaled_float:[-9 TO 9223372036854775807]", getQueryString(scaledFloatQ)); scaledFloatQ = ft.rangeQuery(-0.105, null, false, true, MOCK_QSC); - assertEquals("scaled_float:[-10 TO 9223372036854775807]", scaledFloatQ.toString()); + assertEquals("scaled_float:[-10 TO 9223372036854775807]", getQueryString(scaledFloatQ)); scaledFloatQ = ft.rangeQuery(-0.105, null, true, true, MOCK_QSC); - assertEquals("scaled_float:[-10 TO 9223372036854775807]", scaledFloatQ.toString()); + assertEquals("scaled_float:[-10 TO 9223372036854775807]", getQueryString(scaledFloatQ)); + } + + private String getQueryString(Query query) { + assertTrue(query instanceof IndexOrDocValuesQuery); + return ((IndexOrDocValuesQuery) query).getIndexQuery().toString(); } public void testValueForSearch() { From 7da8b76b0e73386a0e2baeb5df02f0f42f7258c1 Mon Sep 17 00:00:00 2001 From: Marc Handalian Date: Tue, 5 Dec 2023 17:21:31 -0800 Subject: [PATCH 14/26] Fix import path of CSVUtil Signed-off-by: Marc Handalian --- .../org/opensearch/index/analysis/KuromojiTokenizerFactory.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/analysis-kuromoji/src/main/java/org/opensearch/index/analysis/KuromojiTokenizerFactory.java b/plugins/analysis-kuromoji/src/main/java/org/opensearch/index/analysis/KuromojiTokenizerFactory.java index 2939711f6f7e1..76b109932e642 100644 --- a/plugins/analysis-kuromoji/src/main/java/org/opensearch/index/analysis/KuromojiTokenizerFactory.java +++ b/plugins/analysis-kuromoji/src/main/java/org/opensearch/index/analysis/KuromojiTokenizerFactory.java @@ -38,7 +38,7 @@ import org.apache.lucene.analysis.ja.JapaneseTokenizer; import org.apache.lucene.analysis.ja.JapaneseTokenizer.Mode; import org.apache.lucene.analysis.ja.dict.UserDictionary; -import org.apache.lucene.analysis.ja.util.CSVUtil; +import org.apache.lucene.analysis.util.CSVUtil; import org.opensearch.OpenSearchException; import org.opensearch.common.settings.Settings; import org.opensearch.env.Environment; From bbbea8cc2c5d771bf636d27defb771cfd2ab6704 Mon Sep 17 00:00:00 2001 From: Marc Handalian Date: Wed, 6 Dec 2023 11:50:29 -0800 Subject: [PATCH 15/26] bump CompletionFieldMapper to use Completion99PostingsFormat Signed-off-by: Marc Handalian --- .../org/opensearch/index/mapper/CompletionFieldMapper.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/org/opensearch/index/mapper/CompletionFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/CompletionFieldMapper.java index e1413fd9b4bbe..a9d9f6cb35fcb 100644 --- a/server/src/main/java/org/opensearch/index/mapper/CompletionFieldMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/CompletionFieldMapper.java @@ -35,7 +35,7 @@ import org.apache.lucene.document.FieldType; import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.Term; -import org.apache.lucene.search.suggest.document.Completion90PostingsFormat; +import org.apache.lucene.search.suggest.document.Completion99PostingsFormat; import org.apache.lucene.search.suggest.document.CompletionAnalyzer; import org.apache.lucene.search.suggest.document.CompletionQuery; import org.apache.lucene.search.suggest.document.FuzzyCompletionQuery; @@ -330,7 +330,7 @@ public ContextMappings getContextMappings() { */ public static synchronized PostingsFormat postingsFormat() { if (postingsFormat == null) { - postingsFormat = new Completion90PostingsFormat(); + postingsFormat = new Completion99PostingsFormat(); } return postingsFormat; } From ca9f827342026e9cb32d8b6617c30e1b5971f962 Mon Sep 17 00:00:00 2001 From: Marc Handalian Date: Wed, 6 Dec 2023 19:49:14 -0800 Subject: [PATCH 16/26] Fix broken test testExplainDateRangeInQueryString Signed-off-by: Marc Handalian --- .../java/org/opensearch/validate/SimpleValidateQueryIT.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/validate/SimpleValidateQueryIT.java b/server/src/internalClusterTest/java/org/opensearch/validate/SimpleValidateQueryIT.java index e619b93cdd93b..4ac2e1669ca67 100644 --- a/server/src/internalClusterTest/java/org/opensearch/validate/SimpleValidateQueryIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/validate/SimpleValidateQueryIT.java @@ -270,7 +270,10 @@ public void testExplainDateRangeInQueryString() { long twoMonthsAgo = now.minus(2, ChronoUnit.MONTHS).truncatedTo(ChronoUnit.DAYS).toEpochSecond() * 1000; long rangeEnd = (now.plus(1, ChronoUnit.DAYS).truncatedTo(ChronoUnit.DAYS).toEpochSecond() * 1000) - 1; - assertThat(response.getQueryExplanation().get(0).getExplanation(), equalTo("past:[" + twoMonthsAgo + " TO " + rangeEnd + "]")); + assertThat( + response.getQueryExplanation().get(0).getExplanation(), + containsString("past:[" + twoMonthsAgo + " TO " + rangeEnd + "]") + ); assertThat(response.isValid(), equalTo(true)); } From df4a0d7eab3b49db527dc1ef3c6c274ae9de8118 Mon Sep 17 00:00:00 2001 From: Marc Handalian Date: Tue, 12 Dec 2023 10:29:49 -0800 Subject: [PATCH 17/26] Fix test CorruptFileIT#testPrimaryCorruptionDuringReplicationDoesNotFailReplicaShard This test fails because checkIndex in 9.9+ now flags previous corrupt commit points. Signed-off-by: Marc Handalian --- .../org/opensearch/index/store/CorruptedFileIT.java | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/server/src/internalClusterTest/java/org/opensearch/index/store/CorruptedFileIT.java b/server/src/internalClusterTest/java/org/opensearch/index/store/CorruptedFileIT.java index 8291fef5d177b..f46f413f4d23f 100644 --- a/server/src/internalClusterTest/java/org/opensearch/index/store/CorruptedFileIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/index/store/CorruptedFileIT.java @@ -710,6 +710,7 @@ public void testPrimaryCorruptionDuringReplicationDoesNotFailReplicaShard() thro final NodeStats primaryNode = dataNodeStats.get(0); final NodeStats replicaNode = dataNodeStats.get(1); + assertAcked( prepareCreate("test").setSettings( Settings.builder() @@ -795,6 +796,17 @@ public void testPrimaryCorruptionDuringReplicationDoesNotFailReplicaShard() thro // Assert the cluster returns to green status because the replica will be promoted to primary ensureGreen(); + + // After Lucene 9.9 check index will flag corruption with old (not the latest) commit points. + // For this test our previous corrupt commit "segments_2" will remain on the primary. + // This asserts this is the case, and then resets check index status. + assertEquals("Check index has a single failure", 1, checkIndexFailures.size()); + assertTrue( + checkIndexFailures.get(0) + .getMessage() + .contains("could not read old (not latest) commit point segments file \"segments_2\" in directory") + ); + resetCheckIndexStatus(); } private int numShards(String... index) { From f11ad852ebcbf84c7abf23629ea2745ae4b7316b Mon Sep 17 00:00:00 2001 From: Marc Handalian Date: Tue, 12 Dec 2023 11:44:32 -0800 Subject: [PATCH 18/26] Add changelog entry Signed-off-by: Marc Handalian --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 26dee8d29d4a7..b6df19d8a762d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -145,6 +145,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Bump `commons-net:commons-net` from 3.9.0 to 3.10.0 ([#11450](https://github.com/opensearch-project/OpenSearch/pull/11450)) - Bump `org.apache.maven:maven-model` from 3.9.4 to 3.9.6 ([#11445](https://github.com/opensearch-project/OpenSearch/pull/11445)) - Bump `org.apache.xmlbeans:xmlbeans` from 5.1.1 to 5.2.0 ([#11448](https://github.com/opensearch-project/OpenSearch/pull/11448)) +- Bump 'Lucene' from 9.8.0 to 9.9.0 ([11421](https://github.com/opensearch-project/OpenSearch/pull/11421)) ### Changed - Mute the query profile IT with concurrent execution ([#9840](https://github.com/opensearch-project/OpenSearch/pull/9840)) From e1367bbdeaefa0c5a1c85fde0ea861b65d3f361b Mon Sep 17 00:00:00 2001 From: Marc Handalian Date: Tue, 12 Dec 2023 12:09:14 -0800 Subject: [PATCH 19/26] Fix changelog Signed-off-by: Marc Handalian --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4e5207ab9448e..56d9ef85ffbd9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -153,7 +153,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Bump `org.wiremock:wiremock-standalone` from 3.1.0 to 3.3.1 ([#11555](https://github.com/opensearch-project/OpenSearch/pull/11555)) - Bump `org.apache.commons:commons-compress` from 1.24.0 to 1.25.0 ([#11556](https://github.com/opensearch-project/OpenSearch/pull/11556)) - Bump `actions/stale` from 8 to 9 ([#11557](https://github.com/opensearch-project/OpenSearch/pull/11557)) -- Bump `Lucene` from 9.8.0 to 9.9.0 ([11421](https://github.com/opensearch-project/OpenSearch/pull/11421)) +- Bump `Lucene` from 9.8.0 to 9.9.0 ([#11421](https://github.com/opensearch-project/OpenSearch/pull/11421)) ### Changed - Mute the query profile IT with concurrent execution ([#9840](https://github.com/opensearch-project/OpenSearch/pull/9840)) From c330e5ef09ec289631579ed72253c853ae891071 Mon Sep 17 00:00:00 2001 From: Marc Handalian Date: Thu, 28 Dec 2023 11:21:44 -0800 Subject: [PATCH 20/26] bump to 9.9.1 Signed-off-by: Marc Handalian --- buildSrc/version.properties | 2 +- libs/core/src/main/java/org/opensearch/Version.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/buildSrc/version.properties b/buildSrc/version.properties index 223bf5983fb8f..e6adf51d75089 100644 --- a/buildSrc/version.properties +++ b/buildSrc/version.properties @@ -1,5 +1,5 @@ opensearch = 3.0.0 -lucene = 9.9.0 +lucene = 9.9.1 bundled_jdk_vendor = adoptium bundled_jdk = 21.0.1+12 diff --git a/libs/core/src/main/java/org/opensearch/Version.java b/libs/core/src/main/java/org/opensearch/Version.java index 7d6dba1b06890..b2a8e619c9720 100644 --- a/libs/core/src/main/java/org/opensearch/Version.java +++ b/libs/core/src/main/java/org/opensearch/Version.java @@ -99,7 +99,7 @@ public class Version implements Comparable, ToXContentFragment { public static final Version V_2_11_1 = new Version(2110199, org.apache.lucene.util.Version.LUCENE_9_7_0); public static final Version V_2_11_2 = new Version(2110299, org.apache.lucene.util.Version.LUCENE_9_7_0); public static final Version V_2_12_0 = new Version(2120099, org.apache.lucene.util.Version.LUCENE_9_8_0); - public static final Version V_3_0_0 = new Version(3000099, org.apache.lucene.util.Version.LUCENE_9_9_0); + public static final Version V_3_0_0 = new Version(3000099, org.apache.lucene.util.Version.LUCENE_9_9_1); public static final Version CURRENT = V_3_0_0; public static Version fromId(int id) { From ec0961b99e86774f12b9afa0fbce3ec05b8fc721 Mon Sep 17 00:00:00 2001 From: Marc Handalian Date: Tue, 2 Jan 2024 10:57:07 -0800 Subject: [PATCH 21/26] Update SHAs for 9.9.1 Signed-off-by: Marc Handalian --- libs/core/licenses/lucene-core-9.9.0.jar.sha1 | 1 - libs/core/licenses/lucene-core-9.9.1.jar.sha1 | 1 + .../lang-expression/licenses/lucene-expressions-9.9.0.jar.sha1 | 1 - .../lang-expression/licenses/lucene-expressions-9.9.1.jar.sha1 | 1 + plugins/analysis-icu/licenses/lucene-analysis-icu-9.9.0.jar.sha1 | 1 - plugins/analysis-icu/licenses/lucene-analysis-icu-9.9.1.jar.sha1 | 1 + .../licenses/lucene-analysis-kuromoji-9.9.0.jar.sha1 | 1 - .../licenses/lucene-analysis-kuromoji-9.9.1.jar.sha1 | 1 + .../analysis-nori/licenses/lucene-analysis-nori-9.9.0.jar.sha1 | 1 - .../analysis-nori/licenses/lucene-analysis-nori-9.9.1.jar.sha1 | 1 + .../licenses/lucene-analysis-phonetic-9.9.0.jar.sha1 | 1 - .../licenses/lucene-analysis-phonetic-9.9.1.jar.sha1 | 1 + .../licenses/lucene-analysis-smartcn-9.9.0.jar.sha1 | 1 - .../licenses/lucene-analysis-smartcn-9.9.1.jar.sha1 | 1 + .../licenses/lucene-analysis-stempel-9.9.0.jar.sha1 | 1 - .../licenses/lucene-analysis-stempel-9.9.1.jar.sha1 | 1 + .../licenses/lucene-analysis-morfologik-9.9.0.jar.sha1 | 1 - .../licenses/lucene-analysis-morfologik-9.9.1.jar.sha1 | 1 + server/licenses/lucene-analysis-common-9.9.0.jar.sha1 | 1 - server/licenses/lucene-analysis-common-9.9.1.jar.sha1 | 1 + server/licenses/lucene-backward-codecs-9.9.0.jar.sha1 | 1 - server/licenses/lucene-backward-codecs-9.9.1.jar.sha1 | 1 + server/licenses/lucene-core-9.9.0.jar.sha1 | 1 - server/licenses/lucene-core-9.9.1.jar.sha1 | 1 + server/licenses/lucene-grouping-9.9.0.jar.sha1 | 1 - server/licenses/lucene-grouping-9.9.1.jar.sha1 | 1 + server/licenses/lucene-highlighter-9.9.0.jar.sha1 | 1 - server/licenses/lucene-highlighter-9.9.1.jar.sha1 | 1 + server/licenses/lucene-join-9.9.0.jar.sha1 | 1 - server/licenses/lucene-join-9.9.1.jar.sha1 | 1 + server/licenses/lucene-memory-9.9.0.jar.sha1 | 1 - server/licenses/lucene-memory-9.9.1.jar.sha1 | 1 + server/licenses/lucene-misc-9.9.0.jar.sha1 | 1 - server/licenses/lucene-misc-9.9.1.jar.sha1 | 1 + server/licenses/lucene-queries-9.9.0.jar.sha1 | 1 - server/licenses/lucene-queries-9.9.1.jar.sha1 | 1 + server/licenses/lucene-queryparser-9.9.0.jar.sha1 | 1 - server/licenses/lucene-queryparser-9.9.1.jar.sha1 | 1 + server/licenses/lucene-sandbox-9.9.0.jar.sha1 | 1 - server/licenses/lucene-sandbox-9.9.1.jar.sha1 | 1 + server/licenses/lucene-spatial-extras-9.9.0.jar.sha1 | 1 - server/licenses/lucene-spatial-extras-9.9.1.jar.sha1 | 1 + server/licenses/lucene-spatial3d-9.9.0.jar.sha1 | 1 - server/licenses/lucene-spatial3d-9.9.1.jar.sha1 | 1 + server/licenses/lucene-suggest-9.9.0.jar.sha1 | 1 - server/licenses/lucene-suggest-9.9.1.jar.sha1 | 1 + 46 files changed, 23 insertions(+), 23 deletions(-) delete mode 100644 libs/core/licenses/lucene-core-9.9.0.jar.sha1 create mode 100644 libs/core/licenses/lucene-core-9.9.1.jar.sha1 delete mode 100644 modules/lang-expression/licenses/lucene-expressions-9.9.0.jar.sha1 create mode 100644 modules/lang-expression/licenses/lucene-expressions-9.9.1.jar.sha1 delete mode 100644 plugins/analysis-icu/licenses/lucene-analysis-icu-9.9.0.jar.sha1 create mode 100644 plugins/analysis-icu/licenses/lucene-analysis-icu-9.9.1.jar.sha1 delete mode 100644 plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.9.0.jar.sha1 create mode 100644 plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.9.1.jar.sha1 delete mode 100644 plugins/analysis-nori/licenses/lucene-analysis-nori-9.9.0.jar.sha1 create mode 100644 plugins/analysis-nori/licenses/lucene-analysis-nori-9.9.1.jar.sha1 delete mode 100644 plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.9.0.jar.sha1 create mode 100644 plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.9.1.jar.sha1 delete mode 100644 plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.9.0.jar.sha1 create mode 100644 plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.9.1.jar.sha1 delete mode 100644 plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.9.0.jar.sha1 create mode 100644 plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.9.1.jar.sha1 delete mode 100644 plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.9.0.jar.sha1 create mode 100644 plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.9.1.jar.sha1 delete mode 100644 server/licenses/lucene-analysis-common-9.9.0.jar.sha1 create mode 100644 server/licenses/lucene-analysis-common-9.9.1.jar.sha1 delete mode 100644 server/licenses/lucene-backward-codecs-9.9.0.jar.sha1 create mode 100644 server/licenses/lucene-backward-codecs-9.9.1.jar.sha1 delete mode 100644 server/licenses/lucene-core-9.9.0.jar.sha1 create mode 100644 server/licenses/lucene-core-9.9.1.jar.sha1 delete mode 100644 server/licenses/lucene-grouping-9.9.0.jar.sha1 create mode 100644 server/licenses/lucene-grouping-9.9.1.jar.sha1 delete mode 100644 server/licenses/lucene-highlighter-9.9.0.jar.sha1 create mode 100644 server/licenses/lucene-highlighter-9.9.1.jar.sha1 delete mode 100644 server/licenses/lucene-join-9.9.0.jar.sha1 create mode 100644 server/licenses/lucene-join-9.9.1.jar.sha1 delete mode 100644 server/licenses/lucene-memory-9.9.0.jar.sha1 create mode 100644 server/licenses/lucene-memory-9.9.1.jar.sha1 delete mode 100644 server/licenses/lucene-misc-9.9.0.jar.sha1 create mode 100644 server/licenses/lucene-misc-9.9.1.jar.sha1 delete mode 100644 server/licenses/lucene-queries-9.9.0.jar.sha1 create mode 100644 server/licenses/lucene-queries-9.9.1.jar.sha1 delete mode 100644 server/licenses/lucene-queryparser-9.9.0.jar.sha1 create mode 100644 server/licenses/lucene-queryparser-9.9.1.jar.sha1 delete mode 100644 server/licenses/lucene-sandbox-9.9.0.jar.sha1 create mode 100644 server/licenses/lucene-sandbox-9.9.1.jar.sha1 delete mode 100644 server/licenses/lucene-spatial-extras-9.9.0.jar.sha1 create mode 100644 server/licenses/lucene-spatial-extras-9.9.1.jar.sha1 delete mode 100644 server/licenses/lucene-spatial3d-9.9.0.jar.sha1 create mode 100644 server/licenses/lucene-spatial3d-9.9.1.jar.sha1 delete mode 100644 server/licenses/lucene-suggest-9.9.0.jar.sha1 create mode 100644 server/licenses/lucene-suggest-9.9.1.jar.sha1 diff --git a/libs/core/licenses/lucene-core-9.9.0.jar.sha1 b/libs/core/licenses/lucene-core-9.9.0.jar.sha1 deleted file mode 100644 index b6df006f2f947..0000000000000 --- a/libs/core/licenses/lucene-core-9.9.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -2f0876192ee21f2b6f85b9435c99b9f0e04f3fcc \ No newline at end of file diff --git a/libs/core/licenses/lucene-core-9.9.1.jar.sha1 b/libs/core/licenses/lucene-core-9.9.1.jar.sha1 new file mode 100644 index 0000000000000..ae596196d9e6a --- /dev/null +++ b/libs/core/licenses/lucene-core-9.9.1.jar.sha1 @@ -0,0 +1 @@ +55249fa9a0ed321adcf8283c6f3b649a6812b0a9 \ No newline at end of file diff --git a/modules/lang-expression/licenses/lucene-expressions-9.9.0.jar.sha1 b/modules/lang-expression/licenses/lucene-expressions-9.9.0.jar.sha1 deleted file mode 100644 index 5c25ba1e48cfd..0000000000000 --- a/modules/lang-expression/licenses/lucene-expressions-9.9.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -a36fc5b5087b704852a5f228c38cb3a47fea8c36 \ No newline at end of file diff --git a/modules/lang-expression/licenses/lucene-expressions-9.9.1.jar.sha1 b/modules/lang-expression/licenses/lucene-expressions-9.9.1.jar.sha1 new file mode 100644 index 0000000000000..402cc36ba3d68 --- /dev/null +++ b/modules/lang-expression/licenses/lucene-expressions-9.9.1.jar.sha1 @@ -0,0 +1 @@ +1782a69d0e83af9cc3c65db0dcd2e7e7c1e5f90e \ No newline at end of file diff --git a/plugins/analysis-icu/licenses/lucene-analysis-icu-9.9.0.jar.sha1 b/plugins/analysis-icu/licenses/lucene-analysis-icu-9.9.0.jar.sha1 deleted file mode 100644 index 85793f76f884b..0000000000000 --- a/plugins/analysis-icu/licenses/lucene-analysis-icu-9.9.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -fa8ddcddd8ad086d123dbcf1b2804683a14f252f \ No newline at end of file diff --git a/plugins/analysis-icu/licenses/lucene-analysis-icu-9.9.1.jar.sha1 b/plugins/analysis-icu/licenses/lucene-analysis-icu-9.9.1.jar.sha1 new file mode 100644 index 0000000000000..dde9b7c100dc7 --- /dev/null +++ b/plugins/analysis-icu/licenses/lucene-analysis-icu-9.9.1.jar.sha1 @@ -0,0 +1 @@ +147cb42a90a29501d9ca6094ea0db1d213f3076a \ No newline at end of file diff --git a/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.9.0.jar.sha1 b/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.9.0.jar.sha1 deleted file mode 100644 index a95fb85beed94..0000000000000 --- a/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.9.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -70903a94800e3f09898f279987a591e1443e7f11 \ No newline at end of file diff --git a/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.9.1.jar.sha1 b/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.9.1.jar.sha1 new file mode 100644 index 0000000000000..b70a22e9db096 --- /dev/null +++ b/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.9.1.jar.sha1 @@ -0,0 +1 @@ +b034dd3a975763e083c7e11b5d0f7d516ab72590 \ No newline at end of file diff --git a/plugins/analysis-nori/licenses/lucene-analysis-nori-9.9.0.jar.sha1 b/plugins/analysis-nori/licenses/lucene-analysis-nori-9.9.0.jar.sha1 deleted file mode 100644 index e9b373a9c0ae3..0000000000000 --- a/plugins/analysis-nori/licenses/lucene-analysis-nori-9.9.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -521d3e833ffb371dd4712b8241239cae031507e5 \ No newline at end of file diff --git a/plugins/analysis-nori/licenses/lucene-analysis-nori-9.9.1.jar.sha1 b/plugins/analysis-nori/licenses/lucene-analysis-nori-9.9.1.jar.sha1 new file mode 100644 index 0000000000000..323f165c62790 --- /dev/null +++ b/plugins/analysis-nori/licenses/lucene-analysis-nori-9.9.1.jar.sha1 @@ -0,0 +1 @@ +c405f2f7d0fc127d88dfbadd753469b2028fdf52 \ No newline at end of file diff --git a/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.9.0.jar.sha1 b/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.9.0.jar.sha1 deleted file mode 100644 index 1d17b615f88e7..0000000000000 --- a/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.9.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -40d1fb2e36efc071d1b96958f23bfe3d5e3d5758 \ No newline at end of file diff --git a/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.9.1.jar.sha1 b/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.9.1.jar.sha1 new file mode 100644 index 0000000000000..dd659ddf4de95 --- /dev/null +++ b/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.9.1.jar.sha1 @@ -0,0 +1 @@ +970e5775876c2d7e1b9af7421a4b17d96f63faf4 \ No newline at end of file diff --git a/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.9.0.jar.sha1 b/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.9.0.jar.sha1 deleted file mode 100644 index b290b2015a7c0..0000000000000 --- a/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.9.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -4a4f7ba40b82f108a2c3d150624cb46f8281d212 \ No newline at end of file diff --git a/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.9.1.jar.sha1 b/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.9.1.jar.sha1 new file mode 100644 index 0000000000000..ed0e81d8f1f75 --- /dev/null +++ b/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.9.1.jar.sha1 @@ -0,0 +1 @@ +2421e5238e9b8484929291744d709dd743c01da1 \ No newline at end of file diff --git a/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.9.0.jar.sha1 b/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.9.0.jar.sha1 deleted file mode 100644 index 1c5f02aa45451..0000000000000 --- a/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.9.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -067dd0df852c5df001d77505dd048a4e90a4b365 \ No newline at end of file diff --git a/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.9.1.jar.sha1 b/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.9.1.jar.sha1 new file mode 100644 index 0000000000000..fd8e000088180 --- /dev/null +++ b/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.9.1.jar.sha1 @@ -0,0 +1 @@ +a23e7de4cd9ae7af285c89dc1c55e0ac3f157fd3 \ No newline at end of file diff --git a/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.9.0.jar.sha1 b/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.9.0.jar.sha1 deleted file mode 100644 index 9e3d15356d69a..0000000000000 --- a/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.9.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -ac5225e8a5fa4e1d9784b46f64942b75b2716b26 \ No newline at end of file diff --git a/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.9.1.jar.sha1 b/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.9.1.jar.sha1 new file mode 100644 index 0000000000000..d0e7a3b0c751c --- /dev/null +++ b/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.9.1.jar.sha1 @@ -0,0 +1 @@ +8d9bce1ea51db279878c51091dd9aefc7b335da4 \ No newline at end of file diff --git a/server/licenses/lucene-analysis-common-9.9.0.jar.sha1 b/server/licenses/lucene-analysis-common-9.9.0.jar.sha1 deleted file mode 100644 index 52c91a5a7babf..0000000000000 --- a/server/licenses/lucene-analysis-common-9.9.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -445448c09a0c8ba40f81df7d86ac97303ce0b19e \ No newline at end of file diff --git a/server/licenses/lucene-analysis-common-9.9.1.jar.sha1 b/server/licenses/lucene-analysis-common-9.9.1.jar.sha1 new file mode 100644 index 0000000000000..c9e6120da7497 --- /dev/null +++ b/server/licenses/lucene-analysis-common-9.9.1.jar.sha1 @@ -0,0 +1 @@ +24c8401b530308f9568eb7b408c2029c63f564c6 \ No newline at end of file diff --git a/server/licenses/lucene-backward-codecs-9.9.0.jar.sha1 b/server/licenses/lucene-backward-codecs-9.9.0.jar.sha1 deleted file mode 100644 index 62cf8eb86ff5f..0000000000000 --- a/server/licenses/lucene-backward-codecs-9.9.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -fd7a9d02972955bb85896ded80e9bf5ffa173b2b \ No newline at end of file diff --git a/server/licenses/lucene-backward-codecs-9.9.1.jar.sha1 b/server/licenses/lucene-backward-codecs-9.9.1.jar.sha1 new file mode 100644 index 0000000000000..69ecf6aa68200 --- /dev/null +++ b/server/licenses/lucene-backward-codecs-9.9.1.jar.sha1 @@ -0,0 +1 @@ +11c46007366bb037be7d271ab0a5849b1d544662 \ No newline at end of file diff --git a/server/licenses/lucene-core-9.9.0.jar.sha1 b/server/licenses/lucene-core-9.9.0.jar.sha1 deleted file mode 100644 index b6df006f2f947..0000000000000 --- a/server/licenses/lucene-core-9.9.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -2f0876192ee21f2b6f85b9435c99b9f0e04f3fcc \ No newline at end of file diff --git a/server/licenses/lucene-core-9.9.1.jar.sha1 b/server/licenses/lucene-core-9.9.1.jar.sha1 new file mode 100644 index 0000000000000..ae596196d9e6a --- /dev/null +++ b/server/licenses/lucene-core-9.9.1.jar.sha1 @@ -0,0 +1 @@ +55249fa9a0ed321adcf8283c6f3b649a6812b0a9 \ No newline at end of file diff --git a/server/licenses/lucene-grouping-9.9.0.jar.sha1 b/server/licenses/lucene-grouping-9.9.0.jar.sha1 deleted file mode 100644 index 4be6aae4c8be1..0000000000000 --- a/server/licenses/lucene-grouping-9.9.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -becfa739de2d3e29c129ed0347890a2dcab24b9b \ No newline at end of file diff --git a/server/licenses/lucene-grouping-9.9.1.jar.sha1 b/server/licenses/lucene-grouping-9.9.1.jar.sha1 new file mode 100644 index 0000000000000..e7df056400661 --- /dev/null +++ b/server/licenses/lucene-grouping-9.9.1.jar.sha1 @@ -0,0 +1 @@ +2f2785e17c5c823cc8f41a7ddb4647aaca8ee773 \ No newline at end of file diff --git a/server/licenses/lucene-highlighter-9.9.0.jar.sha1 b/server/licenses/lucene-highlighter-9.9.0.jar.sha1 deleted file mode 100644 index 6addb610c1b28..0000000000000 --- a/server/licenses/lucene-highlighter-9.9.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -fc7392800a3baff95efa592b1dca0df047e862e5 \ No newline at end of file diff --git a/server/licenses/lucene-highlighter-9.9.1.jar.sha1 b/server/licenses/lucene-highlighter-9.9.1.jar.sha1 new file mode 100644 index 0000000000000..828c7294aa586 --- /dev/null +++ b/server/licenses/lucene-highlighter-9.9.1.jar.sha1 @@ -0,0 +1 @@ +30928513461bf79a5cb057e84da7d34a1e53227d \ No newline at end of file diff --git a/server/licenses/lucene-join-9.9.0.jar.sha1 b/server/licenses/lucene-join-9.9.0.jar.sha1 deleted file mode 100644 index 8194e3e50a065..0000000000000 --- a/server/licenses/lucene-join-9.9.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -58e2ff4e9dbb7bad804f08a1f482f08b72676421 \ No newline at end of file diff --git a/server/licenses/lucene-join-9.9.1.jar.sha1 b/server/licenses/lucene-join-9.9.1.jar.sha1 new file mode 100644 index 0000000000000..34b44ca8c6ad5 --- /dev/null +++ b/server/licenses/lucene-join-9.9.1.jar.sha1 @@ -0,0 +1 @@ +b9c8cc99632280148f92b4c0a64111c482d5d0ac \ No newline at end of file diff --git a/server/licenses/lucene-memory-9.9.0.jar.sha1 b/server/licenses/lucene-memory-9.9.0.jar.sha1 deleted file mode 100644 index 5d1c37d721951..0000000000000 --- a/server/licenses/lucene-memory-9.9.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -266c6dc593217f606e06903d15d20aaa502d3ad2 \ No newline at end of file diff --git a/server/licenses/lucene-memory-9.9.1.jar.sha1 b/server/licenses/lucene-memory-9.9.1.jar.sha1 new file mode 100644 index 0000000000000..b75fba4c331e9 --- /dev/null +++ b/server/licenses/lucene-memory-9.9.1.jar.sha1 @@ -0,0 +1 @@ +49f820b1b321860fa42a4f7583e8ed8f77b9c1c2 \ No newline at end of file diff --git a/server/licenses/lucene-misc-9.9.0.jar.sha1 b/server/licenses/lucene-misc-9.9.0.jar.sha1 deleted file mode 100644 index 1d892105059e8..0000000000000 --- a/server/licenses/lucene-misc-9.9.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -873e4beb02f028a41c981a86e04c6d86a4e4cd68 \ No newline at end of file diff --git a/server/licenses/lucene-misc-9.9.1.jar.sha1 b/server/licenses/lucene-misc-9.9.1.jar.sha1 new file mode 100644 index 0000000000000..f1e1e056004e9 --- /dev/null +++ b/server/licenses/lucene-misc-9.9.1.jar.sha1 @@ -0,0 +1 @@ +db7c30217602dfcda394a4d0f0a9e68140d385a6 \ No newline at end of file diff --git a/server/licenses/lucene-queries-9.9.0.jar.sha1 b/server/licenses/lucene-queries-9.9.0.jar.sha1 deleted file mode 100644 index 1d93435a9af2a..0000000000000 --- a/server/licenses/lucene-queries-9.9.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -298606babedbd3a67b7539fd1f84ccca7140c4de \ No newline at end of file diff --git a/server/licenses/lucene-queries-9.9.1.jar.sha1 b/server/licenses/lucene-queries-9.9.1.jar.sha1 new file mode 100644 index 0000000000000..888b9b4a05ec8 --- /dev/null +++ b/server/licenses/lucene-queries-9.9.1.jar.sha1 @@ -0,0 +1 @@ +d157547bd24edc8e9d9d59c273107dc3ac5fde5e \ No newline at end of file diff --git a/server/licenses/lucene-queryparser-9.9.0.jar.sha1 b/server/licenses/lucene-queryparser-9.9.0.jar.sha1 deleted file mode 100644 index 2fe4c83bf4a1a..0000000000000 --- a/server/licenses/lucene-queryparser-9.9.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -1968cd1b3ccaff22d30ffa5e5e54ec51b4a4e66e \ No newline at end of file diff --git a/server/licenses/lucene-queryparser-9.9.1.jar.sha1 b/server/licenses/lucene-queryparser-9.9.1.jar.sha1 new file mode 100644 index 0000000000000..1ce8a069a0f4e --- /dev/null +++ b/server/licenses/lucene-queryparser-9.9.1.jar.sha1 @@ -0,0 +1 @@ +12d844fe224f6f97c510ac20d68903ed7f626f6c \ No newline at end of file diff --git a/server/licenses/lucene-sandbox-9.9.0.jar.sha1 b/server/licenses/lucene-sandbox-9.9.0.jar.sha1 deleted file mode 100644 index 349b90f7a259a..0000000000000 --- a/server/licenses/lucene-sandbox-9.9.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -25472d0b679ba12ce34dfe925b7541c8b7d74025 \ No newline at end of file diff --git a/server/licenses/lucene-sandbox-9.9.1.jar.sha1 b/server/licenses/lucene-sandbox-9.9.1.jar.sha1 new file mode 100644 index 0000000000000..14fd86dadc404 --- /dev/null +++ b/server/licenses/lucene-sandbox-9.9.1.jar.sha1 @@ -0,0 +1 @@ +272e588fd3d8c0a401b28a1ac715f27044bf62ec \ No newline at end of file diff --git a/server/licenses/lucene-spatial-extras-9.9.0.jar.sha1 b/server/licenses/lucene-spatial-extras-9.9.0.jar.sha1 deleted file mode 100644 index a801a810e10c6..0000000000000 --- a/server/licenses/lucene-spatial-extras-9.9.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -37b89d9250edf4f2b1940395c053c66869b4449d \ No newline at end of file diff --git a/server/licenses/lucene-spatial-extras-9.9.1.jar.sha1 b/server/licenses/lucene-spatial-extras-9.9.1.jar.sha1 new file mode 100644 index 0000000000000..0efd5a7595bfe --- /dev/null +++ b/server/licenses/lucene-spatial-extras-9.9.1.jar.sha1 @@ -0,0 +1 @@ +e066432e7ab02b2a4914f989bcd8c44adbf340ad \ No newline at end of file diff --git a/server/licenses/lucene-spatial3d-9.9.0.jar.sha1 b/server/licenses/lucene-spatial3d-9.9.0.jar.sha1 deleted file mode 100644 index 702b20318890a..0000000000000 --- a/server/licenses/lucene-spatial3d-9.9.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -e51e7d3ca30c1789b7779b54afce4ab140830718 \ No newline at end of file diff --git a/server/licenses/lucene-spatial3d-9.9.1.jar.sha1 b/server/licenses/lucene-spatial3d-9.9.1.jar.sha1 new file mode 100644 index 0000000000000..7f06466e4c721 --- /dev/null +++ b/server/licenses/lucene-spatial3d-9.9.1.jar.sha1 @@ -0,0 +1 @@ +fa54c9b962778e28ebc0efb9f75297781350361a \ No newline at end of file diff --git a/server/licenses/lucene-suggest-9.9.0.jar.sha1 b/server/licenses/lucene-suggest-9.9.0.jar.sha1 deleted file mode 100644 index 016eb9add1ea3..0000000000000 --- a/server/licenses/lucene-suggest-9.9.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -400c160b7fef8a4d142aa7cfc18f98469a5aec24 \ No newline at end of file diff --git a/server/licenses/lucene-suggest-9.9.1.jar.sha1 b/server/licenses/lucene-suggest-9.9.1.jar.sha1 new file mode 100644 index 0000000000000..06732480d1b6c --- /dev/null +++ b/server/licenses/lucene-suggest-9.9.1.jar.sha1 @@ -0,0 +1 @@ +9554de5b22ae7483b344b94a9a956960b7a5d49c \ No newline at end of file From 5976dc74c8be4478e0a7900cd9a74cec6b6974eb Mon Sep 17 00:00:00 2001 From: Marc Handalian Date: Tue, 2 Jan 2024 17:46:13 -0800 Subject: [PATCH 22/26] Set BestBucketsDeferringCollector as finished by default when there are no leaves With Lucene 9.9.0+ IndexSearcher#search is not invoked if there are no leaves, skipping directly to invoking reduce on the collectorManager. This collector would then throw in prepareSelectedBuckets thinking it has not yet finished collection. Signed-off-by: Marc Handalian --- .../aggregations/bucket/BestBucketsDeferringCollector.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/BestBucketsDeferringCollector.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/BestBucketsDeferringCollector.java index 2e7c4659bcb00..70a8fd91f911c 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/BestBucketsDeferringCollector.java +++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/BestBucketsDeferringCollector.java @@ -90,7 +90,7 @@ static class Entry { protected PackedLongValues.Builder docDeltasBuilder; protected PackedLongValues.Builder bucketsBuilder; protected long maxBucket = -1; - protected boolean finished = false; + protected boolean finished; protected LongHash selectedBuckets; /** @@ -101,6 +101,7 @@ static class Entry { public BestBucketsDeferringCollector(SearchContext context, boolean isGlobal) { this.searchContext = context; this.isGlobal = isGlobal; + this.finished = context.searcher().getLeafContexts().isEmpty(); } @Override From 7ac0dd862f435d76560acb3468784a1b2f593c3a Mon Sep 17 00:00:00 2001 From: Marc Handalian Date: Tue, 2 Jan 2024 17:51:17 -0800 Subject: [PATCH 23/26] PR feedback. Signed-off-by: Marc Handalian --- .../opensearch/search/internal/ContextIndexSearcherTests.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/test/java/org/opensearch/search/internal/ContextIndexSearcherTests.java b/server/src/test/java/org/opensearch/search/internal/ContextIndexSearcherTests.java index b8966b76d0193..a707c8b34e0a4 100644 --- a/server/src/test/java/org/opensearch/search/internal/ContextIndexSearcherTests.java +++ b/server/src/test/java/org/opensearch/search/internal/ContextIndexSearcherTests.java @@ -394,7 +394,7 @@ public void testGetSlicesWithNonNullExecutorButCSDisabled() throws Exception { searchContext ); // Case 1: Verify getSlices returns not null when concurrent segment search is disabled - assertNotNull(searcher.getSlices()); + assertEquals(1, searcher.getSlices().length); // Case 2: Verify the slice count when custom max slice computation is used searcher = new ContextIndexSearcher( From 1ddece1bcbb16f4216e290fb8a7c79496542485a Mon Sep 17 00:00:00 2001 From: Marc Handalian Date: Tue, 2 Jan 2024 17:56:05 -0800 Subject: [PATCH 24/26] Fix changelog Signed-off-by: Marc Handalian --- CHANGELOG.md | 1 - 1 file changed, 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d9ac42f468601..95bb1b774cf18 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -153,7 +153,6 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Bump `org.wiremock:wiremock-standalone` from 3.1.0 to 3.3.1 ([#11555](https://github.com/opensearch-project/OpenSearch/pull/11555)) - Bump `org.apache.commons:commons-compress` from 1.24.0 to 1.25.0 ([#11556](https://github.com/opensearch-project/OpenSearch/pull/11556)) - Bump `actions/stale` from 8 to 9 ([#11557](https://github.com/opensearch-project/OpenSearch/pull/11557)) -- Bump `Lucene` from 9.8.0 to 9.9.0 ([#11421](https://github.com/opensearch-project/OpenSearch/pull/11421)) - Bump `com.netflix.nebula:nebula-publishing-plugin` from 20.3.0 to 21.0.0 ([#11671](https://github.com/opensearch-project/OpenSearch/pull/11671)) - Bump `commons-cli:commons-cli` from 1.5.0 to 1.6.0 ([#10996](https://github.com/opensearch-project/OpenSearch/pull/10996)) - Bump `com.maxmind.geoip2:geoip2` from 4.1.0 to 4.2.0 ([#11559](https://github.com/opensearch-project/OpenSearch/pull/11559)) From 0688dfa008918af0cf11f1ded18a986a472f3638 Mon Sep 17 00:00:00 2001 From: Marc Handalian Date: Tue, 9 Jan 2024 13:49:19 -0800 Subject: [PATCH 25/26] rename correlation950 package to correlation990 Signed-off-by: Marc Handalian --- .../correlation/core/index/codec/CorrelationCodecVersion.java | 4 ++-- .../{correlation950 => correlation990}/CorrelationCodec.java | 2 +- .../PerFieldCorrelationVectorsFormat.java | 2 +- .../{correlation950 => correlation990}/package-info.java | 2 +- .../META-INF/services/org.apache.lucene.codecs.Codec | 2 +- .../CorrelationCodecTests.java | 2 +- 6 files changed, 7 insertions(+), 7 deletions(-) rename plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/core/index/codec/{correlation950 => correlation990}/CorrelationCodec.java (99%) rename plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/core/index/codec/{correlation950 => correlation990}/PerFieldCorrelationVectorsFormat.java (99%) rename plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/core/index/codec/{correlation950 => correlation990}/package-info.java (96%) rename plugins/events-correlation-engine/src/test/java/org/opensearch/plugin/correlation/core/index/codec/{correlation950 => correlation990}/CorrelationCodecTests.java (99%) diff --git a/plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/core/index/codec/CorrelationCodecVersion.java b/plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/core/index/codec/CorrelationCodecVersion.java index 1ef6bd1d8f747..3fcc995fb4199 100644 --- a/plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/core/index/codec/CorrelationCodecVersion.java +++ b/plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/core/index/codec/CorrelationCodecVersion.java @@ -11,8 +11,8 @@ import org.apache.lucene.codecs.Codec; import org.apache.lucene.codecs.lucene99.Lucene99Codec; import org.opensearch.index.mapper.MapperService; -import org.opensearch.plugin.correlation.core.index.codec.correlation950.CorrelationCodec; -import org.opensearch.plugin.correlation.core.index.codec.correlation950.PerFieldCorrelationVectorsFormat; +import org.opensearch.plugin.correlation.core.index.codec.correlation990.CorrelationCodec; +import org.opensearch.plugin.correlation.core.index.codec.correlation990.PerFieldCorrelationVectorsFormat; import java.util.Optional; import java.util.function.BiFunction; diff --git a/plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/core/index/codec/correlation950/CorrelationCodec.java b/plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/core/index/codec/correlation990/CorrelationCodec.java similarity index 99% rename from plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/core/index/codec/correlation950/CorrelationCodec.java rename to plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/core/index/codec/correlation990/CorrelationCodec.java index 6da71bda9ae5c..022972e2e06c3 100644 --- a/plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/core/index/codec/correlation950/CorrelationCodec.java +++ b/plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/core/index/codec/correlation990/CorrelationCodec.java @@ -6,7 +6,7 @@ * compatible open source license. */ -package org.opensearch.plugin.correlation.core.index.codec.correlation950; +package org.opensearch.plugin.correlation.core.index.codec.correlation990; import org.apache.lucene.codecs.Codec; import org.apache.lucene.codecs.FilterCodec; diff --git a/plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/core/index/codec/correlation950/PerFieldCorrelationVectorsFormat.java b/plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/core/index/codec/correlation990/PerFieldCorrelationVectorsFormat.java similarity index 99% rename from plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/core/index/codec/correlation950/PerFieldCorrelationVectorsFormat.java rename to plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/core/index/codec/correlation990/PerFieldCorrelationVectorsFormat.java index 185267c53a231..89cc0b614a1a5 100644 --- a/plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/core/index/codec/correlation950/PerFieldCorrelationVectorsFormat.java +++ b/plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/core/index/codec/correlation990/PerFieldCorrelationVectorsFormat.java @@ -6,7 +6,7 @@ * compatible open source license. */ -package org.opensearch.plugin.correlation.core.index.codec.correlation950; +package org.opensearch.plugin.correlation.core.index.codec.correlation990; import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat; import org.opensearch.index.mapper.MapperService; diff --git a/plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/core/index/codec/correlation950/package-info.java b/plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/core/index/codec/correlation990/package-info.java similarity index 96% rename from plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/core/index/codec/correlation950/package-info.java rename to plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/core/index/codec/correlation990/package-info.java index b4dad34d2718e..fc2a9de58a73a 100644 --- a/plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/core/index/codec/correlation950/package-info.java +++ b/plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/core/index/codec/correlation990/package-info.java @@ -9,4 +9,4 @@ /** * custom Lucene9.5 codec package for events-correlation-engine */ -package org.opensearch.plugin.correlation.core.index.codec.correlation950; +package org.opensearch.plugin.correlation.core.index.codec.correlation990; diff --git a/plugins/events-correlation-engine/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec b/plugins/events-correlation-engine/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec index 598a3b6af73c2..013c17e4a9736 100644 --- a/plugins/events-correlation-engine/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec +++ b/plugins/events-correlation-engine/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec @@ -1 +1 @@ -org.opensearch.plugin.correlation.core.index.codec.correlation950.CorrelationCodec +org.opensearch.plugin.correlation.core.index.codec.correlation990.CorrelationCodec diff --git a/plugins/events-correlation-engine/src/test/java/org/opensearch/plugin/correlation/core/index/codec/correlation950/CorrelationCodecTests.java b/plugins/events-correlation-engine/src/test/java/org/opensearch/plugin/correlation/core/index/codec/correlation990/CorrelationCodecTests.java similarity index 99% rename from plugins/events-correlation-engine/src/test/java/org/opensearch/plugin/correlation/core/index/codec/correlation950/CorrelationCodecTests.java rename to plugins/events-correlation-engine/src/test/java/org/opensearch/plugin/correlation/core/index/codec/correlation990/CorrelationCodecTests.java index df679fd7ca678..7223b450a136c 100644 --- a/plugins/events-correlation-engine/src/test/java/org/opensearch/plugin/correlation/core/index/codec/correlation950/CorrelationCodecTests.java +++ b/plugins/events-correlation-engine/src/test/java/org/opensearch/plugin/correlation/core/index/codec/correlation990/CorrelationCodecTests.java @@ -6,7 +6,7 @@ * compatible open source license. */ -package org.opensearch.plugin.correlation.core.index.codec.correlation950; +package org.opensearch.plugin.correlation.core.index.codec.correlation990; import org.apache.lucene.codecs.Codec; import org.apache.lucene.document.Document; From 018504162bb4e3fda83cad0bc065a6d42f1876df Mon Sep 17 00:00:00 2001 From: Marc Handalian Date: Tue, 9 Jan 2024 13:55:02 -0800 Subject: [PATCH 26/26] add comment to BestBucketsDeferringCollector Signed-off-by: Marc Handalian --- .../aggregations/bucket/BestBucketsDeferringCollector.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/BestBucketsDeferringCollector.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/BestBucketsDeferringCollector.java index 70a8fd91f911c..223be3ba2d1ae 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/BestBucketsDeferringCollector.java +++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/BestBucketsDeferringCollector.java @@ -101,6 +101,8 @@ static class Entry { public BestBucketsDeferringCollector(SearchContext context, boolean isGlobal) { this.searchContext = context; this.isGlobal = isGlobal; + // a postCollection call is not made by the IndexSearcher when there are no segments. + // In this case init the collector as finished. this.finished = context.searcher().getLeafContexts().isEmpty(); }