From b59a357e58611b86e4cc1643794e05dd17d71b08 Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Tue, 17 Sep 2024 14:35:30 +0200 Subject: [PATCH] Change docValuesSkipIndex from a boolean to an enum. (#13784) At the moment, our skip indexes record min/max ordinal/value per range of doc IDs. It would be natural to extend it to other pre-aggregated data such as a sum and value count, which facets could take advantage of. This change switches `docValuesSkipIndex` from a boolean to an enum so that we could release such changes in the future in an additive fashion, by adding constants to this enum and new methods to `DocValuesSkipper`. --- .../lucene60/Lucene60FieldInfosFormat.java | 3 +- .../lucene90/Lucene90FieldInfosFormat.java | 3 +- .../SimpleTextFieldInfosFormat.java | 15 ++++-- .../codecs/uniformsplit/TestBlockWriter.java | 3 +- .../sharedterms/TestSTBlockReader.java | 3 +- .../lucene/codecs/DocValuesProducer.java | 3 +- .../lucene90/Lucene90DocValuesConsumer.java | 9 ++-- .../lucene90/Lucene90DocValuesProducer.java | 3 +- .../lucene94/Lucene94FieldInfosFormat.java | 37 +++++++++++++-- .../org/apache/lucene/document/FieldType.java | 11 +++-- .../document/NumericDocValuesField.java | 5 +- .../lucene/document/SortedDocValuesField.java | 5 +- .../document/SortedNumericDocValuesField.java | 5 +- .../document/SortedSetDocValuesField.java | 5 +- .../org/apache/lucene/index/CheckIndex.java | 2 +- .../org/apache/lucene/index/CodecReader.java | 2 +- .../lucene/index/DocValuesSkipIndexType.java | 46 +++++++++++++++++++ .../apache/lucene/index/DocValuesType.java | 18 +++----- .../org/apache/lucene/index/FieldInfo.java | 22 +++++---- .../org/apache/lucene/index/FieldInfos.java | 18 ++++---- .../lucene/index/IndexableFieldType.java | 2 +- .../apache/lucene/index/IndexingChain.java | 21 +++++---- .../lucene/index/ReadersAndUpdates.java | 2 +- .../org/apache/lucene/index/TestCodecs.java | 2 +- .../apache/lucene/index/TestFieldInfos.java | 6 +-- .../apache/lucene/index/TestFieldsReader.java | 2 +- .../apache/lucene/index/TestIndexWriter.java | 10 ++-- .../lucene/index/TestIndexableField.java | 4 +- .../lucene/index/TestPendingSoftDeletes.java | 10 ++-- .../lucene/search/TestSortOptimization.java | 2 +- .../highlight/TermVectorLeafReader.java | 3 +- .../lucene/index/memory/MemoryIndex.java | 4 +- .../asserting/AssertingDocValuesFormat.java | 3 +- .../tests/index/AssertingLeafReader.java | 5 +- .../index/BaseFieldInfoFormatTestCase.java | 16 ++++--- .../index/BaseIndexFileFormatTestCase.java | 2 +- .../index/BaseKnnVectorsFormatTestCase.java | 3 +- .../tests/index/MismatchedLeafReader.java | 2 +- .../tests/index/RandomPostingsTester.java | 5 +- 39 files changed, 214 insertions(+), 108 deletions(-) create mode 100644 lucene/core/src/java/org/apache/lucene/index/DocValuesSkipIndexType.java diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene60/Lucene60FieldInfosFormat.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene60/Lucene60FieldInfosFormat.java index be57d6161bbd..4e0d7c8e53e6 100644 --- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene60/Lucene60FieldInfosFormat.java +++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene60/Lucene60FieldInfosFormat.java @@ -24,6 +24,7 @@ import org.apache.lucene.codecs.DocValuesFormat; import org.apache.lucene.codecs.FieldInfosFormat; import org.apache.lucene.index.CorruptIndexException; +import org.apache.lucene.index.DocValuesSkipIndexType; import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.FieldInfos; @@ -209,7 +210,7 @@ private FieldInfo[] readFieldInfos(IndexInput input, int version) throws IOExcep storePayloads, indexOptions, docValuesType, - false, + DocValuesSkipIndexType.NONE, dvGen, attributes, pointDataDimensionCount, diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene90/Lucene90FieldInfosFormat.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene90/Lucene90FieldInfosFormat.java index 63a4f0f2bbfa..65c7864fa9dd 100644 --- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene90/Lucene90FieldInfosFormat.java +++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene90/Lucene90FieldInfosFormat.java @@ -23,6 +23,7 @@ import org.apache.lucene.codecs.DocValuesFormat; import org.apache.lucene.codecs.FieldInfosFormat; import org.apache.lucene.index.CorruptIndexException; +import org.apache.lucene.index.DocValuesSkipIndexType; import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.FieldInfos; @@ -186,7 +187,7 @@ public FieldInfos read( storePayloads, indexOptions, docValuesType, - false, + DocValuesSkipIndexType.NONE, dvGen, attributes, pointDataDimensionCount, diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldInfosFormat.java b/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldInfosFormat.java index 90ae65a177c8..fefbb44bd80b 100644 --- a/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldInfosFormat.java +++ b/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldInfosFormat.java @@ -22,6 +22,7 @@ import java.util.HashMap; import java.util.Map; import org.apache.lucene.codecs.FieldInfosFormat; +import org.apache.lucene.index.DocValuesSkipIndexType; import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.FieldInfos; @@ -125,8 +126,8 @@ public FieldInfos read( SimpleTextUtil.readLine(input, scratch); assert StringHelper.startsWith(scratch.get(), DOCVALUES_SKIP_INDEX); - boolean docValueSkipper = - Boolean.parseBoolean(readString(DOCVALUES_SKIP_INDEX.length, scratch)); + DocValuesSkipIndexType docValueSkipper = + docValuesSkipIndexType(readString(DOCVALUES_SKIP_INDEX.length, scratch)); SimpleTextUtil.readLine(input, scratch); assert StringHelper.startsWith(scratch.get(), DOCVALUES_GEN); @@ -221,6 +222,10 @@ public DocValuesType docValuesType(String dvType) { return DocValuesType.valueOf(dvType); } + public DocValuesSkipIndexType docValuesSkipIndexType(String dvSkipIndexType) { + return DocValuesSkipIndexType.valueOf(dvSkipIndexType); + } + public VectorEncoding vectorEncoding(String vectorEncoding) { return VectorEncoding.valueOf(vectorEncoding); } @@ -284,7 +289,7 @@ public void write( SimpleTextUtil.writeNewline(out); SimpleTextUtil.write(out, DOCVALUES_SKIP_INDEX); - SimpleTextUtil.write(out, Boolean.toString(fi.hasDocValuesSkipIndex()), scratch); + SimpleTextUtil.write(out, getDocValuesSkipIndexType(fi.docValuesSkipIndexType()), scratch); SimpleTextUtil.writeNewline(out); SimpleTextUtil.write(out, DOCVALUES_GEN); @@ -355,4 +360,8 @@ public void write( private static String getDocValuesType(DocValuesType type) { return type.toString(); } + + private static String getDocValuesSkipIndexType(DocValuesSkipIndexType type) { + return type.toString(); + } } diff --git a/lucene/codecs/src/test/org/apache/lucene/codecs/uniformsplit/TestBlockWriter.java b/lucene/codecs/src/test/org/apache/lucene/codecs/uniformsplit/TestBlockWriter.java index 24f144481785..8710b846f937 100644 --- a/lucene/codecs/src/test/org/apache/lucene/codecs/uniformsplit/TestBlockWriter.java +++ b/lucene/codecs/src/test/org/apache/lucene/codecs/uniformsplit/TestBlockWriter.java @@ -20,6 +20,7 @@ import java.io.IOException; import java.util.Collections; import org.apache.lucene.codecs.lucene90.tests.MockTermStateFactory; +import org.apache.lucene.index.DocValuesSkipIndexType; import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.IndexOptions; @@ -111,7 +112,7 @@ private static FieldInfo getMockFieldInfo(String fieldName, int number) { true, IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS, DocValuesType.NONE, - false, + DocValuesSkipIndexType.NONE, -1, Collections.emptyMap(), 0, diff --git a/lucene/codecs/src/test/org/apache/lucene/codecs/uniformsplit/sharedterms/TestSTBlockReader.java b/lucene/codecs/src/test/org/apache/lucene/codecs/uniformsplit/sharedterms/TestSTBlockReader.java index 89bee90f3a8e..b2190c19ffeb 100644 --- a/lucene/codecs/src/test/org/apache/lucene/codecs/uniformsplit/sharedterms/TestSTBlockReader.java +++ b/lucene/codecs/src/test/org/apache/lucene/codecs/uniformsplit/sharedterms/TestSTBlockReader.java @@ -34,6 +34,7 @@ import org.apache.lucene.codecs.uniformsplit.FieldMetadata; import org.apache.lucene.codecs.uniformsplit.IndexDictionary; import org.apache.lucene.codecs.uniformsplit.TermBytes; +import org.apache.lucene.index.DocValuesSkipIndexType; import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.FieldInfos; @@ -198,7 +199,7 @@ private static FieldInfo mockFieldInfo(String fieldName, int number) { true, IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS, DocValuesType.NONE, - false, + DocValuesSkipIndexType.NONE, -1, Collections.emptyMap(), 0, diff --git a/lucene/core/src/java/org/apache/lucene/codecs/DocValuesProducer.java b/lucene/core/src/java/org/apache/lucene/codecs/DocValuesProducer.java index 2c90448a39c5..b8b9f68b52dc 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/DocValuesProducer.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/DocValuesProducer.java @@ -19,6 +19,7 @@ import java.io.Closeable; import java.io.IOException; import org.apache.lucene.index.BinaryDocValues; +import org.apache.lucene.index.DocValuesSkipIndexType; import org.apache.lucene.index.DocValuesSkipper; import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.FieldInfo; @@ -77,7 +78,7 @@ protected DocValuesProducer() {} /** * Returns a {@link DocValuesSkipper} for this field. The returned instance need not be * thread-safe: it will only be used by a single thread. The return value is undefined if {@link - * FieldInfo#hasDocValuesSkipIndex()} doesn't return {@code true}. + * FieldInfo#docValuesSkipIndexType()} returns {@link DocValuesSkipIndexType#NONE}. */ public abstract DocValuesSkipper getSkipper(FieldInfo field) throws IOException; diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesConsumer.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesConsumer.java index c9f2e7742333..13f1463d0dbf 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesConsumer.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesConsumer.java @@ -31,6 +31,7 @@ import org.apache.lucene.codecs.DocValuesProducer; import org.apache.lucene.index.BinaryDocValues; import org.apache.lucene.index.DocValues; +import org.apache.lucene.index.DocValuesSkipIndexType; import org.apache.lucene.index.EmptyDocValuesProducer; import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.IndexFileNames; @@ -143,7 +144,7 @@ public SortedNumericDocValues getSortedNumeric(FieldInfo field) throws IOExcepti return DocValues.singleton(valuesProducer.getNumeric(field)); } }; - if (field.hasDocValuesSkipIndex()) { + if (field.docValuesSkipIndexType() != DocValuesSkipIndexType.NONE) { writeSkipIndex(field, producer); } writeValues(field, producer, false); @@ -248,7 +249,7 @@ public static SkipAccumulator merge(List list, int index, int l private void writeSkipIndex(FieldInfo field, DocValuesProducer valuesProducer) throws IOException { - assert field.hasDocValuesSkipIndex(); + assert field.docValuesSkipIndexType() != DocValuesSkipIndexType.NONE; final long start = data.getFilePointer(); final SortedNumericDocValues values = valuesProducer.getSortedNumeric(field); long globalMaxValue = Long.MIN_VALUE; @@ -700,7 +701,7 @@ public long cost() { return DocValues.singleton(sortedOrds); } }; - if (field.hasDocValuesSkipIndex()) { + if (field.docValuesSkipIndexType() != DocValuesSkipIndexType.NONE) { writeSkipIndex(field, producer); } if (addTypeByte) { @@ -873,7 +874,7 @@ public void addSortedNumericField(FieldInfo field, DocValuesProducer valuesProdu private void doAddSortedNumericField( FieldInfo field, DocValuesProducer valuesProducer, boolean ords) throws IOException { - if (field.hasDocValuesSkipIndex()) { + if (field.docValuesSkipIndexType() != DocValuesSkipIndexType.NONE) { writeSkipIndex(field, valuesProducer); } if (ords) { diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesProducer.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesProducer.java index f6222f2a21f1..fb8d578acdf1 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesProducer.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesProducer.java @@ -27,6 +27,7 @@ import org.apache.lucene.index.BinaryDocValues; import org.apache.lucene.index.CorruptIndexException; import org.apache.lucene.index.DocValues; +import org.apache.lucene.index.DocValuesSkipIndexType; import org.apache.lucene.index.DocValuesSkipper; import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.FieldInfos; @@ -191,7 +192,7 @@ private void readFields(IndexInput meta, FieldInfos infos) throws IOException { throw new CorruptIndexException("Invalid field number: " + fieldNumber, meta); } byte type = meta.readByte(); - if (info.hasDocValuesSkipIndex()) { + if (info.docValuesSkipIndexType() != DocValuesSkipIndexType.NONE) { skippers.put(info.number, readDocValueSkipperMeta(meta)); } if (type == Lucene90DocValuesFormat.NUMERIC) { diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene94/Lucene94FieldInfosFormat.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene94/Lucene94FieldInfosFormat.java index 6594c7f275a2..c4cdd722c541 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/lucene94/Lucene94FieldInfosFormat.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene94/Lucene94FieldInfosFormat.java @@ -24,6 +24,7 @@ import org.apache.lucene.codecs.DocValuesFormat; import org.apache.lucene.codecs.FieldInfosFormat; import org.apache.lucene.index.CorruptIndexException; +import org.apache.lucene.index.DocValuesSkipIndexType; import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.FieldInfos; @@ -163,8 +164,6 @@ public FieldInfos read( boolean isSoftDeletesField = (bits & SOFT_DELETES_FIELD) != 0; boolean isParentField = format >= FORMAT_PARENT_FIELD ? (bits & PARENT_FIELD_FIELD) != 0 : false; - boolean hasDocValuesSkipIndex = - format >= FORMAT_DOCVALUE_SKIPPER ? (bits & DOCVALUES_SKIPPER) != 0 : false; if ((bits & 0xC0) != 0) { throw new CorruptIndexException( @@ -187,6 +186,12 @@ public FieldInfos read( // DV Types are packed in one byte final DocValuesType docValuesType = getDocValuesType(input, input.readByte()); + final DocValuesSkipIndexType docValuesSkipIndex; + if (format >= FORMAT_DOCVALUE_SKIPPER) { + docValuesSkipIndex = getDocValuesSkipIndexType(input, input.readByte()); + } else { + docValuesSkipIndex = DocValuesSkipIndexType.NONE; + } final long dvGen = input.readLong(); Map attributes = input.readMapOfStrings(); // just use the last field's map if its the same @@ -217,7 +222,7 @@ public FieldInfos read( storePayloads, indexOptions, docValuesType, - hasDocValuesSkipIndex, + docValuesSkipIndex, dvGen, attributes, pointDataDimensionCount, @@ -270,6 +275,18 @@ private static byte docValuesByte(DocValuesType type) { } } + private static byte docValuesSkipIndexByte(DocValuesSkipIndexType type) { + switch (type) { + case NONE: + return 0; + case RANGE: + return 1; + default: + // BUG + throw new AssertionError("unhandled DocValuesSkipIndexType: " + type); + } + } + private static DocValuesType getDocValuesType(IndexInput input, byte b) throws IOException { switch (b) { case 0: @@ -289,6 +306,18 @@ private static DocValuesType getDocValuesType(IndexInput input, byte b) throws I } } + private static DocValuesSkipIndexType getDocValuesSkipIndexType(IndexInput input, byte b) + throws IOException { + switch (b) { + case 0: + return DocValuesSkipIndexType.NONE; + case 1: + return DocValuesSkipIndexType.RANGE; + default: + throw new CorruptIndexException("invalid docvaluesskipindex byte: " + b, input); + } + } + private static VectorEncoding getVectorEncoding(IndexInput input, byte b) throws IOException { if (b < 0 || b >= VectorEncoding.values().length) { throw new CorruptIndexException("invalid vector encoding: " + b, input); @@ -404,13 +433,13 @@ public void write( if (fi.hasPayloads()) bits |= STORE_PAYLOADS; if (fi.isSoftDeletesField()) bits |= SOFT_DELETES_FIELD; if (fi.isParentField()) bits |= PARENT_FIELD_FIELD; - if (fi.hasDocValuesSkipIndex()) bits |= DOCVALUES_SKIPPER; output.writeByte(bits); output.writeByte(indexOptionsByte(fi.getIndexOptions())); // pack the DV type and hasNorms in one byte output.writeByte(docValuesByte(fi.getDocValuesType())); + output.writeByte(docValuesSkipIndexByte(fi.docValuesSkipIndexType())); output.writeLong(fi.getDocValuesGen()); output.writeMapOfStrings(fi.attributes()); output.writeVInt(fi.getPointDimensionCount()); diff --git a/lucene/core/src/java/org/apache/lucene/document/FieldType.java b/lucene/core/src/java/org/apache/lucene/document/FieldType.java index db4b37f6711c..632c3b24e0dd 100644 --- a/lucene/core/src/java/org/apache/lucene/document/FieldType.java +++ b/lucene/core/src/java/org/apache/lucene/document/FieldType.java @@ -20,6 +20,7 @@ import java.util.Map; import java.util.Objects; import org.apache.lucene.analysis.Analyzer; // javadocs +import org.apache.lucene.index.DocValuesSkipIndexType; import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.IndexWriterConfig; @@ -41,7 +42,7 @@ public class FieldType implements IndexableFieldType { private IndexOptions indexOptions = IndexOptions.NONE; private boolean frozen; private DocValuesType docValuesType = DocValuesType.NONE; - private boolean docValuesSkipIndex; + private DocValuesSkipIndexType docValuesSkipIndex = DocValuesSkipIndexType.NONE; private int dimensionCount; private int indexDimensionCount; private int dimensionNumBytes; @@ -61,7 +62,7 @@ public FieldType(IndexableFieldType ref) { this.omitNorms = ref.omitNorms(); this.indexOptions = ref.indexOptions(); this.docValuesType = ref.docValuesType(); - this.docValuesSkipIndex = ref.hasDocValuesSkipIndex(); + this.docValuesSkipIndex = ref.docValuesSkipIndexType(); this.dimensionCount = ref.pointDimensionCount(); this.indexDimensionCount = ref.pointIndexDimensionCount(); this.dimensionNumBytes = ref.pointNumBytes(); @@ -508,7 +509,7 @@ public void setDocValuesType(DocValuesType type) { } @Override - public boolean hasDocValuesSkipIndex() { + public DocValuesSkipIndexType docValuesSkipIndexType() { return docValuesSkipIndex; } @@ -518,7 +519,7 @@ public boolean hasDocValuesSkipIndex() { * correlate with fields that are part of the index sort, so that values can be expected to be * clustered in the doc ID space. */ - public void setDocValuesSkipIndex(boolean docValuesSkipIndex) { + public void setDocValuesSkipIndexType(DocValuesSkipIndexType docValuesSkipIndex) { checkIfFrozen(); this.docValuesSkipIndex = docValuesSkipIndex; } @@ -531,7 +532,7 @@ public int hashCode() { result = prime * result + indexDimensionCount; result = prime * result + dimensionNumBytes; result = prime * result + ((docValuesType == null) ? 0 : docValuesType.hashCode()); - result = prime * result + Boolean.hashCode(docValuesSkipIndex); + result = prime * result + (docValuesSkipIndex == null ? 0 : docValuesSkipIndex.hashCode()); result = prime * result + indexOptions.hashCode(); result = prime * result + (omitNorms ? 1231 : 1237); result = prime * result + (storeTermVectorOffsets ? 1231 : 1237); diff --git a/lucene/core/src/java/org/apache/lucene/document/NumericDocValuesField.java b/lucene/core/src/java/org/apache/lucene/document/NumericDocValuesField.java index 95ed6eb07115..6f1fb78e70c7 100644 --- a/lucene/core/src/java/org/apache/lucene/document/NumericDocValuesField.java +++ b/lucene/core/src/java/org/apache/lucene/document/NumericDocValuesField.java @@ -16,6 +16,7 @@ */ package org.apache.lucene.document; +import org.apache.lucene.index.DocValuesSkipIndexType; import org.apache.lucene.index.DocValuesType; import org.apache.lucene.search.IndexOrDocValuesQuery; import org.apache.lucene.search.Query; @@ -42,13 +43,13 @@ public class NumericDocValuesField extends Field { TYPE.freeze(); INDEXED_TYPE = new FieldType(TYPE); - INDEXED_TYPE.setDocValuesSkipIndex(true); + INDEXED_TYPE.setDocValuesSkipIndexType(DocValuesSkipIndexType.RANGE); INDEXED_TYPE.freeze(); } /** * Creates a new {@link NumericDocValuesField} with the specified 64-bit long value that also - * creates a {@link FieldType#hasDocValuesSkipIndex() skip index}. + * creates a {@link FieldType#docValuesSkipIndexType() skip index}. * * @param name field name * @param value 64-bit long value diff --git a/lucene/core/src/java/org/apache/lucene/document/SortedDocValuesField.java b/lucene/core/src/java/org/apache/lucene/document/SortedDocValuesField.java index 2ed6956b7170..746f65ae5647 100644 --- a/lucene/core/src/java/org/apache/lucene/document/SortedDocValuesField.java +++ b/lucene/core/src/java/org/apache/lucene/document/SortedDocValuesField.java @@ -17,6 +17,7 @@ package org.apache.lucene.document; import java.util.Collection; +import org.apache.lucene.index.DocValuesSkipIndexType; import org.apache.lucene.index.DocValuesType; import org.apache.lucene.search.IndexOrDocValuesQuery; import org.apache.lucene.search.MultiTermQuery; @@ -48,13 +49,13 @@ public class SortedDocValuesField extends Field { TYPE.freeze(); INDEXED_TYPE = new FieldType(TYPE); - INDEXED_TYPE.setDocValuesSkipIndex(true); + INDEXED_TYPE.setDocValuesSkipIndexType(DocValuesSkipIndexType.RANGE); INDEXED_TYPE.freeze(); } /** * Creates a new {@link SortedDocValuesField} with the specified 64-bit long value that also - * creates a {@link FieldType#hasDocValuesSkipIndex() skip index}. + * creates a {@link FieldType#docValuesSkipIndexType() skip index}. * * @param name field name * @param bytes binary content diff --git a/lucene/core/src/java/org/apache/lucene/document/SortedNumericDocValuesField.java b/lucene/core/src/java/org/apache/lucene/document/SortedNumericDocValuesField.java index 2d635462a226..0efb6f463939 100644 --- a/lucene/core/src/java/org/apache/lucene/document/SortedNumericDocValuesField.java +++ b/lucene/core/src/java/org/apache/lucene/document/SortedNumericDocValuesField.java @@ -16,6 +16,7 @@ */ package org.apache.lucene.document; +import org.apache.lucene.index.DocValuesSkipIndexType; import org.apache.lucene.index.DocValuesType; import org.apache.lucene.search.IndexOrDocValuesQuery; import org.apache.lucene.search.Query; @@ -50,13 +51,13 @@ public class SortedNumericDocValuesField extends Field { TYPE.freeze(); INDEXED_TYPE = new FieldType(TYPE); - INDEXED_TYPE.setDocValuesSkipIndex(true); + INDEXED_TYPE.setDocValuesSkipIndexType(DocValuesSkipIndexType.RANGE); INDEXED_TYPE.freeze(); } /** * Creates a new {@link SortedNumericDocValuesField} with the specified 64-bit long value that - * also creates a {@link FieldType#hasDocValuesSkipIndex() skip index}. + * also creates a {@link FieldType#docValuesSkipIndexType() skip index}. * * @param name field name * @param value 64-bit long value diff --git a/lucene/core/src/java/org/apache/lucene/document/SortedSetDocValuesField.java b/lucene/core/src/java/org/apache/lucene/document/SortedSetDocValuesField.java index 74ae5dc80432..02e5a82d6a6f 100644 --- a/lucene/core/src/java/org/apache/lucene/document/SortedSetDocValuesField.java +++ b/lucene/core/src/java/org/apache/lucene/document/SortedSetDocValuesField.java @@ -17,6 +17,7 @@ package org.apache.lucene.document; import java.util.Collection; +import org.apache.lucene.index.DocValuesSkipIndexType; import org.apache.lucene.index.DocValuesType; import org.apache.lucene.search.IndexOrDocValuesQuery; import org.apache.lucene.search.MultiTermQuery; @@ -49,13 +50,13 @@ public class SortedSetDocValuesField extends Field { TYPE.freeze(); INDEXED_TYPE = new FieldType(TYPE); - INDEXED_TYPE.setDocValuesSkipIndex(true); + INDEXED_TYPE.setDocValuesSkipIndexType(DocValuesSkipIndexType.RANGE); INDEXED_TYPE.freeze(); } /** * Creates a new {@link SortedSetDocValuesField} with the specified 64-bit long value that also - * creates a {@link FieldType#hasDocValuesSkipIndex() skip index}. + * creates a {@link FieldType#docValuesSkipIndexType() skip index}. * * @param name field name * @param bytes binary content diff --git a/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java b/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java index e61501a1244b..b8256ecf5875 100644 --- a/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java +++ b/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java @@ -3731,7 +3731,7 @@ private static void checkNumericDocValues( private static void checkDocValues( FieldInfo fi, DocValuesProducer dvReader, DocValuesStatus status) throws Exception { - if (fi.hasDocValuesSkipIndex()) { + if (fi.docValuesSkipIndexType() != DocValuesSkipIndexType.NONE) { status.totalSkippingIndex++; checkDocValueSkipper(fi, dvReader.getSkipper(fi)); } diff --git a/lucene/core/src/java/org/apache/lucene/index/CodecReader.java b/lucene/core/src/java/org/apache/lucene/index/CodecReader.java index bec27c5176e8..20be7e1a45a8 100644 --- a/lucene/core/src/java/org/apache/lucene/index/CodecReader.java +++ b/lucene/core/src/java/org/apache/lucene/index/CodecReader.java @@ -200,7 +200,7 @@ public final SortedSetDocValues getSortedSetDocValues(String field) throws IOExc public final DocValuesSkipper getDocValuesSkipper(String field) throws IOException { ensureOpen(); FieldInfo fi = getFieldInfos().fieldInfo(field); - if (fi == null || fi.hasDocValuesSkipIndex() == false) { + if (fi == null || fi.docValuesSkipIndexType() == DocValuesSkipIndexType.NONE) { return null; } return getDocValuesReader().getSkipper(fi); diff --git a/lucene/core/src/java/org/apache/lucene/index/DocValuesSkipIndexType.java b/lucene/core/src/java/org/apache/lucene/index/DocValuesSkipIndexType.java new file mode 100644 index 000000000000..fcdb735b3174 --- /dev/null +++ b/lucene/core/src/java/org/apache/lucene/index/DocValuesSkipIndexType.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.index; + +/** Options for skip indexes on doc values. */ +public enum DocValuesSkipIndexType { + /** No skip index should be created. */ + NONE { + @Override + boolean isCompatibleWith(DocValuesType dvType) { + return true; + } + }, + /** + * Record range of values. This is suitable for {@link DocValuesType#NUMERIC}, {@link + * DocValuesType#SORTED_NUMERIC}, {@link DocValuesType#SORTED} and {@link + * DocValuesType#SORTED_SET} doc values, and will record the min/max values per range of doc IDs. + */ + RANGE { + @Override + boolean isCompatibleWith(DocValuesType dvType) { + return dvType == DocValuesType.NUMERIC + || dvType == DocValuesType.SORTED_NUMERIC + || dvType == DocValuesType.SORTED + || dvType == DocValuesType.SORTED_SET; + } + }; + + // TODO: add support for pre-aggregated integer/float/double + + abstract boolean isCompatibleWith(DocValuesType dvType); +} diff --git a/lucene/core/src/java/org/apache/lucene/index/DocValuesType.java b/lucene/core/src/java/org/apache/lucene/index/DocValuesType.java index f40132907a63..5c4f76fdd6fc 100644 --- a/lucene/core/src/java/org/apache/lucene/index/DocValuesType.java +++ b/lucene/core/src/java/org/apache/lucene/index/DocValuesType.java @@ -22,37 +22,31 @@ */ public enum DocValuesType { /** No doc values for this field. */ - NONE(false), + NONE, /** A per-document Number */ - NUMERIC(true), + NUMERIC, /** * A per-document byte[]. Values may be larger than 32766 bytes, but different codecs may enforce * their own limits. */ - BINARY(false), + BINARY, /** * A pre-sorted byte[]. Fields with this type only store distinct byte values and store an * additional offset pointer per document to dereference the shared byte[]. The stored byte[] is * presorted and allows access via document id, ordinal and by-value. Values must be {@code <= * 32766} bytes. */ - SORTED(true), + SORTED, /** * A pre-sorted Number[]. Fields with this type store numeric values in sorted order according to * {@link Long#compare(long, long)}. */ - SORTED_NUMERIC(true), + SORTED_NUMERIC, /** * A pre-sorted Set<byte[]>. Fields with this type only store distinct byte values and store * additional offset pointers per document to dereference the shared byte[]s. The stored byte[] is * presorted and allows access via document id, ordinal and by-value. Values must be {@code <= * 32766} bytes. */ - SORTED_SET(true); - - final boolean supportsSkipIndex; // pkg-private for use in FieldInfo - - DocValuesType(boolean supportsSkipIndex) { - this.supportsSkipIndex = supportsSkipIndex; - } + SORTED_SET; } diff --git a/lucene/core/src/java/org/apache/lucene/index/FieldInfo.java b/lucene/core/src/java/org/apache/lucene/index/FieldInfo.java index edb9bebf900d..e6bdddc9239d 100644 --- a/lucene/core/src/java/org/apache/lucene/index/FieldInfo.java +++ b/lucene/core/src/java/org/apache/lucene/index/FieldInfo.java @@ -37,7 +37,7 @@ public final class FieldInfo { private DocValuesType docValuesType = DocValuesType.NONE; - private final boolean docValuesSkipIndex; + private final DocValuesSkipIndexType docValuesSkipIndex; // True if any document indexed term vectors private boolean storeTermVector; @@ -83,7 +83,7 @@ public FieldInfo( boolean storePayloads, IndexOptions indexOptions, DocValuesType docValues, - boolean hasDocValuesSkipIndex, + DocValuesSkipIndexType docValuesSkipIndex, long dvGen, Map attributes, int pointDimensionCount, @@ -99,7 +99,7 @@ public FieldInfo( this.docValuesType = Objects.requireNonNull( docValues, "DocValuesType must not be null (field: \"" + name + "\")"); - this.docValuesSkipIndex = hasDocValuesSkipIndex; + this.docValuesSkipIndex = docValuesSkipIndex; this.indexOptions = Objects.requireNonNull( indexOptions, "IndexOptions must not be null (field: \"" + name + "\")"); @@ -157,11 +157,13 @@ public void checkConsistency() { if (docValuesType == null) { throw new IllegalArgumentException("DocValuesType must not be null (field: '" + name + "')"); } - if (docValuesType.supportsSkipIndex == false && docValuesSkipIndex) { + if (docValuesSkipIndex.isCompatibleWith(docValuesType) == false) { throw new IllegalArgumentException( "field '" + name - + "' cannot have docValuesSkipIndex set to true with doc values type " + + "' cannot have docValuesSkipIndexType=" + + docValuesSkipIndex + + " with doc values type " + docValuesType); } if (dvGen != -1 && docValuesType == DocValuesType.NONE) { @@ -308,14 +310,16 @@ static void verifySameDocValuesType( * @throws IllegalArgumentException if they are not the same */ static void verifySameDocValuesSkipIndex( - String fieldName, boolean hasDocValuesSkipIndex1, boolean hasDocValuesSkipIndex2) { + String fieldName, + DocValuesSkipIndexType hasDocValuesSkipIndex1, + DocValuesSkipIndexType hasDocValuesSkipIndex2) { if (hasDocValuesSkipIndex1 != hasDocValuesSkipIndex2) { throw new IllegalArgumentException( "cannot change field \"" + fieldName - + "\" from docValuesSkipIndex=" + + "\" from docValuesSkipIndexType=" + hasDocValuesSkipIndex1 - + " to inconsistent docValuesSkipIndex=" + + " to inconsistent docValuesSkipIndexType=" + hasDocValuesSkipIndex2); } } @@ -589,7 +593,7 @@ public DocValuesType getDocValuesType() { } /** Returns true if, and only if, this field has a skip index. */ - public boolean hasDocValuesSkipIndex() { + public DocValuesSkipIndexType docValuesSkipIndexType() { return docValuesSkipIndex; } diff --git a/lucene/core/src/java/org/apache/lucene/index/FieldInfos.java b/lucene/core/src/java/org/apache/lucene/index/FieldInfos.java index fecd17db0c83..5392c102ca91 100644 --- a/lucene/core/src/java/org/apache/lucene/index/FieldInfos.java +++ b/lucene/core/src/java/org/apache/lucene/index/FieldInfos.java @@ -365,7 +365,7 @@ private record FieldProperties( IndexOptions indexOptions, IndexOptionsProperties indexOptionsProperties, DocValuesType docValuesType, - boolean docValuesSkipIndex, + DocValuesSkipIndexType docValuesSkipIndex, FieldDimensions fieldDimensions, FieldVectorProperties fieldVectorProperties) {} @@ -444,7 +444,7 @@ synchronized int addOrGet(FieldInfo fi) { ? new IndexOptionsProperties(fi.hasTermVectors(), fi.omitsNorms()) : null, fi.getDocValuesType(), - fi.hasDocValuesSkipIndex(), + fi.docValuesSkipIndexType(), new FieldDimensions( fi.getPointDimensionCount(), fi.getPointIndexDimensionCount(), @@ -524,9 +524,9 @@ private void verifySameSchema(FieldInfo fi) { DocValuesType currentDVType = fieldProperties.docValuesType; verifySameDocValuesType(fieldName, currentDVType, fi.getDocValuesType()); - boolean currentDocValuesSkipIndex = fieldProperties.docValuesSkipIndex; + DocValuesSkipIndexType currentDocValuesSkipIndex = fieldProperties.docValuesSkipIndex; verifySameDocValuesSkipIndex( - fieldName, currentDocValuesSkipIndex, fi.hasDocValuesSkipIndex()); + fieldName, currentDocValuesSkipIndex, fi.docValuesSkipIndexType()); FieldDimensions dims = fieldProperties.fieldDimensions; verifySamePointsOptions( @@ -582,7 +582,7 @@ synchronized void verifyOrCreateDvOnlyField( false, IndexOptions.NONE, dvType, - false, + DocValuesSkipIndexType.NONE, -1, new HashMap<>(), 0, @@ -609,8 +609,8 @@ synchronized void verifyOrCreateDvOnlyField( + fieldDvType + "]."); } - boolean hasDocValuesSkipIndex = fieldProperties.docValuesSkipIndex; - if (hasDocValuesSkipIndex) { + DocValuesSkipIndexType hasDocValuesSkipIndex = fieldProperties.docValuesSkipIndex; + if (hasDocValuesSkipIndex != DocValuesSkipIndexType.NONE) { throw new IllegalArgumentException( "Can't update [" + dvType @@ -676,7 +676,7 @@ FieldInfo constructFieldInfo(String fieldName, DocValuesType dvType, int newFiel false, IndexOptions.NONE, dvType, - false, + DocValuesSkipIndexType.NONE, -1, new HashMap<>(), 0, @@ -797,7 +797,7 @@ FieldInfo add(FieldInfo fi, long dvGen) { fi.hasPayloads(), fi.getIndexOptions(), fi.getDocValuesType(), - fi.hasDocValuesSkipIndex(), + fi.docValuesSkipIndexType(), dvGen, // original attributes is UnmodifiableMap new HashMap<>(fi.attributes()), diff --git a/lucene/core/src/java/org/apache/lucene/index/IndexableFieldType.java b/lucene/core/src/java/org/apache/lucene/index/IndexableFieldType.java index 006828e98a25..f76956ec5bad 100644 --- a/lucene/core/src/java/org/apache/lucene/index/IndexableFieldType.java +++ b/lucene/core/src/java/org/apache/lucene/index/IndexableFieldType.java @@ -87,7 +87,7 @@ public interface IndexableFieldType { DocValuesType docValuesType(); /** Whether a skip index for doc values should be created on this field. */ - boolean hasDocValuesSkipIndex(); + DocValuesSkipIndexType docValuesSkipIndexType(); /** * If this is positive (representing the number of point dimensions), the field is indexed as a diff --git a/lucene/core/src/java/org/apache/lucene/index/IndexingChain.java b/lucene/core/src/java/org/apache/lucene/index/IndexingChain.java index bb81a1fe5774..f69ff533e665 100644 --- a/lucene/core/src/java/org/apache/lucene/index/IndexingChain.java +++ b/lucene/core/src/java/org/apache/lucene/index/IndexingChain.java @@ -680,7 +680,7 @@ private void initializeFieldInfo(PerField pf) throws IOException { false, s.indexOptions, s.docValuesType, - s.hasDocValuesSkipIndex, + s.docValuesSkipIndex, -1, s.attributes, s.pointDimensionCount, @@ -832,12 +832,14 @@ private static void updateDocFieldSchema( verifyUnIndexedFieldType(fieldName, fieldType); } if (fieldType.docValuesType() != DocValuesType.NONE) { - schema.setDocValues(fieldType.docValuesType(), fieldType.hasDocValuesSkipIndex()); - } else if (fieldType.hasDocValuesSkipIndex()) { + schema.setDocValues(fieldType.docValuesType(), fieldType.docValuesSkipIndexType()); + } else if (fieldType.docValuesSkipIndexType() != DocValuesSkipIndexType.NONE) { throw new IllegalArgumentException( "field '" + schema.name - + "' cannot have docValuesSkipIndex set to true without doc values"); + + "' cannot have docValuesSkipIndexType=" + + fieldType.docValuesSkipIndexType() + + " without doc values"); } if (fieldType.pointDimensionCount() != 0) { schema.setPoints( @@ -1440,7 +1442,7 @@ private static final class FieldSchema { private boolean storeTermVector = false; private IndexOptions indexOptions = IndexOptions.NONE; private DocValuesType docValuesType = DocValuesType.NONE; - private boolean hasDocValuesSkipIndex = false; + private DocValuesSkipIndexType docValuesSkipIndex = DocValuesSkipIndexType.NONE; private int pointDimensionCount = 0; private int pointIndexDimensionCount = 0; private int pointNumBytes = 0; @@ -1506,13 +1508,14 @@ void setIndexOptions( } } - void setDocValues(DocValuesType newDocValuesType, boolean newHasDocValuesSkipIndex) { + void setDocValues( + DocValuesType newDocValuesType, DocValuesSkipIndexType newDocValuesSkipIndex) { if (docValuesType == DocValuesType.NONE) { this.docValuesType = newDocValuesType; - this.hasDocValuesSkipIndex = newHasDocValuesSkipIndex; + this.docValuesSkipIndex = newDocValuesSkipIndex; } else { assertSame("doc values type", docValuesType, newDocValuesType); - assertSame("doc values skip index", hasDocValuesSkipIndex, newHasDocValuesSkipIndex); + assertSame("doc values skip index type", docValuesSkipIndex, newDocValuesSkipIndex); } } @@ -1560,7 +1563,7 @@ void assertSameSchema(FieldInfo fi) { assertSame("omit norms", fi.omitsNorms(), omitNorms); assertSame("store term vector", fi.hasTermVectors(), storeTermVector); assertSame("doc values type", fi.getDocValuesType(), docValuesType); - assertSame("doc values skip index", fi.hasDocValuesSkipIndex(), hasDocValuesSkipIndex); + assertSame("doc values skip index type", fi.docValuesSkipIndexType(), docValuesSkipIndex); assertSame( "vector similarity function", fi.getVectorSimilarityFunction(), vectorSimilarityFunction); assertSame("vector encoding", fi.getVectorEncoding(), vectorEncoding); diff --git a/lucene/core/src/java/org/apache/lucene/index/ReadersAndUpdates.java b/lucene/core/src/java/org/apache/lucene/index/ReadersAndUpdates.java index 6f54ea16d888..e39861a0671b 100644 --- a/lucene/core/src/java/org/apache/lucene/index/ReadersAndUpdates.java +++ b/lucene/core/src/java/org/apache/lucene/index/ReadersAndUpdates.java @@ -713,7 +713,7 @@ private FieldInfo cloneFieldInfo(FieldInfo fi, int fieldNumber) { fi.hasPayloads(), fi.getIndexOptions(), fi.getDocValuesType(), - fi.hasDocValuesSkipIndex(), + fi.docValuesSkipIndexType(), fi.getDocValuesGen(), new HashMap<>(fi.attributes()), fi.getPointDimensionCount(), diff --git a/lucene/core/src/test/org/apache/lucene/index/TestCodecs.java b/lucene/core/src/test/org/apache/lucene/index/TestCodecs.java index 1759271012d1..bdbb2cf8f052 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestCodecs.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestCodecs.java @@ -106,7 +106,7 @@ public FieldData( storePayloads, indexOptions, DocValuesType.NONE, - false, + DocValuesSkipIndexType.NONE, -1, new HashMap<>(), 0, diff --git a/lucene/core/src/test/org/apache/lucene/index/TestFieldInfos.java b/lucene/core/src/test/org/apache/lucene/index/TestFieldInfos.java index e19855bbdda8..a658c8571ec9 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestFieldInfos.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestFieldInfos.java @@ -250,7 +250,7 @@ public void testFieldNumbersAutoIncrement() { false, IndexOptions.NONE, DocValuesType.NONE, - false, + DocValuesSkipIndexType.NONE, -1, new HashMap<>(), 0, @@ -272,7 +272,7 @@ public void testFieldNumbersAutoIncrement() { false, IndexOptions.NONE, DocValuesType.NONE, - false, + DocValuesSkipIndexType.NONE, -1, new HashMap<>(), 0, @@ -296,7 +296,7 @@ public void testFieldNumbersAutoIncrement() { false, IndexOptions.NONE, DocValuesType.NONE, - false, + DocValuesSkipIndexType.NONE, -1, new HashMap<>(), 0, diff --git a/lucene/core/src/test/org/apache/lucene/index/TestFieldsReader.java b/lucene/core/src/test/org/apache/lucene/index/TestFieldsReader.java index 15d6dddcb586..8580691028b5 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestFieldsReader.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestFieldsReader.java @@ -58,7 +58,7 @@ public static void beforeClass() throws Exception { false, ift.indexOptions(), ift.docValuesType(), - ift.hasDocValuesSkipIndex(), + ift.docValuesSkipIndexType(), -1, new HashMap<>(), 0, diff --git a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java index 5a090c131ddd..04adc5035db4 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java @@ -4992,8 +4992,9 @@ public void testDocValuesMixedSkippingIndex() throws Exception { doc2.add(new SortedNumericDocValuesField("test", random().nextLong())); IllegalArgumentException ex = expectThrows(IllegalArgumentException.class, () -> writer.addDocument(doc2)); + ex.printStackTrace(); assertEquals( - "Inconsistency of field data structures across documents for field [test] of doc [1]. doc values skip index: expected 'true', but it has 'false'.", + "Inconsistency of field data structures across documents for field [test] of doc [1]. doc values skip index type: expected 'RANGE', but it has 'NONE'.", ex.getMessage()); } } @@ -5009,7 +5010,7 @@ public void testDocValuesMixedSkippingIndex() throws Exception { IllegalArgumentException ex = expectThrows(IllegalArgumentException.class, () -> writer.addDocument(doc2)); assertEquals( - "Inconsistency of field data structures across documents for field [test] of doc [1]. doc values skip index: expected 'false', but it has 'true'.", + "Inconsistency of field data structures across documents for field [test] of doc [1]. doc values skip index type: expected 'NONE', but it has 'RANGE'.", ex.getMessage()); } } @@ -5021,7 +5022,7 @@ public void testDocValuesSkippingIndexWithoutDocValues() throws Exception { FieldType fieldType = new FieldType(); fieldType.setStored(true); fieldType.setDocValuesType(docValuesType); - fieldType.setDocValuesSkipIndex(true); + fieldType.setDocValuesSkipIndexType(DocValuesSkipIndexType.RANGE); fieldType.freeze(); try (Directory dir = newMockDirectory()) { try (IndexWriter writer = @@ -5031,8 +5032,7 @@ public void testDocValuesSkippingIndexWithoutDocValues() throws Exception { IllegalArgumentException ex = expectThrows(IllegalArgumentException.class, () -> writer.addDocument(doc1)); assertTrue( - ex.getMessage() - .startsWith("field 'test' cannot have docValuesSkipIndex set to true")); + ex.getMessage().startsWith("field 'test' cannot have docValuesSkipIndexType=RANGE")); } } } diff --git a/lucene/core/src/test/org/apache/lucene/index/TestIndexableField.java b/lucene/core/src/test/org/apache/lucene/index/TestIndexableField.java index 19f6bdd74e1b..ddd25dd7682c 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestIndexableField.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestIndexableField.java @@ -96,8 +96,8 @@ public DocValuesType docValuesType() { } @Override - public boolean hasDocValuesSkipIndex() { - return false; + public DocValuesSkipIndexType docValuesSkipIndexType() { + return DocValuesSkipIndexType.NONE; } @Override diff --git a/lucene/core/src/test/org/apache/lucene/index/TestPendingSoftDeletes.java b/lucene/core/src/test/org/apache/lucene/index/TestPendingSoftDeletes.java index cb57e9836919..d3236845a4bc 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestPendingSoftDeletes.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestPendingSoftDeletes.java @@ -191,7 +191,7 @@ public void testApplyUpdates() throws IOException { false, IndexOptions.NONE, DocValuesType.NUMERIC, - false, + DocValuesSkipIndexType.NONE, 0, Collections.emptyMap(), 0, @@ -231,7 +231,7 @@ public void testApplyUpdates() throws IOException { false, IndexOptions.NONE, DocValuesType.NUMERIC, - false, + DocValuesSkipIndexType.NONE, 1, Collections.emptyMap(), 0, @@ -297,7 +297,7 @@ public void testUpdateAppliedOnlyOnce() throws IOException { false, IndexOptions.NONE, DocValuesType.NUMERIC, - false, + DocValuesSkipIndexType.NONE, segmentInfo.getNextDocValuesGen(), Collections.emptyMap(), 0, @@ -368,7 +368,7 @@ public void testResetOnUpdate() throws IOException { false, IndexOptions.NONE, DocValuesType.NUMERIC, - false, + DocValuesSkipIndexType.NONE, segmentInfo.getNextDocValuesGen(), Collections.emptyMap(), 0, @@ -407,7 +407,7 @@ public void testResetOnUpdate() throws IOException { false, IndexOptions.NONE, DocValuesType.NUMERIC, - false, + DocValuesSkipIndexType.NONE, segmentInfo.getNextDocValuesGen(), Collections.emptyMap(), 0, diff --git a/lucene/core/src/test/org/apache/lucene/search/TestSortOptimization.java b/lucene/core/src/test/org/apache/lucene/search/TestSortOptimization.java index 4ddb4dc79c9a..38dbe3399ee8 100644 --- a/lucene/core/src/test/org/apache/lucene/search/TestSortOptimization.java +++ b/lucene/core/src/test/org/apache/lucene/search/TestSortOptimization.java @@ -1313,7 +1313,7 @@ public FieldInfos getFieldInfos() { false, IndexOptions.NONE, fi.getDocValuesType(), - fi.hasDocValuesSkipIndex(), + fi.docValuesSkipIndexType(), fi.getDocValuesGen(), fi.attributes(), 0, diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/highlight/TermVectorLeafReader.java b/lucene/highlighter/src/java/org/apache/lucene/search/highlight/TermVectorLeafReader.java index f60c7966f984..cb8c71a089f0 100644 --- a/lucene/highlighter/src/java/org/apache/lucene/search/highlight/TermVectorLeafReader.java +++ b/lucene/highlighter/src/java/org/apache/lucene/search/highlight/TermVectorLeafReader.java @@ -21,6 +21,7 @@ import java.util.Iterator; import org.apache.lucene.index.BinaryDocValues; import org.apache.lucene.index.ByteVectorValues; +import org.apache.lucene.index.DocValuesSkipIndexType; import org.apache.lucene.index.DocValuesSkipper; import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.FieldInfo; @@ -96,7 +97,7 @@ public int size() { terms.hasPayloads(), indexOptions, DocValuesType.NONE, - false, + DocValuesSkipIndexType.NONE, -1, Collections.emptyMap(), 0, diff --git a/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java b/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java index 8874f4d30c6b..2d46b243d838 100644 --- a/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java +++ b/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java @@ -736,7 +736,7 @@ private FieldInfo createFieldInfo(String fieldName, int ord, IndexableFieldType storePayloads, indexOptions, fieldType.docValuesType(), - false, + fieldType.docValuesSkipIndexType(), -1, Collections.emptyMap(), fieldType.pointDimensionCount(), @@ -841,7 +841,7 @@ private void storeDocValues(Info info, DocValuesType docValuesType, Object docVa info.fieldInfo.hasPayloads(), info.fieldInfo.getIndexOptions(), docValuesType, - false, + DocValuesSkipIndexType.NONE, -1, info.fieldInfo.attributes(), info.fieldInfo.getPointDimensionCount(), diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/asserting/AssertingDocValuesFormat.java b/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/asserting/AssertingDocValuesFormat.java index 3355d925ecae..619b5b02b808 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/asserting/AssertingDocValuesFormat.java +++ b/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/asserting/AssertingDocValuesFormat.java @@ -23,6 +23,7 @@ import org.apache.lucene.codecs.DocValuesFormat; import org.apache.lucene.codecs.DocValuesProducer; import org.apache.lucene.index.BinaryDocValues; +import org.apache.lucene.index.DocValuesSkipIndexType; import org.apache.lucene.index.DocValuesSkipper; import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.FieldInfo; @@ -283,7 +284,7 @@ public SortedSetDocValues getSortedSet(FieldInfo field) throws IOException { @Override public DocValuesSkipper getSkipper(FieldInfo field) throws IOException { - assert field.hasDocValuesSkipIndex(); + assert field.docValuesSkipIndexType() != DocValuesSkipIndexType.NONE; DocValuesSkipper skipper = in.getSkipper(field); assert skipper != null; return new AssertingLeafReader.AssertingDocValuesSkipper(skipper); diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/index/AssertingLeafReader.java b/lucene/test-framework/src/java/org/apache/lucene/tests/index/AssertingLeafReader.java index 02b5c098afdc..50300dc30bcb 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/tests/index/AssertingLeafReader.java +++ b/lucene/test-framework/src/java/org/apache/lucene/tests/index/AssertingLeafReader.java @@ -23,6 +23,7 @@ import java.util.Objects; import org.apache.lucene.index.BinaryDocValues; import org.apache.lucene.index.DocValues; +import org.apache.lucene.index.DocValuesSkipIndexType; import org.apache.lucene.index.DocValuesSkipper; import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.FieldInfo; @@ -1625,10 +1626,10 @@ public DocValuesSkipper getDocValuesSkipper(String field) throws IOException { DocValuesSkipper skipper = super.getDocValuesSkipper(field); FieldInfo fi = getFieldInfos().fieldInfo(field); if (skipper != null) { - assert fi.hasDocValuesSkipIndex(); + assert fi.docValuesSkipIndexType() != DocValuesSkipIndexType.NONE; return new AssertingDocValuesSkipper(skipper); } else { - assert fi == null || fi.hasDocValuesSkipIndex() == false; + assert fi == null || fi.docValuesSkipIndexType() == DocValuesSkipIndexType.NONE; return null; } } diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/index/BaseFieldInfoFormatTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/tests/index/BaseFieldInfoFormatTestCase.java index c7532e1269f7..ba2c6f362a3c 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/tests/index/BaseFieldInfoFormatTestCase.java +++ b/lucene/test-framework/src/java/org/apache/lucene/tests/index/BaseFieldInfoFormatTestCase.java @@ -30,6 +30,7 @@ import org.apache.lucene.document.FieldType; import org.apache.lucene.document.StoredField; import org.apache.lucene.document.TextField; +import org.apache.lucene.index.DocValuesSkipIndexType; import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.FieldInfos; @@ -303,14 +304,14 @@ public void testRandom() throws Exception { storePayloads = random().nextBoolean(); } } - boolean hasDocValuesSkipIndex = false; + DocValuesSkipIndexType docValuesSkipIndexType = DocValuesSkipIndexType.NONE; if (EnumSet.of( DocValuesType.NUMERIC, DocValuesType.SORTED, DocValuesType.SORTED_NUMERIC, DocValuesType.SORTED_SET) .contains(fieldType.docValuesType())) { - hasDocValuesSkipIndex = fieldType.hasDocValuesSkipIndex(); + docValuesSkipIndexType = fieldType.docValuesSkipIndexType(); } FieldInfo fi = new FieldInfo( @@ -321,7 +322,7 @@ public void testRandom() throws Exception { storePayloads, fieldType.indexOptions(), fieldType.docValuesType(), - hasDocValuesSkipIndex, + docValuesSkipIndexType, -1, new HashMap<>(), fieldType.pointDimensionCount(), @@ -374,7 +375,10 @@ private IndexableFieldType randomFieldType(Random r, String fieldName) { || current == DocValuesType.SORTED_NUMERIC || current == DocValuesType.SORTED || current == DocValuesType.SORTED_SET) { - type.setDocValuesSkipIndex(supportDocValuesSkipIndex() && random().nextBoolean()); + type.setDocValuesSkipIndexType( + supportDocValuesSkipIndex() + ? DocValuesSkipIndexType.RANGE + : DocValuesSkipIndexType.NONE); } } @@ -414,7 +418,7 @@ protected void assertEquals(FieldInfo expected, FieldInfo actual) { assertEquals(expected.number, actual.number); assertEquals(expected.name, actual.name); assertEquals(expected.getDocValuesType(), actual.getDocValuesType()); - assertEquals(expected.hasDocValuesSkipIndex(), actual.hasDocValuesSkipIndex()); + assertEquals(expected.docValuesSkipIndexType(), actual.docValuesSkipIndexType()); assertEquals(expected.getIndexOptions(), actual.getIndexOptions()); assertEquals(expected.hasNorms(), actual.hasNorms()); assertEquals(expected.hasPayloads(), actual.hasPayloads()); @@ -455,7 +459,7 @@ private FieldInfo createFieldInfo() { false, TextField.TYPE_STORED.indexOptions(), DocValuesType.NONE, - false, + DocValuesSkipIndexType.NONE, -1, new HashMap<>(), 0, diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/index/BaseIndexFileFormatTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/tests/index/BaseIndexFileFormatTestCase.java index 67d63ad9c652..297c1b777f53 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/tests/index/BaseIndexFileFormatTestCase.java +++ b/lucene/test-framework/src/java/org/apache/lucene/tests/index/BaseIndexFileFormatTestCase.java @@ -357,7 +357,7 @@ public void testMultiClose() throws IOException { proto.hasPayloads(), proto.getIndexOptions(), proto.getDocValuesType(), - proto.hasDocValuesSkipIndex(), + proto.docValuesSkipIndexType(), proto.getDocValuesGen(), new HashMap<>(), proto.getPointDimensionCount(), diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/index/BaseKnnVectorsFormatTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/tests/index/BaseKnnVectorsFormatTestCase.java index ed4abb6f2c62..4c9165e1a10d 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/tests/index/BaseKnnVectorsFormatTestCase.java +++ b/lucene/test-framework/src/java/org/apache/lucene/tests/index/BaseKnnVectorsFormatTestCase.java @@ -46,6 +46,7 @@ import org.apache.lucene.index.CheckIndex; import org.apache.lucene.index.CodecReader; import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.DocValuesSkipIndexType; import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.FieldInfos; @@ -377,7 +378,7 @@ public void testWriterRamEstimate() throws Exception { false, IndexOptions.NONE, DocValuesType.NONE, - false, + DocValuesSkipIndexType.NONE, -1, Map.of(), 0, diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/index/MismatchedLeafReader.java b/lucene/test-framework/src/java/org/apache/lucene/tests/index/MismatchedLeafReader.java index 1e070897592d..eddee35240f7 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/tests/index/MismatchedLeafReader.java +++ b/lucene/test-framework/src/java/org/apache/lucene/tests/index/MismatchedLeafReader.java @@ -103,7 +103,7 @@ static FieldInfos shuffleInfos(FieldInfos infos, Random random) { oldInfo.hasPayloads(), // storePayloads oldInfo.getIndexOptions(), // indexOptions oldInfo.getDocValuesType(), // docValuesType - oldInfo.hasDocValuesSkipIndex(), // hasDocValuesSkipIndex + oldInfo.docValuesSkipIndexType(), // docValuesSkipIndexType oldInfo.getDocValuesGen(), // dvGen oldInfo.attributes(), // attributes oldInfo.getPointDimensionCount(), // data dimension count diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/index/RandomPostingsTester.java b/lucene/test-framework/src/java/org/apache/lucene/tests/index/RandomPostingsTester.java index 5dea79f203a2..78329e889a2e 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/tests/index/RandomPostingsTester.java +++ b/lucene/test-framework/src/java/org/apache/lucene/tests/index/RandomPostingsTester.java @@ -45,6 +45,7 @@ import org.apache.lucene.codecs.FieldsProducer; import org.apache.lucene.codecs.NormsProducer; import org.apache.lucene.index.BaseTermsEnum; +import org.apache.lucene.index.DocValuesSkipIndexType; import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.FieldInfos; @@ -157,7 +158,7 @@ public RandomPostingsTester(Random random) throws IOException { true, IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS, DocValuesType.NONE, - false, + DocValuesSkipIndexType.NONE, -1, new HashMap<>(), 0, @@ -732,7 +733,7 @@ public FieldsProducer buildIndex( doPayloads, indexOptions, DocValuesType.NONE, - false, + DocValuesSkipIndexType.NONE, -1, new HashMap<>(), 0,