From bcc7e120ba7a667be60a947c58fcd2919c866ab5 Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Tue, 19 Dec 2023 11:25:26 +0100 Subject: [PATCH] Modernize LineFileDocs. (#12929) This replaces `StringField`/`SortedDocValuesField` with `KeywordField` and `IntPoint`/`NumericDocValuesField` with `IntField`. --- .../index/TestAllFilesCheckIndexHeader.java | 2 +- .../index/TestAllFilesHaveChecksumFooter.java | 2 +- .../index/TestAllFilesHaveCodecHeader.java | 2 +- .../lucene/tests/util/LineFileDocs.java | 25 +++++++------------ .../apache/lucene/tests/util/TestUtil.java | 16 +++++++++++- 5 files changed, 27 insertions(+), 20 deletions(-) diff --git a/lucene/core/src/test/org/apache/lucene/index/TestAllFilesCheckIndexHeader.java b/lucene/core/src/test/org/apache/lucene/index/TestAllFilesCheckIndexHeader.java index 18ca3e856dae..0fe1465fd0ca 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestAllFilesCheckIndexHeader.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestAllFilesCheckIndexHeader.java @@ -64,7 +64,7 @@ public void test() throws Exception { } if (random().nextInt(15) == 0) { riw.updateNumericDocValue( - new Term("docid", Integer.toString(i)), "docid_intDV", Long.valueOf(i)); + new Term("docid", Integer.toString(i)), "page_views", Long.valueOf(i)); } } diff --git a/lucene/core/src/test/org/apache/lucene/index/TestAllFilesHaveChecksumFooter.java b/lucene/core/src/test/org/apache/lucene/index/TestAllFilesHaveChecksumFooter.java index e937b8d7704a..b9dd3656da4a 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestAllFilesHaveChecksumFooter.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestAllFilesHaveChecksumFooter.java @@ -46,7 +46,7 @@ public void test() throws Exception { } if (random().nextInt(15) == 0) { riw.updateNumericDocValue( - new Term("docid", Integer.toString(i)), "docid_intDV", Long.valueOf(i)); + new Term("docid", Integer.toString(i)), "page_views", Long.valueOf(i)); } } riw.close(); diff --git a/lucene/core/src/test/org/apache/lucene/index/TestAllFilesHaveCodecHeader.java b/lucene/core/src/test/org/apache/lucene/index/TestAllFilesHaveCodecHeader.java index 9c107599965e..cced891b86ec 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestAllFilesHaveCodecHeader.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestAllFilesHaveCodecHeader.java @@ -49,7 +49,7 @@ public void test() throws Exception { } if (random().nextInt(15) == 0) { riw.updateNumericDocValue( - new Term("docid", Integer.toString(i)), "docid_intDV", Long.valueOf(i)); + new Term("docid", Integer.toString(i)), "page_views", Long.valueOf(i)); } } riw.close(); diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/util/LineFileDocs.java b/lucene/test-framework/src/java/org/apache/lucene/tests/util/LineFileDocs.java index 812582fe714c..9d02549c7de9 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/tests/util/LineFileDocs.java +++ b/lucene/test-framework/src/java/org/apache/lucene/tests/util/LineFileDocs.java @@ -37,14 +37,14 @@ import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; import org.apache.lucene.document.FieldType; +import org.apache.lucene.document.IntField; import org.apache.lucene.document.IntPoint; +import org.apache.lucene.document.KeywordField; import org.apache.lucene.document.NumericDocValuesField; -import org.apache.lucene.document.SortedDocValuesField; import org.apache.lucene.document.StringField; import org.apache.lucene.document.TextField; import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.IndexableField; -import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.CloseableThreadLocal; import org.apache.lucene.util.IOUtils; @@ -199,17 +199,16 @@ private static final class DocState { final Document doc; final Field titleTokenized; final Field title; - final Field titleDV; final Field body; final Field id; final Field idNum; - final Field idNumDV; final Field date; + final Field pageViews; public DocState() { doc = new Document(); - title = new StringField("title", "", Field.Store.NO); + title = new KeywordField("title", "", Field.Store.NO); doc.add(title); FieldType ft = new FieldType(TextField.TYPE_STORED); @@ -227,16 +226,15 @@ public DocState() { id = new StringField("docid", "", Field.Store.YES); doc.add(id); - idNum = new IntPoint("docid_int", 0); + idNum = new IntField("docid_int", 0, Field.Store.NO); doc.add(idNum); date = new StringField("date", "", Field.Store.YES); doc.add(date); - titleDV = new SortedDocValuesField("titleDV", new BytesRef()); - idNumDV = new NumericDocValuesField("docid_intDV", 0); - doc.add(titleDV); - doc.add(idNumDV); + // A numeric DV field that can be used for DV updates + pageViews = new NumericDocValuesField("page_views", 0L); + doc.add(pageViews); } } @@ -277,17 +275,12 @@ public Document nextDoc() throws IOException { docState.body.setStringValue(line.substring(1 + spot2, line.length())); final String title = line.substring(0, spot); docState.title.setStringValue(title); - if (docState.titleDV != null) { - docState.titleDV.setBytesValue(new BytesRef(title)); - } docState.titleTokenized.setStringValue(title); docState.date.setStringValue(line.substring(1 + spot, spot2)); final int i = id.getAndIncrement(); docState.id.setStringValue(Integer.toString(i)); docState.idNum.setIntValue(i); - if (docState.idNumDV != null) { - docState.idNumDV.setLongValue(i); - } + docState.pageViews.setLongValue(random.nextInt(10_000)); if (random.nextInt(5) == 4) { // Make some sparse fields diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/util/TestUtil.java b/lucene/test-framework/src/java/org/apache/lucene/tests/util/TestUtil.java index f86bedafef7c..ece42e002bd8 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/tests/util/TestUtil.java +++ b/lucene/test-framework/src/java/org/apache/lucene/tests/util/TestUtil.java @@ -63,6 +63,8 @@ import org.apache.lucene.document.BinaryPoint; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; +import org.apache.lucene.document.IntField; +import org.apache.lucene.document.KeywordField; import org.apache.lucene.document.NumericDocValuesField; import org.apache.lucene.document.SortedDocValuesField; import org.apache.lucene.index.CheckIndex; @@ -1420,7 +1422,19 @@ public static Document cloneDocument(Document doc1) { final Field field2; final DocValuesType dvType = field1.fieldType().docValuesType(); final int dimCount = field1.fieldType().pointDimensionCount(); - if (dvType != DocValuesType.NONE) { + if (f instanceof KeywordField) { + field2 = + new KeywordField( + f.name(), + f.stringValue(), + f.fieldType().stored() ? Field.Store.YES : Field.Store.NO); + } else if (f instanceof IntField) { + field2 = + new IntField( + f.name(), + f.numericValue().intValue(), + f.fieldType().stored() ? Field.Store.YES : Field.Store.NO); + } else if (dvType != DocValuesType.NONE) { switch (dvType) { case NUMERIC: field2 = new NumericDocValuesField(field1.name(), field1.numericValue().longValue());