Skip to content

Commit

Permalink
Add a stored fields test that indexes LineFileDocs. (#12927)
Browse files Browse the repository at this point in the history
Real-world data exhibits patterns that are taken advantage of by the
compression logic, but also hardly reproducible in a randomized way. This makes
this new test introduce interesting coverage.

It takes one second to run on my machine, so I did not mark it `@Nightly`.
  • Loading branch information
jpountz authored Dec 19, 2023
1 parent bf45ab7 commit 5c084fc
Showing 1 changed file with 31 additions and 0 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,7 @@
import org.apache.lucene.tests.analysis.MockAnalyzer;
import org.apache.lucene.tests.store.MockDirectoryWrapper;
import org.apache.lucene.tests.store.MockDirectoryWrapper.Throttling;
import org.apache.lucene.tests.util.LineFileDocs;
import org.apache.lucene.tests.util.TestUtil;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.IOUtils;
Expand Down Expand Up @@ -1006,4 +1007,34 @@ public void testRandomStoredFieldsWithIndexSort() throws Exception {
verifyStoreFields.run();
IOUtils.close(iw, dir);
}

/** Test realistic data, which typically compresses better than random data. */
public void testLineFileDocs() throws IOException {
// Use a FS dir and a non-randomized IWC to not slow down indexing
try (Directory dir = newFSDirectory(createTempDir())) {
try (LineFileDocs docs = new LineFileDocs(random());
IndexWriter w = new IndexWriter(dir, new IndexWriterConfig())) {
final int numDocs = atLeast(10_000);
for (int i = 0; i < numDocs; ++i) {
// Only keep stored fields
Document doc = docs.nextDoc();
Document storedDoc = new Document();
for (IndexableField field : doc.getFields()) {
if (field.fieldType().stored()) {
IndexableField storedField = field;
if (field.stringValue() != null) {
// Disable indexing
storedField = new StoredField(field.name(), field.stringValue());
}
storedDoc.add(storedField);
}
}

w.addDocument(storedDoc);
}
w.forceMerge(1);
}
TestUtil.checkIndex(dir);
}
}
}

0 comments on commit 5c084fc

Please sign in to comment.