Skip to content

Commit

Permalink
add a mapper test case for all synthetic field mapper
Browse files Browse the repository at this point in the history
  • Loading branch information
@jimczi
jimczi committed Nov 14, 2024
1 parent 6b08b40 commit f7024ef
Show file tree
Hide file tree
Showing 4 changed files with 83 additions and 3 deletions.
2 changes: 1 addition & 1 deletion server/src/main/java/org/elasticsearch/index/engine/LuceneChangesSnapshot.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -93,7 +93,7 @@ public int skippedOperations() {
}

@Override
public Translog.Operation nextOperation() throws IOException {
protected Translog.Operation nextOperation() throws IOException {
assert assertAccessingThread();
Translog.Operation op = null;
for (int idx = nextDocIndex(); idx != -1; idx = nextDocIndex()) {
Expand Down
2 changes: 1 addition & 1 deletion ...er/src/main/java/org/elasticsearch/index/engine/LuceneSyntheticSourceChangesSnapshot.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -93,7 +93,7 @@ public int skippedOperations() {
}

@Override
public Translog.Operation nextOperation() throws IOException {
protected Translog.Operation nextOperation() throws IOException {
while (true) {
if (operationQueue.isEmpty()) {
loadNextBatch();
Expand Down
2 changes: 1 addition & 1 deletion server/src/main/java/org/elasticsearch/index/engine/SearchBasedChangesSnapshot.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -111,7 +111,7 @@ protected SearchBasedChangesSnapshot(
* @return The next Translog.Operation in the snapshot.
* @throws IOException If an I/O error occurs.
*/
public abstract Translog.Operation nextOperation() throws IOException;
protected abstract Translog.Operation nextOperation() throws IOException;

/**
* Returns the list of index leaf reader contexts.
Expand Down
80 changes: 80 additions & 0 deletions test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,9 +9,12 @@

package org.elasticsearch.index.mapper;

import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.document.NumericDocValuesField;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.DocValuesType;
import org.apache.lucene.index.IndexOptions;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.IndexableField;
import org.apache.lucene.index.IndexableFieldType;
import org.apache.lucene.index.LeafReader;
Expand All @@ -20,7 +23,11 @@
import org.apache.lucene.search.FieldExistsQuery;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.Sort;
import org.apache.lucene.search.SortField;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.search.UsageTrackingQueryCachingPolicy;
import org.apache.lucene.search.similarities.BM25Similarity;
import org.apache.lucene.store.Directory;
import org.apache.lucene.tests.analysis.MockAnalyzer;
import org.apache.lucene.tests.index.RandomIndexWriter;
Expand All @@ -35,6 +42,8 @@
import org.elasticsearch.index.IndexSettings;
import org.elasticsearch.index.IndexVersion;
import org.elasticsearch.index.IndexVersions;
import org.elasticsearch.index.engine.Engine;
import org.elasticsearch.index.engine.LuceneSyntheticSourceChangesSnapshot;
import org.elasticsearch.index.fielddata.FieldDataContext;
import org.elasticsearch.index.fielddata.IndexFieldData;
import org.elasticsearch.index.fielddata.IndexFieldDataCache;
Expand All @@ -43,6 +52,7 @@
import org.elasticsearch.index.fieldvisitor.StoredFieldLoader;
import org.elasticsearch.index.query.SearchExecutionContext;
import org.elasticsearch.index.termvectors.TermVectorsService;
import org.elasticsearch.index.translog.Translog;
import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
import org.elasticsearch.script.Script;
import org.elasticsearch.script.ScriptContext;
Expand Down Expand Up @@ -1130,6 +1140,11 @@ public final void testSyntheticSource() throws IOException {
assertSyntheticSource(syntheticSourceSupport(shouldUseIgnoreMalformed()).example(5));
}

public final void testSyntheticSourceWithTranslogSnapshot() throws IOException {
assertSyntheticSourceWithTranslogSnapshot(syntheticSourceSupport(shouldUseIgnoreMalformed()), true);
assertSyntheticSourceWithTranslogSnapshot(syntheticSourceSupport(shouldUseIgnoreMalformed()), false);
}

public void testSyntheticSourceIgnoreMalformedExamples() throws IOException {
assumeTrue("type doesn't support ignore_malformed", supportsIgnoreMalformed());
// We need to call this in order to hit the assumption inside so that
Expand All @@ -1155,6 +1170,71 @@ private void assertSyntheticSource(SyntheticSourceExample example) throws IOExce
assertThat(syntheticSource(mapper, example::buildInput), equalTo(example.expected()));
}

private void assertSyntheticSourceWithTranslogSnapshot(SyntheticSourceSupport support, boolean doIndexSort) throws IOException {
var firstExample = support.example(1);
int maxDocs = randomIntBetween(10, 50);
var settings = Settings.builder()
.put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.SYNTHETIC)
.put(IndexSettings.RECOVERY_USE_SYNTHETIC_SOURCE_SETTING.getKey(), true)
.build();
var mapperService = createMapperService(getVersion(), settings, () -> true, mapping(b -> {
b.startObject("field");
firstExample.mapping().accept(b);
b.endObject();
}));
var docMapper = mapperService.documentMapper();
try (var directory = newDirectory()) {
List<SyntheticSourceExample> examples = new ArrayList<>();
IndexWriterConfig config = newIndexWriterConfig(random(), new StandardAnalyzer());
config.setIndexSort(new Sort(new SortField("sort", SortField.Type.LONG)));
try (var iw = new RandomIndexWriter(random(), directory, config)) {
for (int seqNo = 0; seqNo < maxDocs; seqNo++) {
var example = support.example(randomIntBetween(1, 5));
examples.add(example);
var doc = docMapper.parse(source(example::buildInput));
assertNull(doc.dynamicMappingsUpdate());
doc.updateSeqID(seqNo, 1);
doc.version().setLongValue(0);
if (doIndexSort) {
doc.rootDoc().add(new NumericDocValuesField("sort", randomLong()));
}
iw.addDocuments(doc.docs());
if (frequently()) {
iw.flush();
}
}
}
try (var indexReader = wrapInMockESDirectoryReader(DirectoryReader.open(directory))) {
int start = randomBoolean() ? 0 : randomIntBetween(1, maxDocs - 10);
var snapshot = new LuceneSyntheticSourceChangesSnapshot(
mapperService.mappingLookup(),
new Engine.Searcher(
"recovery",
indexReader,
new BM25Similarity(),
null,
new UsageTrackingQueryCachingPolicy(),
() -> {}
),
randomIntBetween(1, maxDocs),
randomLongBetween(1, LuceneSyntheticSourceChangesSnapshot.DEFAULT_MEMORY_SIZE),
start,
maxDocs,
true,
randomBoolean(),
IndexVersion.current()
);
for (int i = start; i < maxDocs; i++) {
var example = examples.get(i);
var op = snapshot.next();
if (op instanceof Translog.Index opIndex) {
assertThat(opIndex.source().utf8ToString(), equalTo(example.expected()));
}
}
}
}
}

protected boolean supportsEmptyInputArray() {
return true;
}
Expand Down

0 comments on commit f7024ef

Please sign in to comment.