Skip to content

Commit

Permalink
Remove usages of MultiTermQuery.setRewriteMethodsetRewriteMethod (#2997)
Browse files Browse the repository at this point in the history 
Remove usages of MultiTermQuery.setRewriteMethod which is removed in 
latest versions of Lucene.

Signed-off-by: ruanwenjun <[email protected]>
  • Loading branch information
@ruanwenjun
ruanwenjun authored Apr 21, 2022
1 parent c7c410a commit a34d11f
Show file tree
Hide file tree
Showing 9 changed files with 89 additions and 65 deletions.
4 changes: 2 additions & 2 deletions ...s/mapper-extras/src/main/java/org/opensearch/index/mapper/SearchAsYouTypeFieldMapper.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,7 @@
import org.apache.lucene.util.automaton.Automaton;
import org.apache.lucene.util.automaton.Operations;
import org.opensearch.common.collect.Iterators;
import org.opensearch.common.lucene.search.AutomatonQueries;
import org.opensearch.index.analysis.AnalyzerScope;
import org.opensearch.index.analysis.IndexAnalyzers;
import org.opensearch.index.analysis.NamedAnalyzer;
Expand Down Expand Up @@ -431,8 +432,7 @@ public Query prefixQuery(String value, MultiTermQuery.RewriteMethod method, bool
automata.add(Automata.makeAnyChar());
}
Automaton automaton = Operations.concatenate(automata);
AutomatonQuery query = new AutomatonQuery(new Term(name(), value + "*"), automaton);
query.setRewriteMethod(method);
AutomatonQuery query = AutomatonQueries.createAutomatonQuery(new Term(name(), value + "*"), automaton, method);
return new BooleanQuery.Builder().add(query, BooleanClause.Occur.SHOULD)
.add(new TermQuery(new Term(parentField, value)), BooleanClause.Occur.SHOULD)
.build();
Expand Down
12 changes: 5 additions & 7 deletions .../plugins/concurrent-search/src/test/java/org/opensearch/search/query/QueryPhaseTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,6 @@
package org.opensearch.search.query;

import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;

import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field.Store;
Expand All @@ -52,6 +51,8 @@
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.NoMergePolicy;
import org.apache.lucene.index.Term;
import org.apache.lucene.queries.spans.SpanNearQuery;
import org.apache.lucene.queries.spans.SpanTermQuery;
import org.apache.lucene.search.BooleanClause;
import org.apache.lucene.search.BooleanClause.Occur;
import org.apache.lucene.search.BooleanQuery;
Expand Down Expand Up @@ -82,8 +83,6 @@
import org.apache.lucene.search.grouping.CollapseTopFieldDocs;
import org.apache.lucene.search.join.BitSetProducer;
import org.apache.lucene.search.join.ScoreMode;
import org.apache.lucene.queries.spans.SpanNearQuery;
import org.apache.lucene.queries.spans.SpanTermQuery;
import org.apache.lucene.store.Directory;
import org.apache.lucene.tests.index.RandomIndexWriter;
import org.apache.lucene.util.BytesRef;
Expand Down Expand Up @@ -122,16 +121,16 @@
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;

import static org.opensearch.search.query.TopDocsCollectorContext.hasInfMaxScore;
import static org.hamcrest.Matchers.anyOf;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.greaterThan;
import static org.hamcrest.Matchers.greaterThanOrEqualTo;
import static org.hamcrest.Matchers.instanceOf;
import static org.hamcrest.Matchers.lessThanOrEqualTo;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.when;
import static org.opensearch.search.query.TopDocsCollectorContext.hasInfMaxScore;

public class QueryPhaseTests extends IndexShardTestCase {

Expand Down Expand Up @@ -1114,8 +1113,7 @@ public void testCancellationDuringPreprocess() throws IOException {
indexShard,
newContextSearcher(reader, executor)
);
PrefixQuery prefixQuery = new PrefixQuery(new Term("foo", "a"));
prefixQuery.setRewriteMethod(MultiTermQuery.SCORING_BOOLEAN_REWRITE);
PrefixQuery prefixQuery = new PrefixQuery(new Term("foo", "a"), MultiTermQuery.SCORING_BOOLEAN_REWRITE);
context.parsedQuery(new ParsedQuery(prefixQuery));
SearchShardTask task = mock(SearchShardTask.class);
when(task.isCancelled()).thenReturn(true);
Expand Down
48 changes: 39 additions & 9 deletions server/src/main/java/org/opensearch/common/lucene/search/AutomatonQueries.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@

import org.apache.lucene.index.Term;
import org.apache.lucene.search.AutomatonQuery;
import org.apache.lucene.search.MultiTermQuery;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.automaton.Automata;
import org.apache.lucene.util.automaton.Automaton;
Expand Down Expand Up @@ -63,29 +64,58 @@ public static Automaton caseInsensitivePrefix(String s) {
return a;
}

/** Build an automaton query accepting all terms with the specified prefix, ASCII case insensitive. */
/**
* Build an automaton query accepting all terms with the specified prefix, ASCII case insensitive.
*/
public static AutomatonQuery caseInsensitivePrefixQuery(Term prefix) {
return new AutomatonQuery(prefix, caseInsensitivePrefix(prefix.text()));
return caseInsensitivePrefixQuery(prefix, MultiTermQuery.CONSTANT_SCORE_REWRITE);
}

/** Build an automaton accepting all terms ASCII case insensitive. */
/**
* Build an automaton query accepting all terms with the specified prefix, ASCII case insensitive.
*/
public static AutomatonQuery caseInsensitivePrefixQuery(Term prefix, MultiTermQuery.RewriteMethod method) {
return createAutomatonQuery(prefix, caseInsensitivePrefix(prefix.text()), method);
}

/**
* Build an automaton accepting all terms ASCII case insensitive.
*/
public static AutomatonQuery caseInsensitiveTermQuery(Term term) {
BytesRef prefix = term.bytes();
return new AutomatonQuery(term, toCaseInsensitiveString(prefix, Integer.MAX_VALUE));
}

/** Build an automaton matching a wildcard pattern, ASCII case insensitive. */
public static AutomatonQuery caseInsensitiveWildcardQuery(Term wildcardquery) {
return new AutomatonQuery(wildcardquery, toCaseInsensitiveWildcardAutomaton(wildcardquery, Integer.MAX_VALUE));
/**
* Build an automaton matching a wildcard pattern, ASCII case insensitive, if the method is null, then will use {@link MultiTermQuery#CONSTANT_SCORE_REWRITE}.
*/
public static AutomatonQuery caseInsensitiveWildcardQuery(Term wildcardquery, MultiTermQuery.RewriteMethod method) {
return createAutomatonQuery(wildcardquery, toCaseInsensitiveWildcardAutomaton(wildcardquery, Integer.MAX_VALUE), method);
}

/**
* Build an automaton matching a given pattern with rewrite method, if the rewrite method is null, then will use {@link MultiTermQuery#CONSTANT_SCORE_REWRITE}.
*/
public static AutomatonQuery createAutomatonQuery(Term term, Automaton automaton, MultiTermQuery.RewriteMethod method) {
if (method == null) {
method = MultiTermQuery.CONSTANT_SCORE_REWRITE;
}
return new AutomatonQuery(term, automaton, Operations.DEFAULT_DETERMINIZE_WORK_LIMIT, false, method);
}

/** String equality with support for wildcards */
/**
* String equality with support for wildcards
*/
public static final char WILDCARD_STRING = '*';

/** Char equality with support for wildcards */
/**
* Char equality with support for wildcards
*/
public static final char WILDCARD_CHAR = '?';

/** Escape character */
/**
* Escape character
*/
public static final char WILDCARD_ESCAPE = '\\';

/**
Expand Down
44 changes: 21 additions & 23 deletions server/src/main/java/org/opensearch/index/mapper/StringFieldType.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,6 @@

import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.AutomatonQuery;
import org.apache.lucene.search.FuzzyQuery;
import org.apache.lucene.search.MultiTermQuery;
import org.apache.lucene.search.PrefixQuery;
Expand All @@ -44,12 +43,12 @@
import org.apache.lucene.search.WildcardQuery;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.BytesRefBuilder;
import org.apache.lucene.util.automaton.Operations;
import org.opensearch.OpenSearchException;
import org.opensearch.common.lucene.BytesRefs;
import org.opensearch.common.lucene.search.AutomatonQueries;
import org.opensearch.common.unit.Fuzziness;
import org.opensearch.index.query.QueryShardContext;
import org.opensearch.index.query.support.QueryParsers;

import java.util.Map;
import java.util.regex.Matcher;
Expand Down Expand Up @@ -111,19 +110,13 @@ public Query prefixQuery(String value, MultiTermQuery.RewriteMethod method, bool
);
}
failIfNotIndexed();
if (caseInsensitive) {
AutomatonQuery query = AutomatonQueries.caseInsensitivePrefixQuery((new Term(name(), indexedValueForSearch(value))));
if (method != null) {
query.setRewriteMethod(method);
}
return query;

if (method == null) {
method = MultiTermQuery.CONSTANT_SCORE_REWRITE;
}
PrefixQuery query = new PrefixQuery(new Term(name(), indexedValueForSearch(value)));
if (method != null) {
query.setRewriteMethod(method);
if (caseInsensitive) {
return AutomatonQueries.caseInsensitivePrefixQuery((new Term(name(), indexedValueForSearch(value))), method);
}
return query;
return new PrefixQuery(new Term(name(), indexedValueForSearch(value)), method);
}

public static final String normalizeWildcardPattern(String fieldname, String value, Analyzer normalizer) {
Expand Down Expand Up @@ -173,13 +166,12 @@ public Query wildcardQuery(String value, MultiTermQuery.RewriteMethod method, bo
term = new Term(name(), indexedValueForSearch(value));
}
if (caseInsensitive) {
AutomatonQuery query = AutomatonQueries.caseInsensitiveWildcardQuery(term);
QueryParsers.setRewriteMethod(query, method);
return query;
return AutomatonQueries.caseInsensitiveWildcardQuery(term, method);
}
WildcardQuery query = new WildcardQuery(term);
QueryParsers.setRewriteMethod(query, method);
return query;
if (method == null) {
method = MultiTermQuery.CONSTANT_SCORE_REWRITE;
}
return new WildcardQuery(term, Operations.DEFAULT_DETERMINIZE_WORK_LIMIT, method);
}

@Override
Expand All @@ -197,11 +189,17 @@ public Query regexpQuery(
);
}
failIfNotIndexed();
RegexpQuery query = new RegexpQuery(new Term(name(), indexedValueForSearch(value)), syntaxFlags, matchFlags, maxDeterminizedStates);
if (method != null) {
query.setRewriteMethod(method);
if (method == null) {
method = MultiTermQuery.CONSTANT_SCORE_REWRITE;
}
return query;
return new RegexpQuery(
new Term(name(), indexedValueForSearch(value)),
syntaxFlags,
matchFlags,
RegexpQuery.DEFAULT_PROVIDER,
maxDeterminizedStates,
method
);
}

@Override
Expand Down
5 changes: 1 addition & 4 deletions server/src/main/java/org/opensearch/index/mapper/TextFieldMapper.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -582,10 +582,7 @@ public Query prefixQuery(String value, MultiTermQuery.RewriteMethod method, bool
automata.add(Automata.makeAnyChar());
}
Automaton automaton = Operations.concatenate(automata);
AutomatonQuery query = new AutomatonQuery(new Term(name(), value + "*"), automaton);
if (method != null) {
query.setRewriteMethod(method);
}
AutomatonQuery query = AutomatonQueries.createAutomatonQuery(new Term(name(), value + "*"), automaton, method);
return new BooleanQuery.Builder().add(query, BooleanClause.Occur.SHOULD)
.add(new TermQuery(new Term(parentField.name(), value)), BooleanClause.Occur.SHOULD)
.build();
Expand Down
13 changes: 7 additions & 6 deletions server/src/main/java/org/opensearch/index/query/RegexpQueryBuilder.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -309,16 +309,17 @@ protected Query doToQuery(QueryShardContext context) throws QueryShardException,
query = fieldType.regexpQuery(value, sanitisedSyntaxFlag, matchFlagsValue, maxDeterminizedStates, method, context);
}
if (query == null) {
RegexpQuery regexpQuery = new RegexpQuery(
if (method == null) {
method = MultiTermQuery.CONSTANT_SCORE_REWRITE;
}
query = new RegexpQuery(
new Term(fieldName, BytesRefs.toBytesRef(value)),
sanitisedSyntaxFlag,
matchFlagsValue,
maxDeterminizedStates
RegexpQuery.DEFAULT_PROVIDER,
maxDeterminizedStates,
method
);
if (method != null) {
regexpQuery.setRewriteMethod(method);
}
query = regexpQuery;
}
return query;
}
Expand Down
11 changes: 6 additions & 5 deletions server/src/main/java/org/opensearch/index/search/QueryStringQueryParser.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,6 @@
import org.opensearch.index.query.ExistsQueryBuilder;
import org.opensearch.index.query.MultiMatchQueryBuilder;
import org.opensearch.index.query.QueryShardContext;
import org.opensearch.index.query.support.QueryParsers;

import java.io.IOException;
import java.time.ZoneId;
Expand Down Expand Up @@ -110,7 +109,7 @@ public class QueryStringQueryParser extends XQueryParser {
private ZoneId timeZone;
private Fuzziness fuzziness = Fuzziness.AUTO;
private int fuzzyMaxExpansions = FuzzyQuery.defaultMaxExpansions;
private MultiTermQuery.RewriteMethod fuzzyRewriteMethod;
private MultiTermQuery.RewriteMethod fuzzyRewriteMethod = MultiTermQuery.CONSTANT_SCORE_REWRITE;
private boolean fuzzyTranspositions = FuzzyQuery.defaultTranspositions;

/**
Expand Down Expand Up @@ -527,9 +526,11 @@ private Query getFuzzyQuerySingle(String field, String termStr, float minSimilar
@Override
protected Query newFuzzyQuery(Term term, float minimumSimilarity, int prefixLength) {
int numEdits = Fuzziness.build(minimumSimilarity).asDistance(term.text());
FuzzyQuery query = new FuzzyQuery(term, numEdits, prefixLength, fuzzyMaxExpansions, fuzzyTranspositions);
QueryParsers.setRewriteMethod(query, fuzzyRewriteMethod);
return query;
if (fuzzyRewriteMethod != null) {
return new FuzzyQuery(term, numEdits, prefixLength, fuzzyMaxExpansions, fuzzyTranspositions, fuzzyRewriteMethod);
} else {
return new FuzzyQuery(term, numEdits, prefixLength, fuzzyMaxExpansions, fuzzyTranspositions);
}
}

@Override
Expand Down
2 changes: 1 addition & 1 deletion server/src/test/java/org/opensearch/deps/lucene/VectorHighlighterTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -121,7 +121,7 @@ public void testVectorHighlighterPrefixQuery() throws Exception {
);
assertThat(fragment, nullValue());

prefixQuery.setRewriteMethod(PrefixQuery.SCORING_BOOLEAN_REWRITE);
prefixQuery = new PrefixQuery(new Term("content", "ba"), PrefixQuery.SCORING_BOOLEAN_REWRITE);
Query rewriteQuery = prefixQuery.rewrite(reader);
fragment = highlighter.getBestFragment(highlighter.getFieldQuery(rewriteQuery), reader, topDocs.scoreDocs[0].doc, "content", 30);
assertThat(fragment, notNullValue());
Expand Down
15 changes: 7 additions & 8 deletions server/src/test/java/org/opensearch/search/query/QueryPhaseTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,9 +49,7 @@
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.NoMergePolicy;
import org.apache.lucene.tests.index.RandomIndexWriter;
import org.apache.lucene.index.Term;
import org.opensearch.lucene.queries.MinDocQuery;
import org.apache.lucene.queries.spans.SpanNearQuery;
import org.apache.lucene.queries.spans.SpanTermQuery;
import org.apache.lucene.search.BooleanClause.Occur;
Expand Down Expand Up @@ -82,6 +80,7 @@
import org.apache.lucene.search.join.BitSetProducer;
import org.apache.lucene.search.join.ScoreMode;
import org.apache.lucene.store.Directory;
import org.apache.lucene.tests.index.RandomIndexWriter;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.FixedBitSet;
import org.opensearch.action.search.SearchShardTask;
Expand All @@ -97,6 +96,7 @@
import org.opensearch.index.search.OpenSearchToParentBlockJoinQuery;
import org.opensearch.index.shard.IndexShard;
import org.opensearch.index.shard.IndexShardTestCase;
import org.opensearch.lucene.queries.MinDocQuery;
import org.opensearch.search.DocValueFormat;
import org.opensearch.search.collapse.CollapseBuilder;
import org.opensearch.search.internal.ContextIndexSearcher;
Expand All @@ -111,16 +111,16 @@
import java.util.Collections;
import java.util.List;

import static org.hamcrest.Matchers.lessThanOrEqualTo;
import static org.hamcrest.Matchers.anyOf;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.greaterThan;
import static org.hamcrest.Matchers.greaterThanOrEqualTo;
import static org.hamcrest.Matchers.anyOf;
import static org.hamcrest.Matchers.instanceOf;
import static org.opensearch.search.query.TopDocsCollectorContext.hasInfMaxScore;
import static org.hamcrest.Matchers.lessThanOrEqualTo;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.when;
import static org.opensearch.search.query.TopDocsCollectorContext.hasInfMaxScore;

public class QueryPhaseTests extends IndexShardTestCase {

Expand Down Expand Up @@ -1069,8 +1069,7 @@ public void testCancellationDuringPreprocess() throws IOException {

try (IndexReader reader = DirectoryReader.open(dir)) {
TestSearchContext context = new TestSearchContextWithRewriteAndCancellation(null, indexShard, newContextSearcher(reader));
PrefixQuery prefixQuery = new PrefixQuery(new Term("foo", "a"));
prefixQuery.setRewriteMethod(MultiTermQuery.SCORING_BOOLEAN_REWRITE);
PrefixQuery prefixQuery = new PrefixQuery(new Term("foo", "a"), MultiTermQuery.SCORING_BOOLEAN_REWRITE);
context.parsedQuery(new ParsedQuery(prefixQuery));
SearchShardTask task = mock(SearchShardTask.class);
when(task.isCancelled()).thenReturn(true);
Expand Down

0 comments on commit a34d11f

Please sign in to comment.