From 5f5317d8577564c36cbae52bd583f1ab3f6103bc Mon Sep 17 00:00:00 2001 From: Michael Froh Date: Mon, 28 Oct 2024 14:21:07 -0700 Subject: [PATCH] [NOCOMMIT] Revert "Flat object field should delegate to keyword field for most query types (#14383)" This reverts commit 9ddee61b1b4eafebe9b4d30e997b40178c939a5e. Trying to identify the source of a benchmark regression for big5 keyword-in-range query. Signed-off-by: Michael Froh --- .../test/index/90_flat_object.yml | 32 + .../92_flat_object_support_doc_values.yml | 788 ------------- .../xcontent/JsonToStringXContentParser.java | 8 +- .../index/mapper/FlatObjectFieldMapper.java | 310 ++--- .../index/mapper/KeywordFieldMapper.java | 62 +- .../mapper/FlatObjectFieldMapperTests.java | 45 +- .../mapper/FlatObjectFieldTypeTests.java | 1002 +---------------- 7 files changed, 258 insertions(+), 1989 deletions(-) delete mode 100644 rest-api-spec/src/main/resources/rest-api-spec/test/index/92_flat_object_support_doc_values.yml diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/index/90_flat_object.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/index/90_flat_object.yml index 2a469aa5ff04d..e8da81d7bee41 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/index/90_flat_object.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/index/90_flat_object.yml @@ -671,6 +671,38 @@ teardown: - match: { error.root_cause.0.reason: "Mapping definition for [data] has unsupported parameters: [analyzer : standard]"} - match: { status: 400 } + # Wildcard Query with dot path. + - do: + catch: bad_request + search: + body: { + _source: true, + query: { + "wildcard": { + "catalog.title": "Mock*" + } + } + } + - match: { error.root_cause.0.type: "query_shard_exception" } + - match: { error.root_cause.0.reason: "Can only use wildcard queries on keyword and text fields - not on [catalog.title] which is of type [flat_object]"} + - match: { status: 400 } + + # Wildcard Query without dot path. + - do: + catch: bad_request + search: + body: { + _source: true, + query: { + "wildcard": { + "catalog": "Mock*" + } + } + } + - match: { error.root_cause.0.type: "query_shard_exception" } + - match: { error.root_cause.0.reason: "Can only use wildcard queries on keyword and text fields - not on [catalog] which is of type [flat_object]" } + - match: { status: 400 } + # Aggregation and Match Query with dot path. - do: catch: bad_request diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/index/92_flat_object_support_doc_values.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/index/92_flat_object_support_doc_values.yml deleted file mode 100644 index 9ec39660a4928..0000000000000 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/index/92_flat_object_support_doc_values.yml +++ /dev/null @@ -1,788 +0,0 @@ ---- -# The test setup includes: -# - Create flat_object mapping for flat_object_doc_values_test index -# - Index 9 example documents -# - Search tests about doc_values and index - -setup: - - skip: - version: " - 2.99.99" - reason: "introduced in 3.0.0 " - - - do: - indices.create: - index: flat_object_doc_values_test - body: - mappings: - properties: - issue: - properties: - labels: - type: "flat_object" - order: - type: "keyword" - - - do: - bulk: - refresh: true - body: | - {"index":{"_index":"flat_object_doc_values_test","_id":"0"}} - {"order":"order0","issue":{"labels":{"number":1,"name":"abc0","status":1}}} - {"index":{"_index":"flat_object_doc_values_test","_id":"1"}} - {"order":"order1","issue":{"labels":{"number":2,"name":"abc1","status":1}}} - {"index":{"_index":"flat_object_doc_values_test","_id":"2"}} - {"order":"order2","issue":{"labels":{"number":2,"name":"abc2","status":1}}} - {"index":{"_index":"flat_object_doc_values_test","_id":"3"}} - {"order":"order3","issue":{"labels":{"number":3,"name":"abc3","status":1}}} - {"index":{"_index":"flat_object_doc_values_test","_id":"4"}} - {"order":"order4","issue":{"labels":{"number":4,"name":"abc4","status":1}}} - {"index":{"_index":"flat_object_doc_values_test","_id":"5"}} - {"order":"order5","issue":{"labels":{"number":5,"name":"abc5","status":1}}} - {"index":{"_index":"flat_object_doc_values_test","_id":"6"}} - {"order":"order6","issue":{"labels":{"number":6,"name":"abc6","status":1}}} - {"index":{"_index":"flat_object_doc_values_test","_id":"7"}} - {"order":"order7","issue":{"labels":{"number":7,"name":"abc7","status":1}}} - {"index":{"_index":"flat_object_doc_values_test","_id":"8"}} - {"order":"order8","issue":{"labels":{"number":8,"name":"abc8","status":1}}} - ---- -# Delete Index when connection is teardown -teardown: - - do: - indices.delete: - index: flat_object_doc_values_test - ---- -"Supported queries": - - skip: - version: " - 2.99.99" - reason: "introduced in 3.0.0 " - - # Verify Document Count - - do: - search: - body: { - query: { - match_all: { } - } - } - - - length: { hits.hits: 9 } - - # Term Query with exact dot path. - - do: - search: - body: { - _source: true, - query: { - bool: { - must: [ - { - term: { - order: "order8" - } - }, - { - term: { - issue.labels.status: 1 - } - } - ] - } - } - } - - - length: { hits.hits: 1 } - - match: { hits.hits.0._source.order: "order8" } - - - do: - search: - body: { - _source: true, - query: { - bool: { - must: [ - { - term: { - issue.labels.name: "abc8" - } - } - ] - } - } - } - - - length: { hits.hits: 1 } - - match: { hits.hits.0._source.order: "order8" } - - # Term Query with no path. - - do: - search: - body: { - _source: true, - query: { - bool: { - must: [ - { - term: { - order: "order8" - } - }, - { - term: { - issue.labels: 1 - } - } - ] - } - } - } - - - length: { hits.hits: 1 } - - match: { hits.hits.0._source.order: "order8" } - - - do: - search: - body: { - _source: true, - query: { - bool: { - must: [ - { - term: { - issue.labels: "abc8" - } - } - ] - } - } - } - - - length: { hits.hits: 1 } - - match: { hits.hits.0._source.order: "order8" } - - # Terms Query with exact dot path. - - do: - search: - body: { - _source: true, - query: { - bool: { - must: [ - { - term: { - order: "order8" - } - }, - { - terms: { - issue.labels.status: [0,1] - } - } - ] - } - } - } - - - length: { hits.hits: 1 } - - match: { hits.hits.0._source.order: "order8" } - - - do: - search: - body: { - _source: true, - query: { - bool: { - must: [ - { - terms: { - issue.labels.name: ["abc8"] - } - } - ] - } - } - } - - - length: { hits.hits: 1 } - - match: { hits.hits.0._source.order: "order8" } - - # Terms Query with no path. - - do: - search: - body: { - _source: true, - query: { - bool: { - must: [ - { - term: { - order: "order8" - } - }, - { - terms: { - issue.labels: [ 0,1 ] - } - } - ] - } - } - } - - - length: { hits.hits: 1 } - - match: { hits.hits.0._source.order: "order8" } - - - do: - search: - body: { - _source: true, - query: { - bool: { - must: [ - { - terms: { - issue.labels.name: ["abc8"] - } - } - ] - } - } - } - - - length: { hits.hits: 1 } - - match: { hits.hits.0._source.order: "order8" } - - # Prefix Query with exact dot path. - - do: - search: - body: { - _source: true, - query: { - bool: { - must: [ - { - term: { - order: "order8" - } - }, - { - prefix: { - issue.labels.name: "ab" - } - } - ] - } - } - } - - - length: { hits.hits: 1 } - - match: { hits.hits.0._source.order: "order8" } - - - do: - search: - body: { - _source: true, - query: { - bool: { - must: [ - { - prefix: { - issue.labels.name: "abc8" - } - } - ] - } - } - } - - - length: { hits.hits: 1 } - - match: { hits.hits.0._source.order: "order8" } - - # Prefix Query with no path. - - do: - search: - body: { - _source: true, - query: { - bool: { - must: [ - { - term: { - order: "order8" - } - }, - { - prefix: { - issue.labels: "ab" - } - } - ] - } - } - } - - - length: { hits.hits: 1 } - - match: { hits.hits.0._source.order: "order8" } - - - do: - search: - body: { - _source: true, - query: { - bool: { - must: [ - { - prefix: { - issue.labels: "abc8" - } - } - ] - } - } - } - - - length: { hits.hits: 1 } - - match: { hits.hits.0._source.order: "order8" } - - # Regexp Query with exact dot path. - - do: - search: - body: { - _source: true, - query: { - bool: { - must: [ - { - term: { - order: "order8" - } - }, - { - regexp: { - issue.labels.name: "ab.*" - } - } - ] - } - } - } - - - length: { hits.hits: 1 } - - match: { hits.hits.0._source.order: "order8" } - - match: { hits.hits.0._source.issue.labels.name: "abc8" } - - - do: - search: - body: { - _source: true, - query: { - bool: { - must: [ - { - regexp: { - issue.labels.name: "a.*c8" - } - } - ] - } - } - } - - - length: { hits.hits: 1 } - - match: { hits.hits.0._source.order: "order8" } - - # Regexp Query with no path. - - do: - search: - body: { - _source: true, - query: { - bool: { - must: [ - { - term: { - order: "order8" - } - }, - { - regexp: { - issue.labels: "ab.*" - } - } - ] - } - } - } - - - length: { hits.hits: 1 } - - match: { hits.hits.0._source.order: "order8" } - - - do: - search: - body: { - _source: true, - query: { - bool: { - must: [ - { - regexp: { - issue.labels: "a.*c8" - } - } - ] - } - } - } - - - length: { hits.hits: 1 } - - match: { hits.hits.0._source.order: "order8" } - - # Fuzzy Query with exact dot path. - - do: - search: - body: { - _source: true, - query: { - bool: { - must: [ - { - term: { - order: "order8" - } - }, - { - fuzzy: { - issue.labels.name: { - value: "abcx", - fuzziness: 1 - } - } - } - ] - } - } - } - - - length: { hits.hits: 1 } - - match: { hits.hits.0._source.order: "order8" } - - - do: - search: - body: { - _source: true, - query: { - bool: { - must: [ - { - fuzzy: { - issue.labels.name: { - value: "abc8", - fuzziness: 0 - } - } - } - ] - } - } - } - - - length: { hits.hits: 1 } - - match: { hits.hits.0._source.order: "order8" } - - # Fuzzy Query with no path. - - do: - search: - body: { - _source: true, - query: { - bool: { - must: [ - { - term: { - order: "order8" - } - }, - { - fuzzy: { - issue.labels: { - value: "abcx", - fuzziness: 1 - } - } - } - ] - } - } - } - - - length: { hits.hits: 1 } - - match: { hits.hits.0._source.order: "order8" } - - - do: - search: - body: { - _source: true, - query: { - bool: { - must: [ - { - fuzzy: { - issue.labels: { - value: "abc8", - fuzziness: 0 - } - } - } - ] - } - } - } - - - length: { hits.hits: 1 } - - match: { hits.hits.0._source.order: "order8" } - - # Range Query with exact dot path. - - do: - search: - body: { - _source: true, - query: { - bool: { - must: [ - { - term: { - order: "order8" - } - }, - { - range: { - issue.labels.status: { - from: 0 - } - } - } - ] - } - } - } - - - length: { hits.hits: 1 } - - match: { hits.hits.0._source.order: "order8" } - - - do: - search: - body: { - _source: true, - query: { - bool: { - must: [ - { - range: { - issue.labels.name: { - from: "abc8" - } - } - } - ] - } - } - } - - - length: { hits.hits: 1 } - - match: { hits.hits.0._source.order: "order8" } - - # Range Query with no path. - - do: - search: - body: { - _source: true, - query: { - bool: { - must: [ - { - term: { - order: "order8" - } - }, - { - range: { - issue.labels: { - from: 0 - } - } - } - ] - } - } - } - - - length: { hits.hits: 1 } - - match: { hits.hits.0._source.order: "order8" } - - - do: - search: - body: { - _source: true, - query: { - bool: { - must: [ - { - range: { - issue.labels: { - from: "abc8" - } - } - } - ] - } - } - } - - - length: { hits.hits: 1 } - - match: { hits.hits.0._source.order: "order8" } - - # Exists Query with exact dot path. - - do: - search: - body: { - _source: true, - query: { - bool: { - must: [ - { - term: { - order: "order8" - } - }, - { - exists: { - field: "issue.labels.status" - } - } - ] - } - } - } - - - length: { hits.hits: 1 } - - match: { hits.hits.0._source.order: "order8" } - - # Exists Query with no path. - - do: - search: - body: { - _source: true, - query: { - bool: { - must: [ - { - term: { - order: "order8" - } - }, - { - exists: { - field: "issue.labels" - } - } - ] - } - } - } - - - length: { hits.hits: 1 } - - match: { hits.hits.0._source.order: "order8" } - - - # Wildcard Query with exact dot path. - - do: - search: - body: { - _source: true, - query: { - bool: { - must: [ - { - term: { - order: "order8" - } - }, - { - wildcard: { - issue.labels.name: "abc*" - } - } - ] - } - } - } - - - length: { hits.hits: 1 } - - match: { hits.hits.0._source.order: "order8" } - - - do: - search: - body: { - _source: true, - query: { - bool: { - must: [ - { - wildcard: { - issue.labels.name: "abc8*" - } - } - ] - } - } - } - - - length: { hits.hits: 1 } - - match: { hits.hits.0._source.order: "order8" } - - # Wildcard Query with no path. - - do: - search: - body: { - _source: true, - query: { - bool: { - must: [ - { - term: { - order: "order8" - } - }, - { - wildcard: { - issue.labels: "abc*" - } - } - ] - } - } - } - - - length: { hits.hits: 1 } - - match: { hits.hits.0._source.order: "order8" } - - - do: - search: - body: { - _source: true, - query: { - bool: { - must: [ - { - wildcard: { - issue.labels: "abc8*" - } - } - ] - } - } - } - - - length: { hits.hits: 1 } - - match: { hits.hits.0._source.order: "order8" } diff --git a/server/src/main/java/org/opensearch/common/xcontent/JsonToStringXContentParser.java b/server/src/main/java/org/opensearch/common/xcontent/JsonToStringXContentParser.java index 21270b4241b15..95a8d9c9495f2 100644 --- a/server/src/main/java/org/opensearch/common/xcontent/JsonToStringXContentParser.java +++ b/server/src/main/java/org/opensearch/common/xcontent/JsonToStringXContentParser.java @@ -37,10 +37,6 @@ * @opensearch.internal */ public class JsonToStringXContentParser extends AbstractXContentParser { - public static final String VALUE_AND_PATH_SUFFIX = "._valueAndPath"; - public static final String VALUE_SUFFIX = "._value"; - public static final String DOT_SYMBOL = "."; - public static final String EQUAL_SYMBOL = "="; private final String fieldTypeName; private final XContentParser parser; @@ -54,6 +50,10 @@ public class JsonToStringXContentParser extends AbstractXContentParser { private final DeprecationHandler deprecationHandler; + private static final String VALUE_AND_PATH_SUFFIX = "._valueAndPath"; + private static final String VALUE_SUFFIX = "._value"; + private static final String EQUAL_SYMBOL = "="; + public JsonToStringXContentParser( NamedXContentRegistry xContentRegistry, DeprecationHandler deprecationHandler, diff --git a/server/src/main/java/org/opensearch/index/mapper/FlatObjectFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/FlatObjectFieldMapper.java index 0ccdb40f9d33a..738efcfafdca1 100644 --- a/server/src/main/java/org/opensearch/index/mapper/FlatObjectFieldMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/FlatObjectFieldMapper.java @@ -16,16 +16,19 @@ import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.Term; import org.apache.lucene.search.BoostQuery; -import org.apache.lucene.search.FieldExistsQuery; import org.apache.lucene.search.MultiTermQuery; +import org.apache.lucene.search.PrefixQuery; import org.apache.lucene.search.Query; +import org.apache.lucene.search.TermInSetQuery; import org.apache.lucene.search.TermQuery; +import org.apache.lucene.search.TermRangeQuery; import org.apache.lucene.util.BytesRef; +import org.opensearch.OpenSearchException; import org.opensearch.Version; import org.opensearch.common.Nullable; import org.opensearch.common.collect.Iterators; import org.opensearch.common.lucene.Lucene; -import org.opensearch.common.unit.Fuzziness; +import org.opensearch.common.lucene.search.AutomatonQueries; import org.opensearch.common.xcontent.JsonToStringXContentParser; import org.opensearch.core.common.ParsingException; import org.opensearch.core.xcontent.DeprecationHandler; @@ -34,8 +37,8 @@ import org.opensearch.index.analysis.NamedAnalyzer; import org.opensearch.index.fielddata.IndexFieldData; import org.opensearch.index.fielddata.plain.SortedSetOrdinalsIndexFieldData; -import org.opensearch.index.mapper.KeywordFieldMapper.KeywordFieldType; import org.opensearch.index.query.QueryShardContext; +import org.opensearch.index.query.QueryShardException; import org.opensearch.search.aggregations.support.CoreValuesSourceType; import org.opensearch.search.lookup.SearchLookup; @@ -49,11 +52,7 @@ import java.util.function.BiFunction; import java.util.function.Supplier; -import static org.opensearch.common.xcontent.JsonToStringXContentParser.DOT_SYMBOL; -import static org.opensearch.common.xcontent.JsonToStringXContentParser.EQUAL_SYMBOL; -import static org.opensearch.common.xcontent.JsonToStringXContentParser.VALUE_AND_PATH_SUFFIX; -import static org.opensearch.common.xcontent.JsonToStringXContentParser.VALUE_SUFFIX; -import static org.opensearch.index.mapper.FlatObjectFieldMapper.FlatObjectFieldType.getKeywordFieldType; +import static org.opensearch.search.SearchService.ALLOW_EXPENSIVE_QUERIES; /** * A field mapper for flat_objects. @@ -63,6 +62,10 @@ public final class FlatObjectFieldMapper extends DynamicKeyFieldMapper { public static final String CONTENT_TYPE = "flat_object"; + private static final String VALUE_AND_PATH_SUFFIX = "._valueAndPath"; + private static final String VALUE_SUFFIX = "._value"; + private static final String DOT_SYMBOL = "."; + private static final String EQUAL_SYMBOL = "="; /** * In flat_object field mapper, field type is similar to keyword field type @@ -83,12 +86,7 @@ public static class Defaults { @Override public MappedFieldType keyedFieldType(String key) { - return new FlatObjectFieldType( - this.name() + DOT_SYMBOL + key, - this.name(), - (KeywordFieldType) valueFieldMapper.fieldType(), - (KeywordFieldType) valueAndPathFieldMapper.fieldType() - ); + return new FlatObjectFieldType(this.name() + DOT_SYMBOL + key, this.name()); } /** @@ -113,12 +111,20 @@ public Builder(String name) { builder = this; } + private FlatObjectFieldType buildFlatObjectFieldType(BuilderContext context, FieldType fieldType) { + return new FlatObjectFieldType(buildFullName(context), fieldType); + } + /** * ValueFieldMapper is the subfield type for values in the Json. * use a {@link KeywordFieldMapper.KeywordField} */ - private ValueFieldMapper buildValueFieldMapper(FieldType fieldType, KeywordFieldType valueFieldType) { + private ValueFieldMapper buildValueFieldMapper(BuilderContext context, FieldType fieldType, FlatObjectFieldType fft) { + String fullName = buildFullName(context); FieldType vft = new FieldType(fieldType); + KeywordFieldMapper.KeywordFieldType valueFieldType = new KeywordFieldMapper.KeywordFieldType(fullName + VALUE_SUFFIX, vft); + + fft.setValueFieldType(valueFieldType); return new ValueFieldMapper(vft, valueFieldType); } @@ -126,30 +132,27 @@ private ValueFieldMapper buildValueFieldMapper(FieldType fieldType, KeywordField * ValueAndPathFieldMapper is the subfield type for path=value format in the Json. * also use a {@link KeywordFieldMapper.KeywordField} */ - private ValueAndPathFieldMapper buildValueAndPathFieldMapper(FieldType fieldType, KeywordFieldType valueAndPathFieldType) { + private ValueAndPathFieldMapper buildValueAndPathFieldMapper(BuilderContext context, FieldType fieldType, FlatObjectFieldType fft) { + String fullName = buildFullName(context); FieldType vft = new FieldType(fieldType); - return new ValueAndPathFieldMapper(vft, valueAndPathFieldType); + KeywordFieldMapper.KeywordFieldType ValueAndPathFieldType = new KeywordFieldMapper.KeywordFieldType( + fullName + VALUE_AND_PATH_SUFFIX, + vft + ); + fft.setValueAndPathFieldType(ValueAndPathFieldType); + return new ValueAndPathFieldMapper(vft, ValueAndPathFieldType); } @Override public FlatObjectFieldMapper build(BuilderContext context) { - boolean isSearchable = true; - boolean hasDocValue = true; - KeywordFieldType valueFieldType = getKeywordFieldType(buildFullName(context), VALUE_SUFFIX, isSearchable, hasDocValue); - KeywordFieldType valueAndPathFieldType = getKeywordFieldType( - buildFullName(context), - VALUE_AND_PATH_SUFFIX, - isSearchable, - hasDocValue - ); - FlatObjectFieldType fft = new FlatObjectFieldType(buildFullName(context), null, valueFieldType, valueAndPathFieldType); - + FieldType fieldtype = new FieldType(Defaults.FIELD_TYPE); + FlatObjectFieldType fft = buildFlatObjectFieldType(context, fieldtype); return new FlatObjectFieldMapper( name, Defaults.FIELD_TYPE, fft, - buildValueFieldMapper(Defaults.FIELD_TYPE, valueFieldType), - buildValueAndPathFieldMapper(Defaults.FIELD_TYPE, valueAndPathFieldType), + buildValueFieldMapper(context, fieldtype, fft), + buildValueAndPathFieldMapper(context, fieldtype, fft), CopyTo.empty(), this ); @@ -186,70 +189,66 @@ public static final class FlatObjectFieldType extends StringFieldType { private final String mappedFieldTypeName; - private final KeywordFieldType valueFieldType; + private KeywordFieldMapper.KeywordFieldType valueFieldType; - private final KeywordFieldType valueAndPathFieldType; + private KeywordFieldMapper.KeywordFieldType valueAndPathFieldType; - public FlatObjectFieldType( - String name, - String mappedFieldTypeName, - boolean isSearchable, - boolean hasDocValues, - NamedAnalyzer analyzer, - Map meta - ) { + public FlatObjectFieldType(String name, boolean isSearchable, boolean hasDocValues, Map meta) { + super(name, isSearchable, false, true, TextSearchInfo.SIMPLE_MATCH_ONLY, meta); + setIndexAnalyzer(Lucene.KEYWORD_ANALYZER); + this.ignoreAbove = Integer.MAX_VALUE; + this.nullValue = null; + this.mappedFieldTypeName = null; + } + + public FlatObjectFieldType(String name, FieldType fieldType) { super( name, - isSearchable, + fieldType.indexOptions() != IndexOptions.NONE, false, - hasDocValues, - analyzer == null ? TextSearchInfo.SIMPLE_MATCH_ONLY : new TextSearchInfo(Defaults.FIELD_TYPE, null, analyzer, analyzer), - meta + true, + new TextSearchInfo(fieldType, null, Lucene.KEYWORD_ANALYZER, Lucene.KEYWORD_ANALYZER), + Collections.emptyMap() ); - setIndexAnalyzer(Lucene.KEYWORD_ANALYZER); this.ignoreAbove = Integer.MAX_VALUE; this.nullValue = null; - this.mappedFieldTypeName = mappedFieldTypeName; - this.valueFieldType = getKeywordFieldType(name, VALUE_SUFFIX, isSearchable, hasDocValues); - this.valueAndPathFieldType = getKeywordFieldType(name, VALUE_AND_PATH_SUFFIX, isSearchable, hasDocValues); + this.mappedFieldTypeName = null; } - public FlatObjectFieldType( - String name, - String mappedFieldTypeName, - KeywordFieldType valueFieldType, - KeywordFieldType valueAndPathFieldType - ) { + public FlatObjectFieldType(String name, NamedAnalyzer analyzer) { + super(name, true, false, true, new TextSearchInfo(Defaults.FIELD_TYPE, null, analyzer, analyzer), Collections.emptyMap()); + this.ignoreAbove = Integer.MAX_VALUE; + this.nullValue = null; + this.mappedFieldTypeName = null; + } + + public FlatObjectFieldType(String name, String mappedFieldTypeName) { super( name, - valueFieldType.isSearchable(), + true, false, - valueFieldType.hasDocValues(), + true, new TextSearchInfo(Defaults.FIELD_TYPE, null, Lucene.KEYWORD_ANALYZER, Lucene.KEYWORD_ANALYZER), Collections.emptyMap() ); this.ignoreAbove = Integer.MAX_VALUE; this.nullValue = null; this.mappedFieldTypeName = mappedFieldTypeName; + } + + void setValueFieldType(KeywordFieldMapper.KeywordFieldType valueFieldType) { this.valueFieldType = valueFieldType; - this.valueAndPathFieldType = valueAndPathFieldType; } - static KeywordFieldType getKeywordFieldType(String fullName, String valueType, boolean isSearchable, boolean hasDocValue) { - return new KeywordFieldType(fullName + valueType, isSearchable, hasDocValue, Collections.emptyMap()) { - @Override - protected String rewriteForDocValue(Object value) { - assert value instanceof String; - return fullName + DOT_SYMBOL + value; - } - }; + void setValueAndPathFieldType(KeywordFieldMapper.KeywordFieldType ValueAndPathFieldType) { + this.valueAndPathFieldType = ValueAndPathFieldType; } - public KeywordFieldType getValueFieldType() { + public KeywordFieldMapper.KeywordFieldType getValueFieldType() { return this.valueFieldType; } - public KeywordFieldType getValueAndPathFieldType() { + public KeywordFieldMapper.KeywordFieldType getValueAndPathFieldType() { return this.valueAndPathFieldType; } @@ -332,10 +331,6 @@ protected BytesRef indexedValueForSearch(Object value) { return getTextSearchInfo().getSearchAnalyzer().normalize(name(), value.toString()); } - private KeywordFieldType valueFieldType() { - return (mappedFieldTypeName == null) ? valueFieldType : valueAndPathFieldType; - } - /** * redirect queries with rewrite value to rewriteSearchValue and directSubFieldName */ @@ -357,12 +352,17 @@ public Query termQuery(Object value, @Nullable QueryShardContext context) { @Override public Query termsQuery(List values, QueryShardContext context) { - List parsedValues = new ArrayList<>(values.size()); - for (Object value : values) { - parsedValues.add(rewriteValue(inputToString(value))); + failIfNotIndexed(); + String directedSearchFieldName = directSubfield(); + BytesRef[] bytesRefs = new BytesRef[values.size()]; + for (int i = 0; i < bytesRefs.length; i++) { + String rewriteValues = rewriteValue(inputToString(values.get(i))); + + bytesRefs[i] = indexedValueForSearch(new BytesRef(rewriteValues)); + } - return valueFieldType().termsQuery(parsedValues, context); + return new TermInSetQuery(directedSearchFieldName, bytesRefs); } /** @@ -395,7 +395,7 @@ public String rewriteValue(String searchValueString) { } - boolean hasMappedFieldTyeNameInQueryFieldName(String input) { + private boolean hasMappedFieldTyeNameInQueryFieldName(String input) { String prefix = this.mappedFieldTypeName; if (prefix == null) { return false; @@ -413,9 +413,6 @@ boolean hasMappedFieldTyeNameInQueryFieldName(String input) { } private String inputToString(Object inputValue) { - if (inputValue == null) { - return null; - } if (inputValue instanceof Integer) { String inputToString = Integer.toString((Integer) inputValue); return inputToString; @@ -451,50 +448,46 @@ private String inputToString(Object inputValue) { @Override public Query prefixQuery(String value, MultiTermQuery.RewriteMethod method, boolean caseInsensitive, QueryShardContext context) { - return valueFieldType().prefixQuery(rewriteValue(value), method, caseInsensitive, context); - } - - @Override - public Query regexpQuery( - String value, - int syntaxFlags, - int matchFlags, - int maxDeterminizedStates, - @Nullable MultiTermQuery.RewriteMethod method, - QueryShardContext context - ) { - return valueFieldType().regexpQuery(rewriteValue(value), syntaxFlags, matchFlags, maxDeterminizedStates, method, context); - } - - @Override - public Query fuzzyQuery( - Object value, - Fuzziness fuzziness, - int prefixLength, - int maxExpansions, - boolean transpositions, - @Nullable MultiTermQuery.RewriteMethod method, - QueryShardContext context - ) { - return valueFieldType().fuzzyQuery( - rewriteValue(inputToString(value)), - fuzziness, - prefixLength, - maxExpansions, - transpositions, - method, - context - ); + String directSubfield = directSubfield(); + String rewriteValue = rewriteValue(value); + + if (context.allowExpensiveQueries() == false) { + throw new OpenSearchException( + "[prefix] queries cannot be executed when '" + + ALLOW_EXPENSIVE_QUERIES.getKey() + + "' is set to false. For optimised prefix queries on text " + + "fields please enable [index_prefixes]." + ); + } + failIfNotIndexed(); + if (method == null) { + method = MultiTermQuery.CONSTANT_SCORE_REWRITE; + } + if (caseInsensitive) { + return AutomatonQueries.caseInsensitivePrefixQuery((new Term(directSubfield, indexedValueForSearch(rewriteValue))), method); + } + return new PrefixQuery(new Term(directSubfield, indexedValueForSearch(rewriteValue)), method); } @Override public Query rangeQuery(Object lowerTerm, Object upperTerm, boolean includeLower, boolean includeUpper, QueryShardContext context) { - return valueFieldType().rangeQuery( - rewriteValue(inputToString(lowerTerm)), - rewriteValue(inputToString(upperTerm)), + String directSubfield = directSubfield(); + String rewriteUpperTerm = rewriteValue(inputToString(upperTerm)); + String rewriteLowerTerm = rewriteValue(inputToString(lowerTerm)); + if (context.allowExpensiveQueries() == false) { + throw new OpenSearchException( + "[range] queries on [text] or [keyword] fields cannot be executed when '" + + ALLOW_EXPENSIVE_QUERIES.getKey() + + "' is set to false." + ); + } + failIfNotIndexed(); + return new TermRangeQuery( + directSubfield, + lowerTerm == null ? null : indexedValueForSearch(rewriteLowerTerm), + upperTerm == null ? null : indexedValueForSearch(rewriteUpperTerm), includeLower, - includeUpper, - context + includeUpper ); } @@ -510,12 +503,8 @@ public Query existsQuery(QueryShardContext context) { searchKey = this.mappedFieldTypeName; searchField = name(); } else { - if (hasDocValues()) { - return new FieldExistsQuery(name()); - } else { - searchKey = FieldNamesFieldMapper.NAME; - searchField = name(); - } + searchKey = FieldNamesFieldMapper.NAME; + searchField = name(); } return new TermQuery(new Term(searchKey, indexedValueForSearch(searchField))); } @@ -527,7 +516,13 @@ public Query wildcardQuery( boolean caseInsensitve, QueryShardContext context ) { - return valueFieldType().wildcardQuery(rewriteValue(value), method, caseInsensitve, context); + // flat_object field types are always normalized, so ignore case sensitivity and force normalize the wildcard + // query text + throw new QueryShardException( + context, + "Can only use wildcard queries on keyword and text fields - not on [" + name() + "] which is of type [" + typeName() + "]" + ); + } } @@ -611,6 +606,7 @@ protected void parseCreateField(ParseContext context) throws IOException { } } } + } @Override @@ -641,8 +637,6 @@ public Iterator iterator() { */ private void parseValueAddFields(ParseContext context, String value, String fieldName) throws IOException { - assert valueFieldMapper != null; - assert valueAndPathFieldMapper != null; NamedAnalyzer normalizer = fieldType().normalizer(); if (normalizer != null) { value = normalizeValue(normalizer, name(), value); @@ -653,57 +647,69 @@ private void parseValueAddFields(ParseContext context, String value, String fiel if (fieldType.indexOptions() != IndexOptions.NONE || fieldType.stored()) { // convert to utf8 only once before feeding postings/dv/stored fields + final BytesRef binaryValue = new BytesRef(fieldType().name() + DOT_SYMBOL + value); + Field field = new FlatObjectField(fieldType().name(), binaryValue, fieldType); - if (fieldType().hasDocValues() == false) { + if (fieldType().hasDocValues() == false && fieldType.omitNorms()) { createFieldNamesField(context); } if (fieldName.equals(fieldType().name())) { - Field field = new FlatObjectField(fieldType().name(), binaryValue, fieldType); context.doc().add(field); - } else if (valueType.equals(VALUE_SUFFIX)) { - valueFieldMapper.addField(context, value); - } else if (valueType.equals(VALUE_AND_PATH_SUFFIX)) { - valueAndPathFieldMapper.addField(context, value); + } + if (valueType.equals(VALUE_SUFFIX)) { + if (valueFieldMapper != null) { + valueFieldMapper.addField(context, value); + } + } + if (valueType.equals(VALUE_AND_PATH_SUFFIX)) { + if (valueAndPathFieldMapper != null) { + valueAndPathFieldMapper.addField(context, value); + } } if (fieldType().hasDocValues()) { if (fieldName.equals(fieldType().name())) { context.doc().add(new SortedSetDocValuesField(fieldType().name(), binaryValue)); - } else if (valueType.equals(VALUE_SUFFIX)) { - context.doc().add(new SortedSetDocValuesField(fieldType().name() + VALUE_SUFFIX, binaryValue)); - } else if (valueType.equals(VALUE_AND_PATH_SUFFIX)) { - context.doc().add(new SortedSetDocValuesField(fieldType().name() + VALUE_AND_PATH_SUFFIX, binaryValue)); + } + if (valueType.equals(VALUE_SUFFIX)) { + if (valueFieldMapper != null) { + context.doc().add(new SortedSetDocValuesField(fieldType().name() + VALUE_SUFFIX, binaryValue)); + } + } + if (valueType.equals(VALUE_AND_PATH_SUFFIX)) { + if (valueAndPathFieldMapper != null) { + context.doc().add(new SortedSetDocValuesField(fieldType().name() + VALUE_AND_PATH_SUFFIX, binaryValue)); + } } } + } + } private static String normalizeValue(NamedAnalyzer normalizer, String field, String value) throws IOException { + String normalizerErrorMessage = "The normalization token stream is " + + "expected to produce exactly 1 token, but got 0 for analyzer " + + normalizer + + " and input \"" + + value + + "\""; try (TokenStream ts = normalizer.tokenStream(field, value)) { final CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class); ts.reset(); if (ts.incrementToken() == false) { - throw new IllegalStateException(errorMessage(normalizer, value)); + throw new IllegalStateException(normalizerErrorMessage); } final String newValue = termAtt.toString(); if (ts.incrementToken()) { - throw new IllegalStateException(errorMessage(normalizer, value)); + throw new IllegalStateException(normalizerErrorMessage); } ts.end(); return newValue; } } - private static String errorMessage(NamedAnalyzer normalizer, String value) { - return "The normalization token stream is " - + "expected to produce exactly 1 token, but got 0 for analyzer " - + normalizer - + " and input \"" - + value - + "\""; - } - @Override protected String contentType() { return CONTENT_TYPE; @@ -711,7 +717,7 @@ protected String contentType() { private static final class ValueAndPathFieldMapper extends FieldMapper { - protected ValueAndPathFieldMapper(FieldType fieldType, KeywordFieldType mappedFieldType) { + protected ValueAndPathFieldMapper(FieldType fieldType, KeywordFieldMapper.KeywordFieldType mappedFieldType) { super(mappedFieldType.name(), fieldType, mappedFieldType, MultiFields.empty(), CopyTo.empty()); } @@ -722,7 +728,7 @@ void addField(ParseContext context, String value) { context.doc().add(field); - if (fieldType().hasDocValues() == false) { + if (fieldType().hasDocValues() == false && fieldType.omitNorms()) { createFieldNamesField(context); } } @@ -752,7 +758,7 @@ public String toString() { private static final class ValueFieldMapper extends FieldMapper { - protected ValueFieldMapper(FieldType fieldType, KeywordFieldType mappedFieldType) { + protected ValueFieldMapper(FieldType fieldType, KeywordFieldMapper.KeywordFieldType mappedFieldType) { super(mappedFieldType.name(), fieldType, mappedFieldType, MultiFields.empty(), CopyTo.empty()); } @@ -762,7 +768,7 @@ void addField(ParseContext context, String value) { Field field = new KeywordFieldMapper.KeywordField(fieldType().name(), binaryValue, fieldType); context.doc().add(field); - if (fieldType().hasDocValues() == false) { + if (fieldType().hasDocValues() == false && fieldType.omitNorms()) { createFieldNamesField(context); } } diff --git a/server/src/main/java/org/opensearch/index/mapper/KeywordFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/KeywordFieldMapper.java index 54a1aead5fcc7..11ff601b3fd6d 100644 --- a/server/src/main/java/org/opensearch/index/mapper/KeywordFieldMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/KeywordFieldMapper.java @@ -263,7 +263,7 @@ public KeywordFieldMapper build(BuilderContext context) { * * @opensearch.internal */ - public static class KeywordFieldType extends StringFieldType { + public static final class KeywordFieldType extends StringFieldType { private final int ignoreAbove; private final String nullValue; @@ -387,10 +387,6 @@ protected BytesRef indexedValueForSearch(Object value) { return getTextSearchInfo().getSearchAnalyzer().normalize(name(), value.toString()); } - protected Object rewriteForDocValue(Object value) { - return value; - } - @Override public Query termsQuery(List values, QueryShardContext context) { failIfNotIndexedAndNoDocValues(); @@ -399,21 +395,19 @@ public Query termsQuery(List values, QueryShardContext context) { if (!context.keywordFieldIndexOrDocValuesEnabled()) { return super.termsQuery(values, context); } - BytesRef[] iBytesRefs = new BytesRef[values.size()]; - BytesRef[] dVByteRefs = new BytesRef[values.size()]; - for (int i = 0; i < iBytesRefs.length; i++) { - iBytesRefs[i] = indexedValueForSearch(values.get(i)); - dVByteRefs[i] = indexedValueForSearch(rewriteForDocValue(values.get(i))); + BytesRef[] bytesRefs = new BytesRef[values.size()]; + for (int i = 0; i < bytesRefs.length; i++) { + bytesRefs[i] = indexedValueForSearch(values.get(i)); } - Query indexQuery = new TermInSetQuery(name(), iBytesRefs); - Query dvQuery = new TermInSetQuery(MultiTermQuery.DOC_VALUES_REWRITE, name(), dVByteRefs); + Query indexQuery = new TermInSetQuery(name(), bytesRefs); + Query dvQuery = new TermInSetQuery(MultiTermQuery.DOC_VALUES_REWRITE, name(), bytesRefs); return new IndexOrDocValuesQuery(indexQuery, dvQuery); } // if we only have doc_values enabled, we construct a new query with doc_values re-written if (hasDocValues()) { BytesRef[] bytesRefs = new BytesRef[values.size()]; for (int i = 0; i < bytesRefs.length; i++) { - bytesRefs[i] = indexedValueForSearch(rewriteForDocValue(values.get(i))); + bytesRefs[i] = indexedValueForSearch(values.get(i)); } return new TermInSetQuery(MultiTermQuery.DOC_VALUES_REWRITE, name(), bytesRefs); } @@ -442,25 +436,17 @@ public Query prefixQuery( return super.prefixQuery(value, method, caseInsensitive, context); } Query indexQuery = super.prefixQuery(value, method, caseInsensitive, context); - Query dvQuery = super.prefixQuery( - (String) rewriteForDocValue(value), - MultiTermQuery.DOC_VALUES_REWRITE, - caseInsensitive, - context - ); + Query dvQuery = super.prefixQuery(value, MultiTermQuery.DOC_VALUES_REWRITE, caseInsensitive, context); return new IndexOrDocValuesQuery(indexQuery, dvQuery); } if (hasDocValues()) { if (caseInsensitive) { return AutomatonQueries.caseInsensitivePrefixQuery( - (new Term(name(), indexedValueForSearch(rewriteForDocValue(value)))), + (new Term(name(), indexedValueForSearch(value))), MultiTermQuery.DOC_VALUES_REWRITE ); } - return new PrefixQuery( - new Term(name(), indexedValueForSearch(rewriteForDocValue(value))), - MultiTermQuery.DOC_VALUES_REWRITE - ); + return new PrefixQuery(new Term(name(), indexedValueForSearch(value)), MultiTermQuery.DOC_VALUES_REWRITE); } return super.prefixQuery(value, method, caseInsensitive, context); } @@ -486,7 +472,7 @@ public Query regexpQuery( } Query indexQuery = super.regexpQuery(value, syntaxFlags, matchFlags, maxDeterminizedStates, method, context); Query dvQuery = super.regexpQuery( - (String) rewriteForDocValue(value), + value, syntaxFlags, matchFlags, maxDeterminizedStates, @@ -497,7 +483,7 @@ public Query regexpQuery( } if (hasDocValues()) { return new RegexpQuery( - new Term(name(), indexedValueForSearch(rewriteForDocValue(value))), + new Term(name(), indexedValueForSearch(value)), syntaxFlags, matchFlags, RegexpQuery.DEFAULT_PROVIDER, @@ -528,8 +514,8 @@ public Query rangeQuery(Object lowerTerm, Object upperTerm, boolean includeLower ); Query dvQuery = new TermRangeQuery( name(), - lowerTerm == null ? null : indexedValueForSearch(rewriteForDocValue(lowerTerm)), - upperTerm == null ? null : indexedValueForSearch(rewriteForDocValue(upperTerm)), + lowerTerm == null ? null : indexedValueForSearch(lowerTerm), + upperTerm == null ? null : indexedValueForSearch(upperTerm), includeLower, includeUpper, MultiTermQuery.DOC_VALUES_REWRITE @@ -539,8 +525,8 @@ public Query rangeQuery(Object lowerTerm, Object upperTerm, boolean includeLower if (hasDocValues()) { return new TermRangeQuery( name(), - lowerTerm == null ? null : indexedValueForSearch(rewriteForDocValue(lowerTerm)), - upperTerm == null ? null : indexedValueForSearch(rewriteForDocValue(upperTerm)), + lowerTerm == null ? null : indexedValueForSearch(lowerTerm), + upperTerm == null ? null : indexedValueForSearch(upperTerm), includeLower, includeUpper, MultiTermQuery.DOC_VALUES_REWRITE @@ -577,7 +563,7 @@ public Query fuzzyQuery( } Query indexQuery = super.fuzzyQuery(value, fuzziness, prefixLength, maxExpansions, transpositions, method, context); Query dvQuery = super.fuzzyQuery( - rewriteForDocValue(value), + value, fuzziness, prefixLength, maxExpansions, @@ -589,8 +575,8 @@ public Query fuzzyQuery( } if (hasDocValues()) { return new FuzzyQuery( - new Term(name(), indexedValueForSearch(rewriteForDocValue(value))), - fuzziness.asDistance(BytesRefs.toString(rewriteForDocValue(value))), + new Term(name(), indexedValueForSearch(value)), + fuzziness.asDistance(BytesRefs.toString(value)), prefixLength, maxExpansions, transpositions, @@ -621,19 +607,13 @@ public Query wildcardQuery( return super.wildcardQuery(value, method, caseInsensitive, true, context); } Query indexQuery = super.wildcardQuery(value, method, caseInsensitive, true, context); - Query dvQuery = super.wildcardQuery( - (String) rewriteForDocValue(value), - MultiTermQuery.DOC_VALUES_REWRITE, - caseInsensitive, - true, - context - ); + Query dvQuery = super.wildcardQuery(value, MultiTermQuery.DOC_VALUES_REWRITE, caseInsensitive, true, context); return new IndexOrDocValuesQuery(indexQuery, dvQuery); } if (hasDocValues()) { Term term; value = normalizeWildcardPattern(name(), value, getTextSearchInfo().getSearchAnalyzer()); - term = new Term(name(), (String) rewriteForDocValue(value)); + term = new Term(name(), value); if (caseInsensitive) { return AutomatonQueries.caseInsensitiveWildcardQuery(term, method); } diff --git a/server/src/test/java/org/opensearch/index/mapper/FlatObjectFieldMapperTests.java b/server/src/test/java/org/opensearch/index/mapper/FlatObjectFieldMapperTests.java index afd9e994ce3ae..94d1f501bee51 100644 --- a/server/src/test/java/org/opensearch/index/mapper/FlatObjectFieldMapperTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/FlatObjectFieldMapperTests.java @@ -12,7 +12,6 @@ import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.IndexableField; import org.apache.lucene.index.IndexableFieldType; -import org.apache.lucene.search.FieldExistsQuery; import org.apache.lucene.search.Query; import org.apache.lucene.search.TermQuery; import org.apache.lucene.util.BytesRef; @@ -24,13 +23,14 @@ import java.io.IOException; -import static org.opensearch.common.xcontent.JsonToStringXContentParser.VALUE_AND_PATH_SUFFIX; -import static org.opensearch.common.xcontent.JsonToStringXContentParser.VALUE_SUFFIX; -import static org.opensearch.index.mapper.FlatObjectFieldMapper.CONTENT_TYPE; import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.core.IsEqual.equalTo; public class FlatObjectFieldMapperTests extends MapperTestCase { + private static final String FIELD_TYPE = "flat_object"; + private static final String VALUE_AND_PATH_SUFFIX = "._valueAndPath"; + private static final String VALUE_SUFFIX = "._value"; + protected boolean supportsMeta() { return false; } @@ -41,7 +41,7 @@ protected boolean supportsOrIgnoresBoost() { public void testMapperServiceHasParser() throws IOException { MapperService mapperService = createMapperService(fieldMapping(b -> { minimalMapping(b); })); - Mapper.TypeParser parser = mapperService.mapperRegistry.getMapperParsers().get(CONTENT_TYPE); + Mapper.TypeParser parser = mapperService.mapperRegistry.getMapperParsers().get(FIELD_TYPE); assertNotNull(parser); assertTrue(parser instanceof FlatObjectFieldMapper.TypeParser); } @@ -49,39 +49,28 @@ public void testMapperServiceHasParser() throws IOException { protected void assertExistsQuery(MapperService mapperService) throws IOException { ParseContext.Document fields = mapperService.documentMapper().parse(source(this::writeField)).rootDoc(); QueryShardContext queryShardContext = createQueryShardContext(mapperService); - FlatObjectFieldMapper.FlatObjectFieldType fieldType = (FlatObjectFieldMapper.FlatObjectFieldType) mapperService.fieldType("field"); + MappedFieldType fieldType = mapperService.fieldType("field"); Query query = fieldType.existsQuery(queryShardContext); assertExistsQuery(fieldType, query, fields); - } - protected void assertExistsQuery(FlatObjectFieldMapper.FlatObjectFieldType fieldType, Query query, ParseContext.Document fields) { + } - if (fieldType.hasDocValues() && fieldType.hasMappedFieldTyeNameInQueryFieldName(fieldType.name()) == false) { - assertThat(query, instanceOf(FieldExistsQuery.class)); - FieldExistsQuery fieldExistsQuery = (FieldExistsQuery) query; - assertEquals(fieldType.name(), fieldExistsQuery.getField()); + protected void assertExistsQuery(MappedFieldType fieldType, Query query, ParseContext.Document fields) { + // we always perform a term query against _field_names, even when the field + // is not added to _field_names because it is not indexed nor stored + assertThat(query, instanceOf(TermQuery.class)); + TermQuery termQuery = (TermQuery) query; + assertEquals(FieldNamesFieldMapper.NAME, termQuery.getTerm().field()); + assertEquals("field", termQuery.getTerm().text()); + if (fieldType.isSearchable() || fieldType.isStored()) { + assertNotNull(fields.getField(FieldNamesFieldMapper.NAME)); } else { - assertThat(query, instanceOf(TermQuery.class)); - TermQuery termQuery = (TermQuery) query; - assertEquals(FieldNamesFieldMapper.NAME, termQuery.getTerm().field()); - assertEquals("field", termQuery.getTerm().text()); - } - - if (fieldType.hasDocValues()) { - assertDocValuesField(fields, "field"); assertNoFieldNamesField(fields); - } else { - assertNoDocValuesField(fields, "field"); - if (fieldType.isSearchable()) { - assertNotNull(fields.getField(FieldNamesFieldMapper.NAME)); - } else { - assertNoFieldNamesField(fields); - } } } public void minimalMapping(XContentBuilder b) throws IOException { - b.field("type", CONTENT_TYPE); + b.field("type", FIELD_TYPE); } /** diff --git a/server/src/test/java/org/opensearch/index/mapper/FlatObjectFieldTypeTests.java b/server/src/test/java/org/opensearch/index/mapper/FlatObjectFieldTypeTests.java index 38a6f13777f00..9ec053dc59d10 100644 --- a/server/src/test/java/org/opensearch/index/mapper/FlatObjectFieldTypeTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/FlatObjectFieldTypeTests.java @@ -8,23 +8,11 @@ package org.opensearch.index.mapper; -import org.apache.lucene.document.FieldType; -import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.Term; -import org.apache.lucene.search.FieldExistsQuery; -import org.apache.lucene.search.FuzzyQuery; -import org.apache.lucene.search.IndexOrDocValuesQuery; -import org.apache.lucene.search.MultiTermQuery; -import org.apache.lucene.search.PrefixQuery; -import org.apache.lucene.search.Query; -import org.apache.lucene.search.RegexpQuery; -import org.apache.lucene.search.TermInSetQuery; import org.apache.lucene.search.TermQuery; -import org.apache.lucene.search.TermRangeQuery; -import org.apache.lucene.search.WildcardQuery; -import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.automaton.Operations; -import org.opensearch.common.unit.Fuzziness; +import org.opensearch.Version; +import org.opensearch.cluster.metadata.IndexMetadata; +import org.opensearch.common.settings.Settings; import org.opensearch.index.analysis.AnalyzerScope; import org.opensearch.index.analysis.NamedAnalyzer; @@ -33,41 +21,18 @@ import java.util.Arrays; import java.util.Collections; import java.util.HashMap; -import java.util.List; import java.util.Map; -import static org.opensearch.common.xcontent.JsonToStringXContentParser.VALUE_AND_PATH_SUFFIX; -import static org.opensearch.common.xcontent.JsonToStringXContentParser.VALUE_SUFFIX; -import static org.apache.lucene.search.MultiTermQuery.CONSTANT_SCORE_REWRITE; -import static org.apache.lucene.search.MultiTermQuery.DOC_VALUES_REWRITE; - public class FlatObjectFieldTypeTests extends FieldTypeTestCase { - - private static MappedFieldType getFlatParentFieldType( - String fieldName, - String mappedFieldTypeName, - boolean isSearchable, - boolean hasDocValues - ) { - FlatObjectFieldMapper.Builder builder = new FlatObjectFieldMapper.Builder(fieldName); - FlatObjectFieldMapper.FlatObjectFieldType flatObjectFieldType = new FlatObjectFieldMapper.FlatObjectFieldType( - fieldName, - mappedFieldTypeName, - isSearchable, - hasDocValues, - null, - Collections.emptyMap() - ); - FieldType fieldtype = new FieldType(FlatObjectFieldMapper.Defaults.FIELD_TYPE); - FieldType vft = new FieldType(fieldtype); - if (flatObjectFieldType.isSearchable() == false) { - vft.setIndexOptions(IndexOptions.NONE); - } - return flatObjectFieldType; + private static MappedFieldType getFlatParentFieldType(String fieldName) { + Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT.id).build(); + Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath()); + MappedFieldType flatParentFieldType = new FlatObjectFieldMapper.Builder(fieldName).build(context).fieldType(); + return flatParentFieldType; } public void testFetchSourceValue() throws IOException { - MappedFieldType mapper = getFlatParentFieldType("field", null, true, true); + MappedFieldType mapper = getFlatParentFieldType("field"); Map jsonPoint = new HashMap<>(); jsonPoint.put("type", "flat_object"); @@ -89,48 +54,32 @@ public void testFetchSourceValue() throws IOException { public void testDirectSubfield() { { - FlatObjectFieldMapper.FlatObjectFieldType flatParentFieldType = - (FlatObjectFieldMapper.FlatObjectFieldType) (getFlatParentFieldType("field", null, true, true)); + MappedFieldType flatParentFieldType = getFlatParentFieldType("field"); // when searching for "foo" in "field", the directSubfield is field._value field - String searchFieldName = (flatParentFieldType).directSubfield(); + String searchFieldName = ((FlatObjectFieldMapper.FlatObjectFieldType) flatParentFieldType).directSubfield(); assertEquals("field._value", searchFieldName); - MappedFieldType dynamicMappedFieldType = new FlatObjectFieldMapper.FlatObjectFieldType( - "bar", - flatParentFieldType.name(), - flatParentFieldType.getValueFieldType(), - flatParentFieldType.getValueAndPathFieldType() - ); + MappedFieldType dynamicMappedFieldType = new FlatObjectFieldMapper.FlatObjectFieldType("bar", flatParentFieldType.name()); // when searching for "foo" in "field.bar", the directSubfield is field._valueAndPath field String searchFieldNameDocPath = ((FlatObjectFieldMapper.FlatObjectFieldType) dynamicMappedFieldType).directSubfield(); assertEquals("field._valueAndPath", searchFieldNameDocPath); } { NamedAnalyzer analyzer = new NamedAnalyzer("default", AnalyzerScope.INDEX, null); - MappedFieldType ft = new FlatObjectFieldMapper.FlatObjectFieldType("field", null, true, true, analyzer, Collections.emptyMap()); + MappedFieldType ft = new FlatObjectFieldMapper.FlatObjectFieldType("field", analyzer); assertEquals("field._value", ((FlatObjectFieldMapper.FlatObjectFieldType) ft).directSubfield()); } } public void testRewriteValue() { - FlatObjectFieldMapper.FlatObjectFieldType flatParentFieldType = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( - "field", - null, - true, - true - ); + MappedFieldType flatParentFieldType = getFlatParentFieldType("field"); // when searching for "foo" in "field", the rewrite value is "foo" - String searchValues = (flatParentFieldType).rewriteValue("foo"); + String searchValues = ((FlatObjectFieldMapper.FlatObjectFieldType) flatParentFieldType).rewriteValue("foo"); assertEquals("foo", searchValues); - MappedFieldType dynamicMappedFieldType = new FlatObjectFieldMapper.FlatObjectFieldType( - "field.bar", - flatParentFieldType.name(), - flatParentFieldType.getValueFieldType(), - flatParentFieldType.getValueAndPathFieldType() - ); + MappedFieldType dynamicMappedFieldType = new FlatObjectFieldMapper.FlatObjectFieldType("field.bar", flatParentFieldType.name()); // when searching for "foo" in "field.bar", the rewrite value is "field.bar=foo" String searchFieldNameDocPath = ((FlatObjectFieldMapper.FlatObjectFieldType) dynamicMappedFieldType).directSubfield(); @@ -140,25 +89,15 @@ public void testRewriteValue() { public void testTermQuery() { - FlatObjectFieldMapper.FlatObjectFieldType flatParentFieldType = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( - "field", - null, - true, - true - ); + MappedFieldType flatParentFieldType = getFlatParentFieldType("field"); // when searching for "foo" in "field", the term query is directed to search "foo" in field._value field - String searchFieldName = (flatParentFieldType).directSubfield(); - String searchValues = (flatParentFieldType).rewriteValue("foo"); + String searchFieldName = ((FlatObjectFieldMapper.FlatObjectFieldType) flatParentFieldType).directSubfield(); + String searchValues = ((FlatObjectFieldMapper.FlatObjectFieldType) flatParentFieldType).rewriteValue("foo"); assertEquals("foo", searchValues); assertEquals(new TermQuery(new Term(searchFieldName, searchValues)), flatParentFieldType.termQuery(searchValues, null)); - MappedFieldType dynamicMappedFieldType = new FlatObjectFieldMapper.FlatObjectFieldType( - "field.bar", - flatParentFieldType.name(), - flatParentFieldType.getValueFieldType(), - flatParentFieldType.getValueAndPathFieldType() - ); + MappedFieldType dynamicMappedFieldType = new FlatObjectFieldMapper.FlatObjectFieldType("field.bar", flatParentFieldType.name()); // when searching for "foo" in "field.bar", the term query is directed to search in field._valueAndPath field String searchFieldNameDocPath = ((FlatObjectFieldMapper.FlatObjectFieldType) dynamicMappedFieldType).directSubfield(); @@ -166,919 +105,30 @@ public void testTermQuery() { assertEquals("field.bar=foo", searchValuesDocPath); assertEquals(new TermQuery(new Term(searchFieldNameDocPath, searchValuesDocPath)), dynamicMappedFieldType.termQuery("foo", null)); - MappedFieldType unsearchable = new FlatObjectFieldMapper.FlatObjectFieldType( - "field", - null, - false, - true, - null, - Collections.emptyMap() - ); - IllegalArgumentException e = expectThrows( - IllegalArgumentException.class, - () -> unsearchable.termQuery("bar", MOCK_QSC_ENABLE_INDEX_DOC_VALUES) - ); + MappedFieldType unsearchable = new FlatObjectFieldMapper.FlatObjectFieldType("field", false, true, Collections.emptyMap()); + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> unsearchable.termQuery("bar", null)); assertEquals("Cannot search on field [field] since it is not indexed.", e.getMessage()); } public void testExistsQuery() { { - FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( - "field", - null, - true, - true - ); + MappedFieldType ft = getFlatParentFieldType("field"); // when checking on the flat_object field name "field", check if exist in the field mapper names - assertEquals(new FieldExistsQuery("field"), ft.existsQuery(null)); + assertEquals(new TermQuery(new Term(FieldNamesFieldMapper.NAME, "field")), ft.existsQuery(null)); // when checking if a subfield within the flat_object, for example, "field.bar", use term query in the flat_object field - MappedFieldType dynamicMappedFieldType = new FlatObjectFieldMapper.FlatObjectFieldType( - "field.bar", - ft.name(), - ft.getValueFieldType(), - ft.getValueAndPathFieldType() - ); + MappedFieldType dynamicMappedFieldType = new FlatObjectFieldMapper.FlatObjectFieldType("field.bar", ft.name()); assertEquals(new TermQuery(new Term("field", "field.bar")), dynamicMappedFieldType.existsQuery(null)); } - { FlatObjectFieldMapper.FlatObjectFieldType ft = new FlatObjectFieldMapper.FlatObjectFieldType( "field", - null, true, false, - null, Collections.emptyMap() ); - assertEquals(new TermQuery(new Term(FieldNamesFieldMapper.NAME, "field")), ft.existsQuery(MOCK_QSC_ENABLE_INDEX_DOC_VALUES)); - } - } - - public void testTermsQuery() { - - // 1.test isSearchable=true, hasDocValues=true, mappedFieldTypeName=null - { - FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( - "field", - null, - true, - true - ); - List indexTerms = new ArrayList<>(); - indexTerms.add(new BytesRef("foo")); - indexTerms.add(new BytesRef("bar")); - List docValueterms = new ArrayList<>(); - docValueterms.add(new BytesRef("field.foo")); - docValueterms.add(new BytesRef("field.bar")); - Query expected = new IndexOrDocValuesQuery( - new TermInSetQuery("field" + VALUE_SUFFIX, indexTerms), - new TermInSetQuery(DOC_VALUES_REWRITE, "field" + VALUE_SUFFIX, docValueterms) - ); - - assertEquals(expected, ft.termsQuery(Arrays.asList("foo", "bar"), MOCK_QSC_ENABLE_INDEX_DOC_VALUES)); - } - - // test isSearchable=true, hasDocValues=true, mappedFieldTypeName!=null - { - FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( - "field", - "field", - true, - true - ); - - List indexTerms = new ArrayList<>(); - indexTerms.add(new BytesRef("foo")); - indexTerms.add(new BytesRef("bar")); - List docValueterms = new ArrayList<>(); - docValueterms.add(new BytesRef("field.foo")); - docValueterms.add(new BytesRef("field.bar")); - Query expected = new IndexOrDocValuesQuery( - new TermInSetQuery("field" + VALUE_AND_PATH_SUFFIX, indexTerms), - new TermInSetQuery(DOC_VALUES_REWRITE, "field" + VALUE_AND_PATH_SUFFIX, docValueterms) - ); - - assertEquals(expected, ft.termsQuery(Arrays.asList("foo", "bar"), MOCK_QSC_ENABLE_INDEX_DOC_VALUES)); - } - - // 2.test isSearchable=true, hasDocValues=false, mappedFieldTypeName=null - { - FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( - "field", - null, - true, - false - ); - List indexTerms = new ArrayList<>(); - indexTerms.add(new BytesRef("foo")); - indexTerms.add(new BytesRef("bar")); - Query expected = new TermInSetQuery("field" + VALUE_SUFFIX, indexTerms); - - assertEquals(expected, ft.termsQuery(Arrays.asList("foo", "bar"), MOCK_QSC_ENABLE_INDEX_DOC_VALUES)); - } - - // test isSearchable=true, hasDocValues=false, mappedFieldTypeName!=null - { - FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( - "field", - "field", - true, - false - ); - List indexTerms = new ArrayList<>(); - indexTerms.add(new BytesRef("foo")); - indexTerms.add(new BytesRef("bar")); - Query expected = new TermInSetQuery("field" + VALUE_AND_PATH_SUFFIX, indexTerms); - - assertEquals(expected, ft.termsQuery(Arrays.asList("foo", "bar"), null)); - } - - // 3.test isSearchable=false, hasDocValues=true, mappedFieldTypeName=null - { - FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( - "field", - null, - false, - true - ); - - List indexTerms = new ArrayList<>(); - indexTerms.add(new BytesRef("foo")); - indexTerms.add(new BytesRef("bar")); - List docValueterms = new ArrayList<>(); - docValueterms.add(new BytesRef("field.foo")); - docValueterms.add(new BytesRef("field.bar")); - Query expected = new TermInSetQuery(DOC_VALUES_REWRITE, "field" + VALUE_SUFFIX, docValueterms); - - assertEquals(expected, ft.termsQuery(Arrays.asList("foo", "bar"), MOCK_QSC_ENABLE_INDEX_DOC_VALUES)); - } - - // test isSearchable=false, hasDocValues=true, mappedFieldTypeName!=null - { - FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( - "field", - "field", - false, - true - ); - - List indexTerms = new ArrayList<>(); - indexTerms.add(new BytesRef("foo")); - indexTerms.add(new BytesRef("bar")); - List docValueterms = new ArrayList<>(); - docValueterms.add(new BytesRef("field.foo")); - docValueterms.add(new BytesRef("field.bar")); - Query expected = new TermInSetQuery(DOC_VALUES_REWRITE, "field" + VALUE_AND_PATH_SUFFIX, docValueterms); - - assertEquals(expected, ft.termsQuery(Arrays.asList("foo", "bar"), MOCK_QSC_ENABLE_INDEX_DOC_VALUES)); - } - - // 4.test isSearchable=false, hasDocValues=false, mappedFieldTypeName=null - { - FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( - "field", - null, - false, - false - ); - IllegalArgumentException e = expectThrows( - IllegalArgumentException.class, - () -> ft.termsQuery(Arrays.asList("foo", "bar"), MOCK_QSC_ENABLE_INDEX_DOC_VALUES) - ); - assertEquals( - "Cannot search on field [field._value] since it is both not indexed, and does not have doc_values " + "enabled.", - e.getMessage() - ); - } - - // test isSearchable=false, hasDocValues=false, mappedFieldTypeName!=null - { - FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( - "field", - "field", - false, - false - ); - IllegalArgumentException e = expectThrows( - IllegalArgumentException.class, - () -> ft.termsQuery(Arrays.asList("foo", "bar"), MOCK_QSC_ENABLE_INDEX_DOC_VALUES) - ); - assertEquals( - "Cannot search on field [field._valueAndPath] since it is both not indexed, and does not have doc_values " + "enabled.", - e.getMessage() - ); - } - } - - public void testPrefixQuery() { - // 1.test isSearchable=true, hasDocValues=true, mappedFieldTypeName=null - { - FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( - "field", - null, - true, - true - ); - Query expected = new IndexOrDocValuesQuery( - new PrefixQuery(new Term("field" + VALUE_SUFFIX, "foo"), CONSTANT_SCORE_REWRITE), - new PrefixQuery(new Term("field" + VALUE_SUFFIX, "field.foo"), DOC_VALUES_REWRITE) - ); - assertEquals(expected, ft.prefixQuery("foo", CONSTANT_SCORE_REWRITE, false, MOCK_QSC_ENABLE_INDEX_DOC_VALUES)); - } - - // test isSearchable=true, hasDocValues=true, mappedFieldTypeName!=null - { - FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( - "field", - "field", - true, - true - ); - Query expected = new IndexOrDocValuesQuery( - new PrefixQuery(new Term("field" + VALUE_AND_PATH_SUFFIX, "foo"), CONSTANT_SCORE_REWRITE), - new PrefixQuery(new Term("field" + VALUE_AND_PATH_SUFFIX, "field.foo"), DOC_VALUES_REWRITE) - ); - assertEquals(expected, ft.prefixQuery("foo", CONSTANT_SCORE_REWRITE, false, MOCK_QSC_ENABLE_INDEX_DOC_VALUES)); - } - - // 2.test isSearchable=true, hasDocValues=false, mappedFieldTypeName=null - { - FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( - "field", - null, - true, - false - ); - Query expected = new PrefixQuery(new Term("field" + VALUE_SUFFIX, "foo"), CONSTANT_SCORE_REWRITE); - assertEquals(expected, ft.prefixQuery("foo", CONSTANT_SCORE_REWRITE, false, MOCK_QSC_ENABLE_INDEX_DOC_VALUES)); - } - - // test isSearchable=true, hasDocValues=false, mappedFieldTypeName!=null - { - FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( - "field", - "field", - true, - false - ); - Query expected = new PrefixQuery(new Term("field" + VALUE_AND_PATH_SUFFIX, "foo"), CONSTANT_SCORE_REWRITE); - assertEquals(expected, ft.prefixQuery("foo", CONSTANT_SCORE_REWRITE, false, MOCK_QSC_ENABLE_INDEX_DOC_VALUES)); - } - - // 3.test isSearchable=false, hasDocValues=true, mappedFieldTypeName=null - { - FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( - "field", - null, - false, - true - ); - Query expected = new PrefixQuery(new Term("field" + VALUE_SUFFIX, "field.foo"), DOC_VALUES_REWRITE); - assertEquals(expected, ft.prefixQuery("foo", CONSTANT_SCORE_REWRITE, false, MOCK_QSC_ENABLE_INDEX_DOC_VALUES)); - } - - // test isSearchable=false, hasDocValues=true, mappedFieldTypeName!=null - { - FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( - "field", - "field", - false, - true - ); - Query expected = new PrefixQuery(new Term("field" + VALUE_AND_PATH_SUFFIX, "field.foo"), DOC_VALUES_REWRITE); - assertEquals(expected, ft.prefixQuery("foo", CONSTANT_SCORE_REWRITE, false, MOCK_QSC_ENABLE_INDEX_DOC_VALUES)); - } - - // 4.test isSearchable=false, hasDocValues=false, mappedFieldTypeName=null - { - FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( - "field", - null, - false, - false - ); - IllegalArgumentException e = expectThrows( - IllegalArgumentException.class, - () -> ft.prefixQuery("foo", CONSTANT_SCORE_REWRITE, false, MOCK_QSC_ENABLE_INDEX_DOC_VALUES) - ); - assertEquals( - "Cannot search on field [field._value] since it is both not indexed, and does not have doc_values " + "enabled.", - e.getMessage() - ); - } - - // test isSearchable=false, hasDocValues=false, mappedFieldTypeName!=null - { - FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( - "field", - "field", - false, - false - ); - IllegalArgumentException e = expectThrows( - IllegalArgumentException.class, - () -> ft.prefixQuery("foo", CONSTANT_SCORE_REWRITE, false, MOCK_QSC_ENABLE_INDEX_DOC_VALUES) - ); - assertEquals( - "Cannot search on field [field._valueAndPath] since it is both not indexed, and does not have doc_values " + "enabled.", - e.getMessage() - ); - } - } - - public void testRegexpQuery() { - // 1.test isSearchable=true, hasDocValues=true, mappedFieldTypeName=null - { - FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( - "field", - null, - true, - true - ); - Query expected = new IndexOrDocValuesQuery( - new RegexpQuery( - new Term("field" + VALUE_SUFFIX, new BytesRef("foo")), - 0, - 0, - RegexpQuery.DEFAULT_PROVIDER, - 10, - CONSTANT_SCORE_REWRITE - ), - new RegexpQuery( - new Term("field" + VALUE_SUFFIX, new BytesRef("field.foo")), - 0, - 0, - RegexpQuery.DEFAULT_PROVIDER, - 10, - DOC_VALUES_REWRITE - ) - ); - assertEquals(expected, ft.regexpQuery("foo", 0, 0, 10, null, MOCK_QSC_ENABLE_INDEX_DOC_VALUES)); - } - - // test isSearchable=true, hasDocValues=true, mappedFieldTypeName!=null - { - FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( - "field", - "field", - true, - true - ); - Query expected = new IndexOrDocValuesQuery( - new RegexpQuery( - new Term("field" + VALUE_AND_PATH_SUFFIX, new BytesRef("foo")), - 0, - 0, - RegexpQuery.DEFAULT_PROVIDER, - 10, - CONSTANT_SCORE_REWRITE - ), - new RegexpQuery( - new Term("field" + VALUE_AND_PATH_SUFFIX, new BytesRef("field.foo")), - 0, - 0, - RegexpQuery.DEFAULT_PROVIDER, - 10, - DOC_VALUES_REWRITE - ) - ); - assertEquals(expected, ft.regexpQuery("foo", 0, 0, 10, null, MOCK_QSC_ENABLE_INDEX_DOC_VALUES)); - } - - // 2.test isSearchable=true, hasDocValues=false, mappedFieldTypeName=null - { - FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( - "field", - null, - true, - false - ); - Query expected = new RegexpQuery( - new Term("field" + VALUE_SUFFIX, new BytesRef("foo")), - 0, - 0, - RegexpQuery.DEFAULT_PROVIDER, - 10, - CONSTANT_SCORE_REWRITE - ); - assertEquals(expected, ft.regexpQuery("foo", 0, 0, 10, null, MOCK_QSC_ENABLE_INDEX_DOC_VALUES)); - } - - // test isSearchable=true, hasDocValues=false, mappedFieldTypeName!=null - { - FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( - "field", - "field", - true, - false - ); - Query expected = new RegexpQuery( - new Term("field" + VALUE_AND_PATH_SUFFIX, new BytesRef("foo")), - 0, - 0, - RegexpQuery.DEFAULT_PROVIDER, - 10, - CONSTANT_SCORE_REWRITE - ); - assertEquals(expected, ft.regexpQuery("foo", 0, 0, 10, null, MOCK_QSC_ENABLE_INDEX_DOC_VALUES)); - } - - // 3.test isSearchable=false, hasDocValues=true, mappedFieldTypeName=null - { - FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( - "field", - null, - false, - true - ); - Query expected = new RegexpQuery( - new Term("field" + VALUE_SUFFIX, new BytesRef("field.foo")), - 0, - 0, - RegexpQuery.DEFAULT_PROVIDER, - 10, - DOC_VALUES_REWRITE - ); - assertEquals(expected, ft.regexpQuery("foo", 0, 0, 10, null, MOCK_QSC_ENABLE_INDEX_DOC_VALUES)); - } - - // test isSearchable=false, hasDocValues=true, mappedFieldTypeName!=null - { - FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( - "field", - "field", - false, - true - ); - Query expected = new RegexpQuery( - new Term("field" + VALUE_AND_PATH_SUFFIX, new BytesRef("field.foo")), - 0, - 0, - RegexpQuery.DEFAULT_PROVIDER, - 10, - DOC_VALUES_REWRITE - ); - assertEquals(expected, ft.regexpQuery("foo", 0, 0, 10, null, MOCK_QSC_ENABLE_INDEX_DOC_VALUES)); - } - - // 4.test isSearchable=false, hasDocValues=false, mappedFieldTypeName=null - { - FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( - "field", - null, - false, - false - ); - IllegalArgumentException e = expectThrows( - IllegalArgumentException.class, - () -> ft.regexpQuery("foo", 0, 0, 10, null, MOCK_QSC_ENABLE_INDEX_DOC_VALUES) - ); - assertEquals( - "Cannot search on field [field._value] since it is both not indexed, and does not have doc_values " + "enabled.", - e.getMessage() - ); - } - - // test isSearchable=false, hasDocValues=false, mappedFieldTypeName!=null - { - FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( - "field", - "field", - false, - false - ); - IllegalArgumentException e = expectThrows( - IllegalArgumentException.class, - () -> ft.regexpQuery("foo", 0, 0, 10, null, MOCK_QSC_ENABLE_INDEX_DOC_VALUES) - ); - assertEquals( - "Cannot search on field [field._valueAndPath] since it is both not indexed, and does not have doc_values " + "enabled.", - e.getMessage() - ); - } - } - - public void testFuzzyQuery() { - // 1.test isSearchable=true, hasDocValues=true, mappedFieldTypeName=null - { - FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( - "field", - null, - true, - true - ); - Query expected = new IndexOrDocValuesQuery( - new FuzzyQuery(new Term("field" + VALUE_SUFFIX, "foo"), 2, 1, 50, true), - new FuzzyQuery(new Term("field" + VALUE_SUFFIX, "field.foo"), 2, 1, 50, true, MultiTermQuery.DOC_VALUES_REWRITE) - ); - assertEquals(expected, ft.fuzzyQuery("foo", Fuzziness.fromEdits(2), 1, 50, true, null, MOCK_QSC_ENABLE_INDEX_DOC_VALUES)); - } - - // test isSearchable=true, hasDocValues=true, mappedFieldTypeName!=null - { - FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( - "field", - "field", - true, - true - ); - Query expected = new IndexOrDocValuesQuery( - new FuzzyQuery(new Term("field" + VALUE_AND_PATH_SUFFIX, "foo"), 2, 1, 50, true), - new FuzzyQuery(new Term("field" + VALUE_AND_PATH_SUFFIX, "field.foo"), 2, 1, 50, true, MultiTermQuery.DOC_VALUES_REWRITE) - ); - assertEquals(expected, ft.fuzzyQuery("foo", Fuzziness.fromEdits(2), 1, 50, true, null, MOCK_QSC_ENABLE_INDEX_DOC_VALUES)); - } - - // 2.test isSearchable=true, hasDocValues=false, mappedFieldTypeName=null - { - FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( - "field", - null, - true, - false - ); - Query expected = new FuzzyQuery(new Term("field" + VALUE_SUFFIX, "foo"), 2, 1, 50, true); - assertEquals(expected, ft.fuzzyQuery("foo", Fuzziness.fromEdits(2), 1, 50, true, null, MOCK_QSC_ENABLE_INDEX_DOC_VALUES)); - } - - // test isSearchable=true, hasDocValues=false, mappedFieldTypeName!=null - { - FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( - "field", - "field", - true, - false - ); - Query expected = new FuzzyQuery(new Term("field" + VALUE_AND_PATH_SUFFIX, "foo"), 2, 1, 50, true); - assertEquals(expected, ft.fuzzyQuery("foo", Fuzziness.fromEdits(2), 1, 50, true, null, MOCK_QSC_ENABLE_INDEX_DOC_VALUES)); - } - - // 3.test isSearchable=false, hasDocValues=true, mappedFieldTypeName=null - { - FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( - "field", - null, - false, - true - ); - Query expected = new FuzzyQuery( - new Term("field" + VALUE_SUFFIX, "field.foo"), - 2, - 1, - 50, - true, - MultiTermQuery.DOC_VALUES_REWRITE - ); - assertEquals(expected, ft.fuzzyQuery("foo", Fuzziness.fromEdits(2), 1, 50, true, null, MOCK_QSC_ENABLE_INDEX_DOC_VALUES)); - } - - // test isSearchable=false, hasDocValues=true, mappedFieldTypeName!=null - { - FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( - "field", - "field", - false, - true - ); - Query expected = new FuzzyQuery( - new Term("field" + VALUE_AND_PATH_SUFFIX, "field.foo"), - 2, - 1, - 50, - true, - MultiTermQuery.DOC_VALUES_REWRITE - ); - assertEquals(expected, ft.fuzzyQuery("foo", Fuzziness.fromEdits(2), 1, 50, true, null, MOCK_QSC_ENABLE_INDEX_DOC_VALUES)); - } - - // 4.test isSearchable=false, hasDocValues=false, mappedFieldTypeName=null - { - FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( - "field", - null, - false, - false - ); - IllegalArgumentException e = expectThrows( - IllegalArgumentException.class, - () -> ft.fuzzyQuery("foo", Fuzziness.fromEdits(2), 1, 50, true, null, MOCK_QSC_ENABLE_INDEX_DOC_VALUES) - ); - assertEquals( - "Cannot search on field [field._value] since it is both not indexed, and does not have doc_values " + "enabled.", - e.getMessage() - ); - } - - // test isSearchable=false, hasDocValues=false, mappedFieldTypeName!=null - { - FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( - "field", - "field", - false, - false - ); - IllegalArgumentException e = expectThrows( - IllegalArgumentException.class, - () -> ft.fuzzyQuery("foo", Fuzziness.fromEdits(2), 1, 50, true, null, MOCK_QSC_ENABLE_INDEX_DOC_VALUES) - ); - assertEquals( - "Cannot search on field [field._valueAndPath] since it is both not indexed, and does not have doc_values " + "enabled.", - e.getMessage() - ); - } - } - - public void testRangeQuery() { - // 1.test isSearchable=true, hasDocValues=true, mappedFieldTypeName=null - { - FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( - "field", - null, - true, - true - ); - Query expected = new IndexOrDocValuesQuery( - new TermRangeQuery("field" + VALUE_SUFFIX, new BytesRef("2"), new BytesRef("10"), true, true), - new TermRangeQuery( - "field" + VALUE_SUFFIX, - new BytesRef("field.2"), - new BytesRef("field.10"), - true, - true, - DOC_VALUES_REWRITE - ) - ); - assertEquals(expected, ft.rangeQuery(new BytesRef("2"), new BytesRef("10"), true, true, MOCK_QSC_ENABLE_INDEX_DOC_VALUES)); - } - - // test isSearchable=true, hasDocValues=true, mappedFieldTypeName!=null - { - FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( - "field", - "field", - true, - true - ); - Query expected = new IndexOrDocValuesQuery( - new TermRangeQuery("field" + VALUE_AND_PATH_SUFFIX, new BytesRef("2"), new BytesRef("10"), true, true), - new TermRangeQuery( - "field" + VALUE_AND_PATH_SUFFIX, - new BytesRef("field.2"), - new BytesRef("field.10"), - true, - true, - DOC_VALUES_REWRITE - ) - ); - assertEquals(expected, ft.rangeQuery(new BytesRef("2"), new BytesRef("10"), true, true, MOCK_QSC_ENABLE_INDEX_DOC_VALUES)); - } - - // 2.test isSearchable=true, hasDocValues=false, mappedFieldTypeName=null - { - FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( - "field", - null, - true, - false - ); - Query expected = new TermRangeQuery("field" + VALUE_SUFFIX, new BytesRef("2"), new BytesRef("10"), true, true); - assertEquals(expected, ft.rangeQuery(new BytesRef("2"), new BytesRef("10"), true, true, MOCK_QSC_ENABLE_INDEX_DOC_VALUES)); - } - - // test isSearchable=true, hasDocValues=false, mappedFieldTypeName!=null - { - FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( - "field", - "field", - true, - false - ); - Query expected = new TermRangeQuery("field" + VALUE_AND_PATH_SUFFIX, new BytesRef("2"), new BytesRef("10"), true, true); - assertEquals(expected, ft.rangeQuery(new BytesRef("2"), new BytesRef("10"), true, true, MOCK_QSC_ENABLE_INDEX_DOC_VALUES)); - } - - // 3.test isSearchable=false, hasDocValues=true, mappedFieldTypeName=null - { - FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( - "field", - null, - false, - true - ); - Query expected = new TermRangeQuery( - "field" + VALUE_SUFFIX, - new BytesRef("field.2"), - new BytesRef("field.10"), - true, - true, - DOC_VALUES_REWRITE - ); - assertEquals(expected, ft.rangeQuery(new BytesRef("2"), new BytesRef("10"), true, true, MOCK_QSC_ENABLE_INDEX_DOC_VALUES)); - } - - // test isSearchable=false, hasDocValues=true, mappedFieldTypeName!=null - { - FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( - "field", - "field", - false, - true - ); - Query expected = new TermRangeQuery( - "field" + VALUE_AND_PATH_SUFFIX, - new BytesRef("field.2"), - new BytesRef("field.10"), - true, - true, - DOC_VALUES_REWRITE - ); - assertEquals(expected, ft.rangeQuery(new BytesRef("2"), new BytesRef("10"), true, true, MOCK_QSC_ENABLE_INDEX_DOC_VALUES)); - } - - // 4.test isSearchable=false, hasDocValues=false, mappedFieldTypeName=null - { - FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( - "field", - null, - false, - false - ); - IllegalArgumentException e = expectThrows( - IllegalArgumentException.class, - () -> ft.rangeQuery(new BytesRef("2"), new BytesRef("10"), true, true, MOCK_QSC_ENABLE_INDEX_DOC_VALUES) - ); - assertEquals( - "Cannot search on field [field._value] since it is both not indexed, and does not have doc_values " + "enabled.", - e.getMessage() - ); - } - - // test isSearchable=false, hasDocValues=false, mappedFieldTypeName!=null - { - FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( - "field", - "field", - false, - false - ); - IllegalArgumentException e = expectThrows( - IllegalArgumentException.class, - () -> ft.rangeQuery(new BytesRef("2"), new BytesRef("10"), true, true, MOCK_QSC_ENABLE_INDEX_DOC_VALUES) - ); - assertEquals( - "Cannot search on field [field._valueAndPath] since it is both not indexed, and does not have doc_values " + "enabled.", - e.getMessage() - ); - } - } - - public void testWildcardQuery() { - // 1.test isSearchable=true, hasDocValues=true, mappedFieldTypeName=null - { - FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( - "field", - null, - true, - true - ); - Query expected = new IndexOrDocValuesQuery( - new WildcardQuery( - new Term("field" + VALUE_SUFFIX, "foo*"), - Operations.DEFAULT_DETERMINIZE_WORK_LIMIT, - MultiTermQuery.CONSTANT_SCORE_REWRITE - ), - new WildcardQuery( - new Term("field" + VALUE_SUFFIX, "field.foo*"), - Operations.DEFAULT_DETERMINIZE_WORK_LIMIT, - MultiTermQuery.DOC_VALUES_REWRITE - ) - ); - assertEquals(expected, ft.wildcardQuery("foo*", null, false, MOCK_QSC_ENABLE_INDEX_DOC_VALUES)); - } - - // test isSearchable=true, hasDocValues=true, mappedFieldTypeName!=null - { - FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( - "field", - "field", - true, - true - ); - Query expected = new IndexOrDocValuesQuery( - new WildcardQuery( - new Term("field" + VALUE_AND_PATH_SUFFIX, "foo*"), - Operations.DEFAULT_DETERMINIZE_WORK_LIMIT, - MultiTermQuery.CONSTANT_SCORE_REWRITE - ), - new WildcardQuery( - new Term("field" + VALUE_AND_PATH_SUFFIX, "field.foo*"), - Operations.DEFAULT_DETERMINIZE_WORK_LIMIT, - MultiTermQuery.DOC_VALUES_REWRITE - ) - ); - assertEquals(expected, ft.wildcardQuery("foo*", null, false, MOCK_QSC_ENABLE_INDEX_DOC_VALUES)); - } - - // 2.test isSearchable=true, hasDocValues=false, mappedFieldTypeName=null - { - FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( - "field", - null, - true, - false - ); - Query expected = new WildcardQuery( - new Term("field" + VALUE_SUFFIX, "foo*"), - Operations.DEFAULT_DETERMINIZE_WORK_LIMIT, - MultiTermQuery.CONSTANT_SCORE_REWRITE - ); - assertEquals(expected, ft.wildcardQuery("foo*", null, false, MOCK_QSC_ENABLE_INDEX_DOC_VALUES)); - } - - // test isSearchable=true, hasDocValues=false, mappedFieldTypeName!=null - { - FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( - "field", - "field", - true, - false - ); - Query expected = new WildcardQuery( - new Term("field" + VALUE_AND_PATH_SUFFIX, "foo*"), - Operations.DEFAULT_DETERMINIZE_WORK_LIMIT, - MultiTermQuery.CONSTANT_SCORE_REWRITE - ); - assertEquals(expected, ft.wildcardQuery("foo*", null, false, MOCK_QSC_ENABLE_INDEX_DOC_VALUES)); - } - - // 3.test isSearchable=false, hasDocValues=true, mappedFieldTypeName=null - { - FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( - "field", - null, - false, - true - ); - Query expected = new WildcardQuery( - new Term("field" + VALUE_SUFFIX, "field.foo*"), - Operations.DEFAULT_DETERMINIZE_WORK_LIMIT, - MultiTermQuery.DOC_VALUES_REWRITE - ); - assertEquals(expected, ft.wildcardQuery("foo*", null, false, MOCK_QSC_ENABLE_INDEX_DOC_VALUES)); - } - - // test isSearchable=false, hasDocValues=true, mappedFieldTypeName!=null - { - FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( - "field", - "field", - false, - true - ); - Query expected = new WildcardQuery( - new Term("field" + VALUE_AND_PATH_SUFFIX, "field.foo*"), - Operations.DEFAULT_DETERMINIZE_WORK_LIMIT, - MultiTermQuery.DOC_VALUES_REWRITE - ); - assertEquals(expected, ft.wildcardQuery("foo*", null, false, MOCK_QSC_ENABLE_INDEX_DOC_VALUES)); - } - - // 4.test isSearchable=false, hasDocValues=false, mappedFieldTypeName=null - { - FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( - "field", - null, - false, - false - ); - IllegalArgumentException e = expectThrows( - IllegalArgumentException.class, - () -> ft.wildcardQuery("foo*", null, false, MOCK_QSC_ENABLE_INDEX_DOC_VALUES) - ); - assertEquals( - "Cannot search on field [field._value] since it is both not indexed, and does not have doc_values " + "enabled.", - e.getMessage() - ); - } - - // test isSearchable=false, hasDocValues=false, mappedFieldTypeName!=null - { - FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( - "field", - "field", - false, - false - ); - IllegalArgumentException e = expectThrows( - IllegalArgumentException.class, - () -> ft.wildcardQuery("foo*", null, false, MOCK_QSC_ENABLE_INDEX_DOC_VALUES) - ); - assertEquals( - "Cannot search on field [field._valueAndPath] since it is both not indexed, and does not have doc_values " + "enabled.", - e.getMessage() - ); + assertEquals(new TermQuery(new Term(FieldNamesFieldMapper.NAME, "field")), ft.existsQuery(null)); } } }