From 70d010098b496f0044ed383afd553f611b30fb40 Mon Sep 17 00:00:00 2001 From: Norman Jordan Date: Fri, 8 Nov 2024 13:25:16 -0800 Subject: [PATCH] Add a multivalued property to field mappings (#16420) * Can only be used for field types that support multiple values * If a field has the multivalued property, then new documents must have an array for its value Signed-off-by: Norman Jordan --- .../index/mapper/ScaledFloatFieldMapper.java | 17 ++++++- .../mapper/SearchAsYouTypeFieldMapper.java | 20 ++++++++ .../mapper/ScaledFloatFieldMapperTests.java | 32 +++++++++++++ .../SearchAsYouTypeFieldMapperTests.java | 33 +++++++++++++ .../mapper/AbstractGeometryFieldMapper.java | 39 ++++++++++++++++ .../AbstractPointGeometryFieldMapper.java | 5 +- .../AbstractShapeGeometryFieldMapper.java | 3 +- .../index/mapper/BinaryFieldMapper.java | 20 +++++++- .../index/mapper/BooleanFieldMapper.java | 20 +++++++- .../index/mapper/CompletionFieldMapper.java | 20 +++++++- .../index/mapper/DateFieldMapper.java | 30 +++++++++++- .../index/mapper/DocumentParser.java | 35 +++++++++++--- .../index/mapper/GeoPointFieldMapper.java | 10 +++- .../index/mapper/GeoShapeFieldMapper.java | 8 +++- .../index/mapper/IpFieldMapper.java | 18 +++++++- .../index/mapper/KeywordFieldMapper.java | 21 +++++++++ .../mapper/LegacyGeoShapeFieldMapper.java | 8 +++- .../org/opensearch/index/mapper/Mapper.java | 7 +++ .../mapper/MatchOnlyTextFieldMapper.java | 1 + .../index/mapper/NumberFieldMapper.java | 17 ++++++- .../index/mapper/RangeFieldMapper.java | 11 ++++- .../index/mapper/TextFieldMapper.java | 23 +++++++++- .../index/mapper/WildcardFieldMapper.java | 20 +++++++- .../index/mapper/BinaryFieldMapperTests.java | 37 +++++++++++++++ .../index/mapper/BooleanFieldMapperTests.java | 35 ++++++++++++++ .../mapper/CompletionFieldMapperTests.java | 33 +++++++++++++ .../index/mapper/DateFieldMapperTests.java | 46 +++++++++++++++++++ .../mapper/GeoPointFieldMapperTests.java | 31 +++++++++++++ .../mapper/GeoShapeFieldMapperTests.java | 25 ++++++++++ .../index/mapper/IpFieldMapperTests.java | 36 +++++++++++++++ .../index/mapper/KeywordFieldMapperTests.java | 35 ++++++++++++++ .../LegacyGeoShapeFieldMapperTests.java | 23 ++++++++++ .../index/mapper/NumberFieldMapperTests.java | 32 +++++++++++++ .../index/mapper/RangeFieldMapperTests.java | 32 +++++++++++++ .../index/mapper/TextFieldMapperTests.java | 36 +++++++++++++++ .../mapper/WildcardFieldMapperTests.java | 36 +++++++++++++++ .../AbstractNumericFieldMapperTestCase.java | 8 ++++ 37 files changed, 840 insertions(+), 23 deletions(-) diff --git a/modules/mapper-extras/src/main/java/org/opensearch/index/mapper/ScaledFloatFieldMapper.java b/modules/mapper-extras/src/main/java/org/opensearch/index/mapper/ScaledFloatFieldMapper.java index b46b58f415cfd..163b57e731e70 100644 --- a/modules/mapper-extras/src/main/java/org/opensearch/index/mapper/ScaledFloatFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/opensearch/index/mapper/ScaledFloatFieldMapper.java @@ -120,6 +120,8 @@ public static class Builder extends ParametrizedFieldMapper.Builder { m -> toType(m).nullValue ).acceptsNull(); + private final Parameter> multivalued; + private final Parameter> meta = Parameter.metaParam(); public Builder(String name, Settings settings) { @@ -135,6 +137,7 @@ public Builder(String name, boolean ignoreMalformedByDefault, boolean coerceByDe ignoreMalformedByDefault ); this.coerce = Parameter.explicitBoolParam("coerce", true, m -> toType(m).coerce, coerceByDefault); + this.multivalued = Parameter.explicitBoolParam("multivalued", true, m -> toType(m).multivalued, false); } Builder scalingFactor(double scalingFactor) { @@ -149,7 +152,7 @@ Builder nullValue(double nullValue) { @Override protected List> getParameters() { - return Arrays.asList(indexed, hasDocValues, stored, ignoreMalformed, meta, scalingFactor, coerce, nullValue); + return Arrays.asList(indexed, hasDocValues, stored, ignoreMalformed, meta, scalingFactor, coerce, nullValue, multivalued); } @Override @@ -372,6 +375,8 @@ public double toDoubleValue(long value) { private final boolean ignoreMalformedByDefault; private final boolean coerceByDefault; + private final Explicit multivalued; + private ScaledFloatFieldMapper( String simpleName, ScaledFloatFieldType mappedFieldType, @@ -389,6 +394,7 @@ private ScaledFloatFieldMapper( this.coerce = builder.coerce.getValue(); this.ignoreMalformedByDefault = builder.ignoreMalformed.getDefaultValue().value(); this.coerceByDefault = builder.coerce.getDefaultValue().value(); + this.multivalued = builder.multivalued.getValue(); } boolean coerce() { @@ -399,6 +405,10 @@ boolean ignoreMalformed() { return ignoreMalformed.value(); } + boolean multivalued() { + return multivalued.value(); + } + @Override public ScaledFloatFieldType fieldType() { return (ScaledFloatFieldType) super.fieldType(); @@ -414,6 +424,11 @@ public ParametrizedFieldMapper.Builder getMergeBuilder() { return new Builder(simpleName(), ignoreMalformedByDefault, coerceByDefault).init(this); } + @Override + public boolean isMultivalue() { + return multivalued.explicit() && multivalued.value() != null && multivalued.value(); + } + @Override protected ScaledFloatFieldMapper clone() { return (ScaledFloatFieldMapper) super.clone(); diff --git a/modules/mapper-extras/src/main/java/org/opensearch/index/mapper/SearchAsYouTypeFieldMapper.java b/modules/mapper-extras/src/main/java/org/opensearch/index/mapper/SearchAsYouTypeFieldMapper.java index f08815ebbbd1e..30f1b52c8f083 100644 --- a/modules/mapper-extras/src/main/java/org/opensearch/index/mapper/SearchAsYouTypeFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/opensearch/index/mapper/SearchAsYouTypeFieldMapper.java @@ -59,6 +59,7 @@ import org.apache.lucene.util.automaton.Automata; import org.apache.lucene.util.automaton.Automaton; import org.apache.lucene.util.automaton.Operations; +import org.opensearch.common.Explicit; import org.opensearch.common.collect.Iterators; import org.opensearch.common.lucene.search.AutomatonQueries; import org.opensearch.index.analysis.AnalyzerScope; @@ -156,6 +157,12 @@ public static class Builder extends ParametrizedFieldMapper.Builder { final Parameter indexOptions = TextParams.indexOptions(m -> toType(m).indexOptions); final Parameter norms = TextParams.norms(true, m -> ft(m).getTextSearchInfo().hasNorms()); final Parameter termVectors = TextParams.termVectors(m -> toType(m).termVectors); + final Parameter> multivalued = Parameter.explicitBoolParam( + "multivalued", + true, + m -> toType(m).multivalued, + false + ); private final Parameter> meta = Parameter.metaParam(); @@ -178,6 +185,7 @@ protected List> getParameters() { indexOptions, norms, termVectors, + multivalued, meta ); } @@ -628,6 +636,8 @@ public SpanQuery spanPrefixQuery(String value, SpanMultiTermQueryWrapper.SpanRew private final IndexAnalyzers indexAnalyzers; + private final Explicit multivalued; + public SearchAsYouTypeFieldMapper( String simpleName, SearchAsYouTypeFieldType mappedFieldType, @@ -646,6 +656,7 @@ public SearchAsYouTypeFieldMapper( this.indexOptions = builder.indexOptions.getValue(); this.termVectors = builder.termVectors.getValue(); this.indexAnalyzers = builder.analyzers.indexAnalyzers; + this.multivalued = builder.multivalued.getValue(); } @Override @@ -684,6 +695,15 @@ public SearchAsYouTypeFieldType fieldType() { return (SearchAsYouTypeFieldType) super.fieldType(); } + boolean multivalued() { + return multivalued.value(); + } + + @Override + public boolean isMultivalue() { + return multivalued.explicit() && multivalued.value() != null && multivalued.value(); + } + public int maxShingleSize() { return maxShingleSize; } diff --git a/modules/mapper-extras/src/test/java/org/opensearch/index/mapper/ScaledFloatFieldMapperTests.java b/modules/mapper-extras/src/test/java/org/opensearch/index/mapper/ScaledFloatFieldMapperTests.java index c3d62b088ced7..9de1665af0438 100644 --- a/modules/mapper-extras/src/test/java/org/opensearch/index/mapper/ScaledFloatFieldMapperTests.java +++ b/modules/mapper-extras/src/test/java/org/opensearch/index/mapper/ScaledFloatFieldMapperTests.java @@ -86,6 +86,7 @@ protected void registerParameters(ParameterChecker checker) throws IOException { b -> b.field("ignore_malformed", true), m -> assertTrue(((ScaledFloatFieldMapper) m).ignoreMalformed()) ); + checker.registerUpdateCheck(b -> b.field("multivalued", true), m -> assertTrue(((ScaledFloatFieldMapper) m).multivalued())); } public void testExistsQueryDocValuesDisabled() throws IOException { @@ -359,6 +360,37 @@ private void doTestIgnoreMalformed(Object value, String exceptionMessageContains assertEquals(0, fields.length); } + public void testMultivalued() throws Exception { + DocumentMapper mapper = createDocumentMapper( + fieldMapping(b -> b.field("type", "scaled_float").field("scaling_factor", 10.0).field("multivalued", true)) + ); + ThrowingRunnable runnable = () -> mapper.parse( + new SourceToParse( + "test", + "1", + BytesReference.bytes(XContentFactory.jsonBuilder().startObject().field("field", "1.34").endObject()), + MediaTypeRegistry.JSON + ) + ); + MapperParsingException e = expectThrows(MapperParsingException.class, runnable); + assertThat( + e.getMessage(), + containsString("object mapping [field] trying to serialize a scalar value [1.34] for a multi-valued field") + ); + + ParsedDocument doc = mapper.parse( + new SourceToParse( + "test", + "1", + BytesReference.bytes(XContentFactory.jsonBuilder().startObject().field("field", List.of("1.34", "2.35")).endObject()), + MediaTypeRegistry.JSON + ) + ); + + IndexableField[] fields = doc.rootDoc().getFields("field"); + assertEquals(4, fields.length); + } + public void testNullValue() throws IOException { DocumentMapper mapper = createDocumentMapper(fieldMapping(this::minimalMapping)); ParsedDocument doc = mapper.parse( diff --git a/modules/mapper-extras/src/test/java/org/opensearch/index/mapper/SearchAsYouTypeFieldMapperTests.java b/modules/mapper-extras/src/test/java/org/opensearch/index/mapper/SearchAsYouTypeFieldMapperTests.java index 7746cb714a019..47339be3c1b6b 100644 --- a/modules/mapper-extras/src/test/java/org/opensearch/index/mapper/SearchAsYouTypeFieldMapperTests.java +++ b/modules/mapper-extras/src/test/java/org/opensearch/index/mapper/SearchAsYouTypeFieldMapperTests.java @@ -52,7 +52,9 @@ import org.apache.lucene.search.SynonymQuery; import org.apache.lucene.search.TermQuery; import org.opensearch.common.lucene.search.MultiPhrasePrefixQuery; +import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.common.Strings; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.index.IndexSettings; @@ -129,6 +131,8 @@ protected void registerParameters(ParameterChecker checker) throws IOException { b.field("search_quote_analyzer", "keyword"); }, m -> assertEquals("keyword", m.fieldType().getTextSearchInfo().getSearchQuoteAnalyzer().name())); + checker.registerUpdateCheck(b -> b.field("multivalued", true), m -> assertTrue(((SearchAsYouTypeFieldMapper) m).multivalued())); + } protected void writeFieldValue(XContentBuilder builder) throws IOException { @@ -636,6 +640,35 @@ public void testMultiMatchBoolPrefix() throws IOException { ); } + public void testMultivalued() throws Exception { + DocumentMapper mapper = createDocumentMapper(fieldMapping(b -> b.field("type", "search_as_you_type").field("multivalued", true))); + ThrowingRunnable runnable = () -> mapper.parse( + new SourceToParse( + "test", + "1", + BytesReference.bytes(XContentFactory.jsonBuilder().startObject().field("field", "foo").endObject()), + MediaTypeRegistry.JSON + ) + ); + MapperParsingException e = expectThrows(MapperParsingException.class, runnable); + assertThat( + e.getMessage(), + containsString("object mapping [field] trying to serialize a scalar value [foo] for a multi-valued field") + ); + + ParsedDocument doc = mapper.parse( + new SourceToParse( + "test", + "1", + BytesReference.bytes(XContentFactory.jsonBuilder().startObject().field("field", List.of("foo", "bar")).endObject()), + MediaTypeRegistry.JSON + ) + ); + + IndexableField[] fields = doc.rootDoc().getFields("field"); + assertEquals(2, fields.length); + } + public void testAnalyzerSerialization() throws IOException { MapperService ms = createMapperService(fieldMapping(b -> { b.field("type", "search_as_you_type"); diff --git a/server/src/main/java/org/opensearch/index/mapper/AbstractGeometryFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/AbstractGeometryFieldMapper.java index 3b6782b34feea..734beec91d8dd 100644 --- a/server/src/main/java/org/opensearch/index/mapper/AbstractGeometryFieldMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/AbstractGeometryFieldMapper.java @@ -75,6 +75,7 @@ public abstract class AbstractGeometryFieldMapper extends Fie public static class Names { public static final ParseField IGNORE_MALFORMED = new ParseField("ignore_malformed"); public static final ParseField IGNORE_Z_VALUE = new ParseField("ignore_z_value"); + public static final ParseField MULTIVALUED = new ParseField("multivalued"); } /** @@ -85,6 +86,7 @@ public static class Names { public static class Defaults { public static final Explicit IGNORE_MALFORMED = new Explicit<>(false, false); public static final Explicit IGNORE_Z_VALUE = new Explicit<>(true, false); + public static final Explicit MULTIVALUED = new Explicit<>(false, false); public static final FieldType FIELD_TYPE = new FieldType(); static { FIELD_TYPE.setStored(false); @@ -166,6 +168,7 @@ public abstract static class Builder, FT extends Abstra protected Boolean ignoreMalformed; protected Boolean ignoreZValue; protected boolean indexed = true; + protected Boolean multivalued; public Builder(String name, FieldType fieldType) { super(name, fieldType); @@ -217,6 +220,18 @@ public Builder ignoreZValue(final boolean ignoreZValue) { this.ignoreZValue = ignoreZValue; return this; } + + public Explicit multivalued() { + if (multivalued != null) { + return new Explicit<>(multivalued, true); + } + return Defaults.MULTIVALUED; + } + + public Builder multivalued(boolean multivalued) { + this.multivalued = multivalued; + return this; + } } /** @@ -245,6 +260,12 @@ public T parse(String name, Map node, Map params XContentMapValues.nodeBooleanValue(propNode, name + "." + Names.IGNORE_Z_VALUE.getPreferredName()) ); iterator.remove(); + } else if (Names.MULTIVALUED.getPreferredName().equals(propName)) { + params.put( + Names.MULTIVALUED.getPreferredName(), + XContentMapValues.nodeBooleanValue(propNode, name + "." + Names.MULTIVALUED.getPreferredName()) + ); + iterator.remove(); } } @@ -257,6 +278,10 @@ public T parse(String name, Map node, Map params if (params.containsKey(Names.IGNORE_MALFORMED.getPreferredName())) { builder.ignoreMalformed((Boolean) params.get(Names.IGNORE_MALFORMED.getPreferredName())); } + + if (params.containsKey(Names.MULTIVALUED.getPreferredName())) { + builder.multivalued((Boolean) params.get(Names.MULTIVALUED.getPreferredName())); + } return builder; } @@ -344,6 +369,7 @@ protected Object parseSourceValue(Object value) { protected Explicit ignoreMalformed; protected Explicit ignoreZValue; + protected Explicit multivalued; protected AbstractGeometryFieldMapper( String simpleName, @@ -351,12 +377,14 @@ protected AbstractGeometryFieldMapper( MappedFieldType mappedFieldType, Explicit ignoreMalformed, Explicit ignoreZValue, + Explicit multivalued, MultiFields multiFields, CopyTo copyTo ) { super(simpleName, fieldType, mappedFieldType, multiFields, copyTo); this.ignoreMalformed = ignoreMalformed; this.ignoreZValue = ignoreZValue; + this.multivalued = multivalued; } @Override @@ -369,6 +397,9 @@ protected void mergeOptions(FieldMapper other, List conflicts) { if (gsfm.ignoreZValue.explicit()) { this.ignoreZValue = gsfm.ignoreZValue; } + if (gsfm.multivalued.explicit()) { + this.multivalued = gsfm.multivalued; + } } @Override @@ -450,6 +481,9 @@ public void doXContentBody(XContentBuilder builder, boolean includeDefaults, Par if (includeDefaults || ignoreZValue.explicit()) { builder.field(Names.IGNORE_Z_VALUE.getPreferredName(), ignoreZValue.value()); } + if (includeDefaults || multivalued.explicit()) { + builder.field(Names.MULTIVALUED.getPreferredName(), multivalued.value()); + } } public Explicit ignoreMalformed() { @@ -459,4 +493,9 @@ public Explicit ignoreMalformed() { public Explicit ignoreZValue() { return ignoreZValue; } + + @Override + public boolean isMultivalue() { + return multivalued.explicit() && multivalued.value() != null && multivalued.value(); + } } diff --git a/server/src/main/java/org/opensearch/index/mapper/AbstractPointGeometryFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/AbstractPointGeometryFieldMapper.java index 1f53490de1ce1..deeadb6ba9cc1 100644 --- a/server/src/main/java/org/opensearch/index/mapper/AbstractPointGeometryFieldMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/AbstractPointGeometryFieldMapper.java @@ -104,6 +104,7 @@ public abstract AbstractPointGeometryFieldMapper build( MultiFields multiFields, Explicit ignoreMalformed, Explicit ignoreZValue, + Explicit multivalued, ParsedPoint nullValue, CopyTo copyTo ); @@ -117,6 +118,7 @@ public AbstractPointGeometryFieldMapper build(BuilderContext context) { multiFieldsBuilder.build(this, context), ignoreMalformed(context), ignoreZValue(context), + multivalued(), nullValue, copyTo ); @@ -183,10 +185,11 @@ protected AbstractPointGeometryFieldMapper( MultiFields multiFields, Explicit ignoreMalformed, Explicit ignoreZValue, + Explicit multivalued, ParsedPoint nullValue, CopyTo copyTo ) { - super(simpleName, fieldType, mappedFieldType, ignoreMalformed, ignoreZValue, multiFields, copyTo); + super(simpleName, fieldType, mappedFieldType, ignoreMalformed, ignoreZValue, multivalued, multiFields, copyTo); this.nullValue = nullValue; } diff --git a/server/src/main/java/org/opensearch/index/mapper/AbstractShapeGeometryFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/AbstractShapeGeometryFieldMapper.java index 186e01047dc7d..36010c1be48ef 100644 --- a/server/src/main/java/org/opensearch/index/mapper/AbstractShapeGeometryFieldMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/AbstractShapeGeometryFieldMapper.java @@ -239,10 +239,11 @@ protected AbstractShapeGeometryFieldMapper( Explicit coerce, Explicit ignoreZValue, Explicit orientation, + Explicit multivalued, MultiFields multiFields, CopyTo copyTo ) { - super(simpleName, fieldType, mappedFieldType, ignoreMalformed, ignoreZValue, multiFields, copyTo); + super(simpleName, fieldType, mappedFieldType, ignoreMalformed, ignoreZValue, multivalued, multiFields, copyTo); this.coerce = coerce; this.orientation = orientation; } diff --git a/server/src/main/java/org/opensearch/index/mapper/BinaryFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/BinaryFieldMapper.java index 040491f775357..612c18e6be0f6 100644 --- a/server/src/main/java/org/opensearch/index/mapper/BinaryFieldMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/BinaryFieldMapper.java @@ -38,6 +38,7 @@ import org.apache.lucene.search.Query; import org.apache.lucene.util.BytesRef; import org.opensearch.OpenSearchException; +import org.opensearch.common.Explicit; import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.core.common.bytes.BytesReference; @@ -84,6 +85,12 @@ public static class Builder extends ParametrizedFieldMapper.Builder { private final Parameter stored = Parameter.storeParam(m -> toType(m).stored, false); private final Parameter hasDocValues = Parameter.docValuesParam(m -> toType(m).hasDocValues, false); private final Parameter> meta = Parameter.metaParam(); + private final Parameter> multivalued = Parameter.explicitBoolParam( + "multivalued", + true, + m -> toType(m).multivalued, + false + ); public Builder(String name) { this(name, false); @@ -96,7 +103,7 @@ public Builder(String name, boolean hasDocValues) { @Override public List> getParameters() { - return Arrays.asList(meta, stored, hasDocValues); + return Arrays.asList(meta, stored, hasDocValues, multivalued); } @Override @@ -176,6 +183,7 @@ public Query termQuery(Object value, QueryShardContext context) { private final boolean stored; private final boolean hasDocValues; + private final Explicit multivalued; protected BinaryFieldMapper( String simpleName, @@ -187,6 +195,7 @@ protected BinaryFieldMapper( super(simpleName, mappedFieldType, multiFields, copyTo); this.stored = builder.stored.getValue(); this.hasDocValues = builder.hasDocValues.getValue(); + this.multivalued = builder.multivalued.getValue(); } @Override @@ -225,6 +234,10 @@ protected void parseCreateField(ParseContext context) throws IOException { } } + boolean multivalued() { + return multivalued.value(); + } + @Override public ParametrizedFieldMapper.Builder getMergeBuilder() { return new BinaryFieldMapper.Builder(simpleName()).init(this); @@ -235,6 +248,11 @@ protected String contentType() { return CONTENT_TYPE; } + @Override + public boolean isMultivalue() { + return multivalued.explicit() && multivalued.value() != null && multivalued.value(); + } + /** * Custom binary doc values field for the binary field mapper * diff --git a/server/src/main/java/org/opensearch/index/mapper/BooleanFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/BooleanFieldMapper.java index b4cf585c1329d..eacf8cf1f7100 100644 --- a/server/src/main/java/org/opensearch/index/mapper/BooleanFieldMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/BooleanFieldMapper.java @@ -44,6 +44,7 @@ import org.apache.lucene.search.TermQuery; import org.apache.lucene.util.BytesRef; import org.opensearch.common.Booleans; +import org.opensearch.common.Explicit; import org.opensearch.common.Nullable; import org.opensearch.common.xcontent.support.XContentMapValues; import org.opensearch.core.xcontent.XContentParser; @@ -122,6 +123,13 @@ public static class Builder extends ParametrizedFieldMapper.Builder { m -> toType(m).nullValue ).acceptsNull(); + private final Parameter> multivalued = Parameter.explicitBoolParam( + "multivalued", + true, + m -> toType(m).multivalued, + false + ); + private final Parameter boost = Parameter.boostParam(); private final Parameter> meta = Parameter.metaParam(); @@ -131,7 +139,7 @@ public Builder(String name) { @Override protected List> getParameters() { - return Arrays.asList(meta, boost, docValues, indexed, nullValue, stored); + return Arrays.asList(meta, boost, docValues, indexed, nullValue, stored, multivalued); } @Override @@ -348,6 +356,7 @@ public Query rangeQuery(Object lowerTerm, Object upperTerm, boolean includeLower private final boolean indexed; private final boolean hasDocValues; private final boolean stored; + private final Explicit multivalued; protected BooleanFieldMapper( String simpleName, @@ -361,6 +370,7 @@ protected BooleanFieldMapper( this.stored = builder.stored.getValue(); this.indexed = builder.indexed.getValue(); this.hasDocValues = builder.docValues.getValue(); + this.multivalued = builder.multivalued.getValue(); } @Override @@ -412,4 +422,12 @@ protected String contentType() { return CONTENT_TYPE; } + @Override + public boolean isMultivalue() { + return multivalued.explicit() && multivalued.value() != null && multivalued.value(); + } + + boolean multivalued() { + return multivalued.value(); + } } diff --git a/server/src/main/java/org/opensearch/index/mapper/CompletionFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/CompletionFieldMapper.java index fe48adf3249a3..043afd43ac327 100644 --- a/server/src/main/java/org/opensearch/index/mapper/CompletionFieldMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/CompletionFieldMapper.java @@ -44,6 +44,7 @@ import org.apache.lucene.search.suggest.document.SuggestField; import org.opensearch.Version; import org.opensearch.cluster.metadata.IndexMetadata; +import org.opensearch.common.Explicit; import org.opensearch.common.logging.DeprecationLogger; import org.opensearch.common.unit.Fuzziness; import org.opensearch.common.util.set.Sets; @@ -183,6 +184,12 @@ public static class Builder extends ParametrizedFieldMapper.Builder { m -> toType(m).maxInputLength, Defaults.DEFAULT_MAX_INPUT_LENGTH ).addDeprecatedName("max_input_len").setValidator(Builder::validateInputLength).alwaysSerialize(); + private final Parameter> multivalued = Parameter.explicitBoolParam( + "multivalued", + true, + m -> toType(m).multivalued, + false + ); private final Parameter> meta = Parameter.metaParam(); private final NamedAnalyzer defaultAnalyzer; @@ -209,7 +216,7 @@ private static void validateInputLength(int maxInputLength) { @Override protected List> getParameters() { - return Arrays.asList(analyzer, searchAnalyzer, preserveSeparators, preservePosInc, maxInputLength, contexts, meta); + return Arrays.asList(analyzer, searchAnalyzer, preserveSeparators, preservePosInc, maxInputLength, contexts, multivalued, meta); } @Override @@ -410,6 +417,7 @@ protected List parseSourceValue(Object value) { private final NamedAnalyzer searchAnalyzer; private final ContextMappings contexts; private final Version indexVersionCreated; + private final Explicit multivalued; public CompletionFieldMapper( String simpleName, @@ -429,6 +437,7 @@ public CompletionFieldMapper( this.searchAnalyzer = builder.searchAnalyzer.getValue(); this.contexts = builder.contexts.getValue(); this.indexVersionCreated = indexVersionCreated; + this.multivalued = builder.multivalued.getValue(); } @Override @@ -665,4 +674,13 @@ public void doValidate(MappingLookup mappers) { } } } + + @Override + public boolean isMultivalue() { + return multivalued.explicit() && multivalued.value() != null && multivalued.value(); + } + + boolean multivalued() { + return multivalued.value(); + } } diff --git a/server/src/main/java/org/opensearch/index/mapper/DateFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/DateFieldMapper.java index 7fbb38c47572c..3832950fc7548 100644 --- a/server/src/main/java/org/opensearch/index/mapper/DateFieldMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/DateFieldMapper.java @@ -43,6 +43,7 @@ import org.apache.lucene.search.Query; import org.opensearch.OpenSearchParseException; import org.opensearch.Version; +import org.opensearch.common.Explicit; import org.opensearch.common.Nullable; import org.opensearch.common.geo.ShapeRelation; import org.opensearch.common.logging.DeprecationLogger; @@ -236,6 +237,8 @@ public static class Builder extends ParametrizedFieldMapper.Builder { private final Parameter docValues = Parameter.docValuesParam(m -> toType(m).hasDocValues, true); private final Parameter store = Parameter.storeParam(m -> toType(m).store, false); + private final Parameter> multivalued; + private final Parameter boost = Parameter.boostParam(); private final Parameter> meta = Parameter.metaParam(); @@ -282,6 +285,7 @@ public Builder( this.printFormat.setValue(dateFormatter.printPattern()); this.locale.setValue(dateFormatter.locale()); } + this.multivalued = Parameter.explicitBoolParam("multivalued", true, m -> toType(m).multivalued, false); } private DateFormatter buildFormatter() { @@ -298,7 +302,19 @@ private DateFormatter buildFormatter() { @Override protected List> getParameters() { - return Arrays.asList(index, docValues, store, format, printFormat, locale, nullValue, ignoreMalformed, boost, meta); + return Arrays.asList( + index, + docValues, + store, + format, + printFormat, + locale, + nullValue, + ignoreMalformed, + boost, + multivalued, + meta + ); } private Long parseNullValue(DateFieldType fieldType) { @@ -703,6 +719,8 @@ public DocValueFormat docValueFormat(@Nullable String format, ZoneId timeZone) { private final boolean ignoreMalformedByDefault; private final Version indexCreatedVersion; + private final Explicit multivalued; + private DateFieldMapper( String simpleName, MappedFieldType mappedFieldType, @@ -725,6 +743,7 @@ private DateFieldMapper( this.resolution = resolution; this.ignoreMalformedByDefault = builder.ignoreMalformed.getDefaultValue(); this.indexCreatedVersion = builder.indexCreatedVersion; + this.multivalued = builder.multivalued.getValue(); } @Override @@ -800,4 +819,13 @@ public boolean getIgnoreMalformed() { public Long getNullValue() { return nullValue; } + + boolean multivalued() { + return multivalued.value(); + } + + @Override + public boolean isMultivalue() { + return multivalued.explicit() && multivalued.value() != null && multivalued.value(); + } } diff --git a/server/src/main/java/org/opensearch/index/mapper/DocumentParser.java b/server/src/main/java/org/opensearch/index/mapper/DocumentParser.java index 50ff816695156..df3eee8204423 100644 --- a/server/src/main/java/org/opensearch/index/mapper/DocumentParser.java +++ b/server/src/main/java/org/opensearch/index/mapper/DocumentParser.java @@ -444,7 +444,7 @@ private static void innerParseObject( parser.skipChildren(); } } else if (token == XContentParser.Token.START_OBJECT) { - parseObject(context, mapper, currentFieldName, paths); + parseObject(context, mapper, currentFieldName, paths, true); } else if (token == XContentParser.Token.START_ARRAY) { parseArray(context, mapper, currentFieldName, paths); } else if (token == XContentParser.Token.VALUE_NULL) { @@ -458,7 +458,7 @@ private static void innerParseObject( + "] as object, but got EOF, has a concrete value been provided to it?" ); } else if (token.isValue()) { - parseValue(context, mapper, currentFieldName, token, paths); + parseValue(context, mapper, currentFieldName, token, paths, true); } token = parser.nextToken(); } @@ -535,12 +535,23 @@ private static void parseObjectOrField(ParseContext context, Mapper mapper) thro } } - private static void parseObject(final ParseContext context, ObjectMapper mapper, String currentFieldName, String[] paths) - throws IOException { + private static void parseObject( + final ParseContext context, + ObjectMapper mapper, + String currentFieldName, + String[] paths, + boolean isValueRoot + ) throws IOException { assert currentFieldName != null; Mapper objectMapper = getMapper(context, mapper, currentFieldName, paths); if (objectMapper != null) { + if (isValueRoot && objectMapper.isMultivalue()) { + throw new MapperParsingException( + "object mapping [" + currentFieldName + "] trying to serialize an object value for a multi-valued field" + ); + } + context.path().add(currentFieldName); parseObjectOrField(context, objectMapper); context.path().remove(); @@ -675,7 +686,7 @@ private static void parseNonDynamicArray(ParseContext context, ObjectMapper mapp final String[] paths = splitAndValidatePath(lastFieldName); while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) { if (token == XContentParser.Token.START_OBJECT) { - parseObject(context, mapper, lastFieldName, paths); + parseObject(context, mapper, lastFieldName, paths, false); } else if (token == XContentParser.Token.START_ARRAY) { parseArray(context, mapper, lastFieldName, paths); } else if (token == XContentParser.Token.VALUE_NULL) { @@ -690,7 +701,7 @@ private static void parseNonDynamicArray(ParseContext context, ObjectMapper mapp ); } else { assert token.isValue(); - parseValue(context, mapper, lastFieldName, token, paths); + parseValue(context, mapper, lastFieldName, token, paths, false); } } } @@ -700,7 +711,8 @@ private static void parseValue( ObjectMapper parentMapper, String currentFieldName, XContentParser.Token token, - String[] paths + String[] paths, + boolean isValueRoot ) throws IOException { if (currentFieldName == null) { throw new MapperParsingException( @@ -714,6 +726,15 @@ private static void parseValue( } Mapper mapper = getMapper(context, parentMapper, currentFieldName, paths); if (mapper != null) { + if (isValueRoot && mapper.isMultivalue()) { + throw new MapperParsingException( + "object mapping [" + + currentFieldName + + "] trying to serialize a scalar value [" + + context.parser().textOrNull() + + "] for a multi-valued field" + ); + } parseObjectOrField(context, mapper); } else { currentFieldName = paths[paths.length - 1]; diff --git a/server/src/main/java/org/opensearch/index/mapper/GeoPointFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/GeoPointFieldMapper.java index fcca7e9804bf3..4c4fac0f8676e 100644 --- a/server/src/main/java/org/opensearch/index/mapper/GeoPointFieldMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/GeoPointFieldMapper.java @@ -99,6 +99,7 @@ public GeoPointFieldMapper build( MultiFields multiFields, Explicit ignoreMalformed, Explicit ignoreZValue, + Explicit multivalued, ParsedPoint nullValue, CopyTo copyTo ) { @@ -108,7 +109,7 @@ public GeoPointFieldMapper build( return point; }, (ParsedGeoPoint) nullValue, ignoreZValue.value(), ignoreMalformed.value())); ft.setGeometryIndexer(new GeoPointIndexer(ft)); - return new GeoPointFieldMapper(name, fieldType, ft, multiFields, ignoreMalformed, ignoreZValue, nullValue, copyTo); + return new GeoPointFieldMapper(name, fieldType, ft, multiFields, ignoreMalformed, ignoreZValue, multivalued, nullValue, copyTo); } } @@ -147,10 +148,11 @@ public GeoPointFieldMapper( MultiFields multiFields, Explicit ignoreMalformed, Explicit ignoreZValue, + Explicit multivalued, ParsedPoint nullValue, CopyTo copyTo ) { - super(simpleName, fieldType, mappedFieldType, multiFields, ignoreMalformed, ignoreZValue, nullValue, copyTo); + super(simpleName, fieldType, mappedFieldType, multiFields, ignoreMalformed, ignoreZValue, multivalued, nullValue, copyTo); } @Override @@ -199,6 +201,10 @@ public GeoPointFieldType fieldType() { return (GeoPointFieldType) mappedFieldType; } + boolean multivalued() { + return multivalued.value(); + } + /** * Concrete field type for geo_point * diff --git a/server/src/main/java/org/opensearch/index/mapper/GeoShapeFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/GeoShapeFieldMapper.java index b44b4b75549c3..fa7b21c9d86e3 100644 --- a/server/src/main/java/org/opensearch/index/mapper/GeoShapeFieldMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/GeoShapeFieldMapper.java @@ -122,6 +122,7 @@ public GeoShapeFieldMapper build(BuilderContext context) { coerce(context), ignoreZValue(), orientation(), + multivalued(), multiFieldsBuilder.build(this, context), copyTo ); @@ -190,10 +191,11 @@ public GeoShapeFieldMapper( Explicit coerce, Explicit ignoreZValue, Explicit orientation, + Explicit multivalued, MultiFields multiFields, CopyTo copyTo ) { - super(simpleName, fieldType, mappedFieldType, ignoreMalformed, coerce, ignoreZValue, orientation, multiFields, copyTo); + super(simpleName, fieldType, mappedFieldType, ignoreMalformed, coerce, ignoreZValue, orientation, multivalued, multiFields, copyTo); } @Override @@ -263,4 +265,8 @@ public GeoShapeFieldType fieldType() { protected String contentType() { return CONTENT_TYPE; } + + boolean multivalued() { + return multivalued.value(); + } } diff --git a/server/src/main/java/org/opensearch/index/mapper/IpFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/IpFieldMapper.java index db8da8a949d6f..b84580e6c7210 100644 --- a/server/src/main/java/org/opensearch/index/mapper/IpFieldMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/IpFieldMapper.java @@ -43,6 +43,7 @@ import org.apache.lucene.util.ArrayUtil; import org.apache.lucene.util.BytesRef; import org.opensearch.Version; +import org.opensearch.common.Explicit; import org.opensearch.common.Nullable; import org.opensearch.common.collect.Tuple; import org.opensearch.common.logging.DeprecationLogger; @@ -95,6 +96,8 @@ public static class Builder extends ParametrizedFieldMapper.Builder { private final Parameter nullValue = Parameter.stringParam("null_value", false, m -> toType(m).nullValueAsString, null) .acceptsNull(); + private final Parameter> multivalued; + private final Parameter> meta = Parameter.metaParam(); private final boolean ignoreMalformedByDefault; @@ -105,6 +108,7 @@ public Builder(String name, boolean ignoreMalformedByDefault, Version indexCreat this.ignoreMalformedByDefault = ignoreMalformedByDefault; this.indexCreatedVersion = indexCreatedVersion; this.ignoreMalformed = Parameter.boolParam("ignore_malformed", true, m -> toType(m).ignoreMalformed, ignoreMalformedByDefault); + this.multivalued = Parameter.explicitBoolParam("multivalued", true, m -> toType(m).multivalued, false); } Builder nullValue(String nullValue) { @@ -134,7 +138,7 @@ private InetAddress parseNullValue() { @Override protected List> getParameters() { - return Arrays.asList(indexed, hasDocValues, stored, ignoreMalformed, nullValue, meta); + return Arrays.asList(indexed, hasDocValues, stored, ignoreMalformed, nullValue, multivalued, meta); } @Override @@ -450,6 +454,8 @@ public DocValueFormat docValueFormat(@Nullable String format, ZoneId timeZone) { private final boolean ignoreMalformedByDefault; private final Version indexCreatedVersion; + private final Explicit multivalued; + private IpFieldMapper(String simpleName, MappedFieldType mappedFieldType, MultiFields multiFields, CopyTo copyTo, Builder builder) { super(simpleName, mappedFieldType, multiFields, copyTo); this.ignoreMalformedByDefault = builder.ignoreMalformedByDefault; @@ -460,12 +466,17 @@ private IpFieldMapper(String simpleName, MappedFieldType mappedFieldType, MultiF this.nullValue = builder.parseNullValue(); this.nullValueAsString = builder.nullValue.getValue(); this.indexCreatedVersion = builder.indexCreatedVersion; + this.multivalued = builder.multivalued.getValue(); } boolean ignoreMalformed() { return ignoreMalformed; } + boolean multivalued() { + return multivalued.value(); + } + @Override public IpFieldType fieldType() { return (IpFieldType) super.fieldType(); @@ -532,4 +543,9 @@ protected void parseCreateField(ParseContext context) throws IOException { public ParametrizedFieldMapper.Builder getMergeBuilder() { return new Builder(simpleName(), ignoreMalformedByDefault, indexCreatedVersion).init(this); } + + @Override + public boolean isMultivalue() { + return multivalued.explicit() && multivalued.value() != null && multivalued.value(); + } } diff --git a/server/src/main/java/org/opensearch/index/mapper/KeywordFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/KeywordFieldMapper.java index 54a1aead5fcc7..dfd2c0471e1b8 100644 --- a/server/src/main/java/org/opensearch/index/mapper/KeywordFieldMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/KeywordFieldMapper.java @@ -51,6 +51,7 @@ import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.automaton.Operations; import org.opensearch.OpenSearchException; +import org.opensearch.common.Explicit; import org.opensearch.common.Nullable; import org.opensearch.common.lucene.BytesRefs; import org.opensearch.common.lucene.Lucene; @@ -165,6 +166,13 @@ public static class Builder extends ParametrizedFieldMapper.Builder { false ); + private final Parameter> multivalued = Parameter.explicitBoolParam( + "multivalued", + true, + m -> toType(m).multivalued, + false + ); + private final Parameter> meta = Parameter.metaParam(); private final Parameter boost = Parameter.boostParam(); @@ -214,6 +222,7 @@ protected List> getParameters() { normalizer, splitQueriesOnWhitespace, boost, + multivalued, meta ); } @@ -657,6 +666,8 @@ public Query wildcardQuery( private final IndexAnalyzers indexAnalyzers; + private final Explicit multivalued; + protected KeywordFieldMapper( String simpleName, FieldType fieldType, @@ -679,6 +690,7 @@ protected KeywordFieldMapper( this.splitQueriesOnWhitespace = builder.splitQueriesOnWhitespace.getValue(); this.indexAnalyzers = builder.indexAnalyzers; + this.multivalued = builder.multivalued.getValue(); } /** @@ -689,6 +701,10 @@ public int ignoreAbove() { return ignoreAbove; } + boolean multivalued() { + return multivalued.value(); + } + @Override protected KeywordFieldMapper clone() { return (KeywordFieldMapper) super.clone(); @@ -777,4 +793,9 @@ protected String contentType() { public ParametrizedFieldMapper.Builder getMergeBuilder() { return new Builder(simpleName(), indexAnalyzers).init(this); } + + @Override + public boolean isMultivalue() { + return multivalued.explicit() && multivalued.value() != null && multivalued.value(); + } } diff --git a/server/src/main/java/org/opensearch/index/mapper/LegacyGeoShapeFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/LegacyGeoShapeFieldMapper.java index a5dcb60a86af9..6c76a911c6a1c 100644 --- a/server/src/main/java/org/opensearch/index/mapper/LegacyGeoShapeFieldMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/LegacyGeoShapeFieldMapper.java @@ -344,6 +344,7 @@ public LegacyGeoShapeFieldMapper build(BuilderContext context) { coerce(context), orientation(), ignoreZValue(), + multivalued(), context.indexSettings(), multiFieldsBuilder.build(this, context), copyTo @@ -516,11 +517,12 @@ public LegacyGeoShapeFieldMapper( Explicit coerce, Explicit orientation, Explicit ignoreZValue, + Explicit multivalued, Settings indexSettings, MultiFields multiFields, CopyTo copyTo ) { - super(simpleName, fieldType, mappedFieldType, ignoreMalformed, coerce, ignoreZValue, orientation, multiFields, copyTo); + super(simpleName, fieldType, mappedFieldType, ignoreMalformed, coerce, ignoreZValue, orientation, multivalued, multiFields, copyTo); this.indexCreatedVersion = IndexMetadata.indexCreated(indexSettings); } @@ -549,6 +551,10 @@ protected boolean docValuesByDefault() { return false; } + boolean multivalued() { + return multivalued.value(); + } + @Override public void doXContentBody(XContentBuilder builder, boolean includeDefaults, Params params) throws IOException { super.doXContentBody(builder, includeDefaults, params); diff --git a/server/src/main/java/org/opensearch/index/mapper/Mapper.java b/server/src/main/java/org/opensearch/index/mapper/Mapper.java index 87fdd8266a795..2aeac9074f5f3 100644 --- a/server/src/main/java/org/opensearch/index/mapper/Mapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/Mapper.java @@ -291,6 +291,13 @@ public final String simpleName() { */ public abstract void validate(MappingLookup mappers); + /** + * Does the field mapping require a multi-value (array). + */ + public boolean isMultivalue() { + return false; + } + /** * Check if settings have IndexMetadata.SETTING_INDEX_VERSION_CREATED setting. * @param settings settings diff --git a/server/src/main/java/org/opensearch/index/mapper/MatchOnlyTextFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/MatchOnlyTextFieldMapper.java index fb97f8c309a70..53a5209bfd337 100644 --- a/server/src/main/java/org/opensearch/index/mapper/MatchOnlyTextFieldMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/MatchOnlyTextFieldMapper.java @@ -206,6 +206,7 @@ protected List> getParameters() { indexPhrases, indexPrefixes, boost, + multivalued, meta ); } diff --git a/server/src/main/java/org/opensearch/index/mapper/NumberFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/NumberFieldMapper.java index 43e975f95757b..e7bc30b4a33d5 100644 --- a/server/src/main/java/org/opensearch/index/mapper/NumberFieldMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/NumberFieldMapper.java @@ -121,6 +121,8 @@ public static class Builder extends ParametrizedFieldMapper.Builder { private final Parameter nullValue; + private final Parameter> multivalued; + private final Parameter> meta = Parameter.metaParam(); private final NumberType type; @@ -152,6 +154,7 @@ public Builder(String name, NumberType type, boolean ignoreMalformedByDefault, b (n, c, o) -> o == null ? null : type.parse(o, false), m -> toType(m).nullValue ).acceptsNull(); + this.multivalued = Parameter.explicitBoolParam("multivalued", true, m -> toType(m).multivalued, false); } Builder nullValue(Number number) { @@ -166,7 +169,7 @@ public Builder docValues(boolean hasDocValues) { @Override protected List> getParameters() { - return Arrays.asList(indexed, hasDocValues, stored, ignoreMalformed, coerce, nullValue, meta); + return Arrays.asList(indexed, hasDocValues, stored, ignoreMalformed, coerce, nullValue, multivalued, meta); } @Override @@ -1733,6 +1736,8 @@ public double toDoubleValue(long value) { private final boolean ignoreMalformedByDefault; private final boolean coerceByDefault; + private final Explicit multivalued; + private NumberFieldMapper(String simpleName, MappedFieldType mappedFieldType, MultiFields multiFields, CopyTo copyTo, Builder builder) { super(simpleName, mappedFieldType, multiFields, copyTo); this.type = builder.type; @@ -1744,6 +1749,7 @@ private NumberFieldMapper(String simpleName, MappedFieldType mappedFieldType, Mu this.nullValue = builder.nullValue.getValue(); this.ignoreMalformedByDefault = builder.ignoreMalformed.getDefaultValue().value(); this.coerceByDefault = builder.coerce.getDefaultValue().value(); + this.multivalued = builder.multivalued.getValue(); } boolean coerce() { @@ -1754,6 +1760,10 @@ boolean ignoreMalformed() { return ignoreMalformed.value(); } + boolean multivalued() { + return multivalued.value(); + } + @Override public NumberFieldType fieldType() { return (NumberFieldType) super.fieldType(); @@ -1817,4 +1827,9 @@ protected void parseCreateField(ParseContext context) throws IOException { public ParametrizedFieldMapper.Builder getMergeBuilder() { return new Builder(simpleName(), type, ignoreMalformedByDefault, coerceByDefault).init(this); } + + @Override + public boolean isMultivalue() { + return multivalued.explicit() && multivalued.value() != null && multivalued.value(); + } } diff --git a/server/src/main/java/org/opensearch/index/mapper/RangeFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/RangeFieldMapper.java index 05ca7dee0fe4b..0f0ddd76b38a8 100644 --- a/server/src/main/java/org/opensearch/index/mapper/RangeFieldMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/RangeFieldMapper.java @@ -136,6 +136,7 @@ public static class Builder extends ParametrizedFieldMapper.Builder { private final Version indexCreatedVersion; private final boolean ignoreMalformedByDefault; private final Parameter ignoreMalformed; + private final Parameter> multivalued; public Builder(String name, RangeType type, Settings settings) { this( @@ -168,6 +169,7 @@ public Builder( format.neverSerialize(); locale.neverSerialize(); } + this.multivalued = Parameter.explicitBoolParam("multivalued", true, m -> toType(m).multivalued, false); } public void docValues(boolean hasDocValues) { @@ -181,7 +183,7 @@ Builder format(String format) { @Override protected List> getParameters() { - return Arrays.asList(index, hasDocValues, store, coerce, format, locale, boost, meta, ignoreMalformed); + return Arrays.asList(index, hasDocValues, store, coerce, format, locale, boost, meta, ignoreMalformed, multivalued); } protected RangeFieldType setupFieldType(BuilderContext context) { @@ -416,6 +418,7 @@ public Query rangeQuery( private final Version indexCreatedVersion; private final boolean ignoreMalformed; private final boolean ignoreMalformedByDefault; + private final Explicit multivalued; private RangeFieldMapper( String simpleName, @@ -437,6 +440,7 @@ private RangeFieldMapper( this.indexCreatedVersion = builder.indexCreatedVersion; this.ignoreMalformed = builder.ignoreMalformed.getValue(); this.ignoreMalformedByDefault = builder.ignoreMalformedByDefault; + this.multivalued = builder.multivalued.getValue(); } boolean coerce() { @@ -463,6 +467,11 @@ protected RangeFieldMapper clone() { return (RangeFieldMapper) super.clone(); } + @Override + public boolean isMultivalue() { + return multivalued.explicit() && multivalued.value() != null && multivalued.value(); + } + @Override protected void parseCreateField(ParseContext context) throws IOException { Range range; diff --git a/server/src/main/java/org/opensearch/index/mapper/TextFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/TextFieldMapper.java index ba053a3aeee1d..82ae50eec13bd 100644 --- a/server/src/main/java/org/opensearch/index/mapper/TextFieldMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/TextFieldMapper.java @@ -70,6 +70,7 @@ import org.apache.lucene.util.automaton.Automaton; import org.apache.lucene.util.automaton.Operations; import org.opensearch.Version; +import org.opensearch.common.Explicit; import org.opensearch.common.collect.Iterators; import org.opensearch.common.lucene.Lucene; import org.opensearch.common.lucene.search.AutomatonQueries; @@ -143,7 +144,7 @@ public static class Defaults { public static final int POSITION_INCREMENT_GAP = 100; } - private static TextFieldMapper toType(FieldMapper in) { + protected static TextFieldMapper toType(FieldMapper in) { return (TextFieldMapper) in; } @@ -331,6 +332,13 @@ public static class Builder extends ParametrizedFieldMapper.Builder { .orElse(null) ).acceptsNull(); + protected final Parameter> multivalued = Parameter.explicitBoolParam( + "multivalued", + true, + m -> toType(m).multivalued, + false + ); + protected final Parameter boost = Parameter.boostParam(); protected final Parameter> meta = Parameter.metaParam(); @@ -390,6 +398,7 @@ protected List> getParameters() { indexPhrases, indexPrefixes, boost, + multivalued, meta ); } @@ -989,6 +998,7 @@ public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, S protected final Version indexCreatedVersion; protected final IndexAnalyzers indexAnalyzers; private final FielddataFrequencyFilter freqFilter; + protected final Explicit multivalued; protected TextFieldMapper( String simpleName, @@ -1016,6 +1026,7 @@ protected TextFieldMapper( this.indexCreatedVersion = builder.indexCreatedVersion; this.indexAnalyzers = builder.analyzers.indexAnalyzers; this.freqFilter = builder.freqFilter.getValue(); + this.multivalued = builder.multivalued.getValue(); } @Override @@ -1081,6 +1092,15 @@ public TextFieldType fieldType() { return (TextFieldType) super.fieldType(); } + boolean multivalued() { + return multivalued.value(); + } + + @Override + public boolean isMultivalue() { + return multivalued.explicit() && multivalued.value() != null && multivalued.value(); + } + public static Query createPhraseQuery(TokenStream stream, String field, int slop, boolean enablePositionIncrements) throws IOException { MultiPhraseQuery.Builder mpqb = new MultiPhraseQuery.Builder(); mpqb.setSlop(slop); @@ -1224,5 +1244,6 @@ protected void doXContentBody(XContentBuilder builder, boolean includeDefaults, mapperBuilder.freqFilter.toXContent(builder, includeDefaults); mapperBuilder.indexPrefixes.toXContent(builder, includeDefaults); mapperBuilder.indexPhrases.toXContent(builder, includeDefaults); + mapperBuilder.multivalued.toXContent(builder, includeDefaults); } } diff --git a/server/src/main/java/org/opensearch/index/mapper/WildcardFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/WildcardFieldMapper.java index e43e3bda692e7..8eea56df5d744 100644 --- a/server/src/main/java/org/opensearch/index/mapper/WildcardFieldMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/WildcardFieldMapper.java @@ -38,6 +38,7 @@ import org.apache.lucene.util.automaton.Automaton; import org.apache.lucene.util.automaton.CompiledAutomaton; import org.apache.lucene.util.automaton.RegExp; +import org.opensearch.common.Explicit; import org.opensearch.common.lucene.BytesRefs; import org.opensearch.common.lucene.Lucene; import org.opensearch.common.lucene.search.AutomatonQueries; @@ -82,6 +83,7 @@ public class WildcardFieldMapper extends ParametrizedFieldMapper { private final String normalizerName; private final boolean hasDocValues; private final IndexAnalyzers indexAnalyzers; + private final Explicit multivalued; /** * The builder for the field mapper. @@ -103,6 +105,12 @@ public static final class Builder extends ParametrizedFieldMapper.Builder { private final Parameter> meta = Parameter.metaParam(); private final Parameter hasDocValues = Parameter.docValuesParam(m -> toType(m).hasDocValues, false); private final IndexAnalyzers indexAnalyzers; + protected final Parameter> multivalued = Parameter.explicitBoolParam( + "multivalued", + true, + m -> toType(m).multivalued, + false + ); public Builder(String name, IndexAnalyzers indexAnalyzers) { super(name); @@ -135,7 +143,7 @@ public WildcardFieldMapper.Builder docValues(boolean hasDocValues) { @Override protected List> getParameters() { - return Arrays.asList(nullValue, ignoreAbove, normalizer, hasDocValues, meta); + return Arrays.asList(nullValue, ignoreAbove, normalizer, hasDocValues, meta, multivalued); } @Override @@ -174,6 +182,7 @@ protected WildcardFieldMapper( this.normalizerName = builder.normalizer.getValue(); this.hasDocValues = builder.hasDocValues.getValue(); this.indexAnalyzers = builder.indexAnalyzers; + this.multivalued = builder.multivalued.getValue(); } public int ignoreAbove() { @@ -888,6 +897,15 @@ public ParametrizedFieldMapper.Builder getMergeBuilder() { return new Builder(simpleName(), indexAnalyzers).init(this); } + boolean multivalued() { + return multivalued.value(); + } + + @Override + public boolean isMultivalue() { + return multivalued.explicit() && multivalued.value() != null && multivalued.value(); + } + private static WildcardFieldMapper toType(FieldMapper in) { return (WildcardFieldMapper) in; } diff --git a/server/src/test/java/org/opensearch/index/mapper/BinaryFieldMapperTests.java b/server/src/test/java/org/opensearch/index/mapper/BinaryFieldMapperTests.java index 87b5ad3434944..06d07b4cf2a0f 100644 --- a/server/src/test/java/org/opensearch/index/mapper/BinaryFieldMapperTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/BinaryFieldMapperTests.java @@ -32,17 +32,22 @@ package org.opensearch.index.mapper; +import org.apache.lucene.index.IndexableField; import org.apache.lucene.util.BytesRef; import org.opensearch.common.io.stream.BytesStreamOutput; +import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.compress.CompressorRegistry; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.XContentBuilder; import java.io.IOException; import java.io.OutputStream; import java.util.Arrays; +import java.util.List; +import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.instanceOf; @@ -64,6 +69,7 @@ protected void minimalMapping(XContentBuilder b) throws IOException { protected void registerParameters(ParameterChecker checker) throws IOException { checker.registerConflictCheck("doc_values", b -> b.field("doc_values", true)); checker.registerConflictCheck("store", b -> b.field("store", true)); + checker.registerUpdateCheck(b -> b.field("multivalued", true), m -> assertTrue(((BinaryFieldMapper) m).multivalued())); } public void testExistsQueryDocValuesEnabled() throws IOException { @@ -135,4 +141,35 @@ public void testStoredValue() throws IOException { assertEquals(new BytesArray(value), originalValue); } } + + public void testMultivalued() throws Exception { + DocumentMapper mapper = createDocumentMapper(fieldMapping(b -> b.field("type", "binary").field("multivalued", true))); + ThrowingRunnable runnable = () -> mapper.parse( + new SourceToParse( + "test", + "1", + BytesReference.bytes(XContentFactory.jsonBuilder().startObject().field("field", "bGlkaHQtd29rfx4=").endObject()), + MediaTypeRegistry.JSON + ) + ); + MapperParsingException e = expectThrows(MapperParsingException.class, runnable); + assertThat( + e.getMessage(), + containsString("object mapping [field] trying to serialize a scalar value [bGlkaHQtd29rfx4=] for a multi-valued field") + ); + + ParsedDocument doc = mapper.parse( + new SourceToParse( + "test", + "1", + BytesReference.bytes( + XContentFactory.jsonBuilder().startObject().field("field", List.of("bGlkaHQtd29rfx4=", "bGlkaHQtd29rfx4=")).endObject() + ), + MediaTypeRegistry.JSON + ) + ); + + IndexableField[] fields = doc.rootDoc().getFields("field"); + assertEquals(0, fields.length); + } } diff --git a/server/src/test/java/org/opensearch/index/mapper/BooleanFieldMapperTests.java b/server/src/test/java/org/opensearch/index/mapper/BooleanFieldMapperTests.java index 5392bd6c358d3..72af138be8f89 100644 --- a/server/src/test/java/org/opensearch/index/mapper/BooleanFieldMapperTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/BooleanFieldMapperTests.java @@ -41,11 +41,16 @@ import org.apache.lucene.search.TermQuery; import org.apache.lucene.util.BytesRef; import org.opensearch.common.xcontent.XContentFactory; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.index.mapper.ParseContext.Document; import java.io.IOException; +import java.util.List; + +import static org.hamcrest.Matchers.containsString; public class BooleanFieldMapperTests extends MapperTestCase { @@ -70,6 +75,7 @@ protected void registerParameters(ParameterChecker checker) throws IOException { checker.registerConflictCheck("store", b -> b.field("store", true)); checker.registerConflictCheck("null_value", b -> b.field("null_value", true)); checker.registerUpdateCheck(b -> b.field("boost", 2.0), m -> assertEquals(m.fieldType().boost(), 2.0, 0)); + checker.registerUpdateCheck(b -> b.field("multivalued", true), m -> assertTrue(((BooleanFieldMapper) m).multivalued())); } public void testExistsQueryDocValuesDisabled() throws IOException { @@ -234,4 +240,33 @@ public void testIndexedValueForSearch() throws Exception { assertEquals("Can't parse boolean value [random], expected [true] or [false]", e.getMessage()); } + + public void testMultivalued() throws Exception { + DocumentMapper mapper = createDocumentMapper(fieldMapping(b -> b.field("type", "boolean").field("multivalued", true))); + ThrowingRunnable runnable = () -> mapper.parse( + new SourceToParse( + "test", + "1", + BytesReference.bytes(XContentFactory.jsonBuilder().startObject().field("field", true).endObject()), + MediaTypeRegistry.JSON + ) + ); + MapperParsingException e = expectThrows(MapperParsingException.class, runnable); + assertThat( + e.getMessage(), + containsString("object mapping [field] trying to serialize a scalar value [true] for a multi-valued field") + ); + + ParsedDocument doc = mapper.parse( + new SourceToParse( + "test", + "1", + BytesReference.bytes(XContentFactory.jsonBuilder().startObject().field("field", List.of(true, false)).endObject()), + MediaTypeRegistry.JSON + ) + ); + + IndexableField[] fields = doc.rootDoc().getFields("field"); + assertEquals(4, fields.length); + } } diff --git a/server/src/test/java/org/opensearch/index/mapper/CompletionFieldMapperTests.java b/server/src/test/java/org/opensearch/index/mapper/CompletionFieldMapperTests.java index b1785f5d7b14c..32c021b579a75 100644 --- a/server/src/test/java/org/opensearch/index/mapper/CompletionFieldMapperTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/CompletionFieldMapperTests.java @@ -47,6 +47,7 @@ import org.apache.lucene.util.automaton.Operations; import org.apache.lucene.util.automaton.RegExp; import org.opensearch.common.unit.Fuzziness; +import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.common.xcontent.json.JsonXContent; import org.opensearch.core.common.Strings; import org.opensearch.core.common.bytes.BytesReference; @@ -65,6 +66,7 @@ import java.io.IOException; import java.util.Collections; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.function.Function; @@ -122,6 +124,8 @@ protected void registerParameters(ParameterChecker checker) throws IOException { CompletionFieldMapper cfm = (CompletionFieldMapper) m; assertEquals(30, cfm.getMaxInputLength()); }); + + checker.registerUpdateCheck(b -> b.field("multivalued", true), m -> assertTrue(((CompletionFieldMapper) m).multivalued())); } @Override @@ -737,6 +741,35 @@ public void testRegexQueryType() throws Exception { assertThat(prefixQuery, instanceOf(RegexCompletionQuery.class)); } + public void testMultivalued() throws Exception { + DocumentMapper mapper = createDocumentMapper(fieldMapping(b -> b.field("type", "completion").field("multivalued", true))); + ThrowingRunnable runnable = () -> mapper.parse( + new SourceToParse( + "test", + "1", + BytesReference.bytes(XContentFactory.jsonBuilder().startObject().field("field", "foo").endObject()), + MediaTypeRegistry.JSON + ) + ); + MapperParsingException e = expectThrows(MapperParsingException.class, runnable); + assertThat( + e.getMessage(), + containsString("object mapping [field] trying to serialize a scalar value [foo] for a multi-valued field") + ); + + ParsedDocument doc = mapper.parse( + new SourceToParse( + "test", + "1", + BytesReference.bytes(XContentFactory.jsonBuilder().startObject().field("field", List.of("foo", "bar")).endObject()), + MediaTypeRegistry.JSON + ) + ); + + IndexableField[] fields = doc.rootDoc().getFields("field"); + assertEquals(2, fields.length); + } + private static void assertFieldsOfType(IndexableField[] fields) { int actualFieldCount = 0; for (IndexableField field : fields) { diff --git a/server/src/test/java/org/opensearch/index/mapper/DateFieldMapperTests.java b/server/src/test/java/org/opensearch/index/mapper/DateFieldMapperTests.java index 98bcaa3a1a46b..f86906c1395e3 100644 --- a/server/src/test/java/org/opensearch/index/mapper/DateFieldMapperTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/DateFieldMapperTests.java @@ -36,6 +36,9 @@ import org.apache.lucene.index.IndexableField; import org.opensearch.common.time.DateFormatter; import org.opensearch.common.util.FeatureFlags; +import org.opensearch.common.xcontent.XContentFactory; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.index.termvectors.TermVectorsService; import org.opensearch.search.DocValueFormat; @@ -74,6 +77,7 @@ protected void registerParameters(ParameterChecker checker) throws IOException { checker.registerConflictCheck("null_value", b -> b.field("null_value", "34500000")); checker.registerUpdateCheck(b -> b.field("ignore_malformed", true), m -> assertTrue(((DateFieldMapper) m).getIgnoreMalformed())); checker.registerUpdateCheck(b -> b.field("boost", 2.0), m -> assertEquals(m.fieldType().boost(), 2.0, 0)); + checker.registerUpdateCheck(b -> b.field("multivalued", true), m -> assertTrue(((DateFieldMapper) m).multivalued())); } public void testExistsQueryDocValuesDisabled() throws IOException { @@ -345,4 +349,46 @@ public void testFetchDocValuesNanos() throws IOException { assertEquals(List.of(date), fetchFromDocValues(mapperService, ft, format, date)); assertEquals(List.of("2020-05-15T21:33:02.123Z"), fetchFromDocValues(mapperService, ft, format, 1589578382123L)); } + + public void testMultivaluedMillis() throws Exception { + doTestMultivalued("date"); + } + + public void testMultivaluedNanos() throws Exception { + doTestMultivalued("date_nanos"); + } + + private void doTestMultivalued(String type) throws Exception { + DocumentMapper mapper = createDocumentMapper(fieldMapping(b -> b.field("type", type).field("multivalued", true))); + ThrowingRunnable runnable = () -> mapper.parse( + new SourceToParse( + "test", + "1", + BytesReference.bytes(XContentFactory.jsonBuilder().startObject().field("field", "2020-06-30T11:12:13Z").endObject()), + MediaTypeRegistry.JSON + ) + ); + MapperParsingException e = expectThrows(MapperParsingException.class, runnable); + assertThat( + e.getMessage(), + containsString("object mapping [field] trying to serialize a scalar value [2020-06-30T11:12:13Z] for a multi-valued field") + ); + + ParsedDocument doc = mapper.parse( + new SourceToParse( + "test", + "1", + BytesReference.bytes( + XContentFactory.jsonBuilder() + .startObject() + .field("field", List.of("2020-06-30T11:12:13Z", "2021-11-05T18:32:43Z")) + .endObject() + ), + MediaTypeRegistry.JSON + ) + ); + + IndexableField[] fields = doc.rootDoc().getFields("field"); + assertEquals(4, fields.length); + } } diff --git a/server/src/test/java/org/opensearch/index/mapper/GeoPointFieldMapperTests.java b/server/src/test/java/org/opensearch/index/mapper/GeoPointFieldMapperTests.java index cbb5fc8ce5a22..ce4dd6ab9d3ad 100644 --- a/server/src/test/java/org/opensearch/index/mapper/GeoPointFieldMapperTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/GeoPointFieldMapperTests.java @@ -31,6 +31,7 @@ package org.opensearch.index.mapper; +import org.apache.lucene.index.IndexableField; import org.apache.lucene.util.BytesRef; import org.opensearch.common.geo.GeoPoint; import org.opensearch.common.geo.GeoUtils; @@ -80,6 +81,7 @@ protected void registerParameters(ParameterChecker checker) throws IOException { GeoPointFieldMapper gpfm = (GeoPointFieldMapper) m; assertEquals(gpfm.nullValue, point); }); + checker.registerUpdateCheck(b -> b.field("multivalued", true), m -> assertTrue(((GeoPointFieldMapper) m).multivalued())); } protected void writeFieldValue(XContentBuilder builder) throws IOException { @@ -405,6 +407,35 @@ public void testGeoJsonIgnoreInvalidForm() throws Exception { assertThat(doc.rootDoc().getField("field"), nullValue()); } + public void testMultivalued() throws Exception { + DocumentMapper mapper = createDocumentMapper(fieldMapping(b -> b.field("type", "geo_point").field("multivalued", true))); + Exception e = expectThrows( + MapperParsingException.class, + () -> mapper.parse( + source(b -> b.startObject("field").field("type", "Point").array("coordinates", new double[] { 1.1, 1.2 }).endObject()) + ) + ); + assertThat(e.getMessage(), containsString("object mapping [field] trying to serialize an object value for a multi-valued field")); + + ParsedDocument doc = mapper.parse( + source( + b -> b.startArray("field") + .startObject() + .field("type", "Point") + .array("coordinates", new double[] { 1.1, 1.2 }) + .endObject() + .startObject() + .field("type", "Point") + .array("coordinates", new double[] { 1.3, 1.4 }) + .endObject() + .endArray() + ) + ); + + IndexableField[] fields = doc.rootDoc().getFields("field"); + assertEquals(4, fields.length); + } + @Override protected GeoPointFieldMapper.Builder newBuilder() { return new GeoPointFieldMapper.Builder("geo"); diff --git a/server/src/test/java/org/opensearch/index/mapper/GeoShapeFieldMapperTests.java b/server/src/test/java/org/opensearch/index/mapper/GeoShapeFieldMapperTests.java index 016862e3ffabc..c5c7946001a28 100644 --- a/server/src/test/java/org/opensearch/index/mapper/GeoShapeFieldMapperTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/GeoShapeFieldMapperTests.java @@ -31,6 +31,7 @@ package org.opensearch.index.mapper; +import org.apache.lucene.index.IndexableField; import org.opensearch.common.Explicit; import org.opensearch.common.geo.builders.ShapeBuilder; import org.opensearch.core.common.Strings; @@ -87,6 +88,7 @@ protected void registerParameters(ParameterChecker checker) throws IOException { GeoShapeFieldMapper gpfm = (GeoShapeFieldMapper) m; assertTrue(gpfm.coerce.value()); }); + checker.registerUpdateCheck(b -> b.field("multivalued", true), m -> assertTrue(((GeoShapeFieldMapper) m).multivalued())); } @Before @@ -251,6 +253,29 @@ public void testGeoShapeArrayParsing() throws Exception { assertThat(document.docs().get(0).getFields("field").length, equalTo(4)); } + public void testMultivalued() throws Exception { + DocumentMapper mapper = createDocumentMapper(fieldMapping(b -> b.field("type", "geo_point").field("multivalued", true))); + Exception e = expectThrows( + MapperParsingException.class, + () -> mapper.parse( + source(b -> b.startObject("field").field("type", "Point").array("coordinates", new double[] { 1.1, 1.2 }).endObject()) + ) + ); + assertThat(e.getMessage(), containsString("object mapping [field] trying to serialize an object value for a multi-valued field")); + + ParsedDocument doc = mapper.parse(source(b -> { + b.startArray("field"); + { + b.startObject().field("type", "Point").startArray("coordinates").value(176.0).value(15.0).endArray().endObject(); + b.startObject().field("type", "Point").startArray("coordinates").value(76.0).value(-15.0).endArray().endObject(); + } + b.endArray(); + })); + + IndexableField[] fields = doc.rootDoc().getFields("field"); + assertEquals(4, fields.length); + } + @Override protected boolean supportsMeta() { return false; diff --git a/server/src/test/java/org/opensearch/index/mapper/IpFieldMapperTests.java b/server/src/test/java/org/opensearch/index/mapper/IpFieldMapperTests.java index 0f30e93f0622a..1905d15552ad1 100644 --- a/server/src/test/java/org/opensearch/index/mapper/IpFieldMapperTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/IpFieldMapperTests.java @@ -40,11 +40,15 @@ import org.apache.lucene.search.TermQuery; import org.apache.lucene.util.BytesRef; import org.opensearch.common.network.InetAddresses; +import org.opensearch.common.xcontent.XContentFactory; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.index.termvectors.TermVectorsService; import java.io.IOException; import java.net.InetAddress; +import java.util.List; import static org.hamcrest.Matchers.containsString; @@ -67,6 +71,7 @@ protected void registerParameters(ParameterChecker checker) throws IOException { checker.registerConflictCheck("store", b -> b.field("store", true)); checker.registerConflictCheck("null_value", b -> b.field("null_value", "::1")); checker.registerUpdateCheck(b -> b.field("ignore_malformed", false), m -> assertFalse(((IpFieldMapper) m).ignoreMalformed())); + checker.registerUpdateCheck(b -> b.field("multivalued", true), m -> assertTrue(((IpFieldMapper) m).multivalued())); } public void testExistsQueryDocValuesDisabled() throws IOException { @@ -215,4 +220,35 @@ public void testNullValue() throws IOException { })); assertWarnings("Error parsing [:1] as IP in [null_value] on field [field]); [null_value] will be ignored"); } + + public void testMultivalued() throws Exception { + DocumentMapper mapper = createDocumentMapper(fieldMapping(b -> b.field("type", "ip").field("multivalued", true))); + ThrowingRunnable runnable = () -> mapper.parse( + new SourceToParse( + "test", + "1", + BytesReference.bytes(XContentFactory.jsonBuilder().startObject().field("field", "192.168.0.1").endObject()), + MediaTypeRegistry.JSON + ) + ); + MapperParsingException e = expectThrows(MapperParsingException.class, runnable); + assertThat( + e.getMessage(), + containsString("object mapping [field] trying to serialize a scalar value [192.168.0.1] for a multi-valued field") + ); + + ParsedDocument doc = mapper.parse( + new SourceToParse( + "test", + "1", + BytesReference.bytes( + XContentFactory.jsonBuilder().startObject().field("field", List.of("192.168.0.1", "192.168.0.2")).endObject() + ), + MediaTypeRegistry.JSON + ) + ); + + IndexableField[] fields = doc.rootDoc().getFields("field"); + assertEquals(4, fields.length); + } } diff --git a/server/src/test/java/org/opensearch/index/mapper/KeywordFieldMapperTests.java b/server/src/test/java/org/opensearch/index/mapper/KeywordFieldMapperTests.java index 4da21da40e0d8..a2b1f51e20e02 100644 --- a/server/src/test/java/org/opensearch/index/mapper/KeywordFieldMapperTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/KeywordFieldMapperTests.java @@ -44,6 +44,9 @@ import org.apache.lucene.tests.analysis.MockLowerCaseFilter; import org.apache.lucene.tests.analysis.MockTokenizer; import org.apache.lucene.util.BytesRef; +import org.opensearch.common.xcontent.XContentFactory; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.index.IndexSettings; import org.opensearch.index.analysis.AnalyzerScope; @@ -202,6 +205,7 @@ protected void registerParameters(ParameterChecker checker) throws IOException { }, m -> assertFalse(m.fieldType().getTextSearchInfo().hasNorms())); checker.registerUpdateCheck(b -> b.field("boost", 2.0), m -> assertEquals(m.fieldType().boost(), 2.0, 0)); + checker.registerUpdateCheck(b -> b.field("multivalued", true), m -> assertTrue(((KeywordFieldMapper) m).multivalued())); } public void testDefaults() throws Exception { @@ -474,4 +478,35 @@ public void testSplitQueriesOnWhitespace() throws IOException { new String[] { "hello world" } ); } + + public void testMultivalued() throws Exception { + DocumentMapper mapper = createDocumentMapper(fieldMapping(b -> b.field("type", "keyword").field("multivalued", true))); + ThrowingRunnable runnable = () -> mapper.parse( + new SourceToParse( + "test", + "1", + BytesReference.bytes(XContentFactory.jsonBuilder().startObject().field("field", "Hello World").endObject()), + MediaTypeRegistry.JSON + ) + ); + MapperParsingException e = expectThrows(MapperParsingException.class, runnable); + assertThat( + e.getMessage(), + containsString("object mapping [field] trying to serialize a scalar value [Hello World] for a multi-valued field") + ); + + ParsedDocument doc = mapper.parse( + new SourceToParse( + "test", + "1", + BytesReference.bytes( + XContentFactory.jsonBuilder().startObject().field("field", List.of("Hello World", "abcdef")).endObject() + ), + MediaTypeRegistry.JSON + ) + ); + + IndexableField[] fields = doc.rootDoc().getFields("field"); + assertEquals(4, fields.length); + } } diff --git a/server/src/test/java/org/opensearch/index/mapper/LegacyGeoShapeFieldMapperTests.java b/server/src/test/java/org/opensearch/index/mapper/LegacyGeoShapeFieldMapperTests.java index 048b51d39ca6c..f6e4e3b0617bd 100644 --- a/server/src/test/java/org/opensearch/index/mapper/LegacyGeoShapeFieldMapperTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/LegacyGeoShapeFieldMapperTests.java @@ -122,6 +122,8 @@ protected void registerParameters(ParameterChecker checker) throws IOException { }); // TODO - distance_error_pct ends up being subsumed into a calculated value, how to test checker.registerUpdateCheck(b -> b.field("distance_error_pct", 0.8), m -> {}); + + checker.registerUpdateCheck(b -> b.field("multivalued", true), m -> assertTrue(((LegacyGeoShapeFieldMapper) m).multivalued())); } @Override @@ -668,4 +670,25 @@ public void testGeoShapeArrayParsing() throws Exception { assertThat(fields.length, equalTo(2)); assertFieldWarnings("tree", "strategy"); } + + public void testMultivalued() throws Exception { + DocumentMapper mapper = createDocumentMapper(fieldMapping(b -> b.field("type", "geo_shape").field("multivalued", true))); + ThrowingRunnable runnable = () -> mapper.parse(source(b -> { + b.startObject("field").field("type", "Point").startArray("coordinates").value(176.0).value(15.0).endArray().endObject(); + })); + MapperParsingException e = expectThrows(MapperParsingException.class, runnable); + assertThat(e.getMessage(), containsString("object mapping [field] trying to serialize an object value for a multi-valued field")); + + ParsedDocument doc = mapper.parse(source(b -> { + b.startArray("field"); + { + b.startObject().field("type", "Point").startArray("coordinates").value(176.0).value(15.0).endArray().endObject(); + b.startObject().field("type", "Point").startArray("coordinates").value(76.0).value(-15.0).endArray().endObject(); + } + b.endArray(); + })); + + IndexableField[] fields = doc.rootDoc().getFields("field"); + assertEquals(4, fields.length); + } } diff --git a/server/src/test/java/org/opensearch/index/mapper/NumberFieldMapperTests.java b/server/src/test/java/org/opensearch/index/mapper/NumberFieldMapperTests.java index 610b69a7fdf88..a81f2ececde56 100644 --- a/server/src/test/java/org/opensearch/index/mapper/NumberFieldMapperTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/NumberFieldMapperTests.java @@ -34,7 +34,9 @@ import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.IndexableField; +import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.index.mapper.NumberFieldMapper.NumberType; @@ -75,6 +77,7 @@ protected void registerParameters(ParameterChecker checker) throws IOException { checker.registerConflictCheck("null_value", b -> b.field("null_value", 1)); checker.registerUpdateCheck(b -> b.field("coerce", false), m -> assertFalse(((NumberFieldMapper) m).coerce())); checker.registerUpdateCheck(b -> b.field("ignore_malformed", true), m -> assertTrue(((NumberFieldMapper) m).ignoreMalformed())); + checker.registerUpdateCheck(b -> b.field("multivalued", true), m -> assertTrue(((NumberFieldMapper) m).multivalued())); } protected void writeFieldValue(XContentBuilder builder) throws IOException { @@ -319,4 +322,33 @@ public void testLongIndexingOutOfRange() throws Exception { ); assertEquals(0, doc.rootDoc().getFields("field").length); } + + protected void doTestMultivalued(String type) throws IOException { + DocumentMapper mapper = createDocumentMapper(fieldMapping(b -> b.field("type", type).field("multivalued", true))); + ThrowingRunnable runnable = () -> mapper.parse( + new SourceToParse( + "test", + "1", + BytesReference.bytes(XContentFactory.jsonBuilder().startObject().field("field", "15").endObject()), + MediaTypeRegistry.JSON + ) + ); + MapperParsingException e = expectThrows(MapperParsingException.class, runnable); + assertThat( + e.getMessage(), + containsString("object mapping [field] trying to serialize a scalar value [15] for a multi-valued field") + ); + + ParsedDocument doc = mapper.parse( + new SourceToParse( + "test", + "1", + BytesReference.bytes(XContentFactory.jsonBuilder().startObject().field("field", List.of("15", "117")).endObject()), + MediaTypeRegistry.JSON + ) + ); + + IndexableField[] fields = doc.rootDoc().getFields("field"); + assertEquals(4, fields.length); + } } diff --git a/server/src/test/java/org/opensearch/index/mapper/RangeFieldMapperTests.java b/server/src/test/java/org/opensearch/index/mapper/RangeFieldMapperTests.java index 91eab942c499a..97aacc9ee6b9b 100644 --- a/server/src/test/java/org/opensearch/index/mapper/RangeFieldMapperTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/RangeFieldMapperTests.java @@ -413,4 +413,36 @@ public void testUpdatesWithSameMappings() throws Exception { mapper.merge(mapping, MergeReason.MAPPING_UPDATE); } } + + protected void doTestMultivalued(String type) throws IOException { + DocumentMapper mapper = createDocumentMapper(rangeFieldMapping(type, b -> b.field("multivalued", true))); + ThrowingRunnable runnable = () -> mapper.parse( + source( + b -> b.startObject("field") + .field(GT_FIELD.getPreferredName(), getFrom(type)) + .field(LT_FIELD.getPreferredName(), getTo(type)) + .endObject() + ) + ); + MapperParsingException e = expectThrows(MapperParsingException.class, runnable); + assertThat(e.getMessage(), containsString("object mapping [field] trying to serialize an object value for a multi-valued field")); + + ParsedDocument doc = mapper.parse( + source( + b -> b.startArray("field") + .startObject() + .field(getFromField(), getFrom(type)) + .field(getToField(), getTo(type)) + .endObject() + .startObject() + .field(getFromField(), getFrom(type)) + .field(getToField(), getTo(type)) + .endObject() + .endArray() + ) + ); + + IndexableField[] fields = doc.rootDoc().getFields("field"); + assertEquals(3, fields.length); + } } diff --git a/server/src/test/java/org/opensearch/index/mapper/TextFieldMapperTests.java b/server/src/test/java/org/opensearch/index/mapper/TextFieldMapperTests.java index 0253caea9759d..590b0f6ef3ce8 100644 --- a/server/src/test/java/org/opensearch/index/mapper/TextFieldMapperTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/TextFieldMapperTests.java @@ -64,7 +64,9 @@ import org.apache.lucene.tests.analysis.Token; import org.apache.lucene.util.BytesRef; import org.opensearch.common.lucene.search.MultiPhrasePrefixQuery; +import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.common.Strings; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; @@ -87,6 +89,7 @@ import java.util.Arrays; import java.util.Collections; import java.util.HashMap; +import java.util.List; import java.util.Map; import static org.hamcrest.Matchers.containsString; @@ -208,6 +211,7 @@ protected void registerParameters(ParameterChecker checker) throws IOException { checker.registerUpdateCheck(b -> b.field("boost", 2.0), m -> assertEquals(m.fieldType().boost(), 2.0, 0)); + checker.registerUpdateCheck(b -> b.field("multivalued", true), m -> assertTrue(((TextFieldMapper) m).multivalued())); } @Override @@ -1066,4 +1070,36 @@ public void testSimpleMerge() throws IOException { assertThat(mapperService.documentMapper().mappers().getMapper("field"), instanceOf(TextFieldMapper.class)); assertThat(mapperService.documentMapper().mappers().getMapper("other_field"), instanceOf(KeywordFieldMapper.class)); } + + public void testMultivalued() throws Exception { + DocumentMapper mapper = createDocumentMapper(fieldMapping(b -> b.field("type", textFieldName).field("multivalued", true))); + ThrowingRunnable runnable = () -> mapper.parse( + new SourceToParse( + "test", + "1", + BytesReference.bytes(XContentFactory.jsonBuilder().startObject().field("field", "Hello World").endObject()), + MediaTypeRegistry.JSON + ) + ); + MapperParsingException e = expectThrows(MapperParsingException.class, runnable); + assertThat( + e.getMessage(), + containsString("object mapping [field] trying to serialize a scalar value [Hello World] for a multi-valued field") + ); + + ParsedDocument doc = mapper.parse( + new SourceToParse( + "test", + "1", + BytesReference.bytes( + XContentFactory.jsonBuilder().startObject().field("field", List.of("Hello World", "abcdef")).endObject() + ), + MediaTypeRegistry.JSON + ) + ); + + IndexableField[] fields = doc.rootDoc().getFields("field"); + assertEquals(2, fields.length); + } + } diff --git a/server/src/test/java/org/opensearch/index/mapper/WildcardFieldMapperTests.java b/server/src/test/java/org/opensearch/index/mapper/WildcardFieldMapperTests.java index a93f6b2d47e4f..b95d8251f8eb3 100644 --- a/server/src/test/java/org/opensearch/index/mapper/WildcardFieldMapperTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/WildcardFieldMapperTests.java @@ -23,6 +23,9 @@ import org.opensearch.Version; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.common.settings.Settings; +import org.opensearch.common.xcontent.XContentFactory; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.index.IndexSettings; import org.opensearch.index.analysis.AnalyzerScope; @@ -43,6 +46,7 @@ import static java.util.Collections.singletonMap; import static org.opensearch.index.mapper.FieldTypeTestCase.fetchSourceValue; +import static org.hamcrest.Matchers.containsString; public class WildcardFieldMapperTests extends MapperTestCase { @@ -62,6 +66,7 @@ protected void registerParameters(ParameterChecker checker) throws IOException { checker.registerConflictCheck("doc_values", b -> b.field("doc_values", true)); checker.registerConflictCheck("null_value", b -> b.field("null_value", "foo")); checker.registerUpdateCheck(b -> b.field("ignore_above", 256), m -> assertEquals(256, ((WildcardFieldMapper) m).ignoreAbove())); + checker.registerUpdateCheck(b -> b.field("multivalued", true), m -> assertTrue(((WildcardFieldMapper) m).multivalued())); } public void testTokenizer() throws IOException { @@ -330,4 +335,35 @@ public void testFetchSourceValue() throws IOException { MappedFieldType nullValueMapper = new WildcardFieldMapper.Builder("field").nullValue("NULL").build(context).fieldType(); assertEquals(Collections.singletonList("NULL"), fetchSourceValue(nullValueMapper, null)); } + + public void testMultivalued() throws Exception { + DocumentMapper mapper = createDocumentMapper(fieldMapping(b -> b.field("type", "wildcard").field("multivalued", true))); + ThrowingRunnable runnable = () -> mapper.parse( + new SourceToParse( + "test", + "1", + BytesReference.bytes(XContentFactory.jsonBuilder().startObject().field("field", "Hello World").endObject()), + MediaTypeRegistry.JSON + ) + ); + MapperParsingException e = expectThrows(MapperParsingException.class, runnable); + assertThat( + e.getMessage(), + containsString("object mapping [field] trying to serialize a scalar value [Hello World] for a multi-valued field") + ); + + ParsedDocument doc = mapper.parse( + new SourceToParse( + "test", + "1", + BytesReference.bytes( + XContentFactory.jsonBuilder().startObject().field("field", List.of("Hello World", "abcdef")).endObject() + ), + MediaTypeRegistry.JSON + ) + ); + + IndexableField[] fields = doc.rootDoc().getFields("field"); + assertEquals(2, fields.length); + } } diff --git a/test/framework/src/main/java/org/opensearch/index/mapper/AbstractNumericFieldMapperTestCase.java b/test/framework/src/main/java/org/opensearch/index/mapper/AbstractNumericFieldMapperTestCase.java index 79b826c123a20..20c00daeff7fb 100644 --- a/test/framework/src/main/java/org/opensearch/index/mapper/AbstractNumericFieldMapperTestCase.java +++ b/test/framework/src/main/java/org/opensearch/index/mapper/AbstractNumericFieldMapperTestCase.java @@ -102,4 +102,12 @@ public final void testNullValue() throws IOException { } protected abstract void doTestNullValue(String type) throws IOException; + + public final void testMultivalued() throws IOException { + for (String type : types()) { + doTestMultivalued(type); + } + } + + protected abstract void doTestMultivalued(String type) throws IOException; }