Skip to content

Commit

Permalink
Add a multivalued property to field mappings (#16420)
Browse files Browse the repository at this point in the history 
* Can only be used for field types that support multiple values
* If a field has the multivalued property, then new documents must have an array for its value

Signed-off-by: Norman Jordan <[email protected]>
  • Loading branch information
@normanj-bitquill
normanj-bitquill committed Nov 8, 2024
1 parent 5909e1a commit 70d0100
Show file tree
Hide file tree
Showing 37 changed files with 840 additions and 23 deletions.
17 changes: 16 additions & 1 deletion modules/mapper-extras/src/main/java/org/opensearch/index/mapper/ScaledFloatFieldMapper.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -120,6 +120,8 @@ public static class Builder extends ParametrizedFieldMapper.Builder {
m -> toType(m).nullValue
).acceptsNull();

private final Parameter<Explicit<Boolean>> multivalued;

private final Parameter<Map<String, String>> meta = Parameter.metaParam();

public Builder(String name, Settings settings) {
Expand All @@ -135,6 +137,7 @@ public Builder(String name, boolean ignoreMalformedByDefault, boolean coerceByDe
ignoreMalformedByDefault
);
this.coerce = Parameter.explicitBoolParam("coerce", true, m -> toType(m).coerce, coerceByDefault);
this.multivalued = Parameter.explicitBoolParam("multivalued", true, m -> toType(m).multivalued, false);
}

Builder scalingFactor(double scalingFactor) {
Expand All @@ -149,7 +152,7 @@ Builder nullValue(double nullValue) {

@Override
protected List<Parameter<?>> getParameters() {
return Arrays.asList(indexed, hasDocValues, stored, ignoreMalformed, meta, scalingFactor, coerce, nullValue);
return Arrays.asList(indexed, hasDocValues, stored, ignoreMalformed, meta, scalingFactor, coerce, nullValue, multivalued);
}

@Override
Expand Down Expand Up @@ -372,6 +375,8 @@ public double toDoubleValue(long value) {
private final boolean ignoreMalformedByDefault;
private final boolean coerceByDefault;

private final Explicit<Boolean> multivalued;

private ScaledFloatFieldMapper(
String simpleName,
ScaledFloatFieldType mappedFieldType,
Expand All @@ -389,6 +394,7 @@ private ScaledFloatFieldMapper(
this.coerce = builder.coerce.getValue();
this.ignoreMalformedByDefault = builder.ignoreMalformed.getDefaultValue().value();
this.coerceByDefault = builder.coerce.getDefaultValue().value();
this.multivalued = builder.multivalued.getValue();
}

boolean coerce() {
Expand All @@ -399,6 +405,10 @@ boolean ignoreMalformed() {
return ignoreMalformed.value();
}

boolean multivalued() {
return multivalued.value();
}

@Override
public ScaledFloatFieldType fieldType() {
return (ScaledFloatFieldType) super.fieldType();
Expand All @@ -414,6 +424,11 @@ public ParametrizedFieldMapper.Builder getMergeBuilder() {
return new Builder(simpleName(), ignoreMalformedByDefault, coerceByDefault).init(this);
}

@Override
public boolean isMultivalue() {
return multivalued.explicit() && multivalued.value() != null && multivalued.value();
}

@Override
protected ScaledFloatFieldMapper clone() {
return (ScaledFloatFieldMapper) super.clone();
Expand Down
20 changes: 20 additions & 0 deletions ...s/mapper-extras/src/main/java/org/opensearch/index/mapper/SearchAsYouTypeFieldMapper.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,7 @@
import org.apache.lucene.util.automaton.Automata;
import org.apache.lucene.util.automaton.Automaton;
import org.apache.lucene.util.automaton.Operations;
import org.opensearch.common.Explicit;
import org.opensearch.common.collect.Iterators;
import org.opensearch.common.lucene.search.AutomatonQueries;
import org.opensearch.index.analysis.AnalyzerScope;
Expand Down Expand Up @@ -156,6 +157,12 @@ public static class Builder extends ParametrizedFieldMapper.Builder {
final Parameter<String> indexOptions = TextParams.indexOptions(m -> toType(m).indexOptions);
final Parameter<Boolean> norms = TextParams.norms(true, m -> ft(m).getTextSearchInfo().hasNorms());
final Parameter<String> termVectors = TextParams.termVectors(m -> toType(m).termVectors);
final Parameter<Explicit<Boolean>> multivalued = Parameter.explicitBoolParam(
"multivalued",
true,
m -> toType(m).multivalued,
false
);

private final Parameter<Map<String, String>> meta = Parameter.metaParam();

Expand All @@ -178,6 +185,7 @@ protected List<Parameter<?>> getParameters() {
indexOptions,
norms,
termVectors,
multivalued,
meta
);
}
Expand Down Expand Up @@ -628,6 +636,8 @@ public SpanQuery spanPrefixQuery(String value, SpanMultiTermQueryWrapper.SpanRew

private final IndexAnalyzers indexAnalyzers;

private final Explicit<Boolean> multivalued;

public SearchAsYouTypeFieldMapper(
String simpleName,
SearchAsYouTypeFieldType mappedFieldType,
Expand All @@ -646,6 +656,7 @@ public SearchAsYouTypeFieldMapper(
this.indexOptions = builder.indexOptions.getValue();
this.termVectors = builder.termVectors.getValue();
this.indexAnalyzers = builder.analyzers.indexAnalyzers;
this.multivalued = builder.multivalued.getValue();
}

@Override
Expand Down Expand Up @@ -684,6 +695,15 @@ public SearchAsYouTypeFieldType fieldType() {
return (SearchAsYouTypeFieldType) super.fieldType();
}

boolean multivalued() {
return multivalued.value();
}

@Override
public boolean isMultivalue() {
return multivalued.explicit() && multivalued.value() != null && multivalued.value();
}

public int maxShingleSize() {
return maxShingleSize;
}
Expand Down
32 changes: 32 additions & 0 deletions .../mapper-extras/src/test/java/org/opensearch/index/mapper/ScaledFloatFieldMapperTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,7 @@ protected void registerParameters(ParameterChecker checker) throws IOException {
b -> b.field("ignore_malformed", true),
m -> assertTrue(((ScaledFloatFieldMapper) m).ignoreMalformed())
);
checker.registerUpdateCheck(b -> b.field("multivalued", true), m -> assertTrue(((ScaledFloatFieldMapper) m).multivalued()));
}

public void testExistsQueryDocValuesDisabled() throws IOException {
Expand Down Expand Up @@ -359,6 +360,37 @@ private void doTestIgnoreMalformed(Object value, String exceptionMessageContains
assertEquals(0, fields.length);
}

public void testMultivalued() throws Exception {
DocumentMapper mapper = createDocumentMapper(
fieldMapping(b -> b.field("type", "scaled_float").field("scaling_factor", 10.0).field("multivalued", true))
);
ThrowingRunnable runnable = () -> mapper.parse(
new SourceToParse(
"test",
"1",
BytesReference.bytes(XContentFactory.jsonBuilder().startObject().field("field", "1.34").endObject()),
MediaTypeRegistry.JSON
)
);
MapperParsingException e = expectThrows(MapperParsingException.class, runnable);
assertThat(
e.getMessage(),
containsString("object mapping [field] trying to serialize a scalar value [1.34] for a multi-valued field")
);

ParsedDocument doc = mapper.parse(
new SourceToParse(
"test",
"1",
BytesReference.bytes(XContentFactory.jsonBuilder().startObject().field("field", List.of("1.34", "2.35")).endObject()),
MediaTypeRegistry.JSON
)
);

IndexableField[] fields = doc.rootDoc().getFields("field");
assertEquals(4, fields.length);
}

public void testNullValue() throws IOException {
DocumentMapper mapper = createDocumentMapper(fieldMapping(this::minimalMapping));
ParsedDocument doc = mapper.parse(
Expand Down
33 changes: 33 additions & 0 deletions ...per-extras/src/test/java/org/opensearch/index/mapper/SearchAsYouTypeFieldMapperTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,9 @@
import org.apache.lucene.search.SynonymQuery;
import org.apache.lucene.search.TermQuery;
import org.opensearch.common.lucene.search.MultiPhrasePrefixQuery;
import org.opensearch.common.xcontent.XContentFactory;
import org.opensearch.core.common.Strings;
import org.opensearch.core.common.bytes.BytesReference;
import org.opensearch.core.xcontent.MediaTypeRegistry;
import org.opensearch.core.xcontent.XContentBuilder;
import org.opensearch.index.IndexSettings;
Expand Down Expand Up @@ -129,6 +131,8 @@ protected void registerParameters(ParameterChecker checker) throws IOException {
b.field("search_quote_analyzer", "keyword");
}, m -> assertEquals("keyword", m.fieldType().getTextSearchInfo().getSearchQuoteAnalyzer().name()));

checker.registerUpdateCheck(b -> b.field("multivalued", true), m -> assertTrue(((SearchAsYouTypeFieldMapper) m).multivalued()));

}

protected void writeFieldValue(XContentBuilder builder) throws IOException {
Expand Down Expand Up @@ -636,6 +640,35 @@ public void testMultiMatchBoolPrefix() throws IOException {
);
}

public void testMultivalued() throws Exception {
DocumentMapper mapper = createDocumentMapper(fieldMapping(b -> b.field("type", "search_as_you_type").field("multivalued", true)));
ThrowingRunnable runnable = () -> mapper.parse(
new SourceToParse(
"test",
"1",
BytesReference.bytes(XContentFactory.jsonBuilder().startObject().field("field", "foo").endObject()),
MediaTypeRegistry.JSON
)
);
MapperParsingException e = expectThrows(MapperParsingException.class, runnable);
assertThat(
e.getMessage(),
containsString("object mapping [field] trying to serialize a scalar value [foo] for a multi-valued field")
);

ParsedDocument doc = mapper.parse(
new SourceToParse(
"test",
"1",
BytesReference.bytes(XContentFactory.jsonBuilder().startObject().field("field", List.of("foo", "bar")).endObject()),
MediaTypeRegistry.JSON
)
);

IndexableField[] fields = doc.rootDoc().getFields("field");
assertEquals(2, fields.length);
}

public void testAnalyzerSerialization() throws IOException {
MapperService ms = createMapperService(fieldMapping(b -> {
b.field("type", "search_as_you_type");
Expand Down
39 changes: 39 additions & 0 deletions server/src/main/java/org/opensearch/index/mapper/AbstractGeometryFieldMapper.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,7 @@ public abstract class AbstractGeometryFieldMapper<Parsed, Processed> extends Fie
public static class Names {
public static final ParseField IGNORE_MALFORMED = new ParseField("ignore_malformed");
public static final ParseField IGNORE_Z_VALUE = new ParseField("ignore_z_value");
public static final ParseField MULTIVALUED = new ParseField("multivalued");
}

/**
Expand All @@ -85,6 +86,7 @@ public static class Names {
public static class Defaults {
public static final Explicit<Boolean> IGNORE_MALFORMED = new Explicit<>(false, false);
public static final Explicit<Boolean> IGNORE_Z_VALUE = new Explicit<>(true, false);
public static final Explicit<Boolean> MULTIVALUED = new Explicit<>(false, false);
public static final FieldType FIELD_TYPE = new FieldType();
static {
FIELD_TYPE.setStored(false);
Expand Down Expand Up @@ -166,6 +168,7 @@ public abstract static class Builder<T extends Builder<T, FT>, FT extends Abstra
protected Boolean ignoreMalformed;
protected Boolean ignoreZValue;
protected boolean indexed = true;
protected Boolean multivalued;

public Builder(String name, FieldType fieldType) {
super(name, fieldType);
Expand Down Expand Up @@ -217,6 +220,18 @@ public Builder ignoreZValue(final boolean ignoreZValue) {
this.ignoreZValue = ignoreZValue;
return this;
}

public Explicit<Boolean> multivalued() {
if (multivalued != null) {
return new Explicit<>(multivalued, true);
}
return Defaults.MULTIVALUED;
}

public Builder multivalued(boolean multivalued) {
this.multivalued = multivalued;
return this;
}
}

/**
Expand Down Expand Up @@ -245,6 +260,12 @@ public T parse(String name, Map<String, Object> node, Map<String, Object> params
XContentMapValues.nodeBooleanValue(propNode, name + "." + Names.IGNORE_Z_VALUE.getPreferredName())
);
iterator.remove();
} else if (Names.MULTIVALUED.getPreferredName().equals(propName)) {
params.put(
Names.MULTIVALUED.getPreferredName(),
XContentMapValues.nodeBooleanValue(propNode, name + "." + Names.MULTIVALUED.getPreferredName())
);
iterator.remove();
}
}

Expand All @@ -257,6 +278,10 @@ public T parse(String name, Map<String, Object> node, Map<String, Object> params
if (params.containsKey(Names.IGNORE_MALFORMED.getPreferredName())) {
builder.ignoreMalformed((Boolean) params.get(Names.IGNORE_MALFORMED.getPreferredName()));
}

if (params.containsKey(Names.MULTIVALUED.getPreferredName())) {
builder.multivalued((Boolean) params.get(Names.MULTIVALUED.getPreferredName()));
}
return builder;
}

Expand Down Expand Up @@ -344,19 +369,22 @@ protected Object parseSourceValue(Object value) {

protected Explicit<Boolean> ignoreMalformed;
protected Explicit<Boolean> ignoreZValue;
protected Explicit<Boolean> multivalued;

protected AbstractGeometryFieldMapper(
String simpleName,
FieldType fieldType,
MappedFieldType mappedFieldType,
Explicit<Boolean> ignoreMalformed,
Explicit<Boolean> ignoreZValue,
Explicit<Boolean> multivalued,
MultiFields multiFields,
CopyTo copyTo
) {
super(simpleName, fieldType, mappedFieldType, multiFields, copyTo);
this.ignoreMalformed = ignoreMalformed;
this.ignoreZValue = ignoreZValue;
this.multivalued = multivalued;
}

@Override
Expand All @@ -369,6 +397,9 @@ protected void mergeOptions(FieldMapper other, List<String> conflicts) {
if (gsfm.ignoreZValue.explicit()) {
this.ignoreZValue = gsfm.ignoreZValue;
}
if (gsfm.multivalued.explicit()) {
this.multivalued = gsfm.multivalued;
}
}

@Override
Expand Down Expand Up @@ -450,6 +481,9 @@ public void doXContentBody(XContentBuilder builder, boolean includeDefaults, Par
if (includeDefaults || ignoreZValue.explicit()) {
builder.field(Names.IGNORE_Z_VALUE.getPreferredName(), ignoreZValue.value());
}
if (includeDefaults || multivalued.explicit()) {
builder.field(Names.MULTIVALUED.getPreferredName(), multivalued.value());
}
}

public Explicit<Boolean> ignoreMalformed() {
Expand All @@ -459,4 +493,9 @@ public Explicit<Boolean> ignoreMalformed() {
public Explicit<Boolean> ignoreZValue() {
return ignoreZValue;
}

@Override
public boolean isMultivalue() {
return multivalued.explicit() && multivalued.value() != null && multivalued.value();
}
}
5 changes: 4 additions & 1 deletion server/src/main/java/org/opensearch/index/mapper/AbstractPointGeometryFieldMapper.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -104,6 +104,7 @@ public abstract AbstractPointGeometryFieldMapper build(
MultiFields multiFields,
Explicit<Boolean> ignoreMalformed,
Explicit<Boolean> ignoreZValue,
Explicit<Boolean> multivalued,
ParsedPoint nullValue,
CopyTo copyTo
);
Expand All @@ -117,6 +118,7 @@ public AbstractPointGeometryFieldMapper build(BuilderContext context) {
multiFieldsBuilder.build(this, context),
ignoreMalformed(context),
ignoreZValue(context),
multivalued(),
nullValue,
copyTo
);
Expand Down Expand Up @@ -183,10 +185,11 @@ protected AbstractPointGeometryFieldMapper(
MultiFields multiFields,
Explicit<Boolean> ignoreMalformed,
Explicit<Boolean> ignoreZValue,
Explicit<Boolean> multivalued,
ParsedPoint nullValue,
CopyTo copyTo
) {
super(simpleName, fieldType, mappedFieldType, ignoreMalformed, ignoreZValue, multiFields, copyTo);
super(simpleName, fieldType, mappedFieldType, ignoreMalformed, ignoreZValue, multivalued, multiFields, copyTo);
this.nullValue = nullValue;
}

Expand Down
3 changes: 2 additions & 1 deletion server/src/main/java/org/opensearch/index/mapper/AbstractShapeGeometryFieldMapper.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -239,10 +239,11 @@ protected AbstractShapeGeometryFieldMapper(
Explicit<Boolean> coerce,
Explicit<Boolean> ignoreZValue,
Explicit<Orientation> orientation,
Explicit<Boolean> multivalued,
MultiFields multiFields,
CopyTo copyTo
) {
super(simpleName, fieldType, mappedFieldType, ignoreMalformed, ignoreZValue, multiFields, copyTo);
super(simpleName, fieldType, mappedFieldType, ignoreMalformed, ignoreZValue, multivalued, multiFields, copyTo);
this.coerce = coerce;
this.orientation = orientation;
}
Expand Down
Loading

0 comments on commit 70d0100

Please sign in to comment.