forked from opensearch-project/OpenSearch
-
Notifications
You must be signed in to change notification settings - Fork 2
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
[DerivedFields] PR2 - Implementation for all supported types and Deri…
…vedFieldType (opensearch-project#12808) Implementation for all supported types and DerivedFieldType. We support the following types so far: boolean date geo_point ip keyword long double --------- Signed-off-by: Rishabh Maurya <[email protected]>
- Loading branch information
1 parent
5db84d1
commit 6ddbdcd
Showing
4 changed files
with
614 additions
and
1 deletion.
There are no files selected for viewing
156 changes: 156 additions & 0 deletions
156
server/src/main/java/org/opensearch/index/mapper/DerivedFieldSupportedTypes.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,156 @@ | ||
/* | ||
* SPDX-License-Identifier: Apache-2.0 | ||
* | ||
* The OpenSearch Contributors require contributions made to | ||
* this file be licensed under the Apache-2.0 license or a | ||
* compatible open source license. | ||
*/ | ||
|
||
package org.opensearch.index.mapper; | ||
|
||
import org.apache.lucene.document.DoubleField; | ||
import org.apache.lucene.document.Field; | ||
import org.apache.lucene.document.FieldType; | ||
import org.apache.lucene.document.InetAddressPoint; | ||
import org.apache.lucene.document.KeywordField; | ||
import org.apache.lucene.document.LatLonPoint; | ||
import org.apache.lucene.document.LongField; | ||
import org.apache.lucene.document.LongPoint; | ||
import org.apache.lucene.index.IndexOptions; | ||
import org.apache.lucene.index.IndexableField; | ||
import org.opensearch.Version; | ||
import org.opensearch.common.Booleans; | ||
import org.opensearch.common.lucene.Lucene; | ||
import org.opensearch.common.network.InetAddresses; | ||
|
||
import java.net.InetAddress; | ||
import java.util.Arrays; | ||
import java.util.List; | ||
import java.util.Map; | ||
import java.util.function.BiFunction; | ||
import java.util.function.Function; | ||
import java.util.stream.Collectors; | ||
|
||
/** | ||
* Contains logic to get the FieldMapper for a given type of derived field. Also, for a given type of derived field, | ||
* it is used to create an IndexableField for the provided type and object. It is useful when indexing into | ||
* lucene MemoryIndex in {@link org.opensearch.index.query.DerivedFieldQuery}. | ||
*/ | ||
enum DerivedFieldSupportedTypes { | ||
|
||
BOOLEAN("boolean", (name, context) -> { | ||
BooleanFieldMapper.Builder builder = new BooleanFieldMapper.Builder(name); | ||
return builder.build(context); | ||
}, name -> o -> { | ||
// Trying to mimic the logic for parsing source value as used in BooleanFieldMapper valueFetcher | ||
Boolean value; | ||
if (o instanceof Boolean) { | ||
value = (Boolean) o; | ||
} else { | ||
String textValue = o.toString(); | ||
value = Booleans.parseBooleanStrict(textValue, false); | ||
} | ||
return new Field(name, value ? "T" : "F", BooleanFieldMapper.Defaults.FIELD_TYPE); | ||
}), | ||
DATE("date", (name, context) -> { | ||
// TODO: should we support mapping settings exposed by a given field type from derived fields too? | ||
// for example, support `format` for date type? | ||
DateFieldMapper.Builder builder = new DateFieldMapper.Builder( | ||
name, | ||
DateFieldMapper.Resolution.MILLISECONDS, | ||
DateFieldMapper.getDefaultDateTimeFormatter(), | ||
false, | ||
Version.CURRENT | ||
); | ||
return builder.build(context); | ||
}, name -> o -> new LongPoint(name, (long) o)), | ||
GEO_POINT("geo_point", (name, context) -> { | ||
GeoPointFieldMapper.Builder builder = new GeoPointFieldMapper.Builder(name); | ||
return builder.build(context); | ||
}, name -> o -> { | ||
// convert o to array of double | ||
if (!(o instanceof List) || ((List<?>) o).size() != 2 || !(((List<?>) o).get(0) instanceof Double)) { | ||
throw new ClassCastException("geo_point should be in format emit(double lat, double lon) for derived fields"); | ||
} | ||
return new LatLonPoint(name, (Double) ((List<?>) o).get(0), (Double) ((List<?>) o).get(1)); | ||
}), | ||
IP("ip", (name, context) -> { | ||
IpFieldMapper.Builder builder = new IpFieldMapper.Builder(name, false, Version.CURRENT); | ||
return builder.build(context); | ||
}, name -> o -> { | ||
InetAddress address; | ||
if (o instanceof InetAddress) { | ||
address = (InetAddress) o; | ||
} else { | ||
address = InetAddresses.forString(o.toString()); | ||
} | ||
return new InetAddressPoint(name, address); | ||
}), | ||
KEYWORD("keyword", (name, context) -> { | ||
FieldType dummyFieldType = new FieldType(); | ||
dummyFieldType.setIndexOptions(IndexOptions.DOCS_AND_FREQS); | ||
KeywordFieldMapper.Builder keywordBuilder = new KeywordFieldMapper.Builder(name); | ||
KeywordFieldMapper.KeywordFieldType keywordFieldType = keywordBuilder.buildFieldType(context, dummyFieldType); | ||
keywordFieldType.setIndexAnalyzer(Lucene.KEYWORD_ANALYZER); | ||
return new KeywordFieldMapper( | ||
name, | ||
dummyFieldType, | ||
keywordFieldType, | ||
keywordBuilder.multiFieldsBuilder.build(keywordBuilder, context), | ||
keywordBuilder.copyTo.build(), | ||
keywordBuilder | ||
); | ||
}, name -> o -> new KeywordField(name, (String) o, Field.Store.NO)), | ||
LONG("long", (name, context) -> { | ||
NumberFieldMapper.Builder longBuilder = new NumberFieldMapper.Builder(name, NumberFieldMapper.NumberType.LONG, false, false); | ||
return longBuilder.build(context); | ||
}, name -> o -> new LongField(name, Long.parseLong(o.toString()), Field.Store.NO)), | ||
DOUBLE("double", (name, context) -> { | ||
NumberFieldMapper.Builder doubleBuilder = new NumberFieldMapper.Builder(name, NumberFieldMapper.NumberType.DOUBLE, false, false); | ||
return doubleBuilder.build(context); | ||
}, name -> o -> new DoubleField(name, Double.parseDouble(o.toString()), Field.Store.NO)); | ||
|
||
final String name; | ||
private final BiFunction<String, Mapper.BuilderContext, FieldMapper> builder; | ||
|
||
private final Function<String, Function<Object, IndexableField>> indexableFieldBuilder; | ||
|
||
DerivedFieldSupportedTypes( | ||
String name, | ||
BiFunction<String, Mapper.BuilderContext, FieldMapper> builder, | ||
Function<String, Function<Object, IndexableField>> indexableFieldBuilder | ||
) { | ||
this.name = name; | ||
this.builder = builder; | ||
this.indexableFieldBuilder = indexableFieldBuilder; | ||
} | ||
|
||
public String getName() { | ||
return name; | ||
} | ||
|
||
private FieldMapper getFieldMapper(String name, Mapper.BuilderContext context) { | ||
return builder.apply(name, context); | ||
} | ||
|
||
private Function<Object, IndexableField> getIndexableFieldGenerator(String name) { | ||
return indexableFieldBuilder.apply(name); | ||
} | ||
|
||
private static final Map<String, DerivedFieldSupportedTypes> enumMap = Arrays.stream(DerivedFieldSupportedTypes.values()) | ||
.collect(Collectors.toMap(DerivedFieldSupportedTypes::getName, enumValue -> enumValue)); | ||
|
||
public static FieldMapper getFieldMapperFromType(String type, String name, Mapper.BuilderContext context) { | ||
if (!enumMap.containsKey(type)) { | ||
throw new IllegalArgumentException("Type [" + type + "] isn't supported in Derived field context."); | ||
} | ||
return enumMap.get(type).getFieldMapper(name, context); | ||
} | ||
|
||
public static Function<Object, IndexableField> getIndexableFieldGeneratorType(String type, String name) { | ||
if (!enumMap.containsKey(type)) { | ||
throw new IllegalArgumentException("Type [" + type + "] isn't supported in Derived field context."); | ||
} | ||
return enumMap.get(type).getIndexableFieldGenerator(name); | ||
} | ||
} |
Oops, something went wrong.