-
Notifications
You must be signed in to change notification settings - Fork 1.9k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Signed-off-by: Rishabh Maurya <[email protected]>
- Loading branch information
1 parent
5f5f54d
commit 2f45c5a
Showing
14 changed files
with
930 additions
and
221 deletions.
There are no files selected for viewing
185 changes: 185 additions & 0 deletions
185
server/src/main/java/org/opensearch/index/mapper/DefaultDerivedFieldResolver.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,185 @@ | ||
| /* | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| * | ||
| * The OpenSearch Contributors require contributions made to | ||
| * this file be licensed under the Apache-2.0 license or a | ||
| * compatible open source license. | ||
| */ | ||
|
|
||
| package org.opensearch.index.mapper; | ||
|
|
||
| import org.apache.logging.log4j.LogManager; | ||
| import org.apache.logging.log4j.Logger; | ||
| import org.opensearch.common.regex.Regex; | ||
| import org.opensearch.index.query.QueryShardContext; | ||
| import org.opensearch.script.Script; | ||
|
|
||
| import java.io.IOException; | ||
| import java.util.HashMap; | ||
| import java.util.HashSet; | ||
| import java.util.List; | ||
| import java.util.Map; | ||
| import java.util.Optional; | ||
| import java.util.Set; | ||
| import java.util.concurrent.ConcurrentHashMap; | ||
|
|
||
| /** | ||
| * Accepts definition of DerivedField from search request in both forms: map parsed from SearchRequest and {@link DerivedField} defined using client. | ||
| * The object is initialized per search request and is responsible to resolve {@link DerivedFieldType} given a field name. | ||
| * It uses {@link FieldTypeInference} to infer field type for a nested field within DerivedField of {@link DerivedFieldSupportedTypes#OBJECT} type. | ||
| */ | ||
| public class DefaultDerivedFieldResolver implements DerivedFieldResolver { | ||
| private final QueryShardContext queryShardContext; | ||
| private final Map<String, DerivedFieldType> derivedFieldTypeMap = new ConcurrentHashMap<>(); | ||
| private final FieldTypeInference typeInference; | ||
| private static final Logger logger = LogManager.getLogger(DefaultDerivedFieldResolver.class); | ||
|
|
||
| public DefaultDerivedFieldResolver( | ||
| QueryShardContext queryShardContext, | ||
| Map<String, Object> derivedFieldsObject, | ||
| List<DerivedField> derivedFields | ||
| ) { | ||
| this( | ||
| queryShardContext, | ||
| derivedFieldsObject, | ||
| derivedFields, | ||
| new FieldTypeInference( | ||
| queryShardContext.index().getName(), | ||
| queryShardContext.getMapperService(), | ||
| queryShardContext.getIndexReader() | ||
| ) | ||
| ); | ||
| } | ||
|
|
||
| public DefaultDerivedFieldResolver( | ||
| QueryShardContext queryShardContext, | ||
| Map<String, Object> derivedFieldsObject, | ||
| List<DerivedField> derivedFields, | ||
| FieldTypeInference typeInference | ||
| ) { | ||
| this.queryShardContext = queryShardContext; | ||
| initDerivedFieldTypes(derivedFieldsObject, derivedFields); | ||
| this.typeInference = typeInference; | ||
| } | ||
|
|
||
| @Override | ||
| public Set<String> resolvePattern(String pattern) { | ||
| Set<String> derivedFields = new HashSet<>(); | ||
| for (MappedFieldType fieldType : queryShardContext.getMapperService().fieldTypes()) { | ||
| if (Regex.simpleMatch(pattern, fieldType.name()) && fieldType instanceof DerivedFieldType) { | ||
| derivedFields.add(fieldType.name()); | ||
| } | ||
| } | ||
| for (String fieldName : derivedFieldTypeMap.keySet()) { | ||
| if (Regex.simpleMatch(pattern, fieldName)) { | ||
| derivedFields.add(fieldName); | ||
| } | ||
| } | ||
| return derivedFields; | ||
| } | ||
|
|
||
| /** | ||
| * Resolves the fieldName. The search request definitions are given precedence over derived fields definitions in the index mapping. | ||
| * It caches the response for previously resolved field names | ||
| * @param fieldName name of the field. It also accepts nested derived field | ||
| * @return DerivedFieldType if resolved successfully, a null otherwise. | ||
| */ | ||
| @Override | ||
| public DerivedFieldType resolve(String fieldName) { | ||
| return Optional.ofNullable(resolveUsingSearchDefinitions(fieldName)) | ||
| .orElseGet(() -> (DerivedFieldType) queryShardContext.getMapperService().fieldType(fieldName)); | ||
| } | ||
|
|
||
| private DerivedFieldType resolveUsingSearchDefinitions(String fieldName) { | ||
| if (derivedFieldTypeMap.containsKey(fieldName)) { | ||
| return derivedFieldTypeMap.get(fieldName); | ||
| } | ||
| DerivedFieldType resolvedNestedType = resolveNestedField(fieldName); | ||
| if (resolvedNestedType != null) { | ||
| derivedFieldTypeMap.put(fieldName, resolvedNestedType); | ||
| } | ||
| return resolvedNestedType; | ||
| } | ||
|
|
||
| private DerivedFieldType resolveNestedField(String fieldName) { | ||
| DerivedFieldType parentDerivedField = (DerivedFieldType) getParentDerivedField(fieldName); | ||
| if (parentDerivedField != null) { | ||
| try { | ||
| Script script = parentDerivedField.derivedField.getScript(); | ||
| Mapper inferredFieldMapper = typeInference.infer(getValueFetcher(fieldName, script)); | ||
| if (inferredFieldMapper != null) { | ||
| return getDerivedFieldType( | ||
| new DerivedField( | ||
| fieldName, | ||
| inferredFieldMapper.typeName(), | ||
| script, | ||
| parentDerivedField.derivedField.getSourceIndexedField() | ||
| ) | ||
| ); | ||
| } else { | ||
| logger.warn("Field type cannot be inferred. Ensure the field {} is not rare across entire index", fieldName); | ||
| } | ||
| } catch (IOException e) { | ||
| logger.warn(e); | ||
| } | ||
| } | ||
| return null; | ||
| } | ||
|
|
||
| private MappedFieldType getParentDerivedField(String fieldName) { | ||
| if (fieldName.contains(".")) { | ||
| return resolve(fieldName.split("\\.")[0]); | ||
| } | ||
| return null; | ||
| } | ||
|
|
||
| ValueFetcher getValueFetcher(String fieldName, Script script) { | ||
| String subFieldName = fieldName.substring(fieldName.indexOf(".") + 1); | ||
| return new ObjectDerivedFieldType.ObjectDerivedFieldValueFetcher( | ||
| subFieldName, | ||
| DerivedFieldType.getDerivedFieldLeafFactory(script, queryShardContext, queryShardContext.lookup()), | ||
| o -> o // raw object returned will be used to infer the type without modifying it | ||
| ); | ||
| } | ||
|
|
||
| private void initDerivedFieldTypes(Map<String, Object> derivedFieldsObject, List<DerivedField> derivedFields) { | ||
| if (derivedFieldsObject != null && !derivedFieldsObject.isEmpty()) { | ||
| Map<String, Object> derivedFieldObject = new HashMap<>(); | ||
| derivedFieldObject.put(DerivedFieldMapper.CONTENT_TYPE, derivedFieldsObject); | ||
| derivedFieldTypeMap.putAll(getAllDerivedFieldTypeFromObject(derivedFieldObject)); | ||
| } | ||
| if (derivedFields != null) { | ||
| for (DerivedField derivedField : derivedFields) { | ||
| derivedFieldTypeMap.put(derivedField.getName(), getDerivedFieldType(derivedField)); | ||
| } | ||
| } | ||
| } | ||
|
|
||
| private Map<String, DerivedFieldType> getAllDerivedFieldTypeFromObject(Map<String, Object> derivedFieldObject) { | ||
| Map<String, DerivedFieldType> derivedFieldTypes = new HashMap<>(); | ||
| DocumentMapper documentMapper = queryShardContext.getMapperService() | ||
| .documentMapperParser() | ||
| .parse(DerivedFieldMapper.CONTENT_TYPE, derivedFieldObject); | ||
| if (documentMapper != null && documentMapper.mappers() != null) { | ||
| for (Mapper mapper : documentMapper.mappers()) { | ||
| if (mapper instanceof DerivedFieldMapper) { | ||
| DerivedFieldType derivedFieldType = ((DerivedFieldMapper) mapper).fieldType(); | ||
| derivedFieldTypes.put(derivedFieldType.name(), derivedFieldType); | ||
| } | ||
| } | ||
| } | ||
| return derivedFieldTypes; | ||
| } | ||
|
|
||
| private DerivedFieldType getDerivedFieldType(DerivedField derivedField) { | ||
| Mapper.BuilderContext builderContext = new Mapper.BuilderContext( | ||
| queryShardContext.getMapperService().getIndexSettings().getSettings(), | ||
| new ContentPath(1) | ||
| ); | ||
| DerivedFieldMapper.Builder builder = new DerivedFieldMapper.Builder( | ||
| derivedField, | ||
| queryShardContext.getMapperService().getIndexAnalyzers() | ||
| ); | ||
| return builder.build(builderContext).fieldType(); | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.