-
Notifications
You must be signed in to change notification settings - Fork 1.9k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
[Remove] Type from nested fields using new metadata field mapper (#3004)
* [Remove] Type from nested fields using new metadata field mapper types support is removed yet nested documents use the _type field to store the path for nested documents. A new _nested_path metadata field mapper is added to take the place of the _type field in order to remove the type dependency in nested documents. BWC is handled in the new field mapper to ensure compatibility with older versions. Signed-off-by: Nicholas Walter Knize <[email protected]> * pr fixes Signed-off-by: Nicholas Walter Knize <[email protected]> * add test to merge same mapping with empty index settings Signed-off-by: Nicholas Walter Knize <[email protected]>
- Loading branch information
Showing
21 changed files
with
264 additions
and
101 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
96 changes: 96 additions & 0 deletions
96
server/src/main/java/org/opensearch/index/mapper/NestedPathFieldMapper.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,96 @@ | ||
| /* | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| * | ||
| * The OpenSearch Contributors require contributions made to | ||
| * this file be licensed under the Apache-2.0 license or a | ||
| * compatible open source license. | ||
| */ | ||
|
|
||
| package org.opensearch.index.mapper; | ||
|
|
||
| import org.apache.lucene.document.Field; | ||
| import org.apache.lucene.document.FieldType; | ||
| import org.apache.lucene.index.IndexOptions; | ||
| import org.apache.lucene.index.Term; | ||
| import org.apache.lucene.search.Query; | ||
| import org.apache.lucene.search.TermQuery; | ||
| import org.apache.lucene.util.BytesRef; | ||
| import org.opensearch.Version; | ||
| import org.opensearch.index.query.QueryShardContext; | ||
| import org.opensearch.search.lookup.SearchLookup; | ||
|
|
||
| import java.util.Collections; | ||
|
|
||
| public class NestedPathFieldMapper extends MetadataFieldMapper { | ||
| // OpenSearch version 2.0 removed types; this name is used for bwc | ||
| public static final String LEGACY_NAME = "_type"; | ||
| public static final String NAME = "_nested_path"; | ||
|
|
||
| public static class Defaults { | ||
| public static final FieldType FIELD_TYPE = new FieldType(); | ||
| static { | ||
| FIELD_TYPE.setIndexOptions(IndexOptions.DOCS); | ||
| FIELD_TYPE.setTokenized(false); | ||
| FIELD_TYPE.setStored(false); | ||
| FIELD_TYPE.setOmitNorms(true); | ||
| FIELD_TYPE.freeze(); | ||
| } | ||
| } | ||
|
|
||
| /** private ctor; using SINGLETON to control BWC */ | ||
| private NestedPathFieldMapper(String name) { | ||
| super(new NestedPathFieldType(name)); | ||
| } | ||
|
|
||
| /** returns the field name */ | ||
| public static String name(Version version) { | ||
| if (version.before(Version.V_2_0_0)) { | ||
| return LEGACY_NAME; | ||
| } | ||
| return NAME; | ||
| } | ||
|
|
||
| @Override | ||
| protected String contentType() { | ||
| return NAME; | ||
| } | ||
|
|
||
| private static final NestedPathFieldMapper LEGACY_INSTANCE = new NestedPathFieldMapper(LEGACY_NAME); | ||
| private static final NestedPathFieldMapper INSTANCE = new NestedPathFieldMapper(NAME); | ||
|
|
||
| public static final TypeParser PARSER = new FixedTypeParser( | ||
| c -> c.indexVersionCreated().before(Version.V_2_0_0) ? LEGACY_INSTANCE : INSTANCE | ||
| ); | ||
|
|
||
| /** helper method to create a lucene field based on the opensearch version */ | ||
| public static Field field(Version version, String path) { | ||
| return new Field(name(version), path, Defaults.FIELD_TYPE); | ||
| } | ||
|
|
||
| /** helper method to create a query based on the opensearch version */ | ||
| public static Query filter(Version version, String path) { | ||
| return new TermQuery(new Term(name(version), new BytesRef(path))); | ||
| } | ||
|
|
||
| /** field type for the NestPath field */ | ||
| public static final class NestedPathFieldType extends StringFieldType { | ||
| private NestedPathFieldType(String name) { | ||
| super(name, true, false, false, TextSearchInfo.SIMPLE_MATCH_ONLY, Collections.emptyMap()); | ||
| } | ||
|
|
||
| @Override | ||
| public String typeName() { | ||
| return NAME; | ||
| } | ||
|
|
||
| @Override | ||
| public Query existsQuery(QueryShardContext context) { | ||
| throw new UnsupportedOperationException("Cannot run exists() query against the nested field path"); | ||
| } | ||
|
|
||
| @Override | ||
| public ValueFetcher valueFetcher(QueryShardContext context, SearchLookup searchLookup, String format) { | ||
| throw new UnsupportedOperationException("Cannot fetch values for internal field [" + name() + "]."); | ||
| } | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
47 changes: 47 additions & 0 deletions
47
server/src/test/java/org/opensearch/index/mapper/NestedPathFieldMapperTests.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,47 @@ | ||
| /* | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| * | ||
| * The OpenSearch Contributors require contributions made to | ||
| * this file be licensed under the Apache-2.0 license or a | ||
| * compatible open source license. | ||
| */ | ||
|
|
||
| package org.opensearch.index.mapper; | ||
|
|
||
| import org.apache.lucene.index.IndexableField; | ||
| import org.opensearch.common.bytes.BytesArray; | ||
| import org.opensearch.common.compress.CompressedXContent; | ||
| import org.opensearch.common.settings.Settings; | ||
| import org.opensearch.common.xcontent.XContentType; | ||
| import org.opensearch.test.OpenSearchSingleNodeTestCase; | ||
|
|
||
| import java.io.IOException; | ||
| import java.util.Arrays; | ||
| import java.util.Collections; | ||
|
|
||
| /** tests for {@link org.opensearch.index.mapper.NestedPathFieldMapper} */ | ||
| public class NestedPathFieldMapperTests extends OpenSearchSingleNodeTestCase { | ||
|
|
||
| public void testDefaultConfig() throws IOException { | ||
| Settings indexSettings = Settings.EMPTY; | ||
| MapperService mapperService = createIndex("test", indexSettings).mapperService(); | ||
| DocumentMapper mapper = mapperService.merge( | ||
| MapperService.SINGLE_MAPPING_NAME, | ||
| new CompressedXContent("{\"" + MapperService.SINGLE_MAPPING_NAME + "\":{}}"), | ||
| MapperService.MergeReason.MAPPING_UPDATE | ||
| ); | ||
| ParsedDocument document = mapper.parse(new SourceToParse("index", "id", new BytesArray("{}"), XContentType.JSON)); | ||
| assertEquals(Collections.<IndexableField>emptyList(), Arrays.asList(document.rootDoc().getFields(NestedPathFieldMapper.NAME))); | ||
| } | ||
|
|
||
| public void testUpdatesWithSameMappings() throws IOException { | ||
| Settings indexSettings = Settings.EMPTY; | ||
| MapperService mapperService = createIndex("test", indexSettings).mapperService(); | ||
| DocumentMapper mapper = mapperService.merge( | ||
| MapperService.SINGLE_MAPPING_NAME, | ||
| new CompressedXContent("{\"" + MapperService.SINGLE_MAPPING_NAME + "\":{}}"), | ||
| MapperService.MergeReason.MAPPING_UPDATE | ||
| ); | ||
| mapper.merge(mapper.mapping(), MapperService.MergeReason.MAPPING_UPDATE); | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.