From 8fc692e8ab44fc4b5a5d5bb404bde1dfe6db5f08 Mon Sep 17 00:00:00 2001 From: Nicholas Walter Knize Date: Wed, 20 Apr 2022 13:55:13 -0500 Subject: [PATCH 1/3] [Remove] Type from nested fields using new metadata field mapper types support is removed yet nested documents use the _type field to store the path for nested documents. A new _nested_path metadata field mapper is added to take the place of the _type field in order to remove the type dependency in nested documents. BWC is handled in the new field mapper to ensure compatibility with older versions. Signed-off-by: Nicholas Walter Knize --- .../index/mapper/DocumentParser.java | 12 ++- .../index/mapper/MapperService.java | 14 ++- .../index/mapper/NestedPathFieldMapper.java | 96 +++++++++++++++++++ .../opensearch/index/mapper/ObjectMapper.java | 21 ++-- .../org/opensearch/indices/IndicesModule.java | 2 + .../indices/mapper/MapperRegistry.java | 7 +- .../index/mapper/DocumentParserTests.java | 6 +- .../FieldAliasMapperValidationTests.java | 4 +- .../index/mapper/NestedObjectMapperTests.java | 6 +- .../mapper/NestedPathFieldMapperTests.java | 36 +++++++ .../index/search/NestedHelperTests.java | 7 +- .../search/nested/NestedSortingTests.java | 66 ++++++------- .../index/shard/ShardSplittingQueryTests.java | 10 +- .../indices/IndicesModuleTests.java | 16 +++- .../indices/IndicesServiceTests.java | 9 +- .../bucket/nested/NestedAggregatorTests.java | 30 +++--- .../nested/ReverseNestedAggregatorTests.java | 6 +- .../terms/RareTermsAggregatorTests.java | 4 +- .../bucket/terms/TermsAggregatorTests.java | 4 +- .../search/sort/FieldSortBuilderTests.java | 4 +- .../sort/GeoDistanceSortBuilderTests.java | 4 +- .../search/sort/ScriptSortBuilderTests.java | 4 +- 22 files changed, 266 insertions(+), 102 deletions(-) create mode 100644 server/src/main/java/org/opensearch/index/mapper/NestedPathFieldMapper.java create mode 100644 server/src/test/java/org/opensearch/index/mapper/NestedPathFieldMapperTests.java diff --git a/server/src/main/java/org/opensearch/index/mapper/DocumentParser.java b/server/src/main/java/org/opensearch/index/mapper/DocumentParser.java index bcafddd6d5816..f9d6187d60eb8 100644 --- a/server/src/main/java/org/opensearch/index/mapper/DocumentParser.java +++ b/server/src/main/java/org/opensearch/index/mapper/DocumentParser.java @@ -455,21 +455,23 @@ private static void innerParseObject( private static void nested(ParseContext context, ObjectMapper.Nested nested) { ParseContext.Document nestedDoc = context.doc(); ParseContext.Document parentDoc = nestedDoc.getParent(); + Version indexVersion = context.indexSettings().getIndexVersionCreated(); if (nested.isIncludeInParent()) { - addFields(nestedDoc, parentDoc); + addFields(indexVersion, nestedDoc, parentDoc); } if (nested.isIncludeInRoot()) { ParseContext.Document rootDoc = context.rootDoc(); // don't add it twice, if its included in parent, and we are handling the master doc... if (!nested.isIncludeInParent() || parentDoc != rootDoc) { - addFields(nestedDoc, rootDoc); + addFields(indexVersion, nestedDoc, rootDoc); } } } - private static void addFields(ParseContext.Document nestedDoc, ParseContext.Document rootDoc) { + private static void addFields(Version indexVersion, ParseContext.Document nestedDoc, ParseContext.Document rootDoc) { + String nestedPathFieldName = NestedPathFieldMapper.name(indexVersion); for (IndexableField field : nestedDoc.getFields()) { - if (!field.name().equals(TypeFieldMapper.NAME)) { + if (field.name().equals(nestedPathFieldName) == false) { rootDoc.add(field); } } @@ -498,7 +500,7 @@ private static ParseContext nestedContext(ParseContext context, ObjectMapper map // the type of the nested doc starts with __, so we can identify that its a nested one in filters // note, we don't prefix it with the type of the doc since it allows us to execute a nested query // across types (for example, with similar nested objects) - nestedDoc.add(new Field(TypeFieldMapper.NAME, mapper.nestedTypePathAsString(), TypeFieldMapper.Defaults.NESTED_FIELD_TYPE)); + nestedDoc.add(NestedPathFieldMapper.field(context.indexSettings().getIndexVersionCreated(), mapper.nestedTypePath())); return context; } diff --git a/server/src/main/java/org/opensearch/index/mapper/MapperService.java b/server/src/main/java/org/opensearch/index/mapper/MapperService.java index 819df4a6f396e..954a597b0bb8f 100644 --- a/server/src/main/java/org/opensearch/index/mapper/MapperService.java +++ b/server/src/main/java/org/opensearch/index/mapper/MapperService.java @@ -162,7 +162,19 @@ public enum MergeReason { // Deprecated set of meta-fields, for checking if a field is meta, use an instance method isMetadataField instead @Deprecated public static final Set META_FIELDS_BEFORE_7DOT8 = Collections.unmodifiableSet( - new HashSet<>(Arrays.asList("_id", IgnoredFieldMapper.NAME, "_index", "_routing", "_size", "_timestamp", "_ttl", "_type")) + new HashSet<>( + Arrays.asList( + "_id", + IgnoredFieldMapper.NAME, + NestedPathFieldMapper.NAME, + "_index", + "_routing", + "_size", + "_timestamp", + "_ttl", + "_type" + ) + ) ); private static final DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(MapperService.class); diff --git a/server/src/main/java/org/opensearch/index/mapper/NestedPathFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/NestedPathFieldMapper.java new file mode 100644 index 0000000000000..f420897ca187f --- /dev/null +++ b/server/src/main/java/org/opensearch/index/mapper/NestedPathFieldMapper.java @@ -0,0 +1,96 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.mapper; + +import org.apache.lucene.document.Field; +import org.apache.lucene.document.FieldType; +import org.apache.lucene.index.IndexOptions; +import org.apache.lucene.index.Term; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.TermQuery; +import org.apache.lucene.util.BytesRef; +import org.opensearch.Version; +import org.opensearch.index.query.QueryShardContext; +import org.opensearch.search.lookup.SearchLookup; + +import java.util.Collections; + +public class NestedPathFieldMapper extends MetadataFieldMapper { + // OpenSearch version 2.0 removed types; this name is used for bwc + public static final String LEGACY_NAME = "_type"; + public static final String NAME = "_nested_path"; + + public static class Defaults { + public static final FieldType FIELD_TYPE = new FieldType(); + static { + FIELD_TYPE.setIndexOptions(IndexOptions.DOCS); + FIELD_TYPE.setTokenized(false); + FIELD_TYPE.setStored(false); + FIELD_TYPE.setOmitNorms(true); + FIELD_TYPE.freeze(); + } + } + + /** private ctor; using SINGLETON to control BWC */ + private NestedPathFieldMapper(String name) { + super(new NestedPathFieldType(name)); + } + + /** returns the field name */ + public static String name(Version version) { + if (version.before(Version.V_2_0_0)) { + return LEGACY_NAME; + } + return NAME; + } + + @Override + protected String contentType() { + return NAME; + } + + private static final NestedPathFieldMapper LEGACY_INSTANCE = new NestedPathFieldMapper(LEGACY_NAME); + private static final NestedPathFieldMapper INSTANCE = new NestedPathFieldMapper(NAME); + + public static final TypeParser PARSER = new FixedTypeParser( + c -> c.indexVersionCreated().before(Version.V_2_0_0) ? LEGACY_INSTANCE : INSTANCE + ); + + /** helper method to create a lucene field based on the opensearch version */ + public static Field field(Version version, String path) { + return new Field(name(version), path, Defaults.FIELD_TYPE); + } + + /** helper method to create a query based on the opensearch version */ + public static Query filter(Version version, String path) { + return new TermQuery(new Term(name(version), new BytesRef(path))); + } + + /** field type for the NestPath field */ + public static final class NestedPathFieldType extends StringFieldType { + private NestedPathFieldType(String name) { + super(name, true, false, false, TextSearchInfo.SIMPLE_MATCH_ONLY, Collections.emptyMap()); + } + + @Override + public String typeName() { + return NAME; + } + + @Override + public Query existsQuery(QueryShardContext context) { + throw new UnsupportedOperationException("Cannot run exists() query against the nested field path"); + } + + @Override + public ValueFetcher valueFetcher(QueryShardContext context, SearchLookup searchLookup, String format) { + throw new UnsupportedOperationException("Cannot fetch values for internal field [" + name() + "]."); + } + } +} diff --git a/server/src/main/java/org/opensearch/index/mapper/ObjectMapper.java b/server/src/main/java/org/opensearch/index/mapper/ObjectMapper.java index a9923d7c6d756..d3c2e7f1e5372 100644 --- a/server/src/main/java/org/opensearch/index/mapper/ObjectMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/ObjectMapper.java @@ -32,11 +32,9 @@ package org.opensearch.index.mapper; -import org.apache.lucene.index.Term; import org.apache.lucene.search.Query; -import org.apache.lucene.search.TermQuery; -import org.apache.lucene.util.BytesRef; import org.opensearch.OpenSearchParseException; +import org.opensearch.Version; import org.opensearch.common.Explicit; import org.opensearch.common.Nullable; import org.opensearch.common.collect.CopyOnWriteHashMap; @@ -388,8 +386,7 @@ protected static void parseProperties(ObjectMapper.Builder objBuilder, Map initBuiltInMetadataMa builtInMetadataMappers.put(IndexFieldMapper.NAME, IndexFieldMapper.PARSER); builtInMetadataMappers.put(DataStreamFieldMapper.NAME, DataStreamFieldMapper.PARSER); builtInMetadataMappers.put(SourceFieldMapper.NAME, SourceFieldMapper.PARSER); + builtInMetadataMappers.put(NestedPathFieldMapper.NAME, NestedPathFieldMapper.PARSER); builtInMetadataMappers.put(VersionFieldMapper.NAME, VersionFieldMapper.PARSER); builtInMetadataMappers.put(SeqNoFieldMapper.NAME, SeqNoFieldMapper.PARSER); // _field_names must be added last so that it has a chance to see all the other mappers diff --git a/server/src/main/java/org/opensearch/indices/mapper/MapperRegistry.java b/server/src/main/java/org/opensearch/indices/mapper/MapperRegistry.java index f56b2f98f0f6e..54f3c32208c46 100644 --- a/server/src/main/java/org/opensearch/indices/mapper/MapperRegistry.java +++ b/server/src/main/java/org/opensearch/indices/mapper/MapperRegistry.java @@ -35,6 +35,7 @@ import org.opensearch.Version; import org.opensearch.index.mapper.Mapper; import org.opensearch.index.mapper.MetadataFieldMapper; +import org.opensearch.index.mapper.NestedPathFieldMapper; import org.opensearch.plugins.MapperPlugin; import java.util.Collections; @@ -50,6 +51,7 @@ public final class MapperRegistry { private final Map mapperParsers; private final Map metadataMapperParsers; + private final Map metadataMapperParsersPre20; private final Function> fieldFilter; public MapperRegistry( @@ -59,6 +61,9 @@ public MapperRegistry( ) { this.mapperParsers = Collections.unmodifiableMap(new LinkedHashMap<>(mapperParsers)); this.metadataMapperParsers = Collections.unmodifiableMap(new LinkedHashMap<>(metadataMapperParsers)); + Map tempPre20 = new LinkedHashMap<>(metadataMapperParsers); + tempPre20.remove(NestedPathFieldMapper.NAME); + this.metadataMapperParsersPre20 = Collections.unmodifiableMap(tempPre20); this.fieldFilter = fieldFilter; } @@ -75,7 +80,7 @@ public Map getMapperParsers() { * returned map uses the name of the field as a key. */ public Map getMetadataMapperParsers(Version indexCreatedVersion) { - return metadataMapperParsers; + return indexCreatedVersion.onOrAfter(LegacyESVersion.V_2_0_0) ? metadataMapperParsers : metadataMapperParsersPre20; } /** diff --git a/server/src/test/java/org/opensearch/index/mapper/DocumentParserTests.java b/server/src/test/java/org/opensearch/index/mapper/DocumentParserTests.java index 0ad8dc3f138e0..659042c37d650 100644 --- a/server/src/test/java/org/opensearch/index/mapper/DocumentParserTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/DocumentParserTests.java @@ -248,14 +248,14 @@ public void testNestedHaveIdAndTypeFields() throws Exception { assertNotNull(result.docs().get(0).getField(IdFieldMapper.NAME)); assertEquals(Uid.encodeId("1"), result.docs().get(0).getField(IdFieldMapper.NAME).binaryValue()); assertEquals(IdFieldMapper.Defaults.NESTED_FIELD_TYPE, result.docs().get(0).getField(IdFieldMapper.NAME).fieldType()); - assertNotNull(result.docs().get(0).getField(TypeFieldMapper.NAME)); - assertEquals("__foo", result.docs().get(0).getField(TypeFieldMapper.NAME).stringValue()); + assertNotNull(result.docs().get(0).getField(NestedPathFieldMapper.NAME)); + assertEquals("foo", result.docs().get(0).getField(NestedPathFieldMapper.NAME).stringValue()); assertEquals("value1", result.docs().get(0).getField("foo.bar").binaryValue().utf8ToString()); // Root document: assertNotNull(result.docs().get(1).getField(IdFieldMapper.NAME)); assertEquals(Uid.encodeId("1"), result.docs().get(1).getField(IdFieldMapper.NAME).binaryValue()); assertEquals(IdFieldMapper.Defaults.FIELD_TYPE, result.docs().get(1).getField(IdFieldMapper.NAME).fieldType()); - assertNull(result.docs().get(1).getField(TypeFieldMapper.NAME)); + assertNull(result.docs().get(1).getField(NestedPathFieldMapper.NAME)); assertEquals("value2", result.docs().get(1).getField("baz").binaryValue().utf8ToString()); } diff --git a/server/src/test/java/org/opensearch/index/mapper/FieldAliasMapperValidationTests.java b/server/src/test/java/org/opensearch/index/mapper/FieldAliasMapperValidationTests.java index 7ffc22f92d839..92de2707078f3 100644 --- a/server/src/test/java/org/opensearch/index/mapper/FieldAliasMapperValidationTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/FieldAliasMapperValidationTests.java @@ -220,7 +220,7 @@ private static ObjectMapper createObjectMapper(String name) { ObjectMapper.Nested.NO, ObjectMapper.Dynamic.FALSE, emptyMap(), - Settings.EMPTY + SETTINGS ); } @@ -232,7 +232,7 @@ private static ObjectMapper createNestedObjectMapper(String name) { ObjectMapper.Nested.newNested(), ObjectMapper.Dynamic.FALSE, emptyMap(), - Settings.EMPTY + SETTINGS ); } } diff --git a/server/src/test/java/org/opensearch/index/mapper/NestedObjectMapperTests.java b/server/src/test/java/org/opensearch/index/mapper/NestedObjectMapperTests.java index fe3ce5da6c90a..245ba1404cb5c 100644 --- a/server/src/test/java/org/opensearch/index/mapper/NestedObjectMapperTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/NestedObjectMapperTests.java @@ -149,7 +149,7 @@ public void testSingleNested() throws Exception { ); assertThat(doc.docs().size(), equalTo(2)); - assertThat(doc.docs().get(0).get(TypeFieldMapper.NAME), equalTo(nested1Mapper.nestedTypePathAsString())); + assertThat(doc.docs().get(0).get(NestedPathFieldMapper.NAME), equalTo(nested1Mapper.nestedTypePath())); assertThat(doc.docs().get(0).get("nested1.field1"), equalTo("1")); assertThat(doc.docs().get(0).get("nested1.field2"), equalTo("2")); @@ -180,10 +180,10 @@ public void testSingleNested() throws Exception { ); assertThat(doc.docs().size(), equalTo(3)); - assertThat(doc.docs().get(0).get(TypeFieldMapper.NAME), equalTo(nested1Mapper.nestedTypePathAsString())); + assertThat(doc.docs().get(0).get(NestedPathFieldMapper.NAME), equalTo(nested1Mapper.nestedTypePath())); assertThat(doc.docs().get(0).get("nested1.field1"), equalTo("1")); assertThat(doc.docs().get(0).get("nested1.field2"), equalTo("2")); - assertThat(doc.docs().get(1).get(TypeFieldMapper.NAME), equalTo(nested1Mapper.nestedTypePathAsString())); + assertThat(doc.docs().get(1).get(NestedPathFieldMapper.NAME), equalTo(nested1Mapper.nestedTypePath())); assertThat(doc.docs().get(1).get("nested1.field1"), equalTo("3")); assertThat(doc.docs().get(1).get("nested1.field2"), equalTo("4")); diff --git a/server/src/test/java/org/opensearch/index/mapper/NestedPathFieldMapperTests.java b/server/src/test/java/org/opensearch/index/mapper/NestedPathFieldMapperTests.java new file mode 100644 index 0000000000000..226713960a459 --- /dev/null +++ b/server/src/test/java/org/opensearch/index/mapper/NestedPathFieldMapperTests.java @@ -0,0 +1,36 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.mapper; + +import org.apache.lucene.index.IndexableField; +import org.opensearch.common.bytes.BytesArray; +import org.opensearch.common.compress.CompressedXContent; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.xcontent.XContentType; +import org.opensearch.test.OpenSearchSingleNodeTestCase; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Collections; + +/** tests for {@link org.opensearch.index.mapper.NestedPathFieldMapper} */ +public class NestedPathFieldMapperTests extends OpenSearchSingleNodeTestCase { + + public void testDefaultConfig() throws IOException { + Settings indexSettings = Settings.EMPTY; + MapperService mapperService = createIndex("test", indexSettings).mapperService(); + DocumentMapper mapper = mapperService.merge( + MapperService.SINGLE_MAPPING_NAME, + new CompressedXContent("{\"" + MapperService.SINGLE_MAPPING_NAME + "\":{}}"), + MapperService.MergeReason.MAPPING_UPDATE + ); + ParsedDocument document = mapper.parse(new SourceToParse("index", "id", new BytesArray("{}"), XContentType.JSON)); + assertEquals(Collections.emptyList(), Arrays.asList(document.rootDoc().getFields(NestedPathFieldMapper.NAME))); + } +} diff --git a/server/src/test/java/org/opensearch/index/search/NestedHelperTests.java b/server/src/test/java/org/opensearch/index/search/NestedHelperTests.java index c02df8168afee..7c9895a9e0642 100644 --- a/server/src/test/java/org/opensearch/index/search/NestedHelperTests.java +++ b/server/src/test/java/org/opensearch/index/search/NestedHelperTests.java @@ -47,6 +47,7 @@ import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.index.IndexService; import org.opensearch.index.mapper.MapperService; +import org.opensearch.index.mapper.NestedPathFieldMapper; import org.opensearch.index.query.MatchAllQueryBuilder; import org.opensearch.index.query.NestedQueryBuilder; import org.opensearch.index.query.QueryShardContext; @@ -324,7 +325,7 @@ public void testNested() throws IOException { Query expectedChildQuery = new BooleanQuery.Builder().add(new MatchAllDocsQuery(), Occur.MUST) // we automatically add a filter since the inner query might match non-nested docs - .add(new TermQuery(new Term("_type", "__nested1")), Occur.FILTER) + .add(new TermQuery(new Term(NestedPathFieldMapper.NAME, "nested1")), Occur.FILTER) .build(); assertEquals(expectedChildQuery, query.getChildQuery()); @@ -352,7 +353,7 @@ public void testNested() throws IOException { // we need to add the filter again because of include_in_parent expectedChildQuery = new BooleanQuery.Builder().add(new TermQuery(new Term("nested2.foo", "bar")), Occur.MUST) - .add(new TermQuery(new Term("_type", "__nested2")), Occur.FILTER) + .add(new TermQuery(new Term(NestedPathFieldMapper.NAME, "nested2")), Occur.FILTER) .build(); assertEquals(expectedChildQuery, query.getChildQuery()); @@ -367,7 +368,7 @@ public void testNested() throws IOException { // we need to add the filter again because of include_in_root expectedChildQuery = new BooleanQuery.Builder().add(new TermQuery(new Term("nested3.foo", "bar")), Occur.MUST) - .add(new TermQuery(new Term("_type", "__nested3")), Occur.FILTER) + .add(new TermQuery(new Term(NestedPathFieldMapper.NAME, "nested3")), Occur.FILTER) .build(); assertEquals(expectedChildQuery, query.getChildQuery()); diff --git a/server/src/test/java/org/opensearch/index/search/nested/NestedSortingTests.java b/server/src/test/java/org/opensearch/index/search/nested/NestedSortingTests.java index a8cd6c5411875..726e9f56f98c1 100644 --- a/server/src/test/java/org/opensearch/index/search/nested/NestedSortingTests.java +++ b/server/src/test/java/org/opensearch/index/search/nested/NestedSortingTests.java @@ -68,6 +68,7 @@ import org.opensearch.index.fielddata.NoOrdinalsStringFieldDataTests; import org.opensearch.index.fielddata.fieldcomparator.BytesRefFieldComparatorSource; import org.opensearch.index.fielddata.plain.PagedBytesIndexFieldData; +import org.opensearch.index.mapper.NestedPathFieldMapper; import org.opensearch.index.query.MatchAllQueryBuilder; import org.opensearch.index.query.NestedQueryBuilder; import org.opensearch.index.query.QueryBuilder; @@ -103,14 +104,14 @@ public void testDuel() throws Exception { for (int j = 0; j < numChildren; ++j) { Document doc = new Document(); doc.add(new StringField("f", TestUtil.randomSimpleString(random(), 2), Field.Store.NO)); - doc.add(new StringField("__type", "child", Field.Store.NO)); + doc.add(new StringField(NestedPathFieldMapper.NAME, "child", Field.Store.NO)); docs.add(doc); } if (randomBoolean()) { docs.add(new Document()); } Document parent = new Document(); - parent.add(new StringField("__type", "parent", Field.Store.NO)); + parent.add(new StringField(NestedPathFieldMapper.NAME, "parent", Field.Store.NO)); docs.add(parent); writer.addDocuments(docs); if (rarely()) { // we need to have a bit more segments than what RandomIndexWriter would do by default @@ -149,8 +150,8 @@ private TopDocs getTopDocs( int n, boolean reverse ) throws IOException { - Query parentFilter = new TermQuery(new Term("__type", "parent")); - Query childFilter = new TermQuery(new Term("__type", "child")); + Query parentFilter = new TermQuery(new Term(NestedPathFieldMapper.NAME, "parent")); + Query childFilter = new TermQuery(new Term(NestedPathFieldMapper.NAME, "child")); SortField sortField = indexFieldData.sortField(missingValue, sortMode, createNested(searcher, parentFilter, childFilter), reverse); Query query = new ConstantScoreQuery(parentFilter); Sort sort = new Sort(sortField); @@ -172,7 +173,7 @@ public void testNestedSorting() throws Exception { document.add(new StringField("filter_1", "T", Field.Store.NO)); docs.add(document); document = new Document(); - document.add(new StringField("__type", "parent", Field.Store.NO)); + document.add(new StringField(NestedPathFieldMapper.NAME, "parent", Field.Store.NO)); document.add(new StringField("field1", "a", Field.Store.NO)); docs.add(document); writer.addDocuments(docs); @@ -192,7 +193,7 @@ public void testNestedSorting() throws Exception { document.add(new StringField("filter_1", "T", Field.Store.NO)); docs.add(document); document = new Document(); - document.add(new StringField("__type", "parent", Field.Store.NO)); + document.add(new StringField(NestedPathFieldMapper.NAME, "parent", Field.Store.NO)); document.add(new StringField("field1", "b", Field.Store.NO)); docs.add(document); writer.addDocuments(docs); @@ -211,7 +212,7 @@ public void testNestedSorting() throws Exception { document.add(new StringField("filter_1", "T", Field.Store.NO)); docs.add(document); document = new Document(); - document.add(new StringField("__type", "parent", Field.Store.NO)); + document.add(new StringField(NestedPathFieldMapper.NAME, "parent", Field.Store.NO)); document.add(new StringField("field1", "c", Field.Store.NO)); docs.add(document); writer.addDocuments(docs); @@ -230,7 +231,7 @@ public void testNestedSorting() throws Exception { document.add(new StringField("filter_1", "F", Field.Store.NO)); docs.add(document); document = new Document(); - document.add(new StringField("__type", "parent", Field.Store.NO)); + document.add(new StringField(NestedPathFieldMapper.NAME, "parent", Field.Store.NO)); document.add(new StringField("field1", "d", Field.Store.NO)); docs.add(document); writer.addDocuments(docs); @@ -250,7 +251,7 @@ public void testNestedSorting() throws Exception { document.add(new StringField("filter_1", "F", Field.Store.NO)); docs.add(document); document = new Document(); - document.add(new StringField("__type", "parent", Field.Store.NO)); + document.add(new StringField(NestedPathFieldMapper.NAME, "parent", Field.Store.NO)); document.add(new StringField("field1", "f", Field.Store.NO)); docs.add(document); writer.addDocuments(docs); @@ -269,14 +270,14 @@ public void testNestedSorting() throws Exception { document.add(new StringField("filter_1", "T", Field.Store.NO)); docs.add(document); document = new Document(); - document.add(new StringField("__type", "parent", Field.Store.NO)); + document.add(new StringField(NestedPathFieldMapper.NAME, "parent", Field.Store.NO)); document.add(new StringField("field1", "g", Field.Store.NO)); docs.add(document); writer.addDocuments(docs); // This doc will not be included, because it doesn't have nested docs document = new Document(); - document.add(new StringField("__type", "parent", Field.Store.NO)); + document.add(new StringField(NestedPathFieldMapper.NAME, "parent", Field.Store.NO)); document.add(new StringField("field1", "h", Field.Store.NO)); writer.addDocument(document); @@ -294,7 +295,7 @@ public void testNestedSorting() throws Exception { document.add(new StringField("filter_1", "F", Field.Store.NO)); docs.add(document); document = new Document(); - document.add(new StringField("__type", "parent", Field.Store.NO)); + document.add(new StringField(NestedPathFieldMapper.NAME, "parent", Field.Store.NO)); document.add(new StringField("field1", "i", Field.Store.NO)); docs.add(document); writer.addDocuments(docs); @@ -316,7 +317,7 @@ public void testNestedSorting() throws Exception { reader = OpenSearchDirectoryReader.wrap(reader, new ShardId(indexService.index(), 0)); IndexSearcher searcher = new IndexSearcher(reader); PagedBytesIndexFieldData indexFieldData = getForField("field2"); - Query parentFilter = new TermQuery(new Term("__type", "parent")); + Query parentFilter = new TermQuery(new Term(NestedPathFieldMapper.NAME, "parent")); Query childFilter = Queries.not(parentFilter); BytesRefFieldComparatorSource nestedComparatorSource = new BytesRefFieldComparatorSource( indexFieldData, @@ -472,53 +473,52 @@ public void testMultiLevelNestedSorting() throws IOException { List book = new ArrayList<>(); Document document = new Document(); document.add(new TextField("chapters.paragraphs.header", "Paragraph 1", Field.Store.NO)); - document.add(new StringField("_type", "__chapters.paragraphs", Field.Store.NO)); + document.add(new StringField(NestedPathFieldMapper.NAME, "chapters.paragraphs", Field.Store.NO)); document.add(new TextField("chapters.paragraphs.text", "some text...", Field.Store.NO)); document.add(new SortedNumericDocValuesField("chapters.paragraphs.word_count", 743)); document.add(new IntPoint("chapters.paragraphs.word_count", 743)); book.add(document); document = new Document(); document.add(new TextField("chapters.title", "chapter 3", Field.Store.NO)); - document.add(new StringField("_type", "__chapters", Field.Store.NO)); + document.add(new StringField(NestedPathFieldMapper.NAME, "chapters", Field.Store.NO)); document.add(new IntPoint("chapters.read_time_seconds", 400)); document.add(new NumericDocValuesField("chapters.read_time_seconds", 400)); book.add(document); document = new Document(); document.add(new TextField("chapters.paragraphs.header", "Paragraph 1", Field.Store.NO)); - document.add(new StringField("_type", "__chapters.paragraphs", Field.Store.NO)); + document.add(new StringField(NestedPathFieldMapper.NAME, "chapters.paragraphs", Field.Store.NO)); document.add(new TextField("chapters.paragraphs.text", "some text...", Field.Store.NO)); document.add(new SortedNumericDocValuesField("chapters.paragraphs.word_count", 234)); document.add(new IntPoint("chapters.paragraphs.word_count", 234)); book.add(document); document = new Document(); document.add(new TextField("chapters.title", "chapter 2", Field.Store.NO)); - document.add(new StringField("_type", "__chapters", Field.Store.NO)); + document.add(new StringField(NestedPathFieldMapper.NAME, "chapters", Field.Store.NO)); document.add(new IntPoint("chapters.read_time_seconds", 200)); document.add(new NumericDocValuesField("chapters.read_time_seconds", 200)); book.add(document); document = new Document(); document.add(new TextField("chapters.paragraphs.header", "Paragraph 2", Field.Store.NO)); - document.add(new StringField("_type", "__chapters.paragraphs", Field.Store.NO)); + document.add(new StringField(NestedPathFieldMapper.NAME, "chapters.paragraphs", Field.Store.NO)); document.add(new TextField("chapters.paragraphs.text", "some text...", Field.Store.NO)); document.add(new SortedNumericDocValuesField("chapters.paragraphs.word_count", 478)); document.add(new IntPoint("chapters.paragraphs.word_count", 478)); book.add(document); document = new Document(); document.add(new TextField("chapters.paragraphs.header", "Paragraph 1", Field.Store.NO)); - document.add(new StringField("_type", "__chapters.paragraphs", Field.Store.NO)); + document.add(new StringField(NestedPathFieldMapper.NAME, "chapters.paragraphs", Field.Store.NO)); document.add(new TextField("chapters.paragraphs.text", "some text...", Field.Store.NO)); document.add(new SortedNumericDocValuesField("chapters.paragraphs.word_count", 849)); document.add(new IntPoint("chapters.paragraphs.word_count", 849)); book.add(document); document = new Document(); document.add(new TextField("chapters.title", "chapter 1", Field.Store.NO)); - document.add(new StringField("_type", "__chapters", Field.Store.NO)); + document.add(new StringField(NestedPathFieldMapper.NAME, "chapters", Field.Store.NO)); document.add(new IntPoint("chapters.read_time_seconds", 1400)); document.add(new NumericDocValuesField("chapters.read_time_seconds", 1400)); book.add(document); document = new Document(); document.add(new StringField("genre", "science fiction", Field.Store.NO)); - document.add(new StringField("_type", "_doc", Field.Store.NO)); document.add(new StringField("_id", "1", Field.Store.YES)); document.add(new NumericDocValuesField(PRIMARY_TERM_NAME, 0)); book.add(document); @@ -528,20 +528,19 @@ public void testMultiLevelNestedSorting() throws IOException { List book = new ArrayList<>(); Document document = new Document(); document.add(new TextField("chapters.paragraphs.header", "Introduction", Field.Store.NO)); - document.add(new StringField("_type", "__chapters.paragraphs", Field.Store.NO)); + document.add(new StringField(NestedPathFieldMapper.NAME, "chapters.paragraphs", Field.Store.NO)); document.add(new TextField("chapters.paragraphs.text", "some text...", Field.Store.NO)); document.add(new SortedNumericDocValuesField("chapters.paragraphs.word_count", 76)); document.add(new IntPoint("chapters.paragraphs.word_count", 76)); book.add(document); document = new Document(); document.add(new TextField("chapters.title", "chapter 1", Field.Store.NO)); - document.add(new StringField("_type", "__chapters", Field.Store.NO)); + document.add(new StringField(NestedPathFieldMapper.NAME, "chapters", Field.Store.NO)); document.add(new IntPoint("chapters.read_time_seconds", 20)); document.add(new NumericDocValuesField("chapters.read_time_seconds", 20)); book.add(document); document = new Document(); document.add(new StringField("genre", "romance", Field.Store.NO)); - document.add(new StringField("_type", "_doc", Field.Store.NO)); document.add(new StringField("_id", "2", Field.Store.YES)); document.add(new NumericDocValuesField(PRIMARY_TERM_NAME, 0)); book.add(document); @@ -551,20 +550,19 @@ public void testMultiLevelNestedSorting() throws IOException { List book = new ArrayList<>(); Document document = new Document(); document.add(new TextField("chapters.paragraphs.header", "A bad dream", Field.Store.NO)); - document.add(new StringField("_type", "__chapters.paragraphs", Field.Store.NO)); + document.add(new StringField(NestedPathFieldMapper.NAME, "chapters.paragraphs", Field.Store.NO)); document.add(new TextField("chapters.paragraphs.text", "some text...", Field.Store.NO)); document.add(new SortedNumericDocValuesField("chapters.paragraphs.word_count", 976)); document.add(new IntPoint("chapters.paragraphs.word_count", 976)); book.add(document); document = new Document(); document.add(new TextField("chapters.title", "The beginning of the end", Field.Store.NO)); - document.add(new StringField("_type", "__chapters", Field.Store.NO)); + document.add(new StringField(NestedPathFieldMapper.NAME, "chapters", Field.Store.NO)); document.add(new IntPoint("chapters.read_time_seconds", 1200)); document.add(new NumericDocValuesField("chapters.read_time_seconds", 1200)); book.add(document); document = new Document(); document.add(new StringField("genre", "horror", Field.Store.NO)); - document.add(new StringField("_type", "_doc", Field.Store.NO)); document.add(new StringField("_id", "3", Field.Store.YES)); document.add(new NumericDocValuesField(PRIMARY_TERM_NAME, 0)); book.add(document); @@ -574,47 +572,46 @@ public void testMultiLevelNestedSorting() throws IOException { List book = new ArrayList<>(); Document document = new Document(); document.add(new TextField("chapters.paragraphs.header", "macaroni", Field.Store.NO)); - document.add(new StringField("_type", "__chapters.paragraphs", Field.Store.NO)); + document.add(new StringField(NestedPathFieldMapper.NAME, "chapters.paragraphs", Field.Store.NO)); document.add(new TextField("chapters.paragraphs.text", "some text...", Field.Store.NO)); document.add(new SortedNumericDocValuesField("chapters.paragraphs.word_count", 180)); document.add(new IntPoint("chapters.paragraphs.word_count", 180)); book.add(document); document = new Document(); document.add(new TextField("chapters.paragraphs.header", "hamburger", Field.Store.NO)); - document.add(new StringField("_type", "__chapters.paragraphs", Field.Store.NO)); + document.add(new StringField(NestedPathFieldMapper.NAME, "chapters.paragraphs", Field.Store.NO)); document.add(new TextField("chapters.paragraphs.text", "some text...", Field.Store.NO)); document.add(new SortedNumericDocValuesField("chapters.paragraphs.word_count", 150)); document.add(new IntPoint("chapters.paragraphs.word_count", 150)); book.add(document); document = new Document(); document.add(new TextField("chapters.paragraphs.header", "tosti", Field.Store.NO)); - document.add(new StringField("_type", "__chapters.paragraphs", Field.Store.NO)); + document.add(new StringField(NestedPathFieldMapper.NAME, "chapters.paragraphs", Field.Store.NO)); document.add(new TextField("chapters.paragraphs.text", "some text...", Field.Store.NO)); document.add(new SortedNumericDocValuesField("chapters.paragraphs.word_count", 120)); document.add(new IntPoint("chapters.paragraphs.word_count", 120)); book.add(document); document = new Document(); document.add(new TextField("chapters.title", "easy meals", Field.Store.NO)); - document.add(new StringField("_type", "__chapters", Field.Store.NO)); + document.add(new StringField(NestedPathFieldMapper.NAME, "chapters", Field.Store.NO)); document.add(new IntPoint("chapters.read_time_seconds", 800)); document.add(new NumericDocValuesField("chapters.read_time_seconds", 800)); book.add(document); document = new Document(); document.add(new TextField("chapters.paragraphs.header", "introduction", Field.Store.NO)); - document.add(new StringField("_type", "__chapters.paragraphs", Field.Store.NO)); + document.add(new StringField(NestedPathFieldMapper.NAME, "chapters.paragraphs", Field.Store.NO)); document.add(new TextField("chapters.paragraphs.text", "some text...", Field.Store.NO)); document.add(new SortedNumericDocValuesField("chapters.paragraphs.word_count", 87)); document.add(new IntPoint("chapters.paragraphs.word_count", 87)); book.add(document); document = new Document(); document.add(new TextField("chapters.title", "introduction", Field.Store.NO)); - document.add(new StringField("_type", "__chapters", Field.Store.NO)); + document.add(new StringField(NestedPathFieldMapper.NAME, "chapters", Field.Store.NO)); document.add(new IntPoint("chapters.read_time_seconds", 10)); document.add(new NumericDocValuesField("chapters.read_time_seconds", 10)); book.add(document); document = new Document(); document.add(new StringField("genre", "cooking", Field.Store.NO)); - document.add(new StringField("_type", "_doc", Field.Store.NO)); document.add(new StringField("_id", "4", Field.Store.YES)); document.add(new NumericDocValuesField(PRIMARY_TERM_NAME, 0)); book.add(document); @@ -624,7 +621,6 @@ public void testMultiLevelNestedSorting() throws IOException { List book = new ArrayList<>(); Document document = new Document(); document.add(new StringField("genre", "unknown", Field.Store.NO)); - document.add(new StringField("_type", "_doc", Field.Store.NO)); document.add(new StringField("_id", "5", Field.Store.YES)); document.add(new NumericDocValuesField(PRIMARY_TERM_NAME, 0)); book.add(document); diff --git a/server/src/test/java/org/opensearch/index/shard/ShardSplittingQueryTests.java b/server/src/test/java/org/opensearch/index/shard/ShardSplittingQueryTests.java index b39ff0c9b97b3..04dcea210640c 100644 --- a/server/src/test/java/org/opensearch/index/shard/ShardSplittingQueryTests.java +++ b/server/src/test/java/org/opensearch/index/shard/ShardSplittingQueryTests.java @@ -52,9 +52,9 @@ import org.opensearch.cluster.routing.OperationRouting; import org.opensearch.common.settings.Settings; import org.opensearch.index.mapper.IdFieldMapper; +import org.opensearch.index.mapper.NestedPathFieldMapper; import org.opensearch.index.mapper.RoutingFieldMapper; import org.opensearch.index.mapper.SeqNoFieldMapper; -import org.opensearch.index.mapper.TypeFieldMapper; import org.opensearch.index.mapper.Uid; import org.opensearch.test.OpenSearchTestCase; @@ -88,7 +88,7 @@ public void testSplitOnID() throws IOException { docs.add( Arrays.asList( new StringField(IdFieldMapper.NAME, Uid.encodeId(Integer.toString(j)), Field.Store.YES), - new StringField(TypeFieldMapper.NAME, "__nested", Field.Store.YES), + new StringField(NestedPathFieldMapper.NAME, "__nested", Field.Store.YES), new SortedNumericDocValuesField("shard_id", shardId) ) ); @@ -142,7 +142,7 @@ public void testSplitOnRouting() throws IOException { docs.add( Arrays.asList( new StringField(IdFieldMapper.NAME, Uid.encodeId(Integer.toString(j)), Field.Store.YES), - new StringField(TypeFieldMapper.NAME, "__nested", Field.Store.YES), + new StringField(NestedPathFieldMapper.NAME, "__nested", Field.Store.YES), new SortedNumericDocValuesField("shard_id", shardId) ) ); @@ -215,7 +215,7 @@ public void testSplitOnIdOrRouting() throws IOException { docs.add( Arrays.asList( new StringField(IdFieldMapper.NAME, Uid.encodeId(Integer.toString(j)), Field.Store.YES), - new StringField(TypeFieldMapper.NAME, "__nested", Field.Store.YES), + new StringField(NestedPathFieldMapper.NAME, "__nested", Field.Store.YES), new SortedNumericDocValuesField("shard_id", shardId) ) ); @@ -258,7 +258,7 @@ public void testSplitOnRoutingPartitioned() throws IOException { docs.add( Arrays.asList( new StringField(IdFieldMapper.NAME, Uid.encodeId(Integer.toString(j)), Field.Store.YES), - new StringField(TypeFieldMapper.NAME, "__nested", Field.Store.YES), + new StringField(NestedPathFieldMapper.NAME, "__nested", Field.Store.YES), new SortedNumericDocValuesField("shard_id", shardId) ) ); diff --git a/server/src/test/java/org/opensearch/indices/IndicesModuleTests.java b/server/src/test/java/org/opensearch/indices/IndicesModuleTests.java index 8123f044798bd..afcc6aa006500 100644 --- a/server/src/test/java/org/opensearch/indices/IndicesModuleTests.java +++ b/server/src/test/java/org/opensearch/indices/IndicesModuleTests.java @@ -41,6 +41,7 @@ import org.opensearch.index.mapper.Mapper; import org.opensearch.index.mapper.MapperParsingException; import org.opensearch.index.mapper.MetadataFieldMapper; +import org.opensearch.index.mapper.NestedPathFieldMapper; import org.opensearch.index.mapper.RoutingFieldMapper; import org.opensearch.index.mapper.SeqNoFieldMapper; import org.opensearch.index.mapper.SourceFieldMapper; @@ -94,6 +95,7 @@ public Map getMetadataMappers() { IndexFieldMapper.NAME, DataStreamFieldMapper.NAME, SourceFieldMapper.NAME, + NestedPathFieldMapper.NAME, VersionFieldMapper.NAME, SeqNoFieldMapper.NAME, FieldNamesFieldMapper.NAME }; @@ -101,11 +103,7 @@ public Map getMetadataMappers() { public void testBuiltinMappers() { IndicesModule module = new IndicesModule(Collections.emptyList()); { - Version version = VersionUtils.randomVersionBetween( - random(), - Version.CURRENT.minimumIndexCompatibilityVersion(), - Version.CURRENT - ); + Version version = VersionUtils.randomVersionBetween(random(), Version.V_2_0_0, Version.CURRENT); assertFalse(module.getMapperRegistry().getMapperParsers().isEmpty()); assertFalse(module.getMapperRegistry().getMetadataMapperParsers(version).isEmpty()); Map metadataMapperParsers = module.getMapperRegistry() @@ -116,6 +114,14 @@ public void testBuiltinMappers() { assertEquals(EXPECTED_METADATA_FIELDS[i++], field); } } + { + Version version = VersionUtils.randomVersionBetween( + random(), + Version.V_1_0_0, + VersionUtils.getPreviousVersion(Version.V_2_0_0) + ); + assertEquals(EXPECTED_METADATA_FIELDS.length - 1, module.getMapperRegistry().getMetadataMapperParsers(version).size()); + } } public void testBuiltinWithPlugins() { diff --git a/server/src/test/java/org/opensearch/indices/IndicesServiceTests.java b/server/src/test/java/org/opensearch/indices/IndicesServiceTests.java index 8dd156dfcd0d2..da984084321e1 100644 --- a/server/src/test/java/org/opensearch/indices/IndicesServiceTests.java +++ b/server/src/test/java/org/opensearch/indices/IndicesServiceTests.java @@ -66,6 +66,7 @@ import org.opensearch.index.mapper.KeywordFieldMapper; import org.opensearch.index.mapper.Mapper; import org.opensearch.index.mapper.MapperService; +import org.opensearch.index.mapper.NestedPathFieldMapper; import org.opensearch.index.shard.IllegalIndexShardStateException; import org.opensearch.index.shard.IndexShard; import org.opensearch.index.shard.IndexShardState; @@ -567,7 +568,13 @@ public void testIsMetadataField() { final Version randVersion = VersionUtils.randomIndexCompatibleVersion(random()); assertFalse(indicesService.isMetadataField(randVersion, randomAlphaOfLengthBetween(10, 15))); for (String builtIn : IndicesModule.getBuiltInMetadataFields()) { - assertTrue(indicesService.isMetadataField(randVersion, builtIn)); + if (NestedPathFieldMapper.NAME.equals(builtIn) && randVersion.before(Version.V_2_0_0)) { + continue; // nested field mapper does not exist prior to 2.0 + } + assertTrue( + "Expected " + builtIn + " to be a metadata field for version " + randVersion, + indicesService.isMetadataField(randVersion, builtIn) + ); } } diff --git a/server/src/test/java/org/opensearch/search/aggregations/bucket/nested/NestedAggregatorTests.java b/server/src/test/java/org/opensearch/search/aggregations/bucket/nested/NestedAggregatorTests.java index 8ab0cc0023346..65ce02333bae0 100644 --- a/server/src/test/java/org/opensearch/search/aggregations/bucket/nested/NestedAggregatorTests.java +++ b/server/src/test/java/org/opensearch/search/aggregations/bucket/nested/NestedAggregatorTests.java @@ -57,9 +57,9 @@ import org.opensearch.index.mapper.IdFieldMapper; import org.opensearch.index.mapper.KeywordFieldMapper; import org.opensearch.index.mapper.MappedFieldType; +import org.opensearch.index.mapper.NestedPathFieldMapper; import org.opensearch.index.mapper.NumberFieldMapper; import org.opensearch.index.mapper.SeqNoFieldMapper; -import org.opensearch.index.mapper.TypeFieldMapper; import org.opensearch.index.mapper.Uid; import org.opensearch.index.query.MatchAllQueryBuilder; import org.opensearch.script.MockScriptEngine; @@ -343,15 +343,15 @@ public void testResetRootDocId() throws Exception { // 1 segment with, 1 root document, with 3 nested sub docs Document document = new Document(); document.add(new Field(IdFieldMapper.NAME, Uid.encodeId("1"), IdFieldMapper.Defaults.NESTED_FIELD_TYPE)); - document.add(new Field(TypeFieldMapper.NAME, "__nested_field", TypeFieldMapper.Defaults.NESTED_FIELD_TYPE)); + document.add(new Field(NestedPathFieldMapper.NAME, "nested_field", NestedPathFieldMapper.Defaults.FIELD_TYPE)); documents.add(document); document = new Document(); document.add(new Field(IdFieldMapper.NAME, Uid.encodeId("1"), IdFieldMapper.Defaults.NESTED_FIELD_TYPE)); - document.add(new Field(TypeFieldMapper.NAME, "__nested_field", TypeFieldMapper.Defaults.NESTED_FIELD_TYPE)); + document.add(new Field(NestedPathFieldMapper.NAME, "nested_field", NestedPathFieldMapper.Defaults.FIELD_TYPE)); documents.add(document); document = new Document(); document.add(new Field(IdFieldMapper.NAME, Uid.encodeId("1"), IdFieldMapper.Defaults.NESTED_FIELD_TYPE)); - document.add(new Field(TypeFieldMapper.NAME, "__nested_field", TypeFieldMapper.Defaults.NESTED_FIELD_TYPE)); + document.add(new Field(NestedPathFieldMapper.NAME, "nested_field", NestedPathFieldMapper.Defaults.FIELD_TYPE)); documents.add(document); document = new Document(); document.add(new Field(IdFieldMapper.NAME, Uid.encodeId("1"), IdFieldMapper.Defaults.FIELD_TYPE)); @@ -365,7 +365,7 @@ public void testResetRootDocId() throws Exception { // 1 document, with 1 nested subdoc document = new Document(); document.add(new Field(IdFieldMapper.NAME, Uid.encodeId("2"), IdFieldMapper.Defaults.NESTED_FIELD_TYPE)); - document.add(new Field(TypeFieldMapper.NAME, "__nested_field", TypeFieldMapper.Defaults.NESTED_FIELD_TYPE)); + document.add(new Field(NestedPathFieldMapper.NAME, "nested_field", NestedPathFieldMapper.Defaults.FIELD_TYPE)); documents.add(document); document = new Document(); document.add(new Field(IdFieldMapper.NAME, Uid.encodeId("2"), IdFieldMapper.Defaults.FIELD_TYPE)); @@ -376,7 +376,7 @@ public void testResetRootDocId() throws Exception { // and 1 document, with 1 nested subdoc document = new Document(); document.add(new Field(IdFieldMapper.NAME, Uid.encodeId("3"), IdFieldMapper.Defaults.NESTED_FIELD_TYPE)); - document.add(new Field(TypeFieldMapper.NAME, "__nested_field", TypeFieldMapper.Defaults.NESTED_FIELD_TYPE)); + document.add(new Field(NestedPathFieldMapper.NAME, "nested_field", NestedPathFieldMapper.Defaults.FIELD_TYPE)); documents.add(document); document = new Document(); document.add(new Field(IdFieldMapper.NAME, Uid.encodeId("3"), IdFieldMapper.Defaults.FIELD_TYPE)); @@ -613,13 +613,13 @@ public void testPreGetChildLeafCollectors() throws IOException { List documents = new ArrayList<>(); Document document = new Document(); document.add(new Field(IdFieldMapper.NAME, Uid.encodeId("1"), IdFieldMapper.Defaults.NESTED_FIELD_TYPE)); - document.add(new Field(TypeFieldMapper.NAME, "__nested_field", TypeFieldMapper.Defaults.NESTED_FIELD_TYPE)); + document.add(new Field(NestedPathFieldMapper.NAME, "nested_field", NestedPathFieldMapper.Defaults.FIELD_TYPE)); document.add(new SortedDocValuesField("key", new BytesRef("key1"))); document.add(new SortedDocValuesField("value", new BytesRef("a1"))); documents.add(document); document = new Document(); document.add(new Field(IdFieldMapper.NAME, Uid.encodeId("1"), IdFieldMapper.Defaults.NESTED_FIELD_TYPE)); - document.add(new Field(TypeFieldMapper.NAME, "__nested_field", TypeFieldMapper.Defaults.NESTED_FIELD_TYPE)); + document.add(new Field(NestedPathFieldMapper.NAME, "nested_field", NestedPathFieldMapper.Defaults.FIELD_TYPE)); document.add(new SortedDocValuesField("key", new BytesRef("key2"))); document.add(new SortedDocValuesField("value", new BytesRef("b1"))); documents.add(document); @@ -633,13 +633,13 @@ public void testPreGetChildLeafCollectors() throws IOException { document = new Document(); document.add(new Field(IdFieldMapper.NAME, Uid.encodeId("2"), IdFieldMapper.Defaults.NESTED_FIELD_TYPE)); - document.add(new Field(TypeFieldMapper.NAME, "__nested_field", TypeFieldMapper.Defaults.NESTED_FIELD_TYPE)); + document.add(new Field(NestedPathFieldMapper.NAME, "nested_field", NestedPathFieldMapper.Defaults.FIELD_TYPE)); document.add(new SortedDocValuesField("key", new BytesRef("key1"))); document.add(new SortedDocValuesField("value", new BytesRef("a2"))); documents.add(document); document = new Document(); document.add(new Field(IdFieldMapper.NAME, Uid.encodeId("2"), IdFieldMapper.Defaults.NESTED_FIELD_TYPE)); - document.add(new Field(TypeFieldMapper.NAME, "__nested_field", TypeFieldMapper.Defaults.NESTED_FIELD_TYPE)); + document.add(new Field(NestedPathFieldMapper.NAME, "nested_field", NestedPathFieldMapper.Defaults.FIELD_TYPE)); document.add(new SortedDocValuesField("key", new BytesRef("key2"))); document.add(new SortedDocValuesField("value", new BytesRef("b2"))); documents.add(document); @@ -653,13 +653,13 @@ public void testPreGetChildLeafCollectors() throws IOException { document = new Document(); document.add(new Field(IdFieldMapper.NAME, Uid.encodeId("3"), IdFieldMapper.Defaults.NESTED_FIELD_TYPE)); - document.add(new Field(TypeFieldMapper.NAME, "__nested_field", TypeFieldMapper.Defaults.NESTED_FIELD_TYPE)); + document.add(new Field(NestedPathFieldMapper.NAME, "nested_field", NestedPathFieldMapper.Defaults.FIELD_TYPE)); document.add(new SortedDocValuesField("key", new BytesRef("key1"))); document.add(new SortedDocValuesField("value", new BytesRef("a3"))); documents.add(document); document = new Document(); document.add(new Field(IdFieldMapper.NAME, Uid.encodeId("3"), IdFieldMapper.Defaults.NESTED_FIELD_TYPE)); - document.add(new Field(TypeFieldMapper.NAME, "__nested_field", TypeFieldMapper.Defaults.NESTED_FIELD_TYPE)); + document.add(new Field(NestedPathFieldMapper.NAME, "nested_field", NestedPathFieldMapper.Defaults.FIELD_TYPE)); document.add(new SortedDocValuesField("key", new BytesRef("key2"))); document.add(new SortedDocValuesField("value", new BytesRef("b3"))); documents.add(document); @@ -863,7 +863,7 @@ public static CheckedConsumer buildResellerData( } Document document = new Document(); document.add(new Field(IdFieldMapper.NAME, Uid.encodeId(Integer.toString(p)), IdFieldMapper.Defaults.FIELD_TYPE)); - document.add(new Field(TypeFieldMapper.NAME, "__nested_field", TypeFieldMapper.Defaults.NESTED_FIELD_TYPE)); + document.add(new Field(NestedPathFieldMapper.NAME, "nested_field", NestedPathFieldMapper.Defaults.FIELD_TYPE)); document.add(sequenceIDFields.primaryTerm); document.add(new SortedNumericDocValuesField("product_id", p)); documents.add(document); @@ -891,7 +891,7 @@ private static double[] generateDocuments(List documents, int numNeste for (int nested = 0; nested < numNestedDocs; nested++) { Document document = new Document(); document.add(new Field(IdFieldMapper.NAME, Uid.encodeId(Integer.toString(id)), IdFieldMapper.Defaults.NESTED_FIELD_TYPE)); - document.add(new Field(TypeFieldMapper.NAME, "__" + path, TypeFieldMapper.Defaults.NESTED_FIELD_TYPE)); + document.add(new Field(NestedPathFieldMapper.NAME, path, NestedPathFieldMapper.Defaults.FIELD_TYPE)); long value = randomNonNegativeLong() % 10000; document.add(new SortedNumericDocValuesField(fieldName, value)); documents.add(document); @@ -906,7 +906,7 @@ private List generateBook(String id, String[] authors, int[] numPages) for (int numPage : numPages) { Document document = new Document(); document.add(new Field(IdFieldMapper.NAME, Uid.encodeId(id), IdFieldMapper.Defaults.NESTED_FIELD_TYPE)); - document.add(new Field(TypeFieldMapper.NAME, "__nested_chapters", TypeFieldMapper.Defaults.NESTED_FIELD_TYPE)); + document.add(new Field(NestedPathFieldMapper.NAME, "nested_chapters", NestedPathFieldMapper.Defaults.FIELD_TYPE)); document.add(new SortedNumericDocValuesField("num_pages", numPage)); documents.add(document); } diff --git a/server/src/test/java/org/opensearch/search/aggregations/bucket/nested/ReverseNestedAggregatorTests.java b/server/src/test/java/org/opensearch/search/aggregations/bucket/nested/ReverseNestedAggregatorTests.java index cf0e31bc63467..61df6d01aef64 100644 --- a/server/src/test/java/org/opensearch/search/aggregations/bucket/nested/ReverseNestedAggregatorTests.java +++ b/server/src/test/java/org/opensearch/search/aggregations/bucket/nested/ReverseNestedAggregatorTests.java @@ -42,9 +42,9 @@ import org.apache.lucene.store.Directory; import org.opensearch.index.mapper.IdFieldMapper; import org.opensearch.index.mapper.MappedFieldType; +import org.opensearch.index.mapper.NestedPathFieldMapper; import org.opensearch.index.mapper.NumberFieldMapper; import org.opensearch.index.mapper.SeqNoFieldMapper; -import org.opensearch.index.mapper.TypeFieldMapper; import org.opensearch.index.mapper.Uid; import org.opensearch.search.aggregations.AggregationBuilder; import org.opensearch.search.aggregations.AggregatorTestCase; @@ -133,7 +133,7 @@ public void testMaxFromParentDocs() throws IOException { document.add( new Field(IdFieldMapper.NAME, Uid.encodeId(Integer.toString(i)), IdFieldMapper.Defaults.NESTED_FIELD_TYPE) ); - document.add(new Field(TypeFieldMapper.NAME, "__" + NESTED_OBJECT, TypeFieldMapper.Defaults.NESTED_FIELD_TYPE)); + document.add(new Field(NestedPathFieldMapper.NAME, NESTED_OBJECT, NestedPathFieldMapper.Defaults.FIELD_TYPE)); documents.add(document); expectedNestedDocs++; } @@ -193,7 +193,7 @@ public void testFieldAlias() throws IOException { document.add( new Field(IdFieldMapper.NAME, Uid.encodeId(Integer.toString(i)), IdFieldMapper.Defaults.NESTED_FIELD_TYPE) ); - document.add(new Field(TypeFieldMapper.NAME, "__" + NESTED_OBJECT, TypeFieldMapper.Defaults.NESTED_FIELD_TYPE)); + document.add(new Field(NestedPathFieldMapper.NAME, NESTED_OBJECT, NestedPathFieldMapper.Defaults.FIELD_TYPE)); documents.add(document); } Document document = new Document(); diff --git a/server/src/test/java/org/opensearch/search/aggregations/bucket/terms/RareTermsAggregatorTests.java b/server/src/test/java/org/opensearch/search/aggregations/bucket/terms/RareTermsAggregatorTests.java index 9a9a03e715644..678bc2fc6f536 100644 --- a/server/src/test/java/org/opensearch/search/aggregations/bucket/terms/RareTermsAggregatorTests.java +++ b/server/src/test/java/org/opensearch/search/aggregations/bucket/terms/RareTermsAggregatorTests.java @@ -60,11 +60,11 @@ import org.opensearch.index.mapper.KeywordFieldMapper; import org.opensearch.index.mapper.MappedFieldType; import org.opensearch.index.mapper.MapperService; +import org.opensearch.index.mapper.NestedPathFieldMapper; import org.opensearch.index.mapper.NumberFieldMapper; import org.opensearch.index.mapper.RangeFieldMapper; import org.opensearch.index.mapper.RangeType; import org.opensearch.index.mapper.SeqNoFieldMapper; -import org.opensearch.index.mapper.TypeFieldMapper; import org.opensearch.index.mapper.Uid; import org.opensearch.search.SearchHit; import org.opensearch.search.aggregations.Aggregation; @@ -551,7 +551,7 @@ private List generateDocsWithNested(String id, int value, int[] nested for (int nestedValue : nestedValues) { Document document = new Document(); document.add(new Field(IdFieldMapper.NAME, Uid.encodeId(id), IdFieldMapper.Defaults.NESTED_FIELD_TYPE)); - document.add(new Field(TypeFieldMapper.NAME, "__nested_object", TypeFieldMapper.Defaults.NESTED_FIELD_TYPE)); + document.add(new Field(NestedPathFieldMapper.NAME, "nested_object", NestedPathFieldMapper.Defaults.FIELD_TYPE)); document.add(new SortedNumericDocValuesField("nested_value", nestedValue)); documents.add(document); } diff --git a/server/src/test/java/org/opensearch/search/aggregations/bucket/terms/TermsAggregatorTests.java b/server/src/test/java/org/opensearch/search/aggregations/bucket/terms/TermsAggregatorTests.java index a9e819e7cbaf2..cb47bf6cba6a9 100644 --- a/server/src/test/java/org/opensearch/search/aggregations/bucket/terms/TermsAggregatorTests.java +++ b/server/src/test/java/org/opensearch/search/aggregations/bucket/terms/TermsAggregatorTests.java @@ -66,11 +66,11 @@ import org.opensearch.index.mapper.KeywordFieldMapper; import org.opensearch.index.mapper.MappedFieldType; import org.opensearch.index.mapper.MapperService; +import org.opensearch.index.mapper.NestedPathFieldMapper; import org.opensearch.index.mapper.NumberFieldMapper; import org.opensearch.index.mapper.RangeFieldMapper; import org.opensearch.index.mapper.RangeType; import org.opensearch.index.mapper.SeqNoFieldMapper; -import org.opensearch.index.mapper.TypeFieldMapper; import org.opensearch.index.mapper.Uid; import org.opensearch.index.query.MatchAllQueryBuilder; import org.opensearch.index.query.QueryBuilders; @@ -1464,7 +1464,7 @@ private List generateDocsWithNested(String id, int value, int[] nested for (int nestedValue : nestedValues) { Document document = new Document(); document.add(new Field(IdFieldMapper.NAME, Uid.encodeId(id), IdFieldMapper.Defaults.NESTED_FIELD_TYPE)); - document.add(new Field(TypeFieldMapper.NAME, "__nested_object", TypeFieldMapper.Defaults.NESTED_FIELD_TYPE)); + document.add(new Field(NestedPathFieldMapper.NAME, "nested_object", NestedPathFieldMapper.Defaults.FIELD_TYPE)); document.add(new SortedNumericDocValuesField("nested_value", nestedValue)); documents.add(document); } diff --git a/server/src/test/java/org/opensearch/search/sort/FieldSortBuilderTests.java b/server/src/test/java/org/opensearch/search/sort/FieldSortBuilderTests.java index 44d48e9073e23..bcf458c5028cd 100644 --- a/server/src/test/java/org/opensearch/search/sort/FieldSortBuilderTests.java +++ b/server/src/test/java/org/opensearch/search/sort/FieldSortBuilderTests.java @@ -63,8 +63,8 @@ import org.opensearch.index.mapper.DateFieldMapper; import org.opensearch.index.mapper.KeywordFieldMapper; import org.opensearch.index.mapper.MappedFieldType; +import org.opensearch.index.mapper.NestedPathFieldMapper; import org.opensearch.index.mapper.NumberFieldMapper; -import org.opensearch.index.mapper.TypeFieldMapper; import org.opensearch.index.query.MatchNoneQueryBuilder; import org.opensearch.index.query.QueryBuilder; import org.opensearch.index.query.QueryBuilders; @@ -325,7 +325,7 @@ public void testBuildNested() throws IOException { comparatorSource = (XFieldComparatorSource) sortField.getComparatorSource(); nested = comparatorSource.nested(); assertNotNull(nested); - assertEquals(new TermQuery(new Term(TypeFieldMapper.NAME, "__path")), nested.getInnerQuery()); + assertEquals(new TermQuery(new Term(NestedPathFieldMapper.NAME, "path")), nested.getInnerQuery()); sortBuilder = new FieldSortBuilder("fieldName").setNestedPath("path") .setNestedFilter(QueryBuilders.termQuery(MAPPED_STRING_FIELDNAME, "value")); diff --git a/server/src/test/java/org/opensearch/search/sort/GeoDistanceSortBuilderTests.java b/server/src/test/java/org/opensearch/search/sort/GeoDistanceSortBuilderTests.java index c14deb6add083..87adbd9532665 100644 --- a/server/src/test/java/org/opensearch/search/sort/GeoDistanceSortBuilderTests.java +++ b/server/src/test/java/org/opensearch/search/sort/GeoDistanceSortBuilderTests.java @@ -48,7 +48,7 @@ import org.opensearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested; import org.opensearch.index.mapper.GeoPointFieldMapper; import org.opensearch.index.mapper.MappedFieldType; -import org.opensearch.index.mapper.TypeFieldMapper; +import org.opensearch.index.mapper.NestedPathFieldMapper; import org.opensearch.index.query.GeoValidationMethod; import org.opensearch.index.query.MatchAllQueryBuilder; import org.opensearch.index.query.MatchNoneQueryBuilder; @@ -552,7 +552,7 @@ public void testBuildNested() throws IOException { comparatorSource = (XFieldComparatorSource) sortField.getComparatorSource(); nested = comparatorSource.nested(); assertNotNull(nested); - assertEquals(new TermQuery(new Term(TypeFieldMapper.NAME, "__path")), nested.getInnerQuery()); + assertEquals(new TermQuery(new Term(NestedPathFieldMapper.NAME, "path")), nested.getInnerQuery()); sortBuilder = new GeoDistanceSortBuilder("fieldName", 1.0, 1.0).setNestedPath("path") .setNestedFilter(QueryBuilders.matchAllQuery()); diff --git a/server/src/test/java/org/opensearch/search/sort/ScriptSortBuilderTests.java b/server/src/test/java/org/opensearch/search/sort/ScriptSortBuilderTests.java index c1e430abbe3d2..53e15c1c094ab 100644 --- a/server/src/test/java/org/opensearch/search/sort/ScriptSortBuilderTests.java +++ b/server/src/test/java/org/opensearch/search/sort/ScriptSortBuilderTests.java @@ -43,7 +43,7 @@ import org.opensearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested; import org.opensearch.index.fielddata.fieldcomparator.BytesRefFieldComparatorSource; import org.opensearch.index.fielddata.fieldcomparator.DoubleValuesComparatorSource; -import org.opensearch.index.mapper.TypeFieldMapper; +import org.opensearch.index.mapper.NestedPathFieldMapper; import org.opensearch.index.query.MatchNoneQueryBuilder; import org.opensearch.index.query.QueryBuilder; import org.opensearch.index.query.QueryBuilders; @@ -344,7 +344,7 @@ public void testBuildNested() throws IOException { comparatorSource = (XFieldComparatorSource) sortField.getComparatorSource(); nested = comparatorSource.nested(); assertNotNull(nested); - assertEquals(new TermQuery(new Term(TypeFieldMapper.NAME, "__path")), nested.getInnerQuery()); + assertEquals(new TermQuery(new Term(NestedPathFieldMapper.NAME, "path")), nested.getInnerQuery()); sortBuilder = new ScriptSortBuilder(mockScript(MOCK_SCRIPT_NAME), ScriptSortType.NUMBER).setNestedPath("path") .setNestedFilter(QueryBuilders.matchAllQuery()); From 478a61eec2a7b64ffe97ac86de6c594eb89b4532 Mon Sep 17 00:00:00 2001 From: Nicholas Walter Knize Date: Wed, 20 Apr 2022 22:09:38 -0500 Subject: [PATCH 2/3] pr fixes Signed-off-by: Nicholas Walter Knize --- .../org/opensearch/index/mapper/MapperService.java | 14 +------------- .../opensearch/indices/mapper/MapperRegistry.java | 2 +- 2 files changed, 2 insertions(+), 14 deletions(-) diff --git a/server/src/main/java/org/opensearch/index/mapper/MapperService.java b/server/src/main/java/org/opensearch/index/mapper/MapperService.java index 954a597b0bb8f..819df4a6f396e 100644 --- a/server/src/main/java/org/opensearch/index/mapper/MapperService.java +++ b/server/src/main/java/org/opensearch/index/mapper/MapperService.java @@ -162,19 +162,7 @@ public enum MergeReason { // Deprecated set of meta-fields, for checking if a field is meta, use an instance method isMetadataField instead @Deprecated public static final Set META_FIELDS_BEFORE_7DOT8 = Collections.unmodifiableSet( - new HashSet<>( - Arrays.asList( - "_id", - IgnoredFieldMapper.NAME, - NestedPathFieldMapper.NAME, - "_index", - "_routing", - "_size", - "_timestamp", - "_ttl", - "_type" - ) - ) + new HashSet<>(Arrays.asList("_id", IgnoredFieldMapper.NAME, "_index", "_routing", "_size", "_timestamp", "_ttl", "_type")) ); private static final DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(MapperService.class); diff --git a/server/src/main/java/org/opensearch/indices/mapper/MapperRegistry.java b/server/src/main/java/org/opensearch/indices/mapper/MapperRegistry.java index 54f3c32208c46..23ce1b277aeeb 100644 --- a/server/src/main/java/org/opensearch/indices/mapper/MapperRegistry.java +++ b/server/src/main/java/org/opensearch/indices/mapper/MapperRegistry.java @@ -80,7 +80,7 @@ public Map getMapperParsers() { * returned map uses the name of the field as a key. */ public Map getMetadataMapperParsers(Version indexCreatedVersion) { - return indexCreatedVersion.onOrAfter(LegacyESVersion.V_2_0_0) ? metadataMapperParsers : metadataMapperParsersPre20; + return indexCreatedVersion.onOrAfter(Version.V_2_0_0) ? metadataMapperParsers : metadataMapperParsersPre20; } /** From bf6aeccc25db5596e2890f746cb1e82fa929dbe3 Mon Sep 17 00:00:00 2001 From: Nicholas Walter Knize Date: Mon, 25 Apr 2022 11:39:37 -0500 Subject: [PATCH 3/3] add test to merge same mapping with empty index settings Signed-off-by: Nicholas Walter Knize --- .../index/mapper/NestedPathFieldMapperTests.java | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/server/src/test/java/org/opensearch/index/mapper/NestedPathFieldMapperTests.java b/server/src/test/java/org/opensearch/index/mapper/NestedPathFieldMapperTests.java index 226713960a459..6ad1d0f7f09b9 100644 --- a/server/src/test/java/org/opensearch/index/mapper/NestedPathFieldMapperTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/NestedPathFieldMapperTests.java @@ -33,4 +33,15 @@ public void testDefaultConfig() throws IOException { ParsedDocument document = mapper.parse(new SourceToParse("index", "id", new BytesArray("{}"), XContentType.JSON)); assertEquals(Collections.emptyList(), Arrays.asList(document.rootDoc().getFields(NestedPathFieldMapper.NAME))); } + + public void testUpdatesWithSameMappings() throws IOException { + Settings indexSettings = Settings.EMPTY; + MapperService mapperService = createIndex("test", indexSettings).mapperService(); + DocumentMapper mapper = mapperService.merge( + MapperService.SINGLE_MAPPING_NAME, + new CompressedXContent("{\"" + MapperService.SINGLE_MAPPING_NAME + "\":{}}"), + MapperService.MergeReason.MAPPING_UPDATE + ); + mapper.merge(mapper.mapping(), MapperService.MergeReason.MAPPING_UPDATE); + } }