diff --git a/CHANGELOG.md b/CHANGELOG.md index 34cd4c2097e48..bc3bd5c7b8580 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Add ThreadContextPermission for stashAndMergeHeaders and stashWithOrigin ([#15039](https://github.com/opensearch-project/OpenSearch/pull/15039)) - [Concurrent Segment Search] Support composite aggregations with scripting ([#15072](https://github.com/opensearch-project/OpenSearch/pull/15072)) - Add `rangeQuery` and `regexpQuery` for `constant_keyword` field type ([#14711](https://github.com/opensearch-project/OpenSearch/pull/14711)) +- Add index level setting `index.mapping.total_fields.unmap_fields_beyond_limit` to unmap fields beyond the total fields limit ([#14939](https://github.com/opensearch-project/OpenSearch/pull/14939)) - Add took time to request nodes stats ([#15054](https://github.com/opensearch-project/OpenSearch/pull/15054)) - [Workload Management] QueryGroup resource tracking framework changes ([#13897](https://github.com/opensearch-project/OpenSearch/pull/13897)) - Add slice execution listeners to SearchOperationListener interface ([#15153](https://github.com/opensearch-project/OpenSearch/pull/15153)) diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/bulk/100_unmap_fields_beyond_limit.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/bulk/100_unmap_fields_beyond_limit.yml new file mode 100644 index 0000000000000..42506dea2992f --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/bulk/100_unmap_fields_beyond_limit.yml @@ -0,0 +1,107 @@ +--- +"Test unmap the object type fields beyond the total fields limit": + - skip: + version: " - 2.99.99" + reason: "introduced in 3.0.0" + - do: + indices.create: + index: test_index_1 + body: + settings: + index.mapping.total_fields.limit: 3 + index.mapping.total_fields.unmap_fields_beyond_limit: true + + - do: + index: + index: test_index_1 + id: 1 + body: { + field1: "field1", + field2: [1,2,3], + unmapField: "foo" + } + - do: + get: + index: test_index_1 + id: 1 + - match: { _source: { field1: "field1", field2: [1,2,3], unmapField: "foo" }} + + - do: + indices.get_mapping: + index: test_index_1 + - match: {test_index_1.mappings.properties.field1.type: text} + - match: {test_index_1.mappings.properties.field1.fields.keyword.type: keyword} + - match: {test_index_1.mappings.properties.field2.type: long} + - match: {test_index_1.mappings.properties.unmapField: null} + + - do: + index: + index: test_index_1 + id: 2 + body: { + field3: { + "field4": "field4" + }, + "field5": 100, + "dateField": "2024-07-25T05:11:51.243Z", + "booleanField": true + } + - do: + get: + index: test_index_1 + id: 2 + - match: { _source: { field3: { field4: "field4" }, field5: 100, "dateField": "2024-07-25T05:11:51.243Z", "booleanField": true }} + + - do: + indices.get_mapping: + index: test_index_1 + - match: {test_index_1.mappings.properties.field1.type: text} + - match: {test_index_1.mappings.properties.field1.fields.keyword.type: keyword} + - match: {test_index_1.mappings.properties.field2.type: long} + - match: {test_index_1.mappings.properties.field3: null} + - match: {test_index_1.mappings.properties.field5: null} + - match: {test_index_1.mappings.properties.dateField.type: null} + - match: {test_index_1.mappings.properties.booleanField.type: null} + + - do: + indices.put_settings: + index: test_index_1 + body: + index.mapping.total_fields.limit: 100 + + - do: + indices.get_settings: + index: test_index_1 + - match: {test_index_1.settings.index.mapping.total_fields.limit: "100"} + + - do: + index: + index: test_index_1 + id: 2 + body: { + field1: "field1", + field2: [1,2,3], + field3: { + "field4": "field4" + }, + "field5": 100, + "dateField": "2024-07-25T05:11:51.243Z", + "booleanField": true + } + - do: + get: + index: test_index_1 + id: 2 + - match: { _source: { field1: "field1", field2: [1,2,3], field3: { field4: "field4" }, field5: 100, "dateField": "2024-07-25T05:11:51.243Z", "booleanField": true }} + + - do: + indices.get_mapping: + index: test_index_1 + - match: {test_index_1.mappings.properties.field1.type: text} + - match: {test_index_1.mappings.properties.field1.fields.keyword.type: keyword} + - match: {test_index_1.mappings.properties.field2.type: long} + - match: {test_index_1.mappings.properties.field3.properties.field4.type: text} + - match: {test_index_1.mappings.properties.field3.properties.field4.fields.keyword.type: keyword} + - match: {test_index_1.mappings.properties.field5.type: long} + - match: {test_index_1.mappings.properties.dateField.type: date} + - match: {test_index_1.mappings.properties.booleanField.type: boolean} diff --git a/server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java b/server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java index a4d60bc76127c..aeca3fc98026a 100644 --- a/server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java +++ b/server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java @@ -186,6 +186,7 @@ public final class IndexScopedSettings extends AbstractScopedSettings { MapperService.INDEX_MAPPING_NESTED_FIELDS_LIMIT_SETTING, MapperService.INDEX_MAPPING_NESTED_DOCS_LIMIT_SETTING, MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING, + MapperService.INDEX_MAPPING_TOTAL_FIELDS_UNMAP_FIELDS_BEYONGD_LIMIT_SETTING, MapperService.INDEX_MAPPING_DEPTH_LIMIT_SETTING, MapperService.INDEX_MAPPING_FIELD_NAME_LENGTH_LIMIT_SETTING, BitsetFilterCache.INDEX_LOAD_RANDOM_ACCESS_FILTERS_EAGERLY_SETTING, diff --git a/server/src/main/java/org/opensearch/index/IndexSettings.java b/server/src/main/java/org/opensearch/index/IndexSettings.java index a833d66fab5d9..c3c130e3a364f 100644 --- a/server/src/main/java/org/opensearch/index/IndexSettings.java +++ b/server/src/main/java/org/opensearch/index/IndexSettings.java @@ -75,6 +75,7 @@ import static org.opensearch.index.mapper.MapperService.INDEX_MAPPING_NESTED_DOCS_LIMIT_SETTING; import static org.opensearch.index.mapper.MapperService.INDEX_MAPPING_NESTED_FIELDS_LIMIT_SETTING; import static org.opensearch.index.mapper.MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING; +import static org.opensearch.index.mapper.MapperService.INDEX_MAPPING_TOTAL_FIELDS_UNMAP_FIELDS_BEYONGD_LIMIT_SETTING; import static org.opensearch.index.store.remote.directory.RemoteSnapshotDirectory.SEARCHABLE_SNAPSHOT_EXTENDED_COMPATIBILITY_MINIMUM_VERSION; /** @@ -809,6 +810,7 @@ private void setRetentionLeaseMillis(final TimeValue retentionLease) { private volatile long mappingNestedFieldsLimit; private volatile long mappingNestedDocsLimit; private volatile long mappingTotalFieldsLimit; + private volatile boolean unmapFieldsBeyondTotalFieldsLimit; private volatile long mappingDepthLimit; private volatile long mappingFieldNameLengthLimit; @@ -997,6 +999,7 @@ public IndexSettings(final IndexMetadata indexMetadata, final Settings nodeSetti mappingNestedFieldsLimit = scopedSettings.get(INDEX_MAPPING_NESTED_FIELDS_LIMIT_SETTING); mappingNestedDocsLimit = scopedSettings.get(INDEX_MAPPING_NESTED_DOCS_LIMIT_SETTING); mappingTotalFieldsLimit = scopedSettings.get(INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING); + unmapFieldsBeyondTotalFieldsLimit = scopedSettings.get(INDEX_MAPPING_TOTAL_FIELDS_UNMAP_FIELDS_BEYONGD_LIMIT_SETTING); mappingDepthLimit = scopedSettings.get(INDEX_MAPPING_DEPTH_LIMIT_SETTING); mappingFieldNameLengthLimit = scopedSettings.get(INDEX_MAPPING_FIELD_NAME_LENGTH_LIMIT_SETTING); maxFullFlushMergeWaitTime = scopedSettings.get(INDEX_MERGE_ON_FLUSH_MAX_FULL_FLUSH_MERGE_WAIT_TIME); @@ -1115,6 +1118,10 @@ public IndexSettings(final IndexMetadata indexMetadata, final Settings nodeSetti scopedSettings.addSettingsUpdateConsumer(INDEX_MAPPING_NESTED_FIELDS_LIMIT_SETTING, this::setMappingNestedFieldsLimit); scopedSettings.addSettingsUpdateConsumer(INDEX_MAPPING_NESTED_DOCS_LIMIT_SETTING, this::setMappingNestedDocsLimit); scopedSettings.addSettingsUpdateConsumer(INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING, this::setMappingTotalFieldsLimit); + scopedSettings.addSettingsUpdateConsumer( + INDEX_MAPPING_TOTAL_FIELDS_UNMAP_FIELDS_BEYONGD_LIMIT_SETTING, + this::setMappingUnmapFieldsBeyondTotalFieldsLimit + ); scopedSettings.addSettingsUpdateConsumer(INDEX_MAPPING_DEPTH_LIMIT_SETTING, this::setMappingDepthLimit); scopedSettings.addSettingsUpdateConsumer(INDEX_MAPPING_FIELD_NAME_LENGTH_LIMIT_SETTING, this::setMappingFieldNameLengthLimit); scopedSettings.addSettingsUpdateConsumer(INDEX_MERGE_ON_FLUSH_MAX_FULL_FLUSH_MERGE_WAIT_TIME, this::setMaxFullFlushMergeWaitTime); @@ -1861,6 +1868,14 @@ private void setMappingTotalFieldsLimit(long value) { this.mappingTotalFieldsLimit = value; } + public boolean getUnmapFieldsBeyondTotalFieldsLimit() { + return unmapFieldsBeyondTotalFieldsLimit; + } + + private void setMappingUnmapFieldsBeyondTotalFieldsLimit(boolean value) { + this.unmapFieldsBeyondTotalFieldsLimit = value; + } + public long getMappingDepthLimit() { return mappingDepthLimit; } diff --git a/server/src/main/java/org/opensearch/index/mapper/DocumentParser.java b/server/src/main/java/org/opensearch/index/mapper/DocumentParser.java index b03026d560dbf..a8de4f1e2e23f 100644 --- a/server/src/main/java/org/opensearch/index/mapper/DocumentParser.java +++ b/server/src/main/java/org/opensearch/index/mapper/DocumentParser.java @@ -554,6 +554,12 @@ private static void parseObject(final ParseContext context, ObjectMapper mapper, throw new StrictDynamicMappingException(dynamic.name().toLowerCase(Locale.ROOT), mapper.fullPath(), currentFieldName); case TRUE: case STRICT_ALLOW_TEMPLATES: + // if dynamic is true or strict_allow_templates, we check if we need to unmap the fields beyond the total fields limit + if (checkIfUnmapFieldsBeyondTotalFieldsLimit(context)) { + context.parser().skipChildren(); + break; + } + Mapper.Builder builder = findTemplateBuilder( context, currentFieldName, @@ -568,6 +574,7 @@ private static void parseObject(final ParseContext context, ObjectMapper mapper, Mapper.BuilderContext builderContext = new Mapper.BuilderContext(context.indexSettings().getSettings(), context.path()); objectMapper = builder.build(builderContext); context.addDynamicMapper(objectMapper); + increaseDynamicFieldCountIfNeed(context); context.path().add(currentFieldName); parseObjectOrField(context, objectMapper); context.path().remove(); @@ -576,12 +583,46 @@ private static void parseObject(final ParseContext context, ObjectMapper mapper, // not dynamic, read everything up to end object context.parser().skipChildren(); } + for (int i = 0; i < parentMapperTuple.v1(); i++) { context.path().remove(); } } } + /** + * if the setting `index.mapping.total_fields.unmap_fields_beyond_limit` is true, we check if the current total fields count exceed the + * total fields limit + * @param context the parse context + * @return true if `index.mapping.total_fields.unmap_fields_beyond_limit` is true and the current total fields count exceed the limit + */ + private static boolean checkIfUnmapFieldsBeyondTotalFieldsLimit(ParseContext context) { + return context.getUnmapFieldsBeyondTotalFieldsLimit() + && context.docMapper().mappers().exceedTotalFieldsLimit(context.getTotalFieldsLimit()); + } + + /** + * if the setting `index.mapping.total_fields.unmap_fields_beyond_limit` is true, increase the dynamic field count by 1 + * @param context the parse context + */ + private static void increaseDynamicFieldCountIfNeed(ParseContext context) { + if (context.getUnmapFieldsBeyondTotalFieldsLimit()) { + context.docMapper().mappers().increaseDynamicFieldCount(); + } + } + + /** + * if the setting `index.mapping.total_fields.unmap_fields_beyond_limit` is true, increase the dynamic field count by the specified + * field count + * @param context the parse context + * @param fieldCount the field count + */ + private static void increaseDynamicFieldCountIfNeed(ParseContext context, long fieldCount) { + if (context.getUnmapFieldsBeyondTotalFieldsLimit()) { + context.docMapper().mappers().increaseDynamicFieldCount(fieldCount); + } + } + private static void parseArray(ParseContext context, ObjectMapper parentMapper, String lastFieldName, String[] paths) throws IOException { try { @@ -632,6 +673,7 @@ private static void parseArray(ParseContext context, ObjectMapper parentMapper, assert mapper != null; if (parsesArrayValue(mapper)) { context.addDynamicMapper(mapper); + increaseDynamicFieldCountIfNeed(context); context.path().add(arrayFieldName); parseObjectOrField(context, mapper); context.path().remove(); @@ -864,13 +906,27 @@ private static void parseDynamicValue( if (dynamic == ObjectMapper.Dynamic.STRICT) { throw new StrictDynamicMappingException(dynamic.name().toLowerCase(Locale.ROOT), parentMapper.fullPath(), currentFieldName); } - if (dynamic == ObjectMapper.Dynamic.FALSE) { + // if dynamic is true or strict_allow_templates, and index.mapping.total_fields.unmap_fields_beyond_limit is true, + // then we check if we need to unmap the fields beyond the total fields limit + if (dynamic == ObjectMapper.Dynamic.FALSE || checkIfUnmapFieldsBeyondTotalFieldsLimit(context)) { return; } final Mapper.BuilderContext builderContext = new Mapper.BuilderContext(context.indexSettings().getSettings(), context.path()); final Mapper.Builder builder = createBuilderFromDynamicValue(context, token, currentFieldName, dynamic, parentMapper.fullPath()); Mapper mapper = builder.build(builderContext); + + // edge case, adding a new field may increase the dynamic field count by 2 or more, + // so we check if adding a new field will cause the total field count to exceed the total fields limit, if so we don't add it + long fieldCount = 0; + if (context.getUnmapFieldsBeyondTotalFieldsLimit()) { + fieldCount = context.docMapper().mappers().countFields(mapper); + if (context.docMapper().mappers().exceedTotalFieldsLimitIfAddNewField(context.getTotalFieldsLimit(), fieldCount)) { + return; + } + } + context.addDynamicMapper(mapper); + increaseDynamicFieldCountIfNeed(context, fieldCount); parseObjectOrField(context, mapper); } @@ -959,6 +1015,11 @@ private static Tuple getDynamicParentMapper( throw new StrictDynamicMappingException(dynamic.name().toLowerCase(Locale.ROOT), parent.fullPath(), paths[i]); case STRICT_ALLOW_TEMPLATES: case TRUE: + // if dynamic is true or strict_allow_templates, we check if we need to unmap the fields beyond the total fields + // limit + if (checkIfUnmapFieldsBeyondTotalFieldsLimit(context)) { + return new Tuple<>(pathsAdded, parent); + } Mapper.Builder builder = findTemplateBuilder( context, paths[i], @@ -982,6 +1043,7 @@ private static Tuple getDynamicParentMapper( ); } context.addDynamicMapper(mapper); + increaseDynamicFieldCountIfNeed(context); break; case FALSE: // Should not dynamically create any more mappers so return the last mapper diff --git a/server/src/main/java/org/opensearch/index/mapper/MapperService.java b/server/src/main/java/org/opensearch/index/mapper/MapperService.java index 530a3092a5aa7..89d497a7355c9 100644 --- a/server/src/main/java/org/opensearch/index/mapper/MapperService.java +++ b/server/src/main/java/org/opensearch/index/mapper/MapperService.java @@ -146,6 +146,14 @@ public enum MergeReason { Property.Dynamic, Property.IndexScope ); + // if set to true, the new detected fields from dynamic mapping which beyond the total fields limit will be unmapped, i.e. will not + // be added to the mapping + public static final Setting INDEX_MAPPING_TOTAL_FIELDS_UNMAP_FIELDS_BEYONGD_LIMIT_SETTING = Setting.boolSetting( + "index.mapping.total_fields.unmap_fields_beyond_limit", + false, + Property.Dynamic, + Property.IndexScope + ); public static final Setting INDEX_MAPPING_DEPTH_LIMIT_SETTING = Setting.longSetting( "index.mapping.depth.limit", 20L, diff --git a/server/src/main/java/org/opensearch/index/mapper/MappingLookup.java b/server/src/main/java/org/opensearch/index/mapper/MappingLookup.java index fdebe24327ca0..3745f07b8ba22 100644 --- a/server/src/main/java/org/opensearch/index/mapper/MappingLookup.java +++ b/server/src/main/java/org/opensearch/index/mapper/MappingLookup.java @@ -60,8 +60,9 @@ public final class MappingLookup implements Iterable { private final Map objectMappers; private final boolean hasNested; private final FieldTypeLookup fieldTypeLookup; - private final int metadataFieldCount; private final FieldNameAnalyzer indexAnalyzer; + private final int nonMetadataFieldCount; + private int dynamicFieldCount; private static void put(Map analyzers, String key, Analyzer value, Analyzer defaultValue) { if (value == null) { @@ -138,7 +139,6 @@ public MappingLookup( MappedFieldType fieldType = mapper.fieldType(); put(indexAnalyzers, fieldType.name(), fieldType.indexAnalyzer(), defaultIndex); } - this.metadataFieldCount = metadataFieldCount; for (FieldAliasMapper aliasMapper : aliasMappers) { if (objects.containsKey(aliasMapper.name())) { @@ -154,6 +154,7 @@ public MappingLookup( this.fieldMappers = Collections.unmodifiableMap(fieldMappers); this.indexAnalyzer = new FieldNameAnalyzer(indexAnalyzers); this.objectMappers = Collections.unmodifiableMap(objects); + this.nonMetadataFieldCount = fieldMappers.size() + objectMappers.size() - metadataFieldCount; } /** @@ -190,8 +191,58 @@ public void checkLimits(IndexSettings settings) { checkNestedLimit(settings.getMappingNestedFieldsLimit()); } + /** + * + * @param limit the value of the setting index.mapping.total_fields_limit + * @return true if adding new fields will cause to exceed the total fields limit + */ + public boolean exceedTotalFieldsLimit(long limit) { + return nonMetadataFieldCount + dynamicFieldCount >= limit; + } + + /** + * + * @param limit the value of the setting index.mapping.total_fields_limit + * @param fieldCount the field count in the new detected field + * @return true if adding a new field will cause to exceed the total fields limit + */ + public boolean exceedTotalFieldsLimitIfAddNewField(long limit, long fieldCount) { + return nonMetadataFieldCount + dynamicFieldCount + fieldCount > limit; + } + + /** + * increase the dynamic field count by 1 + */ + public void increaseDynamicFieldCount() { + this.dynamicFieldCount++; + } + + /** + * increase the dynamic field count by the given field count + * @param fieldCount the field count in the new detected field + */ + public void increaseDynamicFieldCount(long fieldCount) { + this.dynamicFieldCount += fieldCount; + } + + /** + * count the total fields in the specified field mapper + * @param mapper the field mapper + * @return the field count in the specified field mapper + */ + public long countFields(Mapper mapper) { + long count = 0; + if (mapper instanceof ObjectMapper || mapper instanceof FieldMapper) { + count++; + } + for (Mapper child : mapper) { + count += countFields(child); + } + return count; + } + private void checkFieldLimit(long limit) { - if (fieldMappers.size() + objectMappers.size() - metadataFieldCount > limit) { + if (nonMetadataFieldCount > limit) { throw new IllegalArgumentException("Limit of total fields [" + limit + "] has been exceeded"); } } diff --git a/server/src/main/java/org/opensearch/index/mapper/ParseContext.java b/server/src/main/java/org/opensearch/index/mapper/ParseContext.java index 5d382ff28bcf9..754abeb62d8c9 100644 --- a/server/src/main/java/org/opensearch/index/mapper/ParseContext.java +++ b/server/src/main/java/org/opensearch/index/mapper/ParseContext.java @@ -346,6 +346,16 @@ public void decrementFieldArrayDepth() { public void checkFieldArrayDepthLimit() { in.checkFieldArrayDepthLimit(); } + + @Override + public boolean getUnmapFieldsBeyondTotalFieldsLimit() { + return in.getUnmapFieldsBeyondTotalFieldsLimit(); + } + + @Override + public long getTotalFieldsLimit() { + return in.getTotalFieldsLimit(); + } } /** @@ -392,6 +402,8 @@ public static class InternalParseContext extends ParseContext { private boolean docsReversed = false; private final Set ignoredFields = new HashSet<>(); + private final boolean unmapFieldsBeyondTotalFieldsLimit; + private final long totalFieldsLimit; public InternalParseContext( IndexSettings indexSettings, @@ -417,6 +429,8 @@ public InternalParseContext( this.currentArrayDepth = 0L; this.maxAllowedFieldDepth = indexSettings.getMappingDepthLimit(); this.maxAllowedArrayDepth = indexSettings.getMappingDepthLimit(); + this.unmapFieldsBeyondTotalFieldsLimit = indexSettings.getUnmapFieldsBeyondTotalFieldsLimit(); + this.totalFieldsLimit = indexSettings.getMappingTotalFieldsLimit(); } @Override @@ -622,6 +636,16 @@ public void checkFieldArrayDepthLimit() { ); } } + + @Override + public boolean getUnmapFieldsBeyondTotalFieldsLimit() { + return this.unmapFieldsBeyondTotalFieldsLimit; + } + + @Override + public long getTotalFieldsLimit() { + return this.totalFieldsLimit; + } } /** @@ -800,4 +824,8 @@ public final T parseExternalValue(Class clazz) { public abstract void checkFieldArrayDepthLimit(); + public abstract boolean getUnmapFieldsBeyondTotalFieldsLimit(); + + public abstract long getTotalFieldsLimit(); + } diff --git a/server/src/test/java/org/opensearch/index/IndexSettingsTests.java b/server/src/test/java/org/opensearch/index/IndexSettingsTests.java index 474ec73d5fe61..26605ab98e617 100644 --- a/server/src/test/java/org/opensearch/index/IndexSettingsTests.java +++ b/server/src/test/java/org/opensearch/index/IndexSettingsTests.java @@ -60,6 +60,7 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Function; +import static org.opensearch.index.mapper.MapperService.INDEX_MAPPING_TOTAL_FIELDS_UNMAP_FIELDS_BEYONGD_LIMIT_SETTING; import static org.opensearch.index.store.remote.directory.RemoteSnapshotDirectory.SEARCHABLE_SNAPSHOT_EXTENDED_COMPATIBILITY_MINIMUM_VERSION; import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.core.StringContains.containsString; @@ -787,6 +788,35 @@ public void testRemoteStoreExplicitSetting() { assertTrue(settings.isRemoteStoreEnabled()); } + public void testUnmapFieldsBeyondTotalFieldsLimitSetting() { + IndexMetadata metadata = newIndexMeta( + "index", + Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT).build() + ); + IndexSettings settings = new IndexSettings(metadata, Settings.EMPTY); + assertFalse(settings.getUnmapFieldsBeyondTotalFieldsLimit()); + + metadata = newIndexMeta( + "index", + Settings.builder() + .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT) + .put(INDEX_MAPPING_TOTAL_FIELDS_UNMAP_FIELDS_BEYONGD_LIMIT_SETTING.getKey(), true) + .build() + ); + settings = new IndexSettings(metadata, Settings.EMPTY); + assertTrue(settings.getUnmapFieldsBeyondTotalFieldsLimit()); + + Settings.Builder newSettings = Settings.builder() + .put( + Settings.builder() + .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT) + .put(INDEX_MAPPING_TOTAL_FIELDS_UNMAP_FIELDS_BEYONGD_LIMIT_SETTING.getKey(), false) + .build() + ); + settings.updateIndexMetadata(newIndexMeta("index", newSettings.build())); + assertFalse(settings.getUnmapFieldsBeyondTotalFieldsLimit()); + } + public void testRemoteTranslogStoreDefaultSetting() { IndexMetadata metadata = newIndexMeta( "index", diff --git a/server/src/test/java/org/opensearch/index/mapper/DocumentParserTests.java b/server/src/test/java/org/opensearch/index/mapper/DocumentParserTests.java index c9763d634979b..bf30c21e12944 100644 --- a/server/src/test/java/org/opensearch/index/mapper/DocumentParserTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/DocumentParserTests.java @@ -1200,6 +1200,295 @@ public void testDynamicStrictAllowTemplatesValue() throws Exception { assertEquals(2, doc.rootDoc().getFields("test1").length); } + public void testDynamicValueWithUnmapFieldsBeyondTotalLimit() throws Exception { + { + Settings settings = Settings.builder() + .put(MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING.getKey(), 3) + .put(MapperService.INDEX_MAPPING_TOTAL_FIELDS_UNMAP_FIELDS_BEYONGD_LIMIT_SETTING.getKey(), true) + .build(); + DocumentMapper mapper = createDocumentMapper(topMapping(b -> { + b.field("dynamic", "true"); + b.startObject("properties"); + { + b.startObject("foo"); + b.field("type", "keyword"); + b.endObject(); + b.startObject("bar"); + b.field("type", "keyword"); + b.endObject(); + b.startObject("zoo"); + b.field("type", "keyword"); + b.endObject(); + } + b.endObject(); + } + + ), settings); + + ParsedDocument doc = mapper.parse(source(b -> b.field("test1", "baz"))); + assertEquals(0, doc.rootDoc().getFields("test1").length); + } + + { + Settings settings = Settings.builder() + .put(MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING.getKey(), 3) + .put(MapperService.INDEX_MAPPING_TOTAL_FIELDS_UNMAP_FIELDS_BEYONGD_LIMIT_SETTING.getKey(), true) + .build(); + DocumentMapper mapper = createDocumentMapper(topMapping(b -> { + b.field("dynamic", "true"); + b.startObject("properties"); + { + b.startObject("foo"); + b.field("type", "keyword"); + b.endObject(); + b.startObject("bar"); + b.field("type", "keyword"); + b.endObject(); + } + b.endObject(); + } + + ), settings); + + // Add a string type field will add two fields into the mapping(text+keyword), so the field `test` + // will not be added to the mapping because of the total fields limit + ParsedDocument doc = mapper.parse(source(b -> b.field("test", "baz"))); + assertEquals(0, doc.rootDoc().getFields("test").length); + } + } + + public void testDynamicLongArrayWithUnmapFieldsBeyondTotalLimit() throws Exception { + { + Settings settings = Settings.builder() + .put(MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING.getKey(), 3) + .put(MapperService.INDEX_MAPPING_TOTAL_FIELDS_UNMAP_FIELDS_BEYONGD_LIMIT_SETTING.getKey(), true) + .build(); + DocumentMapper mapper = createDocumentMapper(topMapping(b -> { + b.field("dynamic", "true"); + b.startObject("properties"); + { + b.startObject("foo"); + b.field("type", "keyword"); + b.endObject(); + b.startObject("bar"); + b.field("type", "keyword"); + b.endObject(); + b.startObject("zoo"); + b.field("type", "keyword"); + b.endObject(); + } + b.endObject(); + }), settings); + + ParsedDocument doc = mapper.parse(source(b -> b.startArray("test").value(0).value(1).endArray())); + assertEquals(0, doc.rootDoc().getFields("test").length); + } + + { + Settings settings = Settings.builder() + .put(MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING.getKey(), 3) + .put(MapperService.INDEX_MAPPING_TOTAL_FIELDS_UNMAP_FIELDS_BEYONGD_LIMIT_SETTING.getKey(), true) + .build(); + DocumentMapper mapper = createDocumentMapper(topMapping(b -> { + b.field("dynamic", "true"); + b.startObject("properties"); + { + b.startObject("foo"); + b.field("type", "keyword"); + b.endObject(); + b.startObject("bar"); + b.field("type", "keyword"); + b.endObject(); + } + b.endObject(); + }), settings); + + ParsedDocument doc = mapper.parse(source(b -> b.startArray("test").value(0).value(1).endArray())); + assertEquals(2, doc.rootDoc().getFields("test").length); + } + } + + public void testDynamicObjectWithUnmapFieldsBeyondTotalLimit() throws Exception { + { + Settings settings = Settings.builder() + .put(MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING.getKey(), 3) + .put(MapperService.INDEX_MAPPING_TOTAL_FIELDS_UNMAP_FIELDS_BEYONGD_LIMIT_SETTING.getKey(), true) + .build(); + DocumentMapper mapper = createDocumentMapper(topMapping(b -> { + b.field("dynamic", "true"); + b.startObject("properties"); + { + b.startObject("foo"); + b.field("type", "keyword"); + b.endObject(); + b.startObject("bar"); + b.field("type", "keyword"); + b.endObject(); + b.startObject("zoo"); + b.field("type", "keyword"); + b.endObject(); + } + b.endObject(); + } + + ), settings); + + ParsedDocument doc = mapper.parse(source(b -> b.startObject("test").field("test1", "baz").endObject())); + assertEquals(0, doc.rootDoc().getFields("test.test1").length); + } + + { + Settings settings = Settings.builder() + .put(MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING.getKey(), 3) + .put(MapperService.INDEX_MAPPING_TOTAL_FIELDS_UNMAP_FIELDS_BEYONGD_LIMIT_SETTING.getKey(), true) + .build(); + DocumentMapper mapper = createDocumentMapper(topMapping(b -> { + b.field("dynamic", "true"); + b.startObject("properties"); + { + b.startObject("foo"); + b.field("type", "keyword"); + b.endObject(); + b.startObject("bar"); + b.field("type", "keyword"); + b.endObject(); + } + b.endObject(); + } + + ), settings); + + ParsedDocument doc = mapper.parse(source(b -> b.startObject("test").field("test1", "baz").endObject())); + assertEquals(0, doc.rootDoc().getFields("test.test1").length); + } + } + + public void testDynamicStrictAllowTemplatesValueWithUnmapFieldsBeyondTotalLimit() throws Exception { + Settings settings = Settings.builder() + .put(MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING.getKey(), 3) + .put(MapperService.INDEX_MAPPING_TOTAL_FIELDS_UNMAP_FIELDS_BEYONGD_LIMIT_SETTING.getKey(), true) + .build(); + DocumentMapper mapper = createDocumentMapper(topMapping(b -> { + b.field("dynamic", "strict_allow_templates"); + b.startArray("dynamic_templates"); + { + b.startObject(); + { + b.startObject("test"); + { + b.field("match", "test*"); + b.field("match_mapping_type", "string"); + b.startObject("mapping").field("type", "keyword").endObject(); + } + b.endObject(); + } + b.endObject(); + } + b.endArray(); + b.startObject("properties"); + { + b.startObject("foo"); + b.field("type", "keyword"); + b.endObject(); + b.startObject("bar"); + b.field("type", "keyword"); + b.endObject(); + b.startObject("zoo"); + b.field("type", "keyword"); + b.endObject(); + } + b.endObject(); + } + + ), settings); + + ParsedDocument doc = mapper.parse(source(b -> b.field("test1", "baz"))); + assertEquals(0, doc.rootDoc().getFields("test1").length); + } + + public void testDynamicAllowTemplatesStrictLongArrayWithUnmapFieldsBeyondTotalLimit() throws Exception { + Settings settings = Settings.builder() + .put(MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING.getKey(), 3) + .put(MapperService.INDEX_MAPPING_TOTAL_FIELDS_UNMAP_FIELDS_BEYONGD_LIMIT_SETTING.getKey(), true) + .build(); + DocumentMapper mapper = createDocumentMapper(topMapping(b -> { + b.field("dynamic", "strict_allow_templates"); + b.startArray("dynamic_templates"); + { + b.startObject(); + { + b.startObject("test"); + { + b.field("match", "test"); + b.startObject("mapping").field("type", "long").endObject(); + } + b.endObject(); + } + b.endObject(); + } + b.endArray(); + b.startObject("properties"); + { + b.startObject("foo"); + b.field("type", "keyword"); + b.endObject(); + b.startObject("bar"); + b.field("type", "keyword"); + b.endObject(); + b.startObject("zoo"); + b.field("type", "keyword"); + b.endObject(); + } + b.endObject(); + }), settings); + + ParsedDocument doc = mapper.parse(source(b -> b.startArray("test").value(0).value(1).endArray())); + assertEquals(0, doc.rootDoc().getFields("test").length); + } + + public void testDynamicStrictAllowTemplatesObjectWithUnmapFieldsBeyondTotalLimit() throws Exception { + Settings settings = Settings.builder() + .put(MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING.getKey(), 3) + .put(MapperService.INDEX_MAPPING_TOTAL_FIELDS_UNMAP_FIELDS_BEYONGD_LIMIT_SETTING.getKey(), true) + .build(); + DocumentMapper mapper = createDocumentMapper(topMapping(b -> { + b.field("dynamic", "strict_allow_templates"); + b.startArray("dynamic_templates"); + { + b.startObject(); + { + b.startObject("test"); + { + b.field("match", "test"); + b.field("match_mapping_type", "object"); + b.startObject("mapping").field("type", "object").endObject(); + } + b.endObject(); + } + b.endObject(); + } + b.endArray(); + b.startObject("properties"); + { + b.startObject("foo"); + b.field("type", "keyword"); + b.endObject(); + b.startObject("bar"); + b.field("type", "keyword"); + b.endObject(); + b.startObject("zoo"); + b.field("type", "keyword"); + b.endObject(); + } + b.endObject(); + } + + ), settings); + + ParsedDocument doc = mapper.parse(source(b -> b.startObject("test").field("test1", "baz").endObject())); + assertEquals(0, doc.rootDoc().getFields("test.test1").length); + } + public void testDynamicStrictAllowTemplatesNull() throws Exception { DocumentMapper mapper = createDocumentMapper(topMapping(b -> b.field("dynamic", "strict_allow_templates"))); StrictDynamicMappingException exception = expectThrows( diff --git a/test/framework/src/main/java/org/opensearch/index/mapper/MapperServiceTestCase.java b/test/framework/src/main/java/org/opensearch/index/mapper/MapperServiceTestCase.java index bd16e4f65e159..98fb4018551af 100644 --- a/test/framework/src/main/java/org/opensearch/index/mapper/MapperServiceTestCase.java +++ b/test/framework/src/main/java/org/opensearch/index/mapper/MapperServiceTestCase.java @@ -110,8 +110,12 @@ protected final DocumentMapper createDocumentMapper(XContentBuilder mappings) th return createMapperService(mappings).documentMapper(); } + protected final DocumentMapper createDocumentMapper(XContentBuilder mappings, Settings settings) throws IOException { + return createMapperService(mappings, settings).documentMapper(); + } + protected final DocumentMapper createDocumentMapper(Version version, XContentBuilder mappings) throws IOException { - return createMapperService(version, mappings).documentMapper(); + return createMapperService(version, mappings, getIndexSettings()).documentMapper(); } protected final DocumentMapper createDocumentMapper(String type, String mappings) throws IOException { @@ -121,7 +125,11 @@ protected final DocumentMapper createDocumentMapper(String type, String mappings } protected MapperService createMapperService(XContentBuilder mappings) throws IOException { - return createMapperService(Version.CURRENT, mappings); + return createMapperService(Version.CURRENT, mappings, getIndexSettings()); + } + + protected MapperService createMapperService(XContentBuilder mappings, Settings settings) throws IOException { + return createMapperService(Version.CURRENT, mappings, settings); } protected final MapperService createMapperService(String type, String mappings) throws IOException { @@ -133,13 +141,13 @@ protected final MapperService createMapperService(String type, String mappings) /** * Create a {@link MapperService} like we would for an index. */ - protected final MapperService createMapperService(Version version, XContentBuilder mapping) throws IOException { + protected final MapperService createMapperService(Version version, XContentBuilder mapping, Settings settings) throws IOException { IndexMetadata meta = IndexMetadata.builder("index") .settings(Settings.builder().put("index.version.created", version)) .numberOfReplicas(0) .numberOfShards(1) .build(); - IndexSettings indexSettings = new IndexSettings(meta, getIndexSettings()); + IndexSettings indexSettings = new IndexSettings(meta, settings); MapperRegistry mapperRegistry = new IndicesModule( getPlugins().stream().filter(p -> p instanceof MapperPlugin).map(p -> (MapperPlugin) p).collect(toList()) ).getMapperRegistry();