Add index level setting to unmap fields beyond total fields limit

CHANGELOG.md

@@ -8,6 +8,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Add fingerprint ingest processor ([#13724](https://github.com/opensearch-project/OpenSearch/pull/13724))
 - [Remote Store] Rate limiter for remote store low priority uploads ([#14374](https://github.com/opensearch-project/OpenSearch/pull/14374/))
 - Apply the date histogram rewrite optimization to range aggregation ([#13865](https://github.com/opensearch-project/OpenSearch/pull/13865))
+- Add index level setting `index.mapping.total_fields.unmap_fields_beyond_limit` to unmap fields beyond the total fields limit ([#14939](https://github.com/opensearch-project/OpenSearch/pull/14939))
 - [Writable Warm] Add composite directory implementation and integrate it with FileCache ([12782](https://github.com/opensearch-project/OpenSearch/pull/12782))
 - [Workload Management] Add QueryGroup schema ([13669](https://github.com/opensearch-project/OpenSearch/pull/13669))
 - Add batching supported processor base type AbstractBatchingProcessor ([#14554](https://github.com/opensearch-project/OpenSearch/pull/14554))

rest-api-spec/src/main/resources/rest-api-spec/test/bulk/100_unmap_fields_beyond_limit.yml

@@ -0,0 +1,99 @@
+---
+"Test unmap the object type fields beyond the total fields limit":
+  - skip:
+      version: " - 2.99.99"
+      reason: "introduced in 3.0.0"
+  - do:
+      indices.create:
+        index: test_index_1
+        body:
+          settings:
+            index.mapping.total_fields.limit: 3
+            index.mapping.total_fields.unmap_fields_beyond_limit: true
+
+  - do:
+      index:
+        index:   test_index_1
+        id: 1
+        body:    {
+          field1: "field1",
+          field2: [1,2,3],
+          unmapField: "foo"
+        }
+  - do:
+      get:
+        index: test_index_1
+        id: 1
+  - match: { _source:  {  field1: "field1", field2: [1,2,3], unmapField: "foo" }}
+
+  - do:
+      indices.get_mapping:
+        index: test_index_1
+  - match: {test_index_1.mappings.properties.field1.type:     text}
+  - match: {test_index_1.mappings.properties.field1.fields.keyword.type:     keyword}
+  - match: {test_index_1.mappings.properties.field2.type:     long}
+  - match: {test_index_1.mappings.properties.unmapField: null}
+
+  - do:
+      index:
+        index:   test_index_1
+        id: 2
+        body:    {
+          field3: {
+            "field4": "field4"
+          },
+          "field5": 100
+        }
+  - do:
+      get:
+        index: test_index_1
+        id: 2
+  - match: { _source:  {  field3: { field4: "field4" }, field5: 100 }}
+
+  - do:
+      indices.get_mapping:
+        index: test_index_1
+  - match: {test_index_1.mappings.properties.field1.type:     text}
+  - match: {test_index_1.mappings.properties.field1.fields.keyword.type:     keyword}
+  - match: {test_index_1.mappings.properties.field2.type:     long}
+  - match: {test_index_1.mappings.properties.field3: null}
+  - match: {test_index_1.mappings.properties.field5: null}
+
+  - do:
+      indices.put_settings:
+        index: test_index_1
+        body:
+          index.mapping.total_fields.limit: 100
+
+  - do:
+      indices.get_settings:
+        index: test_index_1
+  - match: {test_index_1.settings.index.mapping.total_fields.limit:     "100"}
+
+  - do:
+      index:
+        index:   test_index_1
+        id: 2
+        body:    {
+          field1: "field1",
+          field2: [1,2,3],
+          field3: {
+            "field4": "field4"
+          },
+          "field5": 100
+        }
+  - do:
+      get:
+        index: test_index_1
+        id: 2
+  - match: { _source:  {  field1: "field1", field2: [1,2,3], field3: { field4: "field4" }, field5: 100 }}
+
+  - do:
+      indices.get_mapping:
+        index: test_index_1
+  - match: {test_index_1.mappings.properties.field1.type:     text}
+  - match: {test_index_1.mappings.properties.field1.fields.keyword.type:     keyword}
+  - match: {test_index_1.mappings.properties.field2.type:     long}
+  - match: {test_index_1.mappings.properties.field3.properties.field4.type:     text}
+  - match: {test_index_1.mappings.properties.field3.properties.field4.fields.keyword.type:     keyword}
+  - match: {test_index_1.mappings.properties.field5.type:     long}

server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java

@@ -186,6 +186,7 @@ public final class IndexScopedSettings extends AbstractScopedSettings {
                 MapperService.INDEX_MAPPING_NESTED_FIELDS_LIMIT_SETTING,
                 MapperService.INDEX_MAPPING_NESTED_DOCS_LIMIT_SETTING,
                 MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING,
+                MapperService.INDEX_MAPPING_TOTAL_FIELDS_UNMAP_FIELDS_BEYONGD_LIMIT_SETTING,
                 MapperService.INDEX_MAPPING_DEPTH_LIMIT_SETTING,
                 MapperService.INDEX_MAPPING_FIELD_NAME_LENGTH_LIMIT_SETTING,
                 BitsetFilterCache.INDEX_LOAD_RANDOM_ACCESS_FILTERS_EAGERLY_SETTING,

server/src/main/java/org/opensearch/index/IndexSettings.java

@@ -75,6 +75,7 @@
 import static org.opensearch.index.mapper.MapperService.INDEX_MAPPING_NESTED_DOCS_LIMIT_SETTING;
 import static org.opensearch.index.mapper.MapperService.INDEX_MAPPING_NESTED_FIELDS_LIMIT_SETTING;
 import static org.opensearch.index.mapper.MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING;
+import static org.opensearch.index.mapper.MapperService.INDEX_MAPPING_TOTAL_FIELDS_UNMAP_FIELDS_BEYONGD_LIMIT_SETTING;
 import static org.opensearch.index.store.remote.directory.RemoteSnapshotDirectory.SEARCHABLE_SNAPSHOT_EXTENDED_COMPATIBILITY_MINIMUM_VERSION;
 
 /**
@@ -809,6 +810,7 @@
     private volatile long mappingNestedFieldsLimit;
     private volatile long mappingNestedDocsLimit;
     private volatile long mappingTotalFieldsLimit;
+    private volatile boolean unmapFieldsBeyondTotalFieldsLimit;
     private volatile long mappingDepthLimit;
     private volatile long mappingFieldNameLengthLimit;
 
@@ -997,6 +999,7 @@
         mappingNestedFieldsLimit = scopedSettings.get(INDEX_MAPPING_NESTED_FIELDS_LIMIT_SETTING);
         mappingNestedDocsLimit = scopedSettings.get(INDEX_MAPPING_NESTED_DOCS_LIMIT_SETTING);
         mappingTotalFieldsLimit = scopedSettings.get(INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING);
+        unmapFieldsBeyondTotalFieldsLimit = scopedSettings.get(INDEX_MAPPING_TOTAL_FIELDS_UNMAP_FIELDS_BEYONGD_LIMIT_SETTING);
         mappingDepthLimit = scopedSettings.get(INDEX_MAPPING_DEPTH_LIMIT_SETTING);
         mappingFieldNameLengthLimit = scopedSettings.get(INDEX_MAPPING_FIELD_NAME_LENGTH_LIMIT_SETTING);
         maxFullFlushMergeWaitTime = scopedSettings.get(INDEX_MERGE_ON_FLUSH_MAX_FULL_FLUSH_MERGE_WAIT_TIME);
@@ -1115,6 +1118,10 @@
         scopedSettings.addSettingsUpdateConsumer(INDEX_MAPPING_NESTED_FIELDS_LIMIT_SETTING, this::setMappingNestedFieldsLimit);
         scopedSettings.addSettingsUpdateConsumer(INDEX_MAPPING_NESTED_DOCS_LIMIT_SETTING, this::setMappingNestedDocsLimit);
         scopedSettings.addSettingsUpdateConsumer(INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING, this::setMappingTotalFieldsLimit);
+        scopedSettings.addSettingsUpdateConsumer(
+            INDEX_MAPPING_TOTAL_FIELDS_UNMAP_FIELDS_BEYONGD_LIMIT_SETTING,
+            this::setMappingUnmapFieldsBeyondTotalFieldsLimit
+        );
         scopedSettings.addSettingsUpdateConsumer(INDEX_MAPPING_DEPTH_LIMIT_SETTING, this::setMappingDepthLimit);
         scopedSettings.addSettingsUpdateConsumer(INDEX_MAPPING_FIELD_NAME_LENGTH_LIMIT_SETTING, this::setMappingFieldNameLengthLimit);
         scopedSettings.addSettingsUpdateConsumer(INDEX_MERGE_ON_FLUSH_MAX_FULL_FLUSH_MERGE_WAIT_TIME, this::setMaxFullFlushMergeWaitTime);
@@ -1861,6 +1868,14 @@
         this.mappingTotalFieldsLimit = value;
     }
 
+    public boolean getUnmapFieldsBeyondTotalFieldsLimit() {
+        return unmapFieldsBeyondTotalFieldsLimit;
+    }
+
+    private void setMappingUnmapFieldsBeyondTotalFieldsLimit(boolean value) {
+        this.unmapFieldsBeyondTotalFieldsLimit = value;
+    }
+
     public long getMappingDepthLimit() {
         return mappingDepthLimit;
     }

server/src/main/java/org/opensearch/index/mapper/DocumentParser.java

@@ -551,6 +551,12 @@
                     throw new StrictDynamicMappingException(dynamic.name().toLowerCase(Locale.ROOT), mapper.fullPath(), currentFieldName);
                 case TRUE:
                 case STRICT_ALLOW_TEMPLATES:
+                    // if dynamic is true or strict_allow_templates, we check if we need to unmap the fields beyond the total fields limit
+                    if (checkIfUnmapFieldsBeyondTotalFieldsLimit(context)) {
+                        context.parser().skipChildren();
+                        break;
+                    }
+
                     Mapper.Builder builder = findTemplateBuilder(
                         context,
                         currentFieldName,
@@ -573,12 +579,27 @@
                     // not dynamic, read everything up to end object
                     context.parser().skipChildren();
             }
+
             for (int i = 0; i < parentMapperTuple.v1(); i++) {
                 context.path().remove();
             }
         }
     }
 
+    private static boolean checkIfUnmapFieldsBeyondTotalFieldsLimit(ParseContext context) {
+        return context.indexSettings().getUnmapFieldsBeyondTotalFieldsLimit()
+            && context.docMapper()
+                .mappers()
+                .exceedTotalFieldsLimit(context.indexSettings().getMappingTotalFieldsLimit(), context.getDynamicMappers());
+    }
+
+    private static boolean checkUnmapFieldsOrNotIfAddNewField(ParseContext context, Mapper mapper) {
+        return context.indexSettings().getUnmapFieldsBeyondTotalFieldsLimit()
+            && context.docMapper()
+                .mappers()
+                .exceedTotalFieldsLimitIfAddNewField(context.indexSettings().getMappingTotalFieldsLimit(), mapper);
+    }
+
     private static void parseArray(ParseContext context, ObjectMapper parentMapper, String lastFieldName, String[] paths)
         throws IOException {
         try {
@@ -861,12 +882,20 @@
         if (dynamic == ObjectMapper.Dynamic.STRICT) {
             throw new StrictDynamicMappingException(dynamic.name().toLowerCase(Locale.ROOT), parentMapper.fullPath(), currentFieldName);
         }
-        if (dynamic == ObjectMapper.Dynamic.FALSE) {
+        // if dynamic is true or strict_allow_templates, and index.mapping.total_fields.unmap_fields_beyond_limit is true,
+        // then we check if we need to unmap the fields beyond the total fields limit
+        if (dynamic == ObjectMapper.Dynamic.FALSE || checkIfUnmapFieldsBeyondTotalFieldsLimit(context)) {
             return;
         }
         final Mapper.BuilderContext builderContext = new Mapper.BuilderContext(context.indexSettings().getSettings(), context.path());
         final Mapper.Builder<?> builder = createBuilderFromDynamicValue(context, token, currentFieldName, dynamic, parentMapper.fullPath());
         Mapper mapper = builder.build(builderContext);
+
+        // edge case, if index.mapping.total_fields.unmap_fields_beyond_limit is true,
+        // then we check if adding a new field will cause the field count to exceed the total fields limit, if so we don't add it
+        if (checkUnmapFieldsOrNotIfAddNewField(context, mapper)) {
+            return;
+        }
         context.addDynamicMapper(mapper);
 
         parseObjectOrField(context, mapper);
@@ -956,6 +985,11 @@
                         throw new StrictDynamicMappingException(dynamic.name().toLowerCase(Locale.ROOT), parent.fullPath(), paths[i]);
                     case STRICT_ALLOW_TEMPLATES:
                     case TRUE:
+                        // if dynamic is true or strict_allow_templates, we check if we need to unmap the fields beyond the total fields
+                        // limit
+                        if (checkIfUnmapFieldsBeyondTotalFieldsLimit(context)) {
+                            return new Tuple<>(pathsAdded, parent);
+                        }
                         Mapper.Builder builder = findTemplateBuilder(
                             context,
                             paths[i],

server/src/main/java/org/opensearch/index/mapper/MapperService.java

@@ -146,6 +146,14 @@ public enum MergeReason {
         Property.Dynamic,
         Property.IndexScope
     );
+    // if set to true, the new detected fields from dynamic mapping which beyond the total fields limit will be unmapped, i.e. will not
+    // be added to the mapping
+    public static final Setting<Boolean> INDEX_MAPPING_TOTAL_FIELDS_UNMAP_FIELDS_BEYONGD_LIMIT_SETTING = Setting.boolSetting(
+        "index.mapping.total_fields.unmap_fields_beyond_limit",
+        false,
+        Property.Dynamic,
+        Property.IndexScope
+    );
     public static final Setting<Long> INDEX_MAPPING_DEPTH_LIMIT_SETTING = Setting.longSetting(
         "index.mapping.depth.limit",
         20L,

server/src/main/java/org/opensearch/index/mapper/MappingLookup.java

@@ -62,6 +62,9 @@
     private final FieldTypeLookup fieldTypeLookup;
     private final int metadataFieldCount;
     private final FieldNameAnalyzer indexAnalyzer;
+    private final int nonMetadataFieldCount;
+    private int dynamicFieldCount;
+    private int dynamicObjectFieldCount;
 
     private static void put(Map<String, Analyzer> analyzers, String key, Analyzer value, Analyzer defaultValue) {
         if (value == null) {
@@ -154,6 +157,7 @@
         this.fieldMappers = Collections.unmodifiableMap(fieldMappers);
         this.indexAnalyzer = new FieldNameAnalyzer(indexAnalyzers);
         this.objectMappers = Collections.unmodifiableMap(objects);
+        this.nonMetadataFieldCount = fieldMappers.size() + objectMappers.size() - metadataFieldCount;
     }
 
     /**
@@ -190,8 +194,49 @@
         checkNestedLimit(settings.getMappingNestedFieldsLimit());
     }
 
+    /**
+     *
+     * @param limit the value of the setting index.mapping.total_fields_limit
+     * @param dynamicMappers mappers of the new fields detected by dynamic mapping
+     * @return true if adding new fields will cause to exceed the total fields limit
+     */
+    public boolean exceedTotalFieldsLimit(long limit, List<Mapper> dynamicMappers) {
+        if (nonMetadataFieldCount > limit) {
+            return true;
+        }
+
+        List<ObjectMapper> newObjectMappers = new ArrayList<>();
+        List<FieldMapper> newFieldMappers = new ArrayList<>();
+        List<FieldAliasMapper> newFieldAliasMappers = new ArrayList<>();
+        dynamicMappers.forEach(dynamicMapper -> collect(dynamicMapper, newObjectMappers, newFieldMappers, newFieldAliasMappers));
+        this.dynamicFieldCount = newFieldMappers.size();
+        this.dynamicObjectFieldCount = newObjectMappers.size();
+
+        return nonMetadataFieldCount + dynamicFieldCount + dynamicObjectFieldCount >= limit;
+    }
+
+    /**
+     *
+     * @param limit the value of the setting index.mapping.total_fields_limit
+     * @param mapper mapper of the new field detected by dynamic mapping
+     * @return true if adding a new field will cause to exceed the total fields limit
+     */
+    public boolean exceedTotalFieldsLimitIfAddNewField(long limit, Mapper mapper) {
+        if (nonMetadataFieldCount > limit) {
+            return true;
+        }
+
+        List<ObjectMapper> newObjectMappers = new ArrayList<>();
+        List<FieldMapper> newFieldMappers = new ArrayList<>();
+        List<FieldAliasMapper> newFieldAliasMappers = new ArrayList<>();
+        collect(mapper, newObjectMappers, newFieldMappers, newFieldAliasMappers);
+
+        return nonMetadataFieldCount + dynamicFieldCount + dynamicObjectFieldCount + newObjectMappers.size() + newFieldMappers
+            .size() > limit;
+    }
+
     private void checkFieldLimit(long limit) {
-        if (fieldMappers.size() + objectMappers.size() - metadataFieldCount > limit) {
+        if (nonMetadataFieldCount > limit) {
             throw new IllegalArgumentException("Limit of total fields [" + limit + "] has been exceeded");
         }
     }

server/src/test/java/org/opensearch/index/IndexSettingsTests.java

@@ -60,6 +60,7 @@
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.function.Function;
 
+import static org.opensearch.index.mapper.MapperService.INDEX_MAPPING_TOTAL_FIELDS_UNMAP_FIELDS_BEYONGD_LIMIT_SETTING;
 import static org.opensearch.index.store.remote.directory.RemoteSnapshotDirectory.SEARCHABLE_SNAPSHOT_EXTENDED_COMPATIBILITY_MINIMUM_VERSION;
 import static org.hamcrest.CoreMatchers.equalTo;
 import static org.hamcrest.core.StringContains.containsString;
@@ -787,6 +788,25 @@ public void testRemoteStoreExplicitSetting() {
         assertTrue(settings.isRemoteStoreEnabled());
     }
 
+    public void testUnmapFieldsBeyondTotalFieldsLimitSetting() {
+        IndexMetadata metadata = newIndexMeta(
+            "index",
+            Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT).build()
+        );
+        IndexSettings settings = new IndexSettings(metadata, Settings.EMPTY);
+        assertFalse(settings.getUnmapFieldsBeyondTotalFieldsLimit());
+
+        metadata = newIndexMeta(
+            "index",
+            Settings.builder()
+                .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT)
+                .put(INDEX_MAPPING_TOTAL_FIELDS_UNMAP_FIELDS_BEYONGD_LIMIT_SETTING.getKey(), true)
+                .build()
+        );
+        settings = new IndexSettings(metadata, Settings.EMPTY);
+        assertTrue(settings.getUnmapFieldsBeyondTotalFieldsLimit());
+    }
+
     public void testRemoteTranslogStoreDefaultSetting() {
         IndexMetadata metadata = newIndexMeta(
             "index",