Add index level setting to unmap fields beyond total fields limit

CHANGELOG.md

@@ -15,6 +15,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Add ThreadContextPermission for stashAndMergeHeaders and stashWithOrigin ([#15039](https://github.com/opensearch-project/OpenSearch/pull/15039))
 - [Concurrent Segment Search] Support composite aggregations with scripting ([#15072](https://github.com/opensearch-project/OpenSearch/pull/15072))
 - Add `rangeQuery` and `regexpQuery` for `constant_keyword` field type ([#14711](https://github.com/opensearch-project/OpenSearch/pull/14711))
+- Add index level setting `index.mapping.total_fields.unmap_fields_beyond_limit` to unmap fields beyond the total fields limit ([#14939](https://github.com/opensearch-project/OpenSearch/pull/14939))
 - Add took time to request nodes stats ([#15054](https://github.com/opensearch-project/OpenSearch/pull/15054))
 - [Workload Management] QueryGroup resource tracking framework changes ([#13897](https://github.com/opensearch-project/OpenSearch/pull/13897))
 - Add slice execution listeners to SearchOperationListener interface ([#15153](https://github.com/opensearch-project/OpenSearch/pull/15153))

rest-api-spec/src/main/resources/rest-api-spec/test/bulk/100_unmap_fields_beyond_limit.yml

@@ -0,0 +1,107 @@
+---
+"Test unmap the object type fields beyond the total fields limit":
+  - skip:
+      version: " - 2.99.99"
+      reason: "introduced in 3.0.0"
+  - do:
+      indices.create:
+        index: test_index_1
+        body:
+          settings:
+            index.mapping.total_fields.limit: 3
+            index.mapping.total_fields.unmap_fields_beyond_limit: true
+
+  - do:
+      index:
+        index:   test_index_1
+        id: 1
+        body:    {
+          field1: "field1",
+          field2: [1,2,3],
+          unmapField: "foo"
+        }
+  - do:
+      get:
+        index: test_index_1
+        id: 1
+  - match: { _source:  {  field1: "field1", field2: [1,2,3], unmapField: "foo" }}
+
+  - do:
+      indices.get_mapping:
+        index: test_index_1
+  - match: {test_index_1.mappings.properties.field1.type:     text}
+  - match: {test_index_1.mappings.properties.field1.fields.keyword.type:     keyword}
+  - match: {test_index_1.mappings.properties.field2.type:     long}
+  - match: {test_index_1.mappings.properties.unmapField: null}
+
+  - do:
+      index:
+        index:   test_index_1
+        id: 2
+        body:    {
+          field3: {
+            "field4": "field4"
+          },
+          "field5": 100,
+          "dateField": "2024-07-25T05:11:51.243Z",
+          "booleanField": true
+        }
+  - do:
+      get:
+        index: test_index_1
+        id: 2
+  - match: { _source:  {  field3: { field4: "field4" }, field5: 100, "dateField": "2024-07-25T05:11:51.243Z", "booleanField": true }}
+
+  - do:
+      indices.get_mapping:
+        index: test_index_1
+  - match: {test_index_1.mappings.properties.field1.type:     text}
+  - match: {test_index_1.mappings.properties.field1.fields.keyword.type:     keyword}
+  - match: {test_index_1.mappings.properties.field2.type:     long}
+  - match: {test_index_1.mappings.properties.field3: null}
+  - match: {test_index_1.mappings.properties.field5: null}
+  - match: {test_index_1.mappings.properties.dateField.type:     null}
+  - match: {test_index_1.mappings.properties.booleanField.type:     null}
+
+  - do:
+      indices.put_settings:
+        index: test_index_1
+        body:
+          index.mapping.total_fields.limit: 100
+
+  - do:
+      indices.get_settings:
+        index: test_index_1
+  - match: {test_index_1.settings.index.mapping.total_fields.limit:     "100"}
+
+  - do:
+      index:
+        index:   test_index_1
+        id: 2
+        body:    {
+          field1: "field1",
+          field2: [1,2,3],
+          field3: {
+            "field4": "field4"
+          },
+          "field5": 100,
+          "dateField": "2024-07-25T05:11:51.243Z",
+          "booleanField": true
+        }
+  - do:
+      get:
+        index: test_index_1
+        id: 2
+  - match: { _source:  {  field1: "field1", field2: [1,2,3], field3: { field4: "field4" }, field5: 100, "dateField": "2024-07-25T05:11:51.243Z", "booleanField": true }}
+
+  - do:
+      indices.get_mapping:
+        index: test_index_1
+  - match: {test_index_1.mappings.properties.field1.type:     text}
+  - match: {test_index_1.mappings.properties.field1.fields.keyword.type:     keyword}
+  - match: {test_index_1.mappings.properties.field2.type:     long}
+  - match: {test_index_1.mappings.properties.field3.properties.field4.type:     text}
+  - match: {test_index_1.mappings.properties.field3.properties.field4.fields.keyword.type:     keyword}
+  - match: {test_index_1.mappings.properties.field5.type:     long}
+  - match: {test_index_1.mappings.properties.dateField.type:     date}
+  - match: {test_index_1.mappings.properties.booleanField.type:     boolean}

server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java

@@ -186,6 +186,7 @@ public final class IndexScopedSettings extends AbstractScopedSettings {
                 MapperService.INDEX_MAPPING_NESTED_FIELDS_LIMIT_SETTING,
                 MapperService.INDEX_MAPPING_NESTED_DOCS_LIMIT_SETTING,
                 MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING,
+                MapperService.INDEX_MAPPING_TOTAL_FIELDS_UNMAP_FIELDS_BEYONGD_LIMIT_SETTING,
                 MapperService.INDEX_MAPPING_DEPTH_LIMIT_SETTING,
                 MapperService.INDEX_MAPPING_FIELD_NAME_LENGTH_LIMIT_SETTING,
                 BitsetFilterCache.INDEX_LOAD_RANDOM_ACCESS_FILTERS_EAGERLY_SETTING,

server/src/main/java/org/opensearch/index/IndexSettings.java

@@ -75,6 +75,7 @@
 import static org.opensearch.index.mapper.MapperService.INDEX_MAPPING_NESTED_DOCS_LIMIT_SETTING;
 import static org.opensearch.index.mapper.MapperService.INDEX_MAPPING_NESTED_FIELDS_LIMIT_SETTING;
 import static org.opensearch.index.mapper.MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING;
+import static org.opensearch.index.mapper.MapperService.INDEX_MAPPING_TOTAL_FIELDS_UNMAP_FIELDS_BEYONGD_LIMIT_SETTING;
 import static org.opensearch.index.store.remote.directory.RemoteSnapshotDirectory.SEARCHABLE_SNAPSHOT_EXTENDED_COMPATIBILITY_MINIMUM_VERSION;
 
 /**
@@ -809,6 +810,7 @@ private void setRetentionLeaseMillis(final TimeValue retentionLease) {
     private volatile long mappingNestedFieldsLimit;
     private volatile long mappingNestedDocsLimit;
     private volatile long mappingTotalFieldsLimit;
+    private volatile boolean unmapFieldsBeyondTotalFieldsLimit;
     private volatile long mappingDepthLimit;
     private volatile long mappingFieldNameLengthLimit;
 
@@ -997,6 +999,7 @@ public IndexSettings(final IndexMetadata indexMetadata, final Settings nodeSetti
         mappingNestedFieldsLimit = scopedSettings.get(INDEX_MAPPING_NESTED_FIELDS_LIMIT_SETTING);
         mappingNestedDocsLimit = scopedSettings.get(INDEX_MAPPING_NESTED_DOCS_LIMIT_SETTING);
         mappingTotalFieldsLimit = scopedSettings.get(INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING);
+        unmapFieldsBeyondTotalFieldsLimit = scopedSettings.get(INDEX_MAPPING_TOTAL_FIELDS_UNMAP_FIELDS_BEYONGD_LIMIT_SETTING);
         mappingDepthLimit = scopedSettings.get(INDEX_MAPPING_DEPTH_LIMIT_SETTING);
         mappingFieldNameLengthLimit = scopedSettings.get(INDEX_MAPPING_FIELD_NAME_LENGTH_LIMIT_SETTING);
         maxFullFlushMergeWaitTime = scopedSettings.get(INDEX_MERGE_ON_FLUSH_MAX_FULL_FLUSH_MERGE_WAIT_TIME);
@@ -1115,6 +1118,10 @@ public IndexSettings(final IndexMetadata indexMetadata, final Settings nodeSetti
         scopedSettings.addSettingsUpdateConsumer(INDEX_MAPPING_NESTED_FIELDS_LIMIT_SETTING, this::setMappingNestedFieldsLimit);
         scopedSettings.addSettingsUpdateConsumer(INDEX_MAPPING_NESTED_DOCS_LIMIT_SETTING, this::setMappingNestedDocsLimit);
         scopedSettings.addSettingsUpdateConsumer(INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING, this::setMappingTotalFieldsLimit);
+        scopedSettings.addSettingsUpdateConsumer(
+            INDEX_MAPPING_TOTAL_FIELDS_UNMAP_FIELDS_BEYONGD_LIMIT_SETTING,
+            this::setMappingUnmapFieldsBeyondTotalFieldsLimit
+        );
         scopedSettings.addSettingsUpdateConsumer(INDEX_MAPPING_DEPTH_LIMIT_SETTING, this::setMappingDepthLimit);
         scopedSettings.addSettingsUpdateConsumer(INDEX_MAPPING_FIELD_NAME_LENGTH_LIMIT_SETTING, this::setMappingFieldNameLengthLimit);
         scopedSettings.addSettingsUpdateConsumer(INDEX_MERGE_ON_FLUSH_MAX_FULL_FLUSH_MERGE_WAIT_TIME, this::setMaxFullFlushMergeWaitTime);
@@ -1861,6 +1868,14 @@ private void setMappingTotalFieldsLimit(long value) {
         this.mappingTotalFieldsLimit = value;
     }
 
+    public boolean getUnmapFieldsBeyondTotalFieldsLimit() {
+        return unmapFieldsBeyondTotalFieldsLimit;
+    }
+
+    private void setMappingUnmapFieldsBeyondTotalFieldsLimit(boolean value) {
+        this.unmapFieldsBeyondTotalFieldsLimit = value;
+    }
+
     public long getMappingDepthLimit() {
         return mappingDepthLimit;
     }

server/src/main/java/org/opensearch/index/mapper/DocumentParser.java

@@ -554,6 +554,12 @@ private static void parseObject(final ParseContext context, ObjectMapper mapper,
                     throw new StrictDynamicMappingException(dynamic.name().toLowerCase(Locale.ROOT), mapper.fullPath(), currentFieldName);
                 case TRUE:
                 case STRICT_ALLOW_TEMPLATES:
+                    // if dynamic is true or strict_allow_templates, we check if we need to unmap the fields beyond the total fields limit
+                    if (checkIfUnmapFieldsBeyondTotalFieldsLimit(context)) {
+                        context.parser().skipChildren();
+                        break;
+                    }
+
                     Mapper.Builder builder = findTemplateBuilder(
                         context,
                         currentFieldName,
@@ -568,6 +574,7 @@ private static void parseObject(final ParseContext context, ObjectMapper mapper,
                     Mapper.BuilderContext builderContext = new Mapper.BuilderContext(context.indexSettings().getSettings(), context.path());
                     objectMapper = builder.build(builderContext);
                     context.addDynamicMapper(objectMapper);
+                    increaseDynamicFieldCountIfNeed(context);
                     context.path().add(currentFieldName);
                     parseObjectOrField(context, objectMapper);
                     context.path().remove();
@@ -576,12 +583,46 @@ private static void parseObject(final ParseContext context, ObjectMapper mapper,
                     // not dynamic, read everything up to end object
                     context.parser().skipChildren();
             }
+
             for (int i = 0; i < parentMapperTuple.v1(); i++) {
                 context.path().remove();
             }
         }
     }
 
+    /**
+     * if the setting `index.mapping.total_fields.unmap_fields_beyond_limit` is true, we check if the current total fields count exceed the
+     * total fields limit
+     * @param context the parse context
+     * @return true if `index.mapping.total_fields.unmap_fields_beyond_limit` is true and the current total fields count exceed the limit
+     */
+    private static boolean checkIfUnmapFieldsBeyondTotalFieldsLimit(ParseContext context) {
+        return context.getUnmapFieldsBeyondTotalFieldsLimit()
+            && context.docMapper().mappers().exceedTotalFieldsLimit(context.getTotalFieldsLimit());
+    }
+
+    /**
+     * if the setting `index.mapping.total_fields.unmap_fields_beyond_limit` is true, increase the dynamic field count by 1
+     * @param context the parse context
+     */
+    private static void increaseDynamicFieldCountIfNeed(ParseContext context) {
+        if (context.getUnmapFieldsBeyondTotalFieldsLimit()) {
+            context.docMapper().mappers().increaseDynamicFieldCount();
+        }
+    }
+
+    /**
+     * if the setting `index.mapping.total_fields.unmap_fields_beyond_limit` is true, increase the dynamic field count by the specified
+     * field count
+     * @param context the parse context
+     * @param fieldCount the field count
+     */
+    private static void increaseDynamicFieldCountIfNeed(ParseContext context, long fieldCount) {
+        if (context.getUnmapFieldsBeyondTotalFieldsLimit()) {
+            context.docMapper().mappers().increaseDynamicFieldCount(fieldCount);
+        }
+    }
+
     private static void parseArray(ParseContext context, ObjectMapper parentMapper, String lastFieldName, String[] paths)
         throws IOException {
         try {
@@ -632,6 +673,7 @@ private static void parseArray(ParseContext context, ObjectMapper parentMapper,
                             assert mapper != null;
                             if (parsesArrayValue(mapper)) {
                                 context.addDynamicMapper(mapper);
+                                increaseDynamicFieldCountIfNeed(context);
                                 context.path().add(arrayFieldName);
                                 parseObjectOrField(context, mapper);
                                 context.path().remove();
@@ -864,13 +906,27 @@ private static void parseDynamicValue(
         if (dynamic == ObjectMapper.Dynamic.STRICT) {
             throw new StrictDynamicMappingException(dynamic.name().toLowerCase(Locale.ROOT), parentMapper.fullPath(), currentFieldName);
         }
-        if (dynamic == ObjectMapper.Dynamic.FALSE) {
+        // if dynamic is true or strict_allow_templates, and index.mapping.total_fields.unmap_fields_beyond_limit is true,
+        // then we check if we need to unmap the fields beyond the total fields limit
+        if (dynamic == ObjectMapper.Dynamic.FALSE || checkIfUnmapFieldsBeyondTotalFieldsLimit(context)) {
             return;
         }
         final Mapper.BuilderContext builderContext = new Mapper.BuilderContext(context.indexSettings().getSettings(), context.path());
         final Mapper.Builder<?> builder = createBuilderFromDynamicValue(context, token, currentFieldName, dynamic, parentMapper.fullPath());
         Mapper mapper = builder.build(builderContext);
+
+        // edge case, adding a new field may increase the dynamic field count by 2 or more,
+        // so we check if adding a new field will cause the total field count to exceed the total fields limit, if so we don't add it
+        long fieldCount = 0;
+        if (context.getUnmapFieldsBeyondTotalFieldsLimit()) {
+            fieldCount = context.docMapper().mappers().countFields(mapper);
+            if (context.docMapper().mappers().exceedTotalFieldsLimitIfAddNewField(context.getTotalFieldsLimit(), fieldCount)) {
+                return;
+            }
+        }
+
         context.addDynamicMapper(mapper);
+        increaseDynamicFieldCountIfNeed(context, fieldCount);
 
         parseObjectOrField(context, mapper);
     }
@@ -959,6 +1015,11 @@ private static Tuple<Integer, ObjectMapper> getDynamicParentMapper(
                         throw new StrictDynamicMappingException(dynamic.name().toLowerCase(Locale.ROOT), parent.fullPath(), paths[i]);
                     case STRICT_ALLOW_TEMPLATES:
                     case TRUE:
+                        // if dynamic is true or strict_allow_templates, we check if we need to unmap the fields beyond the total fields
+                        // limit
+                        if (checkIfUnmapFieldsBeyondTotalFieldsLimit(context)) {
+                            return new Tuple<>(pathsAdded, parent);
+                        }
                         Mapper.Builder builder = findTemplateBuilder(
                             context,
                             paths[i],
@@ -982,6 +1043,7 @@ private static Tuple<Integer, ObjectMapper> getDynamicParentMapper(
                             );
                         }
                         context.addDynamicMapper(mapper);
+                        increaseDynamicFieldCountIfNeed(context);
                         break;
                     case FALSE:
                         // Should not dynamically create any more mappers so return the last mapper

server/src/main/java/org/opensearch/index/mapper/MapperService.java

@@ -146,6 +146,14 @@ public enum MergeReason {
         Property.Dynamic,
         Property.IndexScope
     );
+    // if set to true, the new detected fields from dynamic mapping which beyond the total fields limit will be unmapped, i.e. will not
+    // be added to the mapping
+    public static final Setting<Boolean> INDEX_MAPPING_TOTAL_FIELDS_UNMAP_FIELDS_BEYONGD_LIMIT_SETTING = Setting.boolSetting(
+        "index.mapping.total_fields.unmap_fields_beyond_limit",
+        false,
+        Property.Dynamic,
+        Property.IndexScope
+    );
     public static final Setting<Long> INDEX_MAPPING_DEPTH_LIMIT_SETTING = Setting.longSetting(
         "index.mapping.depth.limit",
         20L,