Add support of default replica count change (#5610) (#5732)

*[Backport 2.x] Add default replica count config settings

Signed-off-by: Arpit Bandejiya <[email protected]>

CHANGELOG.md

@@ -7,6 +7,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 ## [Unreleased 2.x]
 - Add feature flag for extensions ([#5211](https://github.com/opensearch-project/OpenSearch/pull/5211))
 ### Added
+- Add support of default replica count cluster setting ([#5610](https://github.com/opensearch-project/OpenSearch/pull/5610))
 - Apply reproducible builds configuration for OpenSearch plugins through gradle plugin ([#4746](https://github.com/opensearch-project/OpenSearch/pull/4746))
 - Prevent deletion of snapshots that are backing searchable snapshot indexes ([#5069](https://github.com/opensearch-project/OpenSearch/pull/5069))
 - Update to Gradle 7.6 ([#5382](https://github.com/opensearch-project/OpenSearch/pull/5382))

server/src/internalClusterTest/java/org/opensearch/action/admin/indices/rollover/RolloverIT.java

@@ -39,6 +39,7 @@
 import org.opensearch.ResourceAlreadyExistsException;
 import org.opensearch.action.admin.indices.alias.Alias;
 import org.opensearch.action.admin.indices.settings.get.GetSettingsResponse;
+import org.opensearch.action.admin.indices.template.delete.DeleteIndexTemplateRequestBuilder;
 import org.opensearch.action.admin.indices.template.put.PutIndexTemplateRequestBuilder;
 import org.opensearch.cluster.ClusterState;
 import org.opensearch.cluster.metadata.AutoExpandReplicas;
@@ -299,6 +300,31 @@ public void testRolloverWithIndexSettingsBalancedReplica() throws Exception {
         manageReplicaBalanceSetting(false);
     }
 
+    public void testRolloverWithIndexSettingsBalancedWithUseZoneForReplicaDefaultCount() throws Exception {
+        DeleteIndexTemplateRequestBuilder deleteTemplate = client().admin().indices().prepareDeleteTemplate("random_index_template");
+        assertAcked(deleteTemplate.execute().actionGet());
+
+        Alias testAlias = new Alias("test_alias");
+        boolean explicitWriteIndex = randomBoolean();
+        if (explicitWriteIndex) {
+            testAlias.writeIndex(true);
+        }
+        assertAcked(prepareCreate("test_index-2").addAlias(testAlias).get());
+        manageReplicaSettingForDefaultReplica(true);
+        index("test_index-2", "type1", "1", "field", "value");
+        flush("test_index-2");
+
+        final Settings settings = Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 3).build();
+        client().admin().indices().prepareRolloverIndex("test_alias").settings(settings).alias(new Alias("extra_alias")).get();
+
+        final ClusterState state = client().admin().cluster().prepareState().get().getState();
+        final IndexMetadata newIndex = state.metadata().index("test_index-000003");
+        assertThat(newIndex.getNumberOfShards(), equalTo(3));
+        assertThat(newIndex.getNumberOfReplicas(), equalTo(2));
+        manageReplicaSettingForDefaultReplica(false);
+        randomIndexTemplate();
+    }
+
     public void testRolloverWithIndexSettingsWithoutPrefix() throws Exception {
         Alias testAlias = new Alias("test_alias");
         boolean explicitWriteIndex = randomBoolean();

server/src/internalClusterTest/java/org/opensearch/indices/settings/UpdateNumberOfReplicasIT.java

@@ -33,9 +33,11 @@
 package org.opensearch.indices.settings;
 
 import org.opensearch.action.admin.cluster.health.ClusterHealthResponse;
+import org.opensearch.action.admin.indices.template.delete.DeleteIndexTemplateRequestBuilder;
 import org.opensearch.action.search.SearchResponse;
 import org.opensearch.action.support.ActiveShardCount;
 import org.opensearch.action.support.IndicesOptions;
+import org.opensearch.cluster.ClusterState;
 import org.opensearch.cluster.health.ClusterHealthStatus;
 import org.opensearch.cluster.metadata.IndexMetadata;
 import org.opensearch.common.Priority;
@@ -663,4 +665,78 @@ public void testAwarenessReplicaBalance() {
         }
     }
 
+    public void testAwarenessReplicaBalanceWithUseZoneForDefaultReplicaCount() {
+        createIndex("aware-replica", Settings.builder().put("index.number_of_replicas", 0).build());
+        createIndex(".system-index", Settings.builder().put("index.number_of_replicas", 0).build());
+        DeleteIndexTemplateRequestBuilder deleteTemplate = client().admin().indices().prepareDeleteTemplate("random_index_template");
+        assertAcked(deleteTemplate.execute().actionGet());
+        manageReplicaSettingForDefaultReplica(true);
+        int updated = 0;
+
+        try {
+
+            // replica count should not be changed and we should see the original replica count
+            client().admin()
+                .indices()
+                .prepareUpdateSettings("aware-replica")
+                .setSettings(Settings.builder().put("refresh_interval", "1s"))
+                .execute()
+                .actionGet();
+            updated++;
+
+            final ClusterState state = client().admin().cluster().prepareState().get().getState();
+            final IndexMetadata newIndex = state.metadata().index("aware-replica");
+            assertThat(newIndex.getNumberOfReplicas(), equalTo(0));
+
+            // replica count of 2 is ideal
+            client().admin()
+                .indices()
+                .prepareUpdateSettings("aware-replica")
+                .setSettings(Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 2))
+                .execute()
+                .actionGet();
+            updated++;
+
+            // Since auto expand replica setting take precedence, this should pass
+            client().admin()
+                .indices()
+                .prepareUpdateSettings("aware-replica")
+                .setSettings(
+                    Settings.builder()
+                        .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1)
+                        .put(IndexMetadata.SETTING_AUTO_EXPAND_REPLICAS, "0-2")
+                )
+                .execute()
+                .actionGet();
+            updated++;
+
+            // system index - should be able to update
+            client().admin()
+                .indices()
+                .prepareUpdateSettings(".system-index")
+                .setSettings(Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 3))
+                .execute()
+                .actionGet();
+            updated++;
+
+            client().admin()
+                .indices()
+                .prepareUpdateSettings("aware-replica")
+                .setSettings(Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1))
+                .execute()
+                .actionGet();
+            fail("should have thrown an exception about the replica count");
+
+        } catch (IllegalArgumentException e) {
+            assertEquals(
+                "Validation Failed: 1: expected total copies needs to be a multiple of total awareness attributes [3];",
+                e.getMessage()
+            );
+            assertEquals(4, updated);
+        } finally {
+            manageReplicaSettingForDefaultReplica(false);
+            randomIndexTemplate();
+        }
+    }
+
 }

server/src/internalClusterTest/java/org/opensearch/indices/template/SimpleIndexTemplateIT.java

@@ -1031,22 +1031,30 @@ public void testPartitionedTemplate() throws Exception {
     }
 
     public void testAwarenessReplicaBalance() throws IOException {
-        manageReplicaBalanceSetting(true);
+        manageReplicaSettingForDefaultReplica(true);
         int updated = 0;
         try {
             client().admin()
                 .indices()
                 .preparePutTemplate("template_1")
                 .setPatterns(Arrays.asList("a*", "b*"))
-                .setSettings(Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1))
+                .setSettings(Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 2))
                 .get();
             updated++;
 
             client().admin()
                 .indices()
                 .preparePutTemplate("template_1")
                 .setPatterns(Arrays.asList("a*", "b*"))
-                .setSettings(Settings.builder().put(IndexMetadata.SETTING_AUTO_EXPAND_REPLICAS, "0-1"))
+                .setSettings(Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 4))
+                .get();
+            updated++;
+
+            client().admin()
+                .indices()
+                .preparePutTemplate("template_1")
+                .setPatterns(Arrays.asList("a*", "b*"))
+                .setSettings(Settings.builder().put(IndexMetadata.SETTING_AUTO_EXPAND_REPLICAS, "0-2"))
                 .get();
             updated++;
 
@@ -1060,12 +1068,12 @@ public void testAwarenessReplicaBalance() throws IOException {
             fail("should have thrown an exception about the replica  count");
         } catch (InvalidIndexTemplateException e) {
             assertEquals(
-                "index_template [template_1] invalid, cause [Validation Failed: 1: expected total copies needs to be a multiple of total awareness attributes [2];]",
+                "index_template [template_1] invalid, cause [Validation Failed: 1: expected total copies needs to be a multiple of total awareness attributes [3];]",
                 e.getMessage()
             );
-            assertEquals(2, updated);
+            assertEquals(3, updated);
         } finally {
-            manageReplicaBalanceSetting(false);
+            manageReplicaSettingForDefaultReplica(false);
         }
     }
 

server/src/internalClusterTest/java/org/opensearch/snapshots/RestoreSnapshotIT.java

@@ -36,6 +36,7 @@
 import org.opensearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse;
 import org.opensearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse;
 import org.opensearch.action.admin.indices.settings.get.GetSettingsResponse;
+import org.opensearch.action.admin.indices.template.delete.DeleteIndexTemplateRequestBuilder;
 import org.opensearch.action.admin.indices.template.get.GetIndexTemplatesResponse;
 import org.opensearch.action.index.IndexRequestBuilder;
 import org.opensearch.client.Client;
@@ -977,14 +978,17 @@ public void testForbidDisableSoftDeletesDuringRestore() throws Exception {
     public void testRestoreBalancedReplica() {
         try {
             createRepository("test-repo", "fs");
+            DeleteIndexTemplateRequestBuilder deleteTemplate = client().admin().indices().prepareDeleteTemplate("random_index_template");
+            assertAcked(deleteTemplate.execute().actionGet());
+
             createIndex("test-index", Settings.builder().put("index.number_of_replicas", 0).build());
             createIndex(".system-index", Settings.builder().put("index.number_of_replicas", 0).build());
             ensureGreen();
             clusterAdmin().prepareCreateSnapshot("test-repo", "snapshot-0")
                 .setIndices("test-index", ".system-index")
                 .setWaitForCompletion(true)
                 .get();
-            manageReplicaBalanceSetting(true);
+            manageReplicaSettingForDefaultReplica(true);
 
             final IllegalArgumentException restoreError = expectThrows(
                 IllegalArgumentException.class,
@@ -996,7 +1000,7 @@ public void testRestoreBalancedReplica() {
             );
             assertThat(
                 restoreError.getMessage(),
-                containsString("expected total copies needs to be a multiple of total awareness attributes [2]")
+                containsString("expected total copies needs to be a multiple of total awareness attributes [3]")
             );
 
             final IllegalArgumentException restoreError2 = expectThrows(
@@ -1005,14 +1009,14 @@ public void testRestoreBalancedReplica() {
                     .setRenamePattern("test-index")
                     .setRenameReplacement("new-index-2")
                     .setIndexSettings(
-                        Settings.builder().put(SETTING_NUMBER_OF_REPLICAS, 1).put(IndexMetadata.SETTING_AUTO_EXPAND_REPLICAS, "0-2").build()
+                        Settings.builder().put(SETTING_NUMBER_OF_REPLICAS, 1).put(IndexMetadata.SETTING_AUTO_EXPAND_REPLICAS, "0-1").build()
                     )
                     .setIndices("test-index")
                     .get()
             );
             assertThat(
                 restoreError2.getMessage(),
-                containsString("expected max cap on auto expand to be a multiple of total awareness attributes [2]")
+                containsString("expected max cap on auto expand to be a multiple of total awareness attributes [3]")
             );
 
             RestoreSnapshotResponse restoreSnapshotResponse = clusterAdmin().prepareRestoreSnapshot("test-repo", "snapshot-0")
@@ -1028,7 +1032,7 @@ public void testRestoreBalancedReplica() {
             restoreSnapshotResponse = clusterAdmin().prepareRestoreSnapshot("test-repo", "snapshot-0")
                 .setRenamePattern("test-index")
                 .setRenameReplacement("new-index")
-                .setIndexSettings(Settings.builder().put("index.number_of_replicas", 1).build())
+                .setIndexSettings(Settings.builder().put("index.number_of_replicas", 2).build())
                 .setWaitForCompletion(true)
                 .setIndices("test-index")
                 .execute()
@@ -1038,7 +1042,7 @@ public void testRestoreBalancedReplica() {
                 .setRenamePattern("test-index")
                 .setRenameReplacement("new-index-3")
                 .setIndexSettings(
-                    Settings.builder().put(SETTING_NUMBER_OF_REPLICAS, 0).put(IndexMetadata.SETTING_AUTO_EXPAND_REPLICAS, "0-1").build()
+                    Settings.builder().put(SETTING_NUMBER_OF_REPLICAS, 0).put(IndexMetadata.SETTING_AUTO_EXPAND_REPLICAS, "0-2").build()
                 )
                 .setWaitForCompletion(true)
                 .setIndices("test-index")
@@ -1047,7 +1051,9 @@ public void testRestoreBalancedReplica() {
 
             assertThat(restoreSnapshotResponse.getRestoreInfo().totalShards(), greaterThan(0));
         } finally {
-            manageReplicaBalanceSetting(false);
+            manageReplicaSettingForDefaultReplica(false);
+            randomIndexTemplate();
         }
     }
+
 }

server/src/main/java/org/opensearch/cluster/metadata/Metadata.java

@@ -162,6 +162,13 @@ public interface Custom extends NamedDiffable<Custom>, ToXContentFragment, Clust
         EnumSet<XContentContext> context();
     }
 
+    public static final Setting<Integer> DEFAULT_REPLICA_COUNT_SETTING = Setting.intSetting(
+        "cluster.default_number_of_replicas",
+        1,
+        Property.Dynamic,
+        Property.NodeScope
+    );
+
     public static final Setting<Boolean> SETTING_READ_ONLY_SETTING = Setting.boolSetting(
         "cluster.blocks.read_only",
         false,

server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java

@@ -124,6 +124,7 @@
 import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_INDEX_UUID;
 import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_REPLICAS;
 import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_SHARDS;
+import static org.opensearch.cluster.metadata.Metadata.DEFAULT_REPLICA_COUNT_SETTING;
 
 /**
  * Service responsible for submitting create index requests
@@ -555,7 +556,6 @@ private ClusterState applyCreateIndexRequestWithV1Templates(
                 xContentRegistry
             )
         );
-
         final Settings aggregatedIndexSettings = aggregateIndexSettings(
             currentState,
             request,
@@ -869,7 +869,7 @@ static Settings aggregateIndexSettings(
             indexSettingsBuilder.put(SETTING_NUMBER_OF_SHARDS, numberOfShards);
         }
         if (INDEX_NUMBER_OF_REPLICAS_SETTING.exists(indexSettingsBuilder) == false) {
-            indexSettingsBuilder.put(SETTING_NUMBER_OF_REPLICAS, INDEX_NUMBER_OF_REPLICAS_SETTING.get(settings));
+            indexSettingsBuilder.put(SETTING_NUMBER_OF_REPLICAS, DEFAULT_REPLICA_COUNT_SETTING.get(currentState.metadata().settings()));
         }
         if (settings.get(SETTING_AUTO_EXPAND_REPLICAS) != null && indexSettingsBuilder.get(SETTING_AUTO_EXPAND_REPLICAS) == null) {
             indexSettingsBuilder.put(SETTING_AUTO_EXPAND_REPLICAS, settings.get(SETTING_AUTO_EXPAND_REPLICAS));
@@ -1211,10 +1211,10 @@ List<String> getIndexSettingsValidationErrors(
             validationErrors.addAll(validatePrivateSettingsNotExplicitlySet(settings, indexScopedSettings));
         }
         if (indexName.isEmpty() || indexName.get().charAt(0) != '.') {
-            // Apply aware replica balance only to non system indices
+            // Apply aware replica balance validation only to non system indices
             int replicaCount = settings.getAsInt(
                 IndexMetadata.SETTING_NUMBER_OF_REPLICAS,
-                INDEX_NUMBER_OF_REPLICAS_SETTING.getDefault(Settings.EMPTY)
+                DEFAULT_REPLICA_COUNT_SETTING.get(this.clusterService.state().metadata().settings())
             );
             AutoExpandReplicas autoExpandReplica = AutoExpandReplicas.SETTING.get(settings);
             Optional<String> error = awarenessReplicaBalance.validate(replicaCount, autoExpandReplica);

server/src/main/java/org/opensearch/common/settings/ClusterSettings.java

@@ -259,6 +259,7 @@ public void apply(Settings value, Settings current, Settings previous) {
                 MappingUpdatedAction.INDICES_MAX_IN_FLIGHT_UPDATES_SETTING,
                 Metadata.SETTING_READ_ONLY_SETTING,
                 Metadata.SETTING_READ_ONLY_ALLOW_DELETE_SETTING,
+                Metadata.DEFAULT_REPLICA_COUNT_SETTING,
                 ShardLimitValidator.SETTING_CLUSTER_MAX_SHARDS_PER_NODE,
                 ShardLimitValidator.SETTING_CLUSTER_IGNORE_DOT_INDEXES,
                 RecoverySettings.INDICES_RECOVERY_MAX_BYTES_PER_SEC_SETTING,

server/src/test/java/org/opensearch/action/support/replication/TransportReplicationActionTests.java

@@ -201,6 +201,11 @@ public void setUp() throws Exception {
         transportService.acceptIncomingRequests();
         shardStateAction = new ShardStateAction(clusterService, transportService, null, null, threadPool);
         action = new TestAction(Settings.EMPTY, "internal:testAction", transportService, clusterService, shardStateAction, threadPool);
+        Metadata metadata = Metadata.builder().build();
+        ClusterState clusterState = ClusterState.builder(org.opensearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY))
+            .metadata(metadata)
+            .build();
+        setState(clusterService, clusterState);
     }
 
     @Override

server/src/test/java/org/opensearch/cluster/metadata/MetadataCreateIndexServiceTests.java

@@ -1083,6 +1083,13 @@ public void testParseMappingsWithTypelessTemplate() throws Exception {
 
     public void testvalidateIndexSettings() {
         ClusterService clusterService = mock(ClusterService.class);
+        Metadata metadata = Metadata.builder()
+            .transientSettings(Settings.builder().put(Metadata.DEFAULT_REPLICA_COUNT_SETTING.getKey(), 1).build())
+            .build();
+        ClusterState clusterState = ClusterState.builder(org.opensearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY))
+            .metadata(metadata)
+            .build();
+
         ThreadPool threadPool = new TestThreadPool(getTestName());
         Settings settings = Settings.builder()
             .put(AwarenessAllocationDecider.CLUSTER_ROUTING_ALLOCATION_AWARENESS_ATTRIBUTE_SETTING.getKey(), "zone, rack")
@@ -1093,6 +1100,8 @@ public void testvalidateIndexSettings() {
         ClusterSettings clusterSettings = new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
         when(clusterService.getSettings()).thenReturn(settings);
         when(clusterService.getClusterSettings()).thenReturn(clusterSettings);
+        when(clusterService.state()).thenReturn(clusterState);
+
         MetadataCreateIndexService checkerService = new MetadataCreateIndexService(
             settings,
             clusterService,

server/src/test/java/org/opensearch/cluster/metadata/MetadataIndexTemplateServiceTests.java

@@ -2091,6 +2091,11 @@ private static List<Throwable> putTemplate(NamedXContentRegistry xContentRegistr
         ClusterService clusterService = mock(ClusterService.class);
         Settings settings = Settings.builder().put(PATH_HOME_SETTING.getKey(), "dummy").build();
         ClusterSettings clusterSettings = new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
+        Metadata metadata = Metadata.builder().build();
+        ClusterState clusterState = ClusterState.builder(org.opensearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY))
+            .metadata(metadata)
+            .build();
+        when(clusterService.state()).thenReturn(clusterState);
         when(clusterService.getSettings()).thenReturn(settings);
         when(clusterService.getClusterSettings()).thenReturn(clusterSettings);
         MetadataCreateIndexService createIndexService = new MetadataCreateIndexService(