diff --git a/server/src/main/java/org/opensearch/cluster/routing/allocation/allocator/LocalShardsBalancer.java b/server/src/main/java/org/opensearch/cluster/routing/allocation/allocator/LocalShardsBalancer.java
index ec25d041bda43..f4617bae4fdab 100644
--- a/server/src/main/java/org/opensearch/cluster/routing/allocation/allocator/LocalShardsBalancer.java
+++ b/server/src/main/java/org/opensearch/cluster/routing/allocation/allocator/LocalShardsBalancer.java
@@ -780,10 +780,10 @@ void allocateUnassigned() {
          * the next replica. If we could not find a node to allocate (0,R,IDX1) we move all it's replicas to ignoreUnassigned.
          */
         ShardRouting[] unassignedShards = unassigned.drain();
-        List<ShardRouting> allUnassignedShards = Arrays.stream(unassignedShards).collect(Collectors.toList());
-        List<ShardRouting> localUnassignedShards = allUnassignedShards.stream()
+        Set<ShardRouting> allUnassignedShards = Arrays.stream(unassignedShards).collect(Collectors.toSet());
+        Set<ShardRouting> localUnassignedShards = allUnassignedShards.stream()
             .filter(shard -> RoutingPool.LOCAL_ONLY.equals(RoutingPool.getShardPool(shard, allocation)))
-            .collect(Collectors.toList());
+            .collect(Collectors.toSet());
         allUnassignedShards.removeAll(localUnassignedShards);
         allUnassignedShards.forEach(shard -> routingNodes.unassigned().add(shard));
         unassignedShards = localUnassignedShards.toArray(new ShardRouting[0]);