Merge branch 'master' into fix_hpu_module_ids

microsoft · Aug 14, 2024 · 4847a98 · 4847a98
2 parents 9b7bebd + 24d1d86
commit 4847a98
Showing 1 changed file with 2 additions and 0 deletions.
diff --git a/deepspeed/runtime/zero/stage_1_and_2.py b/deepspeed/runtime/zero/stage_1_and_2.py
@@ -972,6 +972,8 @@ def print_rank_0(self, message):
             logger.info(message)
 
     def gradient_reduction_w_predivide(self, tensor):
+        if tensor.size().numel() == 0:
+            return tensor
 
         dp_world_size = dist.get_world_size(group=self.dp_process_group)