updating autotuner optimizer_mem factor

microsoft · Oct 16, 2023 · c272320 · c272320
1 parent 5bbbf41
commit c272320
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/deepspeed/autotuning/autotuner.py b/deepspeed/autotuning/autotuner.py
@@ -286,7 +286,7 @@ def get_instantiation_memory_required_per_gpu(self, zero_stage):
         # ZeroStageEnum.disabled:
         params_mem = num_params * (2 if fp16_enabled else 4)
         gradients_mem = num_params * (2 if fp16_enabled else 4)
-        optimizer_mem = num_params * (16 if fp16_enabled else 8)
+        optimizer_mem = num_params * (4 if fp16_enabled else 8)
 
         if zero_stage >= ZeroStageEnum.optimizer_states:
             optimizer_mem = optimizer_mem / total_gpus