Skip to content

Commit

Permalink
[pre-commit.ci] auto fixes from pre-commit.com hooks
Browse files Browse the repository at this point in the history
for more information, see https://pre-commit.ci
  • Loading branch information
pre-commit-ci[bot] committed Aug 5, 2024
1 parent 811d5af commit 0e6e488
Showing 1 changed file with 4 additions and 2 deletions.
6 changes: 4 additions & 2 deletions colossalai/zero/low_level/low_level_optim.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,8 @@
)
from colossalai.interface import OptimizerWrapper
from colossalai.logging import get_dist_logger
from colossalai.tensor.moe_tensor.api import is_moe_tensor
from colossalai.quantization.fp8 import all_gather_into_tensor_flat_fp8, all_reduce_fp8, reduce_scatter_fp8
from colossalai.tensor.moe_tensor.api import is_moe_tensor

from ._utils import calculate_global_norm_from_list, has_inf_or_nan, release_param_grad, sync_tensor
from .bookkeeping import BucketStore, GradientStore, TensorBucket
Expand Down Expand Up @@ -576,7 +576,9 @@ def step(self, closure=None):
else:
if param_to_gather.numel() > self.pg_to_tensor_bucket[pg].max_size:
if self._fp8_communication:
all_gather_into_tensor_flat_fp8(padded_working_param, param_to_gather, pg, fp8_format="e4m3")
all_gather_into_tensor_flat_fp8(
padded_working_param, param_to_gather, pg, fp8_format="e4m3"
)
else:
dist.all_gather_into_tensor(padded_working_param, param_to_gather, pg)
continue
Expand Down

0 comments on commit 0e6e488

Please sign in to comment.