moe token drop params

NVIDIA · Jul 9, 2024 · 1c3d475 · 1c3d475
1 parent c631a8a
commit 1c3d475
Show file tree

Hide file tree

Showing 4 changed files with 5 additions and 5 deletions.
diff --git a/auto_configurator/base_configs/mixtral_3b.yaml b/auto_configurator/base_configs/mixtral_3b.yaml
@@ -47,9 +47,6 @@ exp_manager:
 model:
   mcore_gpt: true
   moe_grouped_gemm: true
-  moe_token_dispatcher_type: alltoall
-  moe_pad_expert_input_to_capacity: True
-  moe_expert_capacity_factor: 1.0
   micro_batch_size: 1
   global_batch_size: 128
   rampup_batch_size: null

diff --git a/auto_configurator/base_configs/mixtral_7b.yaml b/auto_configurator/base_configs/mixtral_7b.yaml
@@ -48,8 +48,6 @@ model:
   mcore_gpt: true
   moe_grouped_gemm: true
   moe_token_dispatcher_type: alltoall
-  moe_pad_expert_input_to_capacity: True
-  moe_expert_capacity_factor: 1.0
   moe_aux_loss_coeff: 0.01
   micro_batch_size: 1
   global_batch_size: 256

diff --git a/launcher_scripts/conf/training/mixtral/mixtral_8x3b.yaml b/launcher_scripts/conf/training/mixtral/mixtral_8x3b.yaml
@@ -47,6 +47,9 @@ exp_manager:
 model:
   mcore_gpt: true
   moe_grouped_gemm: true
+  moe_token_dispatcher_type: alltoall
+  moe_pad_expert_input_to_capacity: True
+  moe_expert_capacity_factor: 1.0
   micro_batch_size: 1
   global_batch_size: 128
   rampup_batch_size: null

diff --git a/launcher_scripts/conf/training/mixtral/mixtral_8x7b.yaml b/launcher_scripts/conf/training/mixtral/mixtral_8x7b.yaml
@@ -48,6 +48,8 @@ model:
   mcore_gpt: true
   moe_grouped_gemm: true
   moe_token_dispatcher_type: alltoall
+  moe_pad_expert_input_to_capacity: True
+  moe_expert_capacity_factor: 1.0
   moe_aux_loss_coeff: 0.01
   micro_batch_size: 1
   global_batch_size: 256