From 272c1fb23dd3f1f026ed5dc5263f3316508c3fc8 Mon Sep 17 00:00:00 2001 From: Jan Baczek Date: Thu, 11 Jul 2024 11:59:50 +0200 Subject: [PATCH] Disable cuda graph for mlperf 24n config Signed-off-by: Jan Baczek --- launcher_scripts/conf/training/gpt3/mlperf-24n.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/launcher_scripts/conf/training/gpt3/mlperf-24n.yaml b/launcher_scripts/conf/training/gpt3/mlperf-24n.yaml index e372d9ea3..8e82ea4ca 100644 --- a/launcher_scripts/conf/training/gpt3/mlperf-24n.yaml +++ b/launcher_scripts/conf/training/gpt3/mlperf-24n.yaml @@ -302,7 +302,7 @@ model: name: megatron_gpt_full_te_layer_autocast use_tp_pp_dp_mapping: false fp8_params: true - enable_cuda_graph: 1 + enable_cuda_graph: False # TODO: set to true once cuda graph functionality is fully supported defer_embedding_wgrad_compute: true use_te_rng_tracker: true tp_comm_overlap_ag: true