Update megatron/data/gpt_dataset.py

argonne-lcf · Oct 16, 2024 · 63b1901 · 63b1901
1 parent 52a406c
commit 63b1901
Showing 1 changed file with 4 additions and 0 deletions.
diff --git a/megatron/data/gpt_dataset.py b/megatron/data/gpt_dataset.py
@@ -168,6 +168,10 @@ def _build_indices():
                 for i in range(self.num_datasets):
                     self.desc += dataset_builders[i].prefix + ","
 
+                log.info(
+                    f"[BuildConcatDataset] Caught {shuffle=} across"
+                    f" {self.num_samples} samples"
+                )
                 self.desc += (
                     f"-{self.num_samples}"
                     + f"-{dataset_builders[0].seq_length}"