debug rwth-i6/returnn#1577

rwth-i6 · Jul 9, 2024 · 6151c54 · 6151c54
1 parent e65f6ed
commit 6151c54
Showing 1 changed file with 10 additions and 0 deletions.
diff --git a/users/zeyer/experiments/exp2024_04_23_baselines/ctc.py b/users/zeyer/experiments/exp2024_04_23_baselines/ctc.py
@@ -437,6 +437,12 @@ def py():
                 else None
             ),
         )
+    _train_experiments[
+        (
+            "v6-relPosAttDef-bhv20-11gb-f32-bs15k-accgrad1-mgpu4-pavg100-maxSeqLenAudio19_5-wd1e_2-lrlin1e_5_295k-featBN"
+            "-speedpertV2-spm128"
+        )
+    ].get_training_job().set_env("CUDA_LAUNCH_BLOCKING", "1")
 
     train_exp(  # 5.78 (!!!)
         "v6-relPosAttDef-aedLoss-bhv20-11gb-f32-bs15k-accgrad1-mgpu4-pavg100-wd1e_2-lrlin1e_5_295k-speedpertV2-spm10k",
@@ -453,6 +459,9 @@ def py():
     )
 
 
+_train_experiments: Dict[str, ModelWithCheckpoints] = {}
+
+
 # noinspection PyShadowingNames
 def train_exp(
     name: str,
@@ -523,6 +532,7 @@ def train_exp(
         prefix, task, model_with_checkpoint, recog_def=model_recog, recog_post_proc_funcs=recog_post_proc_funcs
     )
 
+    _train_experiments[name] = model_with_checkpoint
     return model_with_checkpoint