proper paths handling

zenml-io · Jul 3, 2024 · 8ca140b · 8ca140b
1 parent 91e6f82
commit 8ca140b
Showing 1 changed file with 2 additions and 3 deletions.
diff --git a/template/steps/finetune.py b/template/steps/finetune.py
@@ -84,8 +84,8 @@ def finetune(
     if should_print:
         logger.info("Loading datasets...")
     tokenizer = load_tokenizer(base_model_id, use_fast=use_fast)
-    tokenized_train_dataset = load_from_disk(dataset_dir / "train")
-    tokenized_val_dataset = load_from_disk(dataset_dir / "val")
+    tokenized_train_dataset = load_from_disk(str((dataset_dir / "train").absolute()))
+    tokenized_val_dataset = load_from_disk(str((dataset_dir / "val").absolute()))
 
     if should_print:
         logger.info("Loading base model...")
@@ -140,7 +140,6 @@ def finetune(
     if should_print:
         logger.info("Saving model...")
 
-    ft_model_dir = Path(ft_model_dir)
     if not use_accelerate or accelerator.is_main_process:
         ft_model_dir.mkdir(parents=True, exist_ok=True)
     if not use_accelerate: