From 93acccc75c74ed9ab57d681f96cbe3664c900968 Mon Sep 17 00:00:00 2001
From: Santosh Tirunagari <stirunag@codon-gpu-009.ebi.ac.uk>
Date: Tue, 26 Nov 2024 13:13:45 +0000
Subject: [PATCH] code setup for sft on eng

---
 configs/fine_tuning.yaml                      | 22 +++++++++----------
 configs/model/Mistral-Nemo-Instruct-2407.yaml | 11 ++++++++++
 dvc.yaml                                      |  1 +
 3 files changed, 23 insertions(+), 11 deletions(-)
 create mode 100644 configs/model/Mistral-Nemo-Instruct-2407.yaml

diff --git a/configs/fine_tuning.yaml b/configs/fine_tuning.yaml
index 78c4303..be7251a 100644
--- a/configs/fine_tuning.yaml
+++ b/configs/fine_tuning.yaml
@@ -30,19 +30,19 @@ peft_args:
   lora_alpha: 16
   lora_dropout: 0 # Supports any, but = 0 is optimized
   bias: "none" # Supports any, but = "none" is optimized
-  target_modules:
-    [
-      "q_proj",
-      "k_proj",
-      "v_proj",
-      "o_proj",
-      "gate_proj",
-      "up_proj",
-      "down_proj",
-    ]
+  target_modules: "all-linear"
+    #[
+     # "q_proj",
+     # "k_proj",
+     # "v_proj",
+     # "o_proj",
+     # "gate_proj",
+     # "up_proj",
+     # "down_proj",
+    #]
 
 truncate_context: True
-wandb_entity: graph-ml-lab-wust
+wandb_entity: ebi_literature
 wandb_project: juddges-fine-tune
 
 output_dir: data/experiments/fine-tune/${hydra:runtime.choices.model}/${hydra:runtime.choices.dataset}
diff --git a/configs/model/Mistral-Nemo-Instruct-2407.yaml b/configs/model/Mistral-Nemo-Instruct-2407.yaml
new file mode 100644
index 0000000..1deb65c
--- /dev/null
+++ b/configs/model/Mistral-Nemo-Instruct-2407.yaml
@@ -0,0 +1,11 @@
+name: mistralai/Mistral-Nemo-Instruct-2407
+tokenizer_name: ${.name}
+
+adapter_path: null
+
+max_seq_length: 7_900 # can handle larger inputs, but set is equal to llama-3 for better comparison
+padding: longest
+batch_size: 1
+use_4bit: true
+
+use_unsloth: false
diff --git a/dvc.yaml b/dvc.yaml
index 4c286f7..e57fd26 100644
--- a/dvc.yaml
+++ b/dvc.yaml
@@ -101,6 +101,7 @@ stages:
         - Unsloth-Mistral-Nemo-Instruct-2407
         - Bielik-7B-Instruct-v0.1
         - Bielik-11B-v2.2-Instruct
+        - Mistral-Nemo-Instruct-2407
     cmd: >-
       PYTHONPATH=. python scripts/sft/fine_tune_llm.py
       dataset=${item.dataset}