Reproduce Bielik v2

configs/model/Bielik-11B-v2.2-Instruct-fine-tuned.yaml

@@ -0,0 +1,11 @@
+name: speakleash/Bielik-11B-v2.2-Instruct
+tokenizer_name: ${.name}
+
+adapter_path: data/experiments/fine-tune/Bielik-11B-v2.2-Instruct/pl-court-instruct/checkpoint-1500
+
+max_seq_length: 7_900
+batch_size: 1
+padding: longest
+use_4bit: true
+
+use_unsloth: true

data/experiments/fine-tune/Bielik-11B-v2.2-Instruct/.gitignore

@@ -0,0 +1 @@
+/pl-court-instruct

data/experiments/predict/pl-court-instruct/Bielik-11B-v2.2-Instruct-fine-tuned/.gitignore

@@ -0,0 +1,9 @@
+/outputs_42.json
+/outputs_7312.json
+/outputs_997.json
+/metrics_997.json
+/metrics_42.json
+/metrics_7312.json
+/judge_metrics_7312.json
+/judge_metrics_42.json
+/judge_metrics_997.json

data/experiments/predict/pl-court-instruct/Bielik-11B-v2.2-Instruct/.gitignore

@@ -1 +1,9 @@
 /outputs_997.json
+/outputs_42.json
+/outputs_7312.json
+/metrics_997.json
+/metrics_42.json
+/metrics_7312.json
+/judge_metrics_42.json
+/judge_metrics_7312.json
+/judge_metrics_997.json

data/experiments/predict/pl-court-instruct/metrics_judge_summary.md

@@ -1,5 +1,17 @@
 | llm                                           | assessment      | court_name      | date            | department_name   | judges          | legal_bases     | recorder        | signature       |
 |:----------------------------------------------|:----------------|:----------------|:----------------|:------------------|:----------------|:----------------|:----------------|:----------------|
+| Bielik-11B-v2.2-Instruct                      | (Correct)       | 0.868 (± 0.003) | 0.914 (± 0.003) | 0.833 (± 0.003)   | 0.514 (± 0.004) | 0.024 (± 0.001) | 0.829 (± 0.001) | 0.837 (± 0.001) |
+| Bielik-11B-v2.2-Instruct                      | (Disagreement)  | 0.037 (± 0.001) | 0.023 (± 0.000) | 0.067 (± 0.002)   | 0.160 (± 0.001) | 0.599 (± 0.002) | 0.005 (± 0.001) | 0.018 (± 0.001) |
+| Bielik-11B-v2.2-Instruct                      | (Subset)        | 0.012 (± 0.001) | 0.000 (± 0.000) | 0.019 (± 0.001)   | 0.020 (± 0.000) | 0.060 (± 0.000) | 0.041 (± 0.001) | 0.004 (± 0.001) |
+| Bielik-11B-v2.2-Instruct                      | (Superset)      | 0.020 (± 0.001) | 0.000 (± 0.000) | 0.017 (± 0.001)   | 0.242 (± 0.002) | 0.154 (± 0.002) | 0.002 (± 0.001) | 0.007 (± 0.000) |
+| Bielik-11B-v2.2-Instruct                      | (empty-answer)  | 0.064 (± 0.003) | 0.064 (± 0.003) | 0.064 (± 0.003)   | 0.065 (± 0.003) | 0.163 (± 0.004) | 0.124 (± 0.001) | 0.134 (± 0.002) |
+| Bielik-11B-v2.2-Instruct                      | (non-evaluable) | 0.000 (± 0.000) | 0.000 (± 0.000) | 0.000 (± 0.000)   | 0.000 (± 0.000) | 0.000 (± 0.000) | 0.000 (± 0.000) | 0.000 (± 0.000) |
+| Bielik-11B-v2.2-Instruct-fine-tuned           | (Correct)       | 0.859 (± 0.002) | 0.847 (± 0.001) | 0.848 (± 0.001)   | 0.824 (± 0.003) | 0.066 (± 0.003) | 0.647 (± 0.011) | 0.529 (± 0.007) |
+| Bielik-11B-v2.2-Instruct-fine-tuned           | (Disagreement)  | 0.009 (± 0.000) | 0.022 (± 0.001) | 0.009 (± 0.000)   | 0.014 (± 0.001) | 0.544 (± 0.002) | 0.044 (± 0.002) | 0.059 (± 0.007) |
+| Bielik-11B-v2.2-Instruct-fine-tuned           | (Subset)        | 0.000 (± 0.000) | 0.000 (± 0.000) | 0.006 (± 0.000)   | 0.011 (± 0.001) | 0.010 (± 0.001) | 0.053 (± 0.003) | 0.038 (± 0.006) |
+| Bielik-11B-v2.2-Instruct-fine-tuned           | (Superset)      | 0.000 (± 0.000) | 0.000 (± 0.000) | 0.006 (± 0.000)   | 0.020 (± 0.001) | 0.164 (± 0.004) | 0.001 (± 0.001) | 0.001 (± 0.000) |
+| Bielik-11B-v2.2-Instruct-fine-tuned           | (empty-answer)  | 0.132 (± 0.002) | 0.132 (± 0.002) | 0.132 (± 0.002)   | 0.132 (± 0.002) | 0.217 (± 0.002) | 0.255 (± 0.012) | 0.373 (± 0.013) |
+| Bielik-11B-v2.2-Instruct-fine-tuned           | (non-evaluable) | 0.000 (± 0.000) | 0.000 (± 0.000) | 0.000 (± 0.000)   | 0.000 (± 0.000) | 0.000 (± 0.000) | 0.000 (± 0.000) | 0.000 (± 0.000) |
 | Bielik-7B-Instruct-v0.1                       | (Correct)       | 0.000 (± 0.000) | 0.001 (± 0.001) | 0.000 (± 0.000)   | 0.001 (± 0.001) | 0.000 (± 0.000) | 0.000 (± 0.000) | 0.000 (± 0.000) |
 | Bielik-7B-Instruct-v0.1                       | (Disagreement)  | 0.000 (± 0.000) | 0.001 (± 0.000) | 0.001 (± 0.000)   | 0.002 (± 0.002) | 0.002 (± 0.001) | 0.001 (± 0.001) | 0.001 (± 0.000) |
 | Bielik-7B-Instruct-v0.1                       | (Subset)        | 0.000 (± 0.000) | 0.000 (± 0.000) | 0.000 (± 0.000)   | 0.000 (± 0.000) | 0.000 (± 0.000) | 0.000 (± 0.000) | 0.000 (± 0.000) |

data/experiments/predict/pl-court-instruct/metrics_ngram_summary.md

@@ -1,5 +1,7 @@
 | llm                                           | full_text_chrf   | court_name      | date            | department_name   | judges          | legal_bases     | recorder        | signature       |
 |:----------------------------------------------|:-----------------|:----------------|:----------------|:------------------|:----------------|:----------------|:----------------|:----------------|
+| Bielik-11B-v2.2-Instruct                      | 0.679 (± 0.001)  | 0.891 (± 0.002) | 0.921 (± 0.002) | 0.902 (± 0.003)   | 0.858 (± 0.003) | 0.472 (± 0.001) | 0.842 (± 0.001) | 0.790 (± 0.002) |
+| Bielik-11B-v2.2-Instruct-fine-tuned           | 0.749 (± 0.001)  | 0.865 (± 0.001) | 0.856 (± 0.001) | 0.864 (± 0.001)   | 0.848 (± 0.002) | 0.548 (± 0.000) | 0.695 (± 0.011) | 0.589 (± 0.010) |
 | Bielik-7B-Instruct-v0.1                       | 0.354 (± 0.001)  | 0.000 (± 0.000) | 0.001 (± 0.000) | 0.001 (± 0.000)   | 0.001 (± 0.000) | 0.001 (± 0.000) | 0.000 (± 0.000) | 0.000 (± 0.000) |
 | Bielik-7B-Instruct-v0.1-fine-tuned            | 0.717 (± 0.000)  | 0.890 (± 0.007) | 0.863 (± 0.007) | 0.886 (± 0.007)   | 0.879 (± 0.007) | 0.465 (± 0.004) | 0.639 (± 0.001) | 0.459 (± 0.002) |
 | Unsloth-Llama-3-8B-Instruct                   | 0.579 (± 0.001)  | 0.863 (± 0.002) | 0.946 (± 0.002) | 0.909 (± 0.002)   | 0.912 (± 0.003) | 0.362 (± 0.002) | 0.735 (± 0.004) | 0.686 (± 0.004) |