Add script for instruction building dataset

pwr-ai · Dec 12, 2024 · 0387b7a · 0387b7a
1 parent 9b7929d
commit 0387b7a
Show file tree

Hide file tree

Showing 6 changed files with 611 additions and 254 deletions.
diff --git a/configs/dataset/pl-court-frankowe-instruct.yaml b/configs/dataset/pl-court-frankowe-instruct.yaml
@@ -1,6 +1,6 @@
-name: data/datasets/pl/sprawy_frankowe/instructions_08_12_2024
+name: data/datasets/pl/sprawy_frankowe
 prompt_field: prompt
 context_field: context
 output_field: output
 
-max_output_tokens: 800 # 1050 is the real maximum, we take 0.95 quantile value
+max_output_tokens: 800
diff --git a/data/datasets/pl/sprawy_frankowe/.gitignore b/data/datasets/pl/sprawy_frankowe/.gitignore
@@ -0,0 +1,3 @@
+/train.jsonl
+/test.jsonl
+/dataset_info.json