diff --git a/scripts/data_preprocessing.sh b/scripts/data_preprocessing.sh new file mode 100644 index 0000000..362186a --- /dev/null +++ b/scripts/data_preprocessing.sh @@ -0,0 +1,7 @@ +DATA_PATH=datasets/raw_datasets/ +OUTPUT_DIR=datasets/formatted_datasets/ +DATASET_ID=hhrlhf_helpful # summarize, hhrlhf_helpful +python datasets/data_preprocessing.py \ +--data-path $DATA_PATH \ +--output-dir $OUTPUT_DIR \ +--dataset-id $DATASET_ID \ No newline at end of file diff --git a/scripts/evaluate_llm_judges.sh b/scripts/evaluate_llm_judges.sh new file mode 100644 index 0000000..0fcbd08 --- /dev/null +++ b/scripts/evaluate_llm_judges.sh @@ -0,0 +1,37 @@ +# Summarization +#DATASET_DIR=./datasets/formatted_datasets/summarize/data.summarize.2024_10_10.jsonl +#DATASET_ID=summarize +#TEMPLATES="['chen-2023_summarize', 'guo-2024_summarize', 'liusie-2024_summarize', 'rafailov-2023_summarize', 'scheurer-2024_summarize', 'wang-2024_summarize', 'wu-2024_summarize', 'zheng-2023_summarize']" + +# HHRLHF-Helpful +DATASET_DIR=./datasets/formatted_datasets/hhrlhf_helpful/data.hhrlhf_helpful.2024_10_10.jsonl +DATASET_ID=hhrlhf_helpful +TEMPLATES="['bai-2023_hhrlhf_helpful', 'cheng-2024_hhrlhf_helpful', 'guo-2024_hhrlhf_helpful', 'mehta-2023_hhrlhf_helpful', 'rafailov-2023_hhrlhf_helpful', 'shen-2024_hhrlhf_helpful', 'wu-2023_hhrlhf_helpful', 'xu-2024_hhrlhf_helpful', 'zeng-2024_hhrlhf_helpful', 'zheng-2023_hhrlhf_helpful']" + +MODEL_NAMES="['gpt-4o-mini']" +SPLIT_SIZE=200 +NUM_SPLITS=5 +SELF_CONSIST_ID=0 +NUM_RUNS=5 +NUM_EVAL=10 +EXTRACT_RULE=combine +TEMPERATURE=0.1 +NUM_WORKERS=8 +CACHE_DIR=./outputs/ + +python eval/eval_llm_judges.py \ +--processed_data_path $DATASET_DIR \ +--dataset_id $DATASET_ID \ +--split_size $SPLIT_SIZE \ +--num_splits $NUM_SPLITS \ +--self_consist_id $SELF_CONSIST_ID \ +--num_runs $NUM_RUNS \ +--num_eval $NUM_EVAL \ +--models "$MODEL_NAMES" \ +--templates "$TEMPLATES" \ +--extract_rule $EXTRACT_RULE \ +--temperature $TEMPERATURE \ +--num_workers $NUM_WORKERS \ +--use_cache_samples \ +--use_cache_results \ +--cache_dir $CACHE_DIR