Update llama-inference.yaml

eole-nlp · Sep 20, 2024 · cc47c06 · cc47c06
1 parent 4b44fdc
commit cc47c06
Showing 1 changed file with 3 additions and 4 deletions.
diff --git a/recipes/llama3.1/llama-inference.yaml b/recipes/llama3.1/llama-inference.yaml
@@ -5,7 +5,6 @@ model_path: "${EOLE_MODEL_DIR}/llama3.1-8b"
 seed: 42
 max_length: 256
 # max_length: 1
-gpu: 0
 batch_type: sents
 batch_size: 4
 world_size: 1
@@ -16,9 +15,9 @@ gpu_ranks: [0]
 # quant_layers: ['gate_up_proj', 'down_proj', 'up_proj', 'linear_values', 'linear_query', 'linear_keys', 'final_linear']
 # quant_type: "bnb_NF4"
 compute_dtype: bf16
-#random_sampling_topk: 1
-#random_sampling_topp: 0.0
-#random_sampling_temp: 0.9
+#top_k: 1
+#top_p: 0.0
+#temperature: 0.9
 beam_size: 1
 n_best: 1
 report_time: true