Update llama-instruct-inference.yaml

eole-nlp · Sep 20, 2024 · c4255b0 · c4255b0
1 parent cc47c06
commit c4255b0
Showing 1 changed file with 9 additions and 10 deletions.
diff --git a/recipes/llama3.1/llama-instruct-inference.yaml b/recipes/llama3.1/llama-instruct-inference.yaml
@@ -5,20 +5,19 @@ model_path: "${EOLE_MODEL_DIR}/llama3.1-8b-instruct"
 seed: 42
 max_length: 256
 # max_length: 1
-# gpu: 0
 batch_type: tokens
 batch_size: 8192
-#world_size: 1
-#gpu_ranks: [0]
-world_size: 2
-gpu_ranks: [0, 1]
-parallel_mode: "tensor_parallel"
+world_size: 1
+gpu_ranks: [0]
+#world_size: 2
+#gpu_ranks: [0, 1]
+#parallel_mode: "tensor_parallel"
 #quant_layers: ['gate_up_proj', 'down_proj', 'up_proj', 'linear_values', 'linear_query', 'linear_keys', 'final_linear']
 #quant_type: "bnb_NF4"
-compute_dtype: fp16
-#random_sampling_topk: 1
-#random_sampling_topp: 0.0
-#random_sampling_temp: 0.9
+compute_dtype: bf16
+top_k: 1
+top_p: 0.0
+#temperature: 0.9
 beam_size: 1
 n_best: 1
 report_time: true