remove ceval from run_finetune (#9100)

llm/docs/quantization.md

@@ -94,13 +94,6 @@ python  run_finetune.py ./config/llama/ptq_c8_argument.json
 python  run_finetune.py ./config/llama/fp8_ptq_argument.json
 ```
 
-### 2.8 测试量化模型 C-Eval 得分
-
-```shell
-python run_finetune.py ./config/llama/ceval_quant_argument.json
-```
-
-
 ### 2.9 量化参数介绍
 
 <summary>&emsp; 量化参数（QuantArgument）</summary>
@@ -135,8 +128,6 @@ python run_finetune.py ./config/llama/ceval_quant_argument.json
 - `do_gptq`: 是否进行 GPTQ 量化，GPTQ 对模型进行 WINT4量化，相比于普通 PTQ 量化精度更高，量化时间较长。默认为 False。
 - `gptq_step`: GPTQ 量化步数，也即模型前向次数，默认为8。
 - `do_awq`: 是否进行 AWQ 量化，AWQ 对模型进行 WINT4量化，相比于普通 PTQ 量化精度更高。默认为 False。
-- `do_ceval`: 是否启动 C-Eval 测试。默认为 False。
-- `ceval_data_path`: C-Eval 数据集路径，默认为"../dataset/ceval"。
 - `auto_clip`: AWQ 时是否进行自动搜索截断值并对模型权重进行截断操作，截断操作有利于量化模型精度，但搜索速度较慢。默认为 False。
 - `autoclip_step`: AutoClip 步数，也即模型前向次数，采样时默认 concat 每轮数据用来搜索截断值，默认为8。
 

llm/run_finetune.py

@@ -18,7 +18,6 @@
 
 import paddle
 from utils.argument import (
-    CEvalArgument,
     DataArgument,
     GenerateArgument,
     ModelArgument,
@@ -73,13 +72,11 @@
 
 
 def main():
-    parser = PdArgumentParser(
-        (GenerateArgument, QuantArgument, ModelArgument, DataArgument, TrainingArguments, CEvalArgument)
-    )
+    parser = PdArgumentParser((GenerateArgument, QuantArgument, ModelArgument, DataArgument, TrainingArguments))
     if len(sys.argv) >= 2 and sys.argv[1].endswith(".json"):
-        gen_args, quant_args, model_args, data_args, training_args, ceval_args = parser.parse_json_file_and_cmd_lines()
+        gen_args, quant_args, model_args, data_args, training_args = parser.parse_json_file_and_cmd_lines()
     else:
-        gen_args, quant_args, model_args, data_args, training_args, ceval_args = parser.parse_args_into_dataclasses()
+        gen_args, quant_args, model_args, data_args, training_args = parser.parse_args_into_dataclasses()
 
     training_args.print_config(model_args, "Model")
     training_args.print_config(data_args, "Data")
@@ -561,10 +558,6 @@ def compute_metrics_do_generation(eval_preds):
         data_args=data_args,
     )
 
-    # Evaluation dev set
-    if training_args.do_eval:
-        before_eval_result = trainer.evaluate(dev_ds)
-
     # Train
     if training_args.do_train:
         checkpoint = None
@@ -726,21 +719,10 @@ def compute_metrics_do_generation(eval_preds):
     # Evaluation dev set
     if training_args.do_eval:
 
-        logger.info("*** Evaluate result before train/ptq/qat/ etc.***")
-        trainer.log_metrics("eval", before_eval_result)
-
         logger.info("*** Evaluate result after train/ptq/qat/ etc.***")
         eval_result = trainer.evaluate(dev_ds)
         trainer.log_metrics("eval", eval_result)
 
-    # C-Eval after qat/ptq/train
-    if ceval_args.do_ceval:
-        logger.info("*** Evaluate on C-Eval ***")
-        ceval_args.output_dir = training_args.output_dir
-        from experimental.ceval.default.eval import run_eval
-
-        run_eval(tokenizer, trainer.model, ceval_args)
-
 
 if __name__ == "__main__":
     main()

llm/utils/argument.py

@@ -348,21 +348,3 @@ class GenerateArgument:
     top_p: float = field(
         default=1.0, metadata={"help": "The cumulative probability for top-p-filtering in the sampling strategy."}
     )
-
-
-@dataclass
-class CEvalArgument:
-    do_ceval: bool = field(
-        default=False,
-        metadata={"help": "Whether to run C-Eval"},
-    )
-    cot: bool = field(default=False, metadata={"help": "Whether to use chain of thought"})
-    few_shot: bool = field(default=False, metadata={"help": "Whether to use few shot"})
-    ntrain: int = field(default=5, metadata={"help": "Number of few shot"})
-    with_prompt: bool = field(default=False, metadata={"help": "Whether to use prompt"})
-    constrained_decoding: bool = field(default=True, metadata={"help": "Whether to use constrained decoding"})
-    temperature: float = field(default=0.2, metadata={"help": "Temperature for decoding"})
-    n_times: int = field(default=1, metadata={"help": "Number of times to run"})
-    do_save_csv: bool = field(default=False, metadata={"help": "Whether to save csv"})
-    do_test: bool = field(default=False, metadata={"help": "Whether to run test"})
-    ceval_data_path: str = field(default="../dataset/ceval", metadata={"help": "Path to the data for ceval"})