[LLM] fix Qwen-7b-Chat precision issue (#7823)

fix qwen-7b0chat model batch inference precision issue add Qwen-7B-Chat to PaddleNLP unit test
PaddlePaddle · Jan 16, 2024 · e4bd6c3 · e4bd6c3
1 parent c9d22db
commit e4bd6c3
Show file tree

Hide file tree

Showing 4 changed files with 5 additions and 3 deletions.
diff --git a/llm/predictor.py b/llm/predictor.py
@@ -1187,6 +1187,7 @@ def create_predictor(
 
     tensor_parallel_rank, tensor_parallel_degree = init_dist_env()
     if not predictor_args.inference_model:
+        tokenizer.padding_side = "left"
         if predictor_args.mode == "dynamic":
             if model_args.model_type == "gpt-3":
                 sys.path.append("./gpt-3")

diff --git a/paddlenlp/generation/utils.py b/paddlenlp/generation/utils.py
@@ -1059,10 +1059,8 @@ def greedy_search(
 
             # pre-process distribution
             next_token_logits = self.adjust_logits_during_generation(next_token_logits)
-            next_tokens_scores = logits_processors(input_ids, next_token_logits)
+            probs = logits_processors(input_ids, next_token_logits)
             # greedy
-            probs = F.softmax(next_tokens_scores)
-            probs = paddle.log(probs)
             next_tokens = paddle.argmax(probs, axis=-1).unsqueeze(-1)
             next_scores = paddle.index_sample(probs, next_tokens)
 

diff --git a/paddlenlp/transformers/qwen/tokenizer.py b/paddlenlp/transformers/qwen/tokenizer.py
@@ -70,6 +70,7 @@ def __init__(
         self,
         vocab_file,
         errors="replace",
+        padding_side="left",
         **kwargs,
     ):
         super().__init__(**kwargs)

diff --git a/tests/llm/test_predictor.py b/tests/llm/test_predictor.py
@@ -26,6 +26,7 @@
     ChatGLMForCausalLM,
     ChatGLMv2ForCausalLM,
     LlamaForCausalLM,
+    QWenForCausalLM,
 )
 from paddlenlp.utils.downloader import (
     COMMUNITY_MODEL_PREFIX,
@@ -43,6 +44,7 @@
         ["__internal_testing__/tiny-fused-bloom", BloomForCausalLM],
         ["__internal_testing__/tiny-fused-chatglm", ChatGLMForCausalLM],
         ["__internal_testing__/tiny-fused-chatglm2", ChatGLMv2ForCausalLM],
+        ["__internal_testing__/tiny-fused-qwen-inference5.2", QWenForCausalLM],
     ],
 )
 class PredictorTest(LLMTest, unittest.TestCase):