修复 glm 4.1v

Author: shell-nlp

gpt_server/model_backend/vllm_backend.py

@@ -69,12 +69,13 @@ async def stream_chat(self, params: Dict[str, Any]) -> AsyncGenerator:
         request_id = params.get("request_id", "0")
         temperature = float(params.get("temperature", 0.8))
         top_p = float(params.get("top_p", 0.8))
-        top_k = params.get("top_k", -1.0)
+        top_k = int(params.get("top_k", 0))
         max_new_tokens = int(params.get("max_new_tokens", 1024 * 8))
         stop_str = params.get("stop", None)
         stop_token_ids = params.get("stop_words_ids", None) or []
         presence_penalty = float(params.get("presence_penalty", 0.0))
         frequency_penalty = float(params.get("frequency_penalty", 0.0))
+        repetition_penalty = float(params.get("repetition_penalty", 1.0))
         request = params.get("request", None)
         # Handle stop_str
         stop = set()
@@ -145,6 +146,7 @@ async def stream_chat(self, params: Dict[str, Any]) -> AsyncGenerator:
             stop_token_ids=stop_token_ids,
             presence_penalty=presence_penalty,
             frequency_penalty=frequency_penalty,
+            repetition_penalty=repetition_penalty,
             guided_decoding=guided_decoding,
         )
         lora_request = None

gpt_server/model_worker/base/model_worker_base.py

@@ -90,6 +90,9 @@ def __init__(
             # logger.info(f"模型配置：{self.model_config}")
             self.vision_config = getattr(self.model_config, "vision_config", None)
             is_vision = self.vision_config is not None
+            if is_vision:
+                multimodal = True
+                logger.warning(f"{model_names[0]} 是多模态模型")
         super().__init__(
             controller_addr,
             worker_addr,
@@ -98,7 +101,7 @@ def __init__(
             model_names,
             limit_worker_concurrency,
             conv_template,
-            multimodal=multimodal or is_vision,
+            multimodal=multimodal,
         )
         os.environ["WORKER_NAME"] = self.__class__.__name__
         self.worker_name = self.__class__.__name__

gpt_server/model_worker/chatglm.py

@@ -77,7 +77,8 @@ async def generate_stream_gate(self, params):
                 # text = self.tokenizer.decode(input_ids.tolist()[0])
                 params["prompt"] = text
                 # params["input_ids"] = input_ids
-
+            else:  # 多模态模型
+                params["multimodal"] = True
             # ---------------添加额外的参数------------------------
             params["messages"] = messages
             params["stop"].extend(self.stop)

tests/test_openai_vl_chat.py

@@ -1,5 +1,6 @@
 import base64
 from openai import OpenAI
+from pathlib import Path
 
 
 def image_to_base64(image_path):
@@ -11,7 +12,7 @@ def image_to_base64(image_path):
     return base64_prefix + base64_string
 
 
-image_path = "../assets/logo.png"
+image_path = Path(__file__).parent.parent / "assets/logo.png"
 # 使用本地的图片
 url = image_to_base64(image_path)
 # 使用网络图片
@@ -22,7 +23,7 @@ def image_to_base64(image_path):
 
 stream = True
 output = client.chat.completions.create(
-    model="minicpmv",  # internlm chatglm3  qwen  llama3 chatglm4
+    model="glm4.1v",  # internlm chatglm3  qwen  llama3 chatglm4
     messages=[
         {
             "role": "user",