Fix tool_parser

vllm-project · Oct 6, 2024 · 6bdbce6 · 6bdbce6
1 parent 0324847
commit 6bdbce6
Showing 1 changed file with 10 additions and 2 deletions.
diff --git a/vllm/entrypoints/openai/serving_chat.py b/vllm/entrypoints/openai/serving_chat.py
@@ -323,9 +323,11 @@ async def chat_completion_stream_generator(
         # Prepare the tool parser if it's needed
         try:
             if tool_choice_auto and self.tool_parser:
-                tool_parser: Optional[ToolParser] = self.tool_parser(tokenizer)
+                tool_parsers: List[Optional[ToolParser]] = [
+                    self.tool_parser(tokenizer) if self.tool_parser else None
+                ] * num_choices
             else:
-                tool_parser = None
+                tool_parsers = [None] * num_choices
         except RuntimeError as e:
             logger.error("Error in tool parser creation: %s", e)
             data = self.create_streaming_error_response(str(e))
@@ -719,6 +721,12 @@ async def chat_completion_full_generator(
                     logger.error("Error in tool parser creation: %s", e)
                     return self.create_error_response(str(e))
 
+                try:
+                    tool_parser = self.tool_parser(tokenizer)
+                except RuntimeError as e:
+                    logger.error("Error in tool parser creation: %s", e)
+                    return self.create_error_response(str(e))
+
                 tool_call_info = tool_parser.extract_tool_calls(
                     output.text, request=request)
                 tools_called = tool_call_info.tools_called