Update vllm version

Author: xusenlin

api/config.py

@@ -176,6 +176,37 @@ class Settings(BaseModel):
         default=get_env("QUANTIZATION_METHOD", None),
         description="Quantization method for vllm server."
     )
+    enforce_eager: Optional[bool] = Field(
+        default=get_bool_env("ENFORCE_EAGER"),
+        description="Always use eager-mode PyTorch. If False, will use eager mode and CUDA graph in hybrid for maximal performance and flexibility."
+    )
+    max_context_len_to_capture: Optional[int] = Field(
+        default=int(get_env("MAX_CONTEXT_LEN_TO_CAPTURE", 8192)),
+        description="aximum context length covered by CUDA graphs. When a sequence has context length larger than this, we fall back to eager mode."
+    )
+    max_loras: Optional[int] = Field(
+        default=int(get_env("MAX_LORAS", 1)),
+        description="Max number of LoRAs in a single batch."
+    )
+    max_lora_rank: Optional[int] = Field(
+        default=int(get_env("MAX_LORA_RANK", 32)),
+        description="Max LoRA rank."
+    )
+    lora_extra_vocab_size: Optional[int] = Field(
+        default=int(get_env("LORA_EXTRA_VOCAB_SIZE", 256)),
+        description="Maximum size of extra vocabulary that can be present in a LoRA adapter added to the base model vocabulary."
+    )
+    lora_dtype: Optional[str] = Field(
+        default=get_env("LORA_DTYPE", "auto"),
+        description="Data type for LoRA. If auto, will default to base model dtype."
+    )
+    max_cpu_loras: Optional[int] = Field(
+        default=int(get_env("MAX_CPU_LORAS", -1)),
+        ge=-1,
+    )
+    lora_modules: Optional[str] = Field(
+        default=get_env("LORA_MODULES", ""),
+    )
 
     # support for transformers.TextIteratorStreamer
     use_streamer_v2: Optional[bool] = Field(

api/core/vllm_engine.py

@@ -1,54 +1,165 @@
 import asyncio
+import time
+from dataclasses import dataclass
 from typing import (
     Optional,
     List,
     Dict,
     Any,
-    AsyncIterator,
     Union,
 )
 
-from fastapi import HTTPException
 from loguru import logger
 from openai.types.chat import ChatCompletionMessageParam
-from transformers import PreTrainedTokenizer
+from openai.types.completion_choice import Logprobs
+from openai.types.model import Model
+from pydantic import BaseModel
 from vllm.engine.async_llm_engine import AsyncLLMEngine
-from vllm.sampling_params import SamplingParams
+from vllm.transformers_utils.tokenizer import get_tokenizer
 
 from api.adapter import get_prompt_adapter
 from api.generation import build_qwen_chat_input
 
 
+@dataclass
+class LoRA:
+    name: str
+    local_path: str
+
+
+class ModelList(BaseModel):
+    object: str = "list"
+    data: List[Model] = []
+
+
 class VllmEngine:
     def __init__(
         self,
         model: AsyncLLMEngine,
-        tokenizer: PreTrainedTokenizer,
         model_name: str,
         prompt_name: Optional[str] = None,
-        context_len: Optional[int] = -1,
+        lora_modules: Optional[List[LoRA]] = None,
     ):
         """
         Initializes the VLLMEngine object.
 
         Args:
             model: The AsyncLLMEngine object.
-            tokenizer: The PreTrainedTokenizer object.
             model_name: The name of the model.
             prompt_name: The name of the prompt (optional).
-            context_len: The length of the context (optional, default=-1).
         """
         self.model = model
         self.model_name = model_name.lower()
-        self.tokenizer = tokenizer
         self.prompt_name = prompt_name.lower() if prompt_name is not None else None
         self.prompt_adapter = get_prompt_adapter(self.model_name, prompt_name=self.prompt_name)
 
-        model_config = asyncio.run(self.model.get_model_config())
-        if "qwen" in self.model_name:
-            self.max_model_len = context_len if context_len > 0 else 8192
+        if lora_modules is None:
+            self.lora_requests = []
         else:
-            self.max_model_len = model_config.max_model_len
+            try:
+                from vllm.lora.request import LoRARequest
+                self.lora_requests = [
+                    LoRARequest(
+                        lora_name=lora.name,
+                        lora_int_id=i,
+                        lora_local_path=lora.local_path,
+                    ) for i, lora in enumerate(lora_modules, start=1)
+                ]
+            except ImportError:
+                self.lora_requests = []
+
+        try:
+            event_loop = asyncio.get_running_loop()
+        except RuntimeError:
+            event_loop = None
+
+        if event_loop is not None and event_loop.is_running():
+            # If the current is instanced by Ray Serve,
+            # there is already a running event loop
+            event_loop.create_task(self._post_init())
+        else:
+            # When using single vLLM without engine_use_ray
+            asyncio.run(self._post_init())
+
+    async def _post_init(self):
+        engine_model_config = await self.model.get_model_config()
+        self.max_model_len = engine_model_config.max_model_len
+
+        # A separate tokenizer to map token IDs to strings.
+        self.tokenizer = get_tokenizer(
+            engine_model_config.tokenizer,
+            tokenizer_mode=engine_model_config.tokenizer_mode,
+            trust_remote_code=engine_model_config.trust_remote_code,
+        )
+
+    async def show_available_models(self) -> ModelList:
+        """Show available models. Right now we only have one model."""
+        model_cards = [
+            Model(
+                id=self.model_name,
+                object="model",
+                created=int(time.time()),
+                owned_by="vllm"
+            )
+        ]
+        lora_cards = [
+            Model(
+                id=lora.lora_name,
+                object="model",
+                created=int(time.time()),
+                owned_by="vllm"
+            )
+            for lora in self.lora_requests
+        ]
+        model_cards.extend(lora_cards)
+        return ModelList(data=model_cards)
+
+    def create_logprobs(
+        self,
+        token_ids: List[int],
+        top_logprobs: Optional[List[Optional[Any]]] = None,
+        num_output_top_logprobs: Optional[int] = None,
+        initial_text_offset: int = 0,
+    ):
+        """Create OpenAI-style logprobs."""
+        logprobs = Logprobs()
+        last_token_len = 0
+        if num_output_top_logprobs:
+            logprobs.top_logprobs = []
+
+        for i, token_id in enumerate(token_ids):
+            step_top_logprobs = top_logprobs[i]
+            if step_top_logprobs is not None:
+                token_logprob = step_top_logprobs[token_id].logprob
+            else:
+                token_logprob = None
+
+            token = step_top_logprobs[token_id].decoded_token
+            logprobs.tokens.append(token)
+            logprobs.token_logprobs.append(token_logprob)
+
+            if len(logprobs.text_offset) == 0:
+                logprobs.text_offset.append(initial_text_offset)
+            else:
+                logprobs.text_offset.append(logprobs.text_offset[-1] + last_token_len)
+            last_token_len = len(token)
+
+            if num_output_top_logprobs:
+                logprobs.top_logprobs.append(
+                    {
+                        p.decoded_token: p.logprob
+                        for i, p in step_top_logprobs.items()
+                    }
+                    if step_top_logprobs else None
+                )
+        return logprobs
+
+    def _maybe_get_lora(self, model_name):
+        for lora in self.lora_requests:
+            if model_name == lora.lora_name:
+                logger.info(f"Lora request: {model_name}")
+                return lora
+        return None
 
     def apply_chat_template(
         self,
@@ -104,61 +215,6 @@ def convert_to_inputs(
             max_input_tokens = max(self.max_model_len - max_tokens, input_len)
         return input_ids[-max_input_tokens:]
 
-    def generate(self, params: Dict[str, Any], request_id: str) -> AsyncIterator:
-        """
-        Generates text based on the given parameters and request ID.
-
-        Args:
-            params (Dict[str, Any]): A dictionary of parameters for text generation.
-            request_id (str): The ID of the request.
-
-        Yields:
-            Any: The generated text.
-        """
-        max_tokens = params.get("max_tokens", 256)
-        prompt_or_messages = params.get("prompt_or_messages")
-        if isinstance(prompt_or_messages, list):
-            prompt_or_messages = self.apply_chat_template(
-                prompt_or_messages,
-                functions=params.get("functions"),
-                tools=params.get("tools"),
-            )
-
-        if isinstance(prompt_or_messages, list):
-            prompt, token_ids = None, prompt_or_messages
-        else:
-            prompt, token_ids = prompt_or_messages, None
-
-        token_ids = self.convert_to_inputs(prompt, token_ids, max_tokens=max_tokens)
-        try:
-            sampling_params = SamplingParams(
-                n=params.get("n", 1),
-                presence_penalty=params.get("presence_penalty", 0.),
-                frequency_penalty=params.get("frequency_penalty", 0.),
-                temperature=params.get("temperature", 0.9),
-                top_p=params.get("top_p", 0.8),
-                stop=params.get("stop", []),
-                stop_token_ids=params.get("stop_token_ids", []),
-                max_tokens=params.get("max_tokens", 256),
-                repetition_penalty=params.get("repetition_penalty", 1.03),
-                min_p=params.get("min_p", 0.0),
-                best_of=params.get("best_of", 1),
-                ignore_eos=params.get("ignore_eos", False),
-                use_beam_search=params.get("use_beam_search", False),
-                skip_special_tokens=params.get("skip_special_tokens", True),
-                spaces_between_special_tokens=params.get("spaces_between_special_tokens", True),
-            )
-            result_generator = self.model.generate(
-                prompt_or_messages if isinstance(prompt_or_messages, str) else None,
-                sampling_params,
-                request_id,
-                token_ids,
-            )
-        except ValueError as e:
-            raise HTTPException(status_code=400, detail=str(e)) from e
-
-        return result_generator
-
     @property
     def stop(self):
         """

api/models.py

@@ -76,8 +76,7 @@ def create_vllm_engine():
     try:
         from vllm.engine.arg_utils import AsyncEngineArgs
         from vllm.engine.async_llm_engine import AsyncLLMEngine
-        from vllm.transformers_utils.tokenizer import get_tokenizer
-        from api.core.vllm_engine import VllmEngine
+        from api.core.vllm_engine import VllmEngine, LoRA
     except ImportError:
         return None
 
@@ -88,32 +87,36 @@ def create_vllm_engine():
         "dtype",
         "gpu_memory_utilization",
         "max_num_seqs",
+        "enforce_eager",
+        "max_context_len_to_capture",
+        "max_loras",
+        "max_lora_rank",
+        "lora_extra_vocab_size",
     }
     kwargs = model_dump(SETTINGS, include=include)
     engine_args = AsyncEngineArgs(
         model=SETTINGS.model_path,
         max_num_batched_tokens=SETTINGS.max_num_batched_tokens if SETTINGS.max_num_batched_tokens > 0 else None,
         max_model_len=SETTINGS.context_length if SETTINGS.context_length > 0 else None,
         quantization=SETTINGS.quantization_method,
+        max_cpu_loras=SETTINGS.max_cpu_loras if SETTINGS.max_cpu_loras > 0 else None,
         **kwargs,
     )
     engine = AsyncLLMEngine.from_engine_args(engine_args)
 
-    # A separate tokenizer to map token IDs to strings.
-    tokenizer = get_tokenizer(
-        engine_args.tokenizer,
-        tokenizer_mode=engine_args.tokenizer_mode,
-        trust_remote_code=True,
-    )
-
     logger.info("Using vllm engine")
 
+    lora_modules = []
+    for item in SETTINGS.lora_modules.strip().split("+"):
+        if "=" in item:
+            name, path = item.split("=")
+            lora_modules.append(LoRA(name, path))
+
     return VllmEngine(
         engine,
-        tokenizer,
         SETTINGS.model_name,
         SETTINGS.chat_template,
-        SETTINGS.context_length,
+        lora_modules=lora_modules,
     )
 
 

api/routes/model.py

@@ -6,6 +6,7 @@
 from pydantic import BaseModel
 
 from api.config import SETTINGS
+from api.models import GENERATE_ENGINE
 from api.utils.request import check_api_key
 
 model_router = APIRouter()
@@ -30,7 +31,7 @@ class ModelList(BaseModel):
 
 @model_router.get("/models", dependencies=[Depends(check_api_key)])
 async def show_available_models():
-    return available_models
+    return await GENERATE_ENGINE.show_available_models() if SETTINGS.engine == "vllm" else available_models
 
 
 @model_router.get("/models/{model}", dependencies=[Depends(check_api_key)])

api/utils/protocol.py

@@ -216,6 +216,16 @@ class ChatCompletionCreateParams(BaseModel):
 
     min_p: Optional[float] = 0.0
 
+    include_stop_str_in_output: Optional[bool] = False
+
+    length_penalty: Optional[float] = 1.0
+
+    guided_json: Optional[Union[str, dict, BaseModel]] = None
+
+    guided_regex: Optional[str] = None
+
+    guided_choice: Optional[List[str]] = None
+
 
 class CompletionCreateParams(BaseModel):
     model: str
@@ -396,6 +406,16 @@ class CompletionCreateParams(BaseModel):
 
     min_p: Optional[float] = 0.0
 
+    include_stop_str_in_output: Optional[bool] = False
+
+    length_penalty: Optional[float] = 1.0
+
+    guided_json: Optional[Union[str, dict, BaseModel]] = None
+
+    guided_regex: Optional[str] = None
+
+    guided_choice: Optional[List[str]] = None
+
 
 class EmbeddingCreateParams(BaseModel):
     input: Union[str, List[str], List[int], List[List[int]]]