[Model][Bugfix] Implicit model flags and reenable Phi-3-Vision (vllm-project#5896)

Author: DarkLight1337

vllm/model_executor/models/baichuan.py

@@ -295,8 +295,6 @@ def forward(
 
 
 class BaiChuanBaseForCausalLM(nn.Module, SupportsLoRA):
-    supports_lora = True
-
     packed_modules_mapping = {
         "W_pack": ["W_pack"],
         "gate_up_proj": [

vllm/model_executor/models/chatglm.py

@@ -325,8 +325,6 @@ def forward(
 
 
 class ChatGLMForCausalLM(nn.Module, SupportsLoRA):
-    supports_lora = True
-
     packed_modules_mapping = {
         "query_key_value": ["query_key_value"],
         "dense_h_to_4h": ["dense_h_to_4h"]

vllm/model_executor/models/gemma.py

@@ -291,8 +291,6 @@ def forward(
 
 
 class GemmaForCausalLM(nn.Module, SupportsLoRA):
-    supports_lora = True
-
     packed_modules_mapping = {
         "qkv_proj": [
             "q_proj",

vllm/model_executor/models/gpt_bigcode.py

@@ -233,8 +233,6 @@ def forward(
 
 
 class GPTBigCodeForCausalLM(nn.Module, SupportsLoRA):
-    supports_lora = True
-
     packed_modules_mapping = {"c_attn": ["c_attn"]}
 
     supported_lora_modules = ["c_fc", "c_proj", "wte", "lm_head", "c_attn"]

vllm/model_executor/models/interfaces.py

@@ -13,7 +13,14 @@
 class SupportsVision(Protocol):
     """The interface required for all vision language models (VLMs)."""
 
-    supports_vision: ClassVar[Literal[True]]
+    supports_vision: ClassVar[Literal[True]] = True
+    """
+    A flag that indicates this model supports vision inputs.
+
+    Note:
+        There is no need to redefine this flag if this class is in the
+        MRO of your model class.
+    """
 
     def __init__(self, *, vlm_config: VisionLanguageConfig) -> None:
         ...
@@ -52,7 +59,14 @@ def supports_vision(
 class SupportsLoRA(Protocol):
     """The interface required for all models that support LoRA."""
 
-    supports_lora: ClassVar[Literal[True]]
+    supports_lora: ClassVar[Literal[True]] = True
+    """
+    A flag that indicates this model supports LoRA.
+
+    Note:
+        There is no need to redefine this flag if this class is in the
+        MRO of your model class.
+    """
 
     packed_modules_mapping: ClassVar[Dict[str, List[str]]]
     supported_lora_modules: ClassVar[List[str]]

vllm/model_executor/models/llama.py

@@ -299,8 +299,6 @@ def forward(
 
 
 class LlamaForCausalLM(nn.Module, SupportsLoRA):
-    supports_lora = True
-
     packed_modules_mapping = {
         "qkv_proj": [
             "q_proj",

vllm/model_executor/models/llava.py

@@ -88,8 +88,6 @@ class LlavaImageFeatureInputs(TypedDict):
 @MULTIMODAL_REGISTRY.register_dummy_data(get_dummy_image_data)
 class LlavaForConditionalGeneration(nn.Module, SupportsVision):
 
-    supports_vision = True
-
     def __init__(self,
                  config: LlavaConfig,
                  vlm_config: VisionLanguageConfig,

vllm/model_executor/models/llava_next.py

@@ -108,8 +108,6 @@ def _image_pixel_processor(
 @MULTIMODAL_REGISTRY.register_dummy_data(_get_dummy_image_data)
 class LlavaNextForConditionalGeneration(nn.Module, SupportsVision):
 
-    supports_vision = True
-
     def __init__(self,
                  config: LlavaNextConfig,
                  vlm_config: VisionLanguageConfig,

vllm/model_executor/models/minicpm.py

@@ -392,8 +392,6 @@ def forward(
 
 
 class MiniCPMForCausalLM(nn.Module, SupportsLoRA):
-    supports_lora = True
-
     packed_modules_mapping = {
         "qkv_proj": [
             "q_proj",

vllm/model_executor/models/mixtral.py

@@ -475,8 +475,6 @@ def forward(
 
 
 class MixtralForCausalLM(nn.Module, SupportsLoRA):
-    supports_lora = True
-
     fall_back_to_pt_during_load = False
 
     packed_modules_mapping = {