Move DummyImageDataFactories into CLIP model file

vllm/model_executor/models/clip.py

@@ -4,19 +4,80 @@
 
 import torch
 import torch.nn as nn
+from PIL import Image
 from transformers import CLIPVisionConfig
 from transformers.models.clip.modeling_clip import CLIPAttention
 
 from vllm.model_executor.layers.activation import get_act_fn
 from vllm.model_executor.layers.linear import (ColumnParallelLinear,
                                                RowParallelLinear)
-from vllm.model_executor.layers.quantization.base_config import (
-    QuantizationConfig)
+from vllm.model_executor.layers.quantization import QuantizationConfig
+from vllm.multimodal.image import ImageFeatureData, ImagePixelData
+from vllm.sequence import SequenceData
 
 
-def get_clip_num_patches(*, image_size: int, patch_size: int) -> int:
+def get_clip_patch_grid_length(*, image_size: int, patch_size: int) -> int:
     assert image_size % patch_size == 0
-    return (image_size // patch_size)**2
+    return image_size // patch_size
+
+
+def get_clip_num_patches(*, image_size: int, patch_size: int) -> int:
+    grid_length = get_clip_patch_grid_length(image_size=image_size,
+                                             patch_size=patch_size)
+    return grid_length * grid_length
+
+
+def get_clip_image_feature_size(hf_config: CLIPVisionConfig) -> int:
+    return get_clip_num_patches(image_size=hf_config.image_size,
+                                patch_size=hf_config.patch_size)
+
+
+def dummy_seq_data_for_clip(
+    hf_config: CLIPVisionConfig,
+    seq_len: int,
+    *,
+    image_token_id: int,
+    image_feature_size_override: Optional[int] = None,
+):
+    if image_feature_size_override is None:
+        image_feature_size = get_clip_image_feature_size(hf_config)
+    else:
+        image_feature_size = image_feature_size_override
+
+    token_ids = [image_token_id] * image_feature_size
+    token_ids += [0] * (seq_len - image_feature_size)
+    return SequenceData(token_ids)
+
+
+def dummy_pixel_data_for_clip(
+    hf_config: CLIPVisionConfig,
+    *,
+    image_width_override: Optional[int] = None,
+    image_height_override: Optional[int] = None,
+):
+    width = height = hf_config.image_size
+    if image_width_override is not None:
+        width = image_width_override
+    if image_height_override is not None:
+        height = image_height_override
+
+    image = Image.new("RGB", (width, height), color=0)
+    return ImagePixelData(image)
+
+
+def dummy_feature_data_for_clip(
+    hf_config: CLIPVisionConfig,
+    *,
+    image_feature_size_override: Optional[int] = None,
+):
+    if image_feature_size_override is None:
+        image_feature_size = get_clip_image_feature_size(hf_config)
+    else:
+        image_feature_size = image_feature_size_override
+
+    values = torch.zeros((1, image_feature_size, hf_config.hidden_size),
+                         dtype=torch.float16)
+    return ImageFeatureData(values)
 
 
 # Adapted from https://github.com/huggingface/transformers/blob/v4.39.0/src/transformers/models/clip/modeling_clip.py#L164 # noqa

vllm/model_executor/models/llava.py

@@ -18,9 +18,10 @@
 from vllm.model_executor.models.llama import LlamaModel
 from vllm.model_executor.sampling_metadata import SamplingMetadata
 from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalData
-from vllm.multimodal.image import DummyImageDataFactories
 from vllm.sequence import SamplerOutput
 
+from .clip import (dummy_feature_data_for_clip, dummy_pixel_data_for_clip,
+                   dummy_seq_data_for_clip)
 from .vlm_base import VisionLanguageModelBase
 
 _KEYS_TO_MODIFY_MAPPING = {
@@ -90,7 +91,7 @@ def dummy_data_for_llava(ctx: InputContext, seq_len: int):
     vision_config = hf_config.vision_config
 
     if isinstance(vision_config, CLIPVisionConfig):
-        seq_data = DummyImageDataFactories.dummy_seq_data_for_clip(
+        seq_data = dummy_seq_data_for_clip(
             vision_config,
             seq_len,
             image_token_id=hf_config.image_token_index,
@@ -100,11 +101,9 @@ def dummy_data_for_llava(ctx: InputContext, seq_len: int):
         ImageInputType = VisionLanguageConfig.ImageInputType
         mm_data: MultiModalData
         if image_input_type == ImageInputType.PIXEL_VALUES:
-            mm_data = DummyImageDataFactories.dummy_pixel_data_for_clip(
-                vision_config)
+            mm_data = dummy_pixel_data_for_clip(vision_config)
         elif image_input_type == ImageInputType.IMAGE_FEATURES:
-            mm_data = DummyImageDataFactories.dummy_feature_data_for_clip(
-                vision_config)
+            mm_data = dummy_feature_data_for_clip(vision_config)
 
         return seq_data, mm_data
 

vllm/model_executor/models/llava_next.py

@@ -22,10 +22,11 @@
 from vllm.model_executor.models.llama import LlamaModel
 from vllm.model_executor.sampling_metadata import SamplingMetadata
 from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalData
-from vllm.multimodal.image import (DummyImageDataFactories, ImagePixelData,
-                                   get_clip_num_patches)
+from vllm.multimodal.image import ImagePixelData
 from vllm.sequence import SamplerOutput
 
+from .clip import (dummy_feature_data_for_clip, dummy_pixel_data_for_clip,
+                   dummy_seq_data_for_clip, get_clip_patch_grid_length)
 from .llava import LlavaMultiModalProjector, merge_vision_embeddings
 from .vlm_base import VisionLanguageModelBase
 
@@ -93,7 +94,10 @@ def _get_llava_next_image_feature_size(
     vision_config = hf_config.vision_config
 
     if isinstance(vision_config, CLIPVisionConfig):
-        num_patches = get_clip_num_patches(vision_config)
+        num_patches = get_clip_patch_grid_length(
+            image_size=vision_config.image_size,
+            patch_size=vision_config.patch_size,
+        )
         base_feature_size = num_patches * num_patches
 
         num_patch_height, num_patch_width = get_anyres_image_grid_shape(
@@ -127,7 +131,7 @@ def dummy_data_for_llava_next(ctx: InputContext, seq_len: int):
         hf_config, input_height=dummy_height, input_width=dummy_width)
 
     if isinstance(vision_config, CLIPVisionConfig):
-        seq_data = DummyImageDataFactories.dummy_seq_data_for_clip(
+        seq_data = dummy_seq_data_for_clip(
             vision_config,
             seq_len,
             image_token_id=hf_config.image_token_index,
@@ -138,13 +142,13 @@ def dummy_data_for_llava_next(ctx: InputContext, seq_len: int):
         ImageInputType = VisionLanguageConfig.ImageInputType
         mm_data: MultiModalData
         if image_input_type == ImageInputType.PIXEL_VALUES:
-            mm_data = DummyImageDataFactories.dummy_pixel_data_for_clip(
+            mm_data = dummy_pixel_data_for_clip(
                 vision_config,
                 image_width_override=dummy_width,
                 image_height_override=dummy_height,
             )
         elif image_input_type == ImageInputType.IMAGE_FEATURES:
-            mm_data = DummyImageDataFactories.dummy_feature_data_for_clip(
+            mm_data = dummy_feature_data_for_clip(
                 vision_config,
                 image_feature_size_override=image_feature_size,
             )

vllm/model_executor/models/phi3v.py

@@ -36,9 +36,11 @@
 from vllm.model_executor.models.vlm_base import VisionLanguageModelBase
 from vllm.model_executor.sampling_metadata import SamplingMetadata
 from vllm.multimodal import MULTIMODAL_REGISTRY
-from vllm.multimodal.image import DummyImageDataFactories, ImagePixelData
+from vllm.multimodal.image import ImagePixelData
 from vllm.sequence import SamplerOutput
 
+from .clip import dummy_pixel_data_for_clip, dummy_seq_data_for_clip
+
 logger = init_logger(__name__)
 
 _KEYS_TO_MODIFY_MAPPING = {
@@ -275,13 +277,13 @@ class Phi3VImagePixelInputs(TypedDict):
 
 
 def dummy_data_for_phi3v(ctx: InputContext, seq_len: int):
-    seq_data = DummyImageDataFactories.dummy_seq_data_for_clip(
+    seq_data = dummy_seq_data_for_clip(
         CLIP_VIT_LARGE_PATCH14_336_CONFIG,
         seq_len,
         image_token_id=32044,
         image_feature_size_override=1921,
     )
-    mm_data = DummyImageDataFactories.dummy_pixel_data_for_clip(
+    mm_data = dummy_pixel_data_for_clip(
         CLIP_VIT_LARGE_PATCH14_336_CONFIG,
         image_width_override=1344,
         image_height_override=1008,

vllm/multimodal/image.py

@@ -1,87 +1,19 @@
 from functools import lru_cache
-from typing import Dict, Optional, Type, Union
+from typing import Dict, Type, Union
 
 import torch
 from PIL import Image
-from transformers import CLIPVisionConfig
 
 from vllm.config import ModelConfig
 from vllm.inputs.registry import InputContext
 from vllm.logger import init_logger
-from vllm.model_executor.models.clip import get_clip_num_patches
-from vllm.sequence import SequenceData
 from vllm.transformers_utils.image_processor import get_image_processor
 
 from .base import MultiModalData, MultiModalPlugin
 
 logger = init_logger(__name__)
 
-_cached_get_image_processor = lru_cache(get_image_processor)
-
-
-def get_clip_image_feature_size(hf_config: CLIPVisionConfig) -> int:
-    return get_clip_num_patches(image_size=hf_config.image_size,
-                                patch_size=hf_config.patch_size)
-
-
-class DummyImageDataFactories:
-    """
-    Contains factories for dummy image data factories.
-
-    See Also:
-        :data:`vllm.inputs.registry.DummyDataFactory`
-    """
-
-    @classmethod
-    def dummy_seq_data_for_clip(
-        cls,
-        hf_config: CLIPVisionConfig,
-        seq_len: int,
-        *,
-        image_token_id: int,
-        image_feature_size_override: Optional[int] = None,
-    ):
-        if image_feature_size_override is None:
-            image_feature_size = get_clip_image_feature_size(hf_config)
-        else:
-            image_feature_size = image_feature_size_override
-
-        token_ids = [image_token_id] * image_feature_size
-        token_ids += [0] * (seq_len - image_feature_size)
-        return SequenceData(token_ids)
-
-    @classmethod
-    def dummy_pixel_data_for_clip(
-        cls,
-        hf_config: CLIPVisionConfig,
-        *,
-        image_width_override: Optional[int] = None,
-        image_height_override: Optional[int] = None,
-    ):
-        width = height = hf_config.image_size
-        if image_width_override is not None:
-            width = image_width_override
-        if image_height_override is not None:
-            height = image_height_override
-
-        image = Image.new("RGB", (width, height), color=0)
-        return ImagePixelData(image)
-
-    @classmethod
-    def dummy_feature_data_for_clip(
-        cls,
-        hf_config: CLIPVisionConfig,
-        *,
-        image_feature_size_override: Optional[int] = None,
-    ):
-        if image_feature_size_override is None:
-            image_feature_size = get_clip_image_feature_size(hf_config)
-        else:
-            image_feature_size = image_feature_size_override
-
-        values = torch.zeros((1, image_feature_size, hf_config.hidden_size),
-                             dtype=torch.float16)
-        return ImageFeatureData(values)
+cached_get_image_processor = lru_cache(get_image_processor)
 
 
 class ImagePixelData(MultiModalData):
@@ -120,7 +52,7 @@ def _get_hf_image_processor(self, model_config: ModelConfig):
         if vlm_config is None or vlm_config.image_processor is None:
             return None
 
-        return _cached_get_image_processor(
+        return cached_get_image_processor(
             vlm_config.image_processor,
             trust_remote_code=model_config.trust_remote_code,
             revision=vlm_config.image_processor_revision,