Revert "Fix after #16229, mm (#286)"

This reverts commit 18ead2d.

Author: xuechendi

tests/full_tests/ci_gsm8k_tests.sh

@@ -10,12 +10,14 @@ set -e
 VLLM_GAUDI_PREFIX=${VLLM_GAUDI_PREFIX:-"vllm-gaudi"}
 echo $VLLM_GAUDI_PREFIX
 
-# Gemma3 with image input
-run_gemma3_test() {
-    echo "➡️ Testing gemma-3-4b-it..."
-    VLLM_SKIP_WARMUP=true PT_HPU_LAZY_MODE=1 VLLM_USE_V1=1 python -u "${VLLM_GAUDI_PREFIX}/tests/models/language/generation/generation_mm.py" --model-card-path "${VLLM_GAUDI_PREFIX}/tests/full_tests/model_cards/gemma-3-4b-it.yaml"
-    echo "✅ Test with multimodal-support with gemma-3-4b-it passed."
-}
+# NOTE(Chendi): temporarily disable gemma3 test due to upstream change.
+# Expect fixing from https://github.com/vllm-project/vllm-gaudi/pull/286
+# # Gemma3 with image input
+# run_gemma3_test() {
+#     echo "➡️ Testing gemma-3-4b-it..."
+#     VLLM_SKIP_WARMUP=true PT_HPU_LAZY_MODE=1 VLLM_USE_V1=1 python -u "${VLLM_GAUDI_PREFIX}/tests/models/language/generation/generation_mm.py" --model-card-path "${VLLM_GAUDI_PREFIX}/tests/full_tests/model_cards/gemma-3-4b-it.yaml"
+#     echo "✅ Test with multimodal-support with gemma-3-4b-it passed."
+# }
 
 # Basic model test
 run_basic_model_test() {
@@ -182,13 +184,15 @@ run_gsm8k_qwen3_30b_test() {
     echo "✅ Test with QWEN3-30B-A3B passed."
 }
 
-# Multimodal-support with qwen2.5-vl
-run_qwen2_5_vl_test() {
-    echo "➡️ Testing Qwen2.5-VL-7B..."
-    VLLM_SKIP_WARMUP=true VLLM_CONTIGUOUS_PA=False PT_HPU_LAZY_MODE=1 VLLM_USE_V1=1 \
-    python -u "${VLLM_GAUDI_PREFIX}/tests/models/language/generation/generation_mm.py" --model-card-path "${VLLM_GAUDI_PREFIX}/tests/full_tests/model_cards/qwen2.5-vl-7b.yaml"
-    echo "✅ Test with multimodal-support with qwen2.5-vl-7b passed."
-}
+# NOTE(Chendi): Disabled due to upstream change #16229
+# Expect fixing from https://github.com/vllm-project/vllm-gaudi/pull/286
+# # Multimodal-support with qwen2.5-vl
+# run_qwen2_5_vl_test() {
+#     echo "➡️ Testing Qwen2.5-VL-7B..."
+#     VLLM_SKIP_WARMUP=true VLLM_CONTIGUOUS_PA=False PT_HPU_LAZY_MODE=1 VLLM_USE_V1=1 \
+#     python -u "${VLLM_GAUDI_PREFIX}/tests/models/language/generation/generation_mm.py" --model-card-path "${VLLM_GAUDI_PREFIX}/tests/full_tests/model_cards/qwen2.5-vl-7b.yaml"
+#     echo "✅ Test with multimodal-support with qwen2.5-vl-7b passed."
+# }
 
 # Spec decode with ngram
 run_spec_decode_ngram_test() {
@@ -213,7 +217,7 @@ run_embedding_model_test() {
 # Function to run all tests sequentially
 launch_all_tests() {
     echo "🚀 Starting all test suites..."
-    run_gemma3_test
+    # run_gemma3_test
     run_basic_model_test
     run_tp2_test
     run_mla_moe_test
@@ -233,7 +237,7 @@ launch_all_tests() {
     run_gsm8k_granite_async_test
     run_gsm8k_deepseek_test
     run_gsm8k_qwen3_30b_test
-    run_qwen2_5_vl_test
+    #run_qwen2_5_vl_test
     run_spec_decode_ngram_test
     #run_embedding_model_test
     echo "🎉 All test suites passed successfully!"

vllm_gaudi/v1/worker/hpu_model_runner.py

@@ -53,7 +53,6 @@
                              AsyncModelRunnerOutput, KVConnectorOutput)
 from vllm.v1.sample.metadata import SamplingMetadata
 from vllm.v1.worker.utils import bind_kv_cache
-from vllm.v1.utils import CpuGpuBuffer
 from vllm_gaudi.v1.worker.hpu_input_batch import InputBatch, CachedRequestState
 from vllm.distributed.parallel_state import get_pp_group
 from vllm.model_executor.models.interfaces import (SupportsMultiModal, supports_eagle3, supports_transcription)
@@ -543,10 +542,8 @@ def forward(self, *args, **kwargs):
                 self._reset_rotary_cos_sin()
         return hidden_states
 
-    def get_input_embeddings(self, input_ids, multimodal_embeddings=None, is_multimodal=False):
-        return self.model.get_input_embeddings(input_ids=input_ids,
-                                               multimodal_embeddings=multimodal_embeddings,
-                                               is_multimodal=is_multimodal)
+    def get_input_embeddings(self, input_ids, multimodal_embeddings=None):
+        return self.model.get_input_embeddings(input_ids=input_ids, multimodal_embeddings=multimodal_embeddings)
 
     def get_multimodal_embeddings(self, **batched_mm_inputs):
         return self.model.get_multimodal_embeddings(**batched_mm_inputs)
@@ -741,8 +738,6 @@ def __init__(
         self.mm_registry = MULTIMODAL_REGISTRY
         self.uses_mrope = model_config.uses_mrope
         self.supports_mm_inputs = self.mm_registry.supports_multimodal_inputs(model_config)
-        if self.supports_mm_inputs:
-            self.is_mm_embed = self._make_buffer(self.max_num_tokens, dtype=torch.bool)
         self.is_multimodal_raw_input_supported = (model_config.is_multimodal_raw_input_only_model)
 
         # Lazy initialization
@@ -853,9 +848,6 @@ def __init__(
         assert not (self.unified_attn and not self.use_contiguous_pa), 'Unified attn requires contiguous_pa!'
         assert not (self.unified_attn and not self.use_merged_prefill), 'Unified attn requires merged_prefill!'
 
-    def _make_buffer(self, *size: Union[int, torch.SymInt], dtype: torch.dtype, numpy: bool = True) -> CpuGpuBuffer:
-        return CpuGpuBuffer(*size, dtype=dtype, device=self.device, pin_memory=self.pin_memory, with_numpy=numpy)
-
     def unified_bucketing_fn(self, is_causal, query_len, shared_blocks, unique_blocks, logits):
         if not get_config().use_bucketing:
             return query_len, shared_blocks, unique_blocks, logits
@@ -1319,15 +1311,8 @@ def _gather_mm_embeddings(
         scheduler_output: "SchedulerOutput",
         req_ids: list[str],
         shift_computed_tokens: int = 0,
-        total_num_scheduled_tokens: Optional[int] = None,
-    ) -> tuple[list[torch.Tensor], torch.Tensor]:
-        total_num_scheduled_tokens = total_num_scheduled_tokens or scheduler_output.total_num_scheduled_tokens
-
-        mm_embeds = list[torch.Tensor]()
-        is_mm_embed = self.is_mm_embed.cpu
-        is_mm_embed[:total_num_scheduled_tokens] = False
-
-        req_start_idx = 0
+    ) -> list[torch.Tensor]:
+        mm_embeds: list[torch.Tensor] = []
         for req_id in req_ids:
             num_scheduled_tokens = scheduler_output.num_scheduled_tokens[req_id]
             req_state = self.requests[req_id]
@@ -1366,17 +1351,8 @@ def _gather_mm_embeddings(
                     encoder_output[start_idx:end_idx],
                     is_embed=is_embed,
                 )
-                req_start_pos = req_start_idx + start_pos - num_computed_tokens
-                is_mm_embed[req_start_pos+start_idx:req_start_pos + end_idx] \
-                    = True
-
-                # Only whole mm items are processed
                 mm_embeds.append(mm_embeds_item)
-            req_start_idx += num_scheduled_tokens
-
-        is_mm_embed = self.is_mm_embed.copy_to_gpu(total_num_scheduled_tokens)
-
-        return mm_embeds, is_mm_embed
+        return mm_embeds
 
     def get_model(self) -> torch.nn.Module:
         assert self.model is not None
@@ -2963,16 +2939,13 @@ def execute_model(
                     with self.profiler.record_event('internal', 'prepare_input_encoders'):
                         self._execute_mm_encoder(scheduler_output, req_id)
 
-                    mm_embeds, is_mm_embed = self._gather_mm_embeddings(scheduler_output,
-                                                                        req_id,
-                                                                        total_num_scheduled_tokens=token_ids.shape[-1])
+                    mm_embeds = self._gather_mm_embeddings(scheduler_output, req_id)
                     # TODO: Only get embeddings for valid token_ids. Ignore token_ids[<pad_idxs>] # noqa E501
                     # This may require moving multimodal input preps into _prepare_inputs,        # noqa E501
                     # to avoid padding issues.
                     inputs_embeds = self.model.get_input_embeddings(
-                        token_ids,
-                        multimodal_embeddings=mm_embeds,
-                        is_multimodal=is_mm_embed,
+                        input_ids=token_ids,
+                        multimodal_embeddings=mm_embeds or None,
                     )
 
                     model_mm_kwargs = self._extract_mm_kwargs(scheduler_output)