rebase code

Signed-off-by: David Chen <530634352@qq.com>

Author: david6666666

docs/configuration/optimization.md

@@ -174,6 +174,7 @@ Regardless, you need to set `mm_encoder_tp_mode="data"` in engine arguments to u
 
 Known supported models:
 
+- GLM-4.5V GLM-4.1V (<gh-pr:23168>)
 - Kimi-VL (<gh-pr:23817>)
 - Llama4 (<gh-pr:18368>)
 - MiniCPM-V-2.5 or above (<gh-pr:23327>, <gh-pr:23948>)

vllm/model_executor/models/glm4_1v.py

@@ -804,8 +804,13 @@ def compute_attn_mask_seqlen(
     def forward(
         self,
         x: torch.Tensor,
-        grid_thw: torch.Tensor,
+        grid_thw: Union[torch.Tensor, list[list[int]]],
     ) -> torch.Tensor:
+        # Convert grid_thw to tensor if it's a list (for compatibility with 
+        # run_dp_sharded_mrope_vision_model)
+        if isinstance(grid_thw, list):
+            grid_thw = torch.tensor(grid_thw, device=x.device, dtype=torch.long)
+        
         # patchify
         x = x.to(device=self.device, dtype=self.dtype)
         x = self.patch_embed(x)
@@ -1467,10 +1472,11 @@ def _process_image_input(
                 # run_dp_sharded_mrope_vision_model already
                 # returns split embeddings
                 return run_dp_sharded_mrope_vision_model(
-                    self.visual, pixel_values, grid_thw)
+                    self.visual, pixel_values, grid_thw.tolist(),
+                    rope_type="rope_3d")
             else:
-                # Non-data parallel mode: self.visual expects tensor format
-                image_embeds = self.visual(pixel_values, grid_thw=grid_thw)
+                # Non-data parallel mode: pass list format for consistency
+                image_embeds = self.visual(pixel_values, grid_thw=grid_thw.tolist())
                 merge_size = self.visual.spatial_merge_size
                 sizes = grid_thw.prod(-1) // merge_size // merge_size
                 return image_embeds.split(sizes.tolist())
@@ -1493,11 +1499,12 @@ def _process_video_input(
                 # run_dp_sharded_mrope_vision_model already
                 # returns split embeddings
                 return run_dp_sharded_mrope_vision_model(
-                    self.visual, pixel_values_videos, grid_thw)
+                    self.visual, pixel_values_videos, grid_thw.tolist(),
+                    rope_type="rope_3d")
             else:
-                # Non-data parallel mode: self.visual expects tensor format
+                # Non-data parallel mode: pass list format for consistency
                 video_embeds = self.visual(pixel_values_videos,
-                                           grid_thw=grid_thw)
+                                           grid_thw=grid_thw.tolist())
                 # Split concatenated embeddings for each video item.
                 merge_size = self.visual.spatial_merge_size
                 sizes = grid_thw.prod(-1) // merge_size // merge_size