[Bugfix] multi_modal_kwargs broadcast for CPU tensor parallel (vllm…

…-project#10541) Signed-off-by: Isotr0py <2037008807@qq.com> Signed-off-by: Maxime Fournioux <55544262+mfournioux@users.noreply.github.com>
mfournioux · Nov 28, 2024 · cbf06f7 · cbf06f7
1 parent a6f54da
commit cbf06f7
Show file tree

Hide file tree

Showing 2 changed files with 2 additions and 0 deletions.
diff --git a/vllm/worker/cpu_enc_dec_model_runner.py b/vllm/worker/cpu_enc_dec_model_runner.py
@@ -35,6 +35,7 @@ def as_broadcastable_tensor_dict(self) -> Dict[str, Any]:
             "input_positions": self.input_positions,
             "encoder_input_tokens": self.encoder_input_tokens,
             "encoder_input_positions": self.encoder_input_positions,
+            "multi_modal_kwargs": self.multi_modal_kwargs,
         }
         _add_attn_metadata_broadcastable_dict(tensor_dict, self.attn_metadata)
         _add_sampling_metadata_broadcastable_dict(tensor_dict,

diff --git a/vllm/worker/cpu_model_runner.py b/vllm/worker/cpu_model_runner.py
@@ -83,6 +83,7 @@ def as_broadcastable_tensor_dict(self) -> Dict[str, Any]:
         tensor_dict = {
             "input_tokens": self.input_tokens,
             "input_positions": self.input_positions,
+            "multi_modal_kwargs": self.multi_modal_kwargs,
         }
         _add_attn_metadata_broadcastable_dict(tensor_dict, self.attn_metadata)
         _add_sampling_metadata_broadcastable_dict(tensor_dict,