honor --mm_encoder_attn_backend when used (#27124)

Summary:
Pull Request resolved: #27124

In #26104, some changes were made in layer.py that resulted in always trying to switch to FA backend for ViT, even when  `VLLM_ATTENTION_BACKEND` is set.

This broke Meta's internal AMD pipelines as it is not desired nor expected behavior. With this change, the models that were changed in the offending PR can explicitly opt-in to this behavior.

Reviewed By: Prowindy

Differential Revision: D84946967

Author: bradleyhd

vllm/attention/layer.py

@@ -3,7 +3,7 @@
 """Attention layer."""
 
 from collections.abc import Callable
-from typing import cast
+from typing import cast, Optional
 
 import torch
 import torch.nn as nn
@@ -93,12 +93,13 @@ def check_upstream_fa_availability(dtype: torch.dtype):
 
 
 def maybe_get_vit_flash_attn_backend(
-    attn_backend: _Backend, use_upstream_fa: bool
+    attn_backend: _Backend, use_upstream_fa: bool, attn_backend_override: Optional[_Backend] = None
 ) -> tuple[_Backend, Callable]:
     if (
         attn_backend != _Backend.FLASH_ATTN
         and attn_backend != _Backend.ROCM_AITER_FA
         and check_upstream_fa_availability(torch.get_default_dtype())
+        and attn_backend_override is None
     ):
         attn_backend = _Backend.FLASH_ATTN
         use_upstream_fa = True
@@ -499,6 +500,7 @@ def __init__(
             maybe_get_vit_flash_attn_backend(
                 self.attn_backend,
                 use_upstream_fa,
+                attn_backend_override=attn_backend_override,
             )
         )
 

vllm/model_executor/models/dots_ocr.py

@@ -299,6 +299,7 @@ def __init__(
             maybe_get_vit_flash_attn_backend(
                 self.attn_backend,
                 self.use_upstream_fa,
+                attn_backend_override=attn_backend_override,
             )
         )
         if self.attn_backend not in {

vllm/model_executor/models/ernie45_vl.py

@@ -206,6 +206,7 @@ def __init__(
             maybe_get_vit_flash_attn_backend(
                 self.attn_backend,
                 self.use_upstream_fa,
+                attn_backend_override=attn_backend_override,
             )
         )
 

vllm/model_executor/models/glm4_1v.py

@@ -296,6 +296,7 @@ def __init__(
             maybe_get_vit_flash_attn_backend(
                 self.attn_backend,
                 self.use_upstream_fa,
+                attn_backend_override=attn_backend_override,
             )
         )
 

vllm/model_executor/models/qwen2_vl.py

@@ -364,6 +364,7 @@ def __init__(
             maybe_get_vit_flash_attn_backend(
                 self.attn_backend,
                 self.use_upstream_fa,
+                attn_backend_override=attn_backend_override,
             )
         )
 

vllm/model_executor/models/siglip2navit.py

@@ -259,6 +259,7 @@ def __init__(
             maybe_get_vit_flash_attn_backend(
                 self.attn_backend,
                 self.use_upstream_fa,
+                attn_backend_override=attn_backend_override,
             )
         )