diff --git a/vllm/model_executor/layers/pooler.py b/vllm/model_executor/layers/pooler.py
index f9f627b796320..1c9772b41cbef 100644
--- a/vllm/model_executor/layers/pooler.py
+++ b/vllm/model_executor/layers/pooler.py
@@ -56,7 +56,9 @@ def from_config_with_defaults(
         softmax: bool,
         step_tag_id: Optional[int] = None,
         returned_token_ids: Optional[List[int]] = None,
-    ) -> "Pooler":
+    ) -> Optional["Pooler"]:
+        if pooler_config is None:
+            return None
         return cls(
             pooling_type=PoolingType[pooler_config.pooling_type]
             if pooler_config.pooling_type is not None else pooling_type,