diff --git a/tests/models/decoder_only/vision_language/test_internvl.py b/tests/models/decoder_only/vision_language/test_internvl.py
index a756f8214edee..49cab75d8ea53 100644
--- a/tests/models/decoder_only/vision_language/test_internvl.py
+++ b/tests/models/decoder_only/vision_language/test_internvl.py
@@ -97,7 +97,8 @@ def __init__(self, hf_runner: HfRunner):
             self.tokenizer = hf_runner.tokenizer
             self.dtype = hf_runner.model.dtype
 
-            self.config = AutoConfig.from_pretrained(hf_runner.model_name)
+            self.config = AutoConfig.from_pretrained(hf_runner.model_name,
+                                                     trust_remote_code=True)
             self.vision_config = self.config.vision_config
             self.use_thumbnail = self.config.use_thumbnail
             self.min_num = self.config.min_dynamic_patch
diff --git a/tests/models/encoder_decoder/vision_language/test_mllama.py b/tests/models/encoder_decoder/vision_language/test_mllama.py
index 254185537e403..78a5c8158e16e 100644
--- a/tests/models/encoder_decoder/vision_language/test_mllama.py
+++ b/tests/models/encoder_decoder/vision_language/test_mllama.py
@@ -195,11 +195,6 @@ def _run_test(
     def process(hf_inputs: BatchEncoding):
         return hf_inputs
 
-    from transformers.models.mllama import MllamaConfig as MllamaConfigHf
-
-    # use transformer's MllamaConfig for hf_runner
-    # and vllm's MllamaConfig for vllm_runner
-    AutoConfig.register("mllama", MllamaConfigHf, exist_ok=True)
     with hf_runner(model,
                    dtype=dtype,
                    model_kwargs={"device_map": "auto"},
@@ -213,8 +208,6 @@ def process(hf_inputs: BatchEncoding):
             for prompts, images in inputs
         ]
 
-    from vllm.transformers_utils.configs.mllama import MllamaConfig
-    AutoConfig.register("mllama", MllamaConfig, exist_ok=True)
     for hf_outputs, vllm_outputs in zip(hf_outputs_per_image,
                                         vllm_outputs_per_image):
         check_logprobs_close(
diff --git a/vllm/transformers_utils/config.py b/vllm/transformers_utils/config.py
index 0f20e8d0c8213..bfba4ca77e1fe 100644
--- a/vllm/transformers_utils/config.py
+++ b/vllm/transformers_utils/config.py
@@ -1,4 +1,3 @@
-import contextlib
 import enum
 import json
 from pathlib import Path
@@ -61,13 +60,6 @@
     **_CONFIG_REGISTRY_OVERRIDE_HF
 }
 
-for name, cls in _CONFIG_REGISTRY.items():
-    with contextlib.suppress(ValueError):
-        if name in _CONFIG_REGISTRY_OVERRIDE_HF:
-            AutoConfig.register(name, cls, exist_ok=True)
-        else:
-            AutoConfig.register(name, cls)
-
 
 class ConfigFormat(str, enum.Enum):
     AUTO = "auto"