Merge pull request #140 from OpenPipe/potential_fix

Fix training stability issues with new vLLM version

Author: saum7800

pyproject.toml

@@ -13,7 +13,7 @@ dependencies = [
     "torchao>=0.9.0",
     "unsloth==2025.5.1 ; sys_platform == 'linux'",
     "unsloth-zoo==2025.5.1 ; sys_platform == 'linux'",
-    "vllm==0.7.3",
+    "vllm>=0.8.5",
     "wandb>=0.19.8",
     "peft>=0.14.0",
     "typer>=0.15.2",

src/art/dev/model.py

@@ -43,6 +43,7 @@ def get_model_config(
         # which is the fallback for devices with compute capability < 8.0
         num_scheduler_steps=16 if torch.cuda.get_device_capability()[0] >= 8 else 1,
         enable_sleep_mode=enable_sleep_mode,
+        generation_config="vllm",
     )
     engine_args.update(config.get("engine_args", {}))
     init_args.update(config.get("init_args", {}))

src/art/dev/openai_server.py

@@ -27,6 +27,7 @@ def get_openai_server_config(
         num_scheduler_steps=16,
         served_model_name=base_model,
         disable_log_requests=True,
+        generation_config="vllm",
     )
     engine_args.update(config.get("engine_args", {}))
     return OpenAIServerConfig(

src/art/local/vllm.py

@@ -265,7 +265,7 @@ def patch_get_lora_tokenizer_async() -> None:
     Specifically, Unsloth patches get_lora_tokenizer_async with a non-async function, which causes issues.
     """
     import vllm.transformers_utils.tokenizer
-    import vllm.transformers_utils.tokenizer_group.tokenizer_group
+    import vllm.transformers_utils.tokenizer_group
 
     async def _return_nothing(*_, **__) -> None:
         return None
@@ -274,10 +274,10 @@ async def get_self_lora_tokenizer_async(self, *args, **kwargs):
         return self.tokenizer
 
     vllm.transformers_utils.tokenizer.get_lora_tokenizer_async = _return_nothing  # type: ignore
-    vllm.transformers_utils.tokenizer_group.tokenizer_group.get_lora_tokenizer_async = (
+    vllm.transformers_utils.tokenizer_group.get_lora_tokenizer_async = (
         _return_nothing  # type: ignore
     )
-    vllm.transformers_utils.tokenizer_group.tokenizer_group.TokenizerGroup.get_lora_tokenizer_async = get_self_lora_tokenizer_async  # type: ignore
+    vllm.transformers_utils.tokenizer_group.TokenizerGroup.get_lora_tokenizer_async = get_self_lora_tokenizer_async  # type: ignore
 
 
 def patch_listen_for_disconnect() -> None: