[Bugfix] Fix using -O[0,3] with LLM entrypoint (vllm-project#10677)

Signed-off-by: mgoin <michael@neuralmagic.com>
neuralmagic · Nov 26, 2024 · 9a99273 · 9a99273
1 parent f5792c7
commit 9a99273
Show file tree

Hide file tree

Showing 2 changed files with 12 additions and 3 deletions.
diff --git a/vllm/engine/arg_utils.py b/vllm/engine/arg_utils.py
@@ -206,7 +206,10 @@ def __post_init__(self):
         # support `EngineArgs(compilation_config={...})`
         # without having to manually construct a
         # CompilationConfig object
-        if isinstance(self.compilation_config, (int, dict)):
+        if isinstance(self.compilation_config, (int)):
+            self.compilation_config = CompilationConfig.from_cli(
+                str(self.compilation_config))
+        elif isinstance(self.compilation_config, (dict)):
             self.compilation_config = CompilationConfig.from_cli(
                 json.dumps(self.compilation_config))
 

diff --git a/vllm/entrypoints/llm.py b/vllm/entrypoints/llm.py
@@ -185,8 +185,14 @@ def __init__(
             kwargs["disable_log_stats"] = True
 
         if compilation_config is not None:
-            compilation_config_instance = CompilationConfig.from_cli(
-                json.dumps(compilation_config))
+            if isinstance(compilation_config, (int)):
+                compilation_config_instance = CompilationConfig.from_cli(
+                    str(compilation_config))
+            elif isinstance(compilation_config, (dict)):
+                compilation_config_instance = CompilationConfig.from_cli(
+                    json.dumps(compilation_config))
+            else:
+                compilation_config_instance = compilation_config
         else:
             compilation_config_instance = None