[Fix] Auto-detect XGrammar compiler threads based on CPU cores.

docs/source/features/structured_outputs.md

@@ -273,3 +273,22 @@ print(outputs[0].outputs[0].text)
 ```
 
 Full example: <gh-file:examples/offline_inference/structured_outputs.py>
+
+## Performance Tuning: XGrammar Compiler Threads
+
+When using the xgrammar backend for structured outputs, the number of threads
+used by the grammar compiler can affect the time to first token, especially for
+complex schemas.
+
+You can control the thread pool size for the xgrammar grammar compiler using the
+environment variable `VLLM_XGRAMMAR_COMPILER_THREADS`. By default, this is set
+to `8`. Increasing this value may help reduce first token latency in some
+environments, particularly when compiling large or complex schemas. For example:
+
+```bash
+export VLLM_XGRAMMAR_COMPILER_THREADS=16
+```
+
+Set this variable before starting vLLM. Note that increasing the thread count
+may increase CPU usage during grammar compilation, and may not always improve
+performance.

vllm/envs.py

@@ -114,6 +114,7 @@
     VLLM_ALLOW_INSECURE_SERIALIZATION: bool = False
     VLLM_NIXL_SIDE_CHANNEL_HOST: str = "localhost"
     VLLM_NIXL_SIDE_CHANNEL_PORT: int = 5557
+    VLLM_XGRAMMAR_COMPILER_THREADS: int = 8
 
 
 def get_default_cache_root():
@@ -757,6 +758,11 @@ def maybe_convert_int(value: Optional[str]) -> Optional[int]:
     # Port used for NIXL handshake between remote agents.
     "VLLM_NIXL_SIDE_CHANNEL_PORT":
     lambda: int(os.getenv("VLLM_NIXL_SIDE_CHANNEL_PORT", "5557")),
+
+    # Number of threads to use for the XGrammar grammar compiler.
+    # If not set, defaults to None and the backend will auto-detect.
+    "VLLM_XGRAMMAR_COMPILER_THREADS":
+    lambda: int(os.getenv("VLLM_XGRAMMAR_COMPILER_THREADS", "8")),
 }
 
 # end-env-vars-definition

vllm/v1/structured_output/backend_xgrammar.py

@@ -87,7 +87,7 @@ def __init__(self, vllm_config: VllmConfig):
             )
         self.compiler = xgr.GrammarCompiler(
             tokenizer_info,
-            max_threads=8,
+            max_threads=vllm.envs.VLLM_XGRAMMAR_COMPILER_THREADS,
             cache_enabled=True,
             cache_limit_bytes=vllm.envs.VLLM_XGRAMMAR_CACHE_MB * 1024 * 1024,
         )