File tree Expand file tree Collapse file tree 1 file changed +2
-4
lines changed Expand file tree Collapse file tree 1 file changed +2
-4
lines changed Original file line number Diff line number Diff line change @@ -146,10 +146,9 @@ python3 vllm/benchmarks/benchmark_serving.py \
146146
147147``` bash
148148VLLM_USE_V1=1 vllm serve meta-llama/Meta-Llama-3-8B-Instruct \
149- --speculative-model " [ngram]" \
150149 --ngram_prompt_lookup_min 2 \
151150 --ngram-prompt-lookup-max 5 \
152- --num_speculative_tokens 5
151+ --speculative_config ' {"model": "[ngram]", " num_speculative_tokens": 5}
153152```
154153
155154``` bash
@@ -274,10 +273,9 @@ python3 vllm/benchmarks/benchmark_throughput.py \
274273 --output-len=100 \
275274 --num-prompts=2048 \
276275 --async-engine \
277- --speculative-model=" [ngram]" \
278276 --ngram_prompt_lookup_min=2 \
279277 --ngram-prompt-lookup-max=5 \
280- --num_speculative_tokens=5
278+ --speculative_config ' { " model " : " [ngram] " , " num_speculative_tokens" : 5}
281279```
282280
283281```
You can’t perform that action at this time.
0 commit comments