match openai's json serializable behavior for very unlikely token log…

…probs (-9999.0)
team-telnyx · Apr 17, 2024 · e3cf0fa · e3cf0fa
1 parent fe3b5bb
commit e3cf0fa
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/vllm/entrypoints/openai/serving_engine.py b/vllm/entrypoints/openai/serving_engine.py
@@ -115,7 +115,7 @@ def _create_logprobs(
 
                 if num_output_top_logprobs:
                     logprobs.top_logprobs.append({
-                        p.decoded_token: p.logprob
+                        p.decoded_token: max(p.logprob, -9999.0)
                         for i, p in step_top_logprobs.items()
                     } if step_top_logprobs else None)