[Bugfix] Include encoder prompts len to non-stream api usage response (…

…vllm-project#8861) Signed-off-by: Amit Garg <mitgarg17495@gmail.com>
garg-amit · Oct 28, 2024 · 188dfb9 · 188dfb9
1 parent 1042b88
commit 188dfb9
Showing 1 changed file with 2 additions and 0 deletions.
diff --git a/vllm/entrypoints/openai/serving_chat.py b/vllm/entrypoints/openai/serving_chat.py
@@ -726,6 +726,8 @@ async def chat_completion_full_generator(
 
         assert final_res.prompt_token_ids is not None
         num_prompt_tokens = len(final_res.prompt_token_ids)
+        if final_res.encoder_prompt_token_ids is not None:
+            num_prompt_tokens += len(final_res.encoder_prompt_token_ids)
         num_generated_tokens = sum(
             len(output.token_ids) for output in final_res.outputs)
         usage = UsageInfo(