[Bugfix] Include encoder prompts len to non-stream api usage response (…

…vllm-project#8861) Signed-off-by: Sumit Dubey <sumit.dubey2@ibm.com>
sumitd2 · Nov 14, 2024 · 79ad521 · 79ad521
1 parent 8abaf98
commit 79ad521
Showing 1 changed file with 2 additions and 0 deletions.
diff --git a/vllm/entrypoints/openai/serving_chat.py b/vllm/entrypoints/openai/serving_chat.py
@@ -726,6 +726,8 @@ async def chat_completion_full_generator(
 
         assert final_res.prompt_token_ids is not None
         num_prompt_tokens = len(final_res.prompt_token_ids)
+        if final_res.encoder_prompt_token_ids is not None:
+            num_prompt_tokens += len(final_res.encoder_prompt_token_ids)
         num_generated_tokens = sum(
             len(output.token_ids) for output in final_res.outputs)
         usage = UsageInfo(