Skip to content

Commit 34a7db2

Browse files
authored
Added native support for Gemini models generation completion signals in LangchainLLMWrapper class / is_finished method (#1727)
# Add Gemini model completion detection support ## Description This PR adds proper completion detection support for Google's Vertex AI Gemini models in the LangchainLLMWrapper class. Currently, Ragas systematically raises `LLMDidNotFinishException` with Gemini models because it doesn't correctly interpret Gemini's completion signals. ## Problem The current implementation in `LangchainLLMWrapper` doesn't properly handle Gemini's completion signals: - Gemini uses "STOP" and "MAX_TOKENS" as valid completion reasons - The completion status can be found in either generation_info or response_metadata - The current logic doesn't account for these Gemini-specific patterns ## Solution 1. Modified is_finished class to support completion detection for Gemini models. Added proper handling of Gemini's completion signals 2.
1 parent 8307f66 commit 34a7db2

File tree

1 file changed

+12
-8
lines changed

1 file changed

+12
-8
lines changed

src/ragas/llms/base.py

+12-8
Original file line numberDiff line numberDiff line change
@@ -134,7 +134,7 @@ def __init__(
134134
def is_finished(self, response: LLMResult) -> bool:
135135
"""
136136
Parse the response to check if the LLM finished by checking the finish_reason
137-
or stop_reason.
137+
or stop_reason. Supports OpenAI and Vertex AI models.
138138
"""
139139
if self.is_finished_parser is not None:
140140
return self.is_finished_parser(response)
@@ -145,30 +145,34 @@ def is_finished(self, response: LLMResult) -> bool:
145145
resp = g.generations[0][0]
146146
if resp.generation_info is not None:
147147
# generation_info is provided - so we parse that
148-
149-
# OpenAI uses "stop" to indicate that the generation is finished
150-
# and is stored in 'finish_reason' key in generation_info
151-
if resp.generation_info.get("finish_reason") is not None:
148+
finish_reason = resp.generation_info.get("finish_reason")
149+
if finish_reason is not None:
150+
# OpenAI uses "stop"
151+
# Vertex AI uses "STOP" or "MAX_TOKENS"
152152
is_finished_list.append(
153-
resp.generation_info.get("finish_reason") == "stop"
153+
finish_reason in ["stop", "STOP", "MAX_TOKENS"]
154154
)
155+
155156
# provied more conditions here
156157
# https://github.com/explodinggradients/ragas/issues/1548
157158

158159
# if generation_info is empty, we parse the response_metadata
159160
# this is less reliable
161+
160162
elif (
161163
isinstance(resp, ChatGeneration)
162164
and t.cast(ChatGeneration, resp).message is not None
163165
):
164166
resp_message: BaseMessage = t.cast(ChatGeneration, resp).message
165167
if resp_message.response_metadata.get("finish_reason") is not None:
168+
finish_reason = resp_message.response_metadata.get("finish_reason")
166169
is_finished_list.append(
167-
resp_message.response_metadata.get("finish_reason") == "stop"
170+
finish_reason in ["stop", "STOP", "MAX_TOKENS"]
168171
)
169172
elif resp_message.response_metadata.get("stop_reason") is not None:
173+
stop_reason = resp_message.response_metadata.get("stop_reason")
170174
is_finished_list.append(
171-
resp_message.response_metadata.get("stop_reason") == "end_turn"
175+
stop_reason in ["end_turn", "STOP", "MAX_TOKENS"]
172176
)
173177
# default to True
174178
else:

0 commit comments

Comments
 (0)