[trtllm] translate tgi stop to stop_sequences (#1930)

deepjavalibrary · May 20, 2024 · f8d162b · f8d162b
1 parent 9572272
commit f8d162b
Showing 1 changed file with 4 additions and 0 deletions.
diff --git a/engines/python/setup/djl_python/rolling_batch/trtllm_rolling_batch.py b/engines/python/setup/djl_python/rolling_batch/trtllm_rolling_batch.py
@@ -77,6 +77,10 @@ def translate_triton_params(self, parameters: dict) -> dict:
             parameters['len_penalty'] = parameters.pop('length_penalty')
         parameters["streaming"] = parameters.pop(
             "stream", parameters.get("streaming", True))
+        stop = parameters.pop("stop", None)
+        if stop:
+            # stop_sequences is translated to stop_words_list in tensorrt_llm_toolkit
+            parameters["stop_sequences"] = stop
         return parameters
 
     @stop_on_any_exception