Support for Text Generation Inference (TGI)

xusenlin · xusenlin · commit f83d4b5d1651 · 2023-12-10T14:36:09.000+08:00
diff --git a/api/llama_cpp_routes/chat.py b/api/llama_cpp_routes/chat.py
@@ -35,8 +35,8 @@ async def create_chat_completion(
 
     prompt = engine.apply_chat_template(request.messages, request.functions, request.tools)
     include = {
-        "temperature", "top_p", "stream", "stop",
-        "max_tokens", "presence_penalty", "frequency_penalty", "model"
+        "temperature", "top_p", "stream", "stop", "model",
+        "max_tokens", "presence_penalty", "frequency_penalty",
     }
     kwargs = model_dump(request, include=include)
     logger.debug(f"==== request ====\n{kwargs}")
diff --git a/api/llama_cpp_routes/completion.py b/api/llama_cpp_routes/completion.py
@@ -33,8 +33,8 @@ async def create_completion(
     request = await handle_request(request, engine.stop)
 
     include = {
-        "temperature", "top_p", "stream", "stop",
-        "max_tokens", "presence_penalty", "frequency_penalty", "model"
+        "temperature", "top_p", "stream", "stop", "model",
+        "max_tokens", "presence_penalty", "frequency_penalty",
     }
     kwargs = model_dump(request, include=include)
     logger.debug(f"==== request ====\n{kwargs}")
diff --git a/api/tgi_routes/chat.py b/api/tgi_routes/chat.py
@@ -56,8 +56,7 @@ async def create_chat_completion(
 
     prompt = engine.apply_chat_template(request.messages)
     include = {
-        "temperature", "stop_token_ids", "stream", "model",
-        "repetition_penalty", "typical_p", "watermark", "best_of"
+        "temperature", "best_of", "repetition_penalty", "typical_p", "watermark",
     }
     params = model_dump(request, include=include)
     params |= dict(
diff --git a/api/tgi_routes/completion.py b/api/tgi_routes/completion.py
@@ -50,8 +50,7 @@ async def create_completion(
 
     request_id: str = f"cmpl-{str(uuid.uuid4())}"
     include = {
-        "temperature", "stop_token_ids", "stream", "model",
-        "repetition_penalty", "typical_p", "watermark", "best_of",
+        "temperature", "best_of", "repetition_penalty", "typical_p", "watermark",
     }
     params = model_dump(request, include=include)
     params |= dict(

Original file line number	Diff line number	Diff line change
`@@ -56,8 +56,7 @@ async def create_chat_completion(`
`56`	`56`
`57`	`57`	`prompt = engine.apply_chat_template(request.messages)`
`58`	`58`	`include = {`
`59`		`- "temperature", "stop_token_ids", "stream", "model",`
`60`		`- "repetition_penalty", "typical_p", "watermark", "best_of"`
	`59`	`+ "temperature", "best_of", "repetition_penalty", "typical_p", "watermark",`
`61`	`60`	`}`
`62`	`61`	`params = model_dump(request, include=include)`
`63`	`62`	`params \|= dict(`
Original file line number	Diff line number	Diff line change
`@@ -50,8 +50,7 @@ async def create_completion(`
`50`	`50`
`51`	`51`	`request_id: str = f"cmpl-{str(uuid.uuid4())}"`
`52`	`52`	`include = {`
`53`		`- "temperature", "stop_token_ids", "stream", "model",`
`54`		`- "repetition_penalty", "typical_p", "watermark", "best_of",`
	`53`	`+ "temperature", "best_of", "repetition_penalty", "typical_p", "watermark",`
`55`	`54`	`}`
`56`	`55`	`params = model_dump(request, include=include)`
`57`	`56`	`params \|= dict(`