Skip to content

Add remove_from_body to OpenAIHTTPBackend #184

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 2 commits into
base: main
Choose a base branch
from
Open

Add remove_from_body to OpenAIHTTPBackend #184

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
8b5ec4f
Add remove_from_body to OpenAIHTTPBackend
jackcook Jun 12, 2025
e3be0ef
Fix typing issue
jackcook Jun 12, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 19 additions & 7 deletions src/guidellm/backend/openai.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,14 @@ class OpenAIHTTPBackend(Backend):
the values of these keys will be used as the parameters for the respective
endpoint.
If not provided, no extra query parameters are added.
:param extra_body: Body parameters to include in requests to the OpenAI server.
If "chat_completions", "models", or "text_completions" are included as keys,
the values of these keys will be included in the body for the respective
endpoint.
If not provided, no extra body parameters are added.
:param remove_from_body: Parameters that should be removed from the body of each
request.
If not provided, no parameters are removed from the body.
"""

def __init__(
Expand All @@ -85,6 +93,7 @@ def __init__(
max_output_tokens: Optional[int] = None,
extra_query: Optional[dict] = None,
extra_body: Optional[dict] = None,
remove_from_body: Optional[list[str]] = None,
):
super().__init__(type_="openai_http")
self._target = target or settings.openai.base_url
Expand Down Expand Up @@ -122,6 +131,7 @@ def __init__(
)
self.extra_query = extra_query
self.extra_body = extra_body
self.remove_from_body = remove_from_body
self._async_client: Optional[httpx.AsyncClient] = None

@property
Expand Down Expand Up @@ -244,9 +254,8 @@ async def text_completions( # type: ignore[override]

headers = self._headers()
params = self._params(TEXT_COMPLETIONS)
body = self._body(TEXT_COMPLETIONS)
payload = self._completions_payload(
body=body,
endpoint_type=TEXT_COMPLETIONS,
orig_kwargs=kwargs,
max_output_tokens=output_token_count,
prompt=prompt,
Expand Down Expand Up @@ -321,12 +330,11 @@ async def chat_completions( # type: ignore[override]
logger.debug("{} invocation with args: {}", self.__class__.__name__, locals())
headers = self._headers()
params = self._params(CHAT_COMPLETIONS)
body = self._body(CHAT_COMPLETIONS)
messages = (
content if raw_content else self._create_chat_messages(content=content)
)
payload = self._completions_payload(
body=body,
endpoint_type=CHAT_COMPLETIONS,
orig_kwargs=kwargs,
max_output_tokens=output_token_count,
messages=messages,
Expand Down Expand Up @@ -402,7 +410,7 @@ def _params(self, endpoint_type: EndpointType) -> dict[str, str]:

return self.extra_query

def _body(self, endpoint_type: EndpointType) -> dict[str, str]:
def _extra_body(self, endpoint_type: EndpointType) -> dict[str, Any]:
if self.extra_body is None:
return {}

Expand All @@ -417,12 +425,12 @@ def _body(self, endpoint_type: EndpointType) -> dict[str, str]:

def _completions_payload(
self,
body: Optional[dict],
endpoint_type: EndpointType,
orig_kwargs: Optional[dict],
max_output_tokens: Optional[int],
**kwargs,
) -> dict:
payload = body or {}
payload = self._extra_body(endpoint_type)
payload.update(orig_kwargs or {})
payload.update(kwargs)
payload["model"] = self.model
Expand All @@ -446,6 +454,10 @@ def _completions_payload(
payload["stop"] = None
payload["ignore_eos"] = True

if self.remove_from_body:
for key in self.remove_from_body:
payload.pop(key, None)

return payload

@staticmethod
Expand Down