Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[Serving] Remove mandatory model check in server #2195

Merged
merged 1 commit into from
Apr 23, 2024
Merged

[Serving] Remove mandatory model check in server #2195

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 5 additions & 5 deletions python/mlc_llm/protocol/openai_api_protocol.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -84,7 +84,7 @@ class CompletionRequest(BaseModel):
API reference: https://platform.openai.com/docs/api-reference/completions/create
"""

model: str
model: Optional[str] = None
prompt: Union[str, List[int]]
best_of: int = 1
echo: bool = False
Expand Down Expand Up @@ -154,7 +154,7 @@ class CompletionResponse(BaseModel):
id: str
choices: List[CompletionResponseChoice]
created: int = Field(default_factory=lambda: int(time.time()))
model: str
model: Optional[str] = None
object: str = "text_completion"
usage: UsageInfo = Field(
default_factory=lambda: UsageInfo() # pylint: disable=unnecessary-lambda
Expand Down Expand Up @@ -200,7 +200,7 @@ class ChatCompletionRequest(BaseModel):
"""

messages: List[ChatCompletionMessage]
model: str
model: Optional[str] = None
frequency_penalty: Optional[float] = None
presence_penalty: Optional[float] = None
logprobs: bool = False
Expand Down Expand Up @@ -343,7 +343,7 @@ class ChatCompletionResponse(BaseModel):
id: str
choices: List[ChatCompletionResponseChoice]
created: int = Field(default_factory=lambda: int(time.time()))
model: str
model: Optional[str] = None
system_fingerprint: str
object: Literal["chat.completion"] = "chat.completion"
usage: UsageInfo = Field(
Expand All @@ -359,7 +359,7 @@ class ChatCompletionStreamResponse(BaseModel):
id: str
choices: List[ChatCompletionStreamResponseChoice]
created: int = Field(default_factory=lambda: int(time.time()))
model: str
model: Optional[str] = None
system_fingerprint: str
object: Literal["chat.completion.chunk"] = "chat.completion.chunk"
usage: UsageInfo = Field(
Expand Down
32 changes: 16 additions & 16 deletions python/mlc_llm/serve/engine.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,8 +61,8 @@ async def create( # pylint: disable=too-many-arguments,too-many-locals
self,
*,
messages: List[Dict[str, Any]],
model: str,
stream: Literal[True],
model: Optional[str] = None,
frequency_penalty: float = 0.0,
presence_penalty: float = 0.0,
logprobs: bool = False,
Expand Down Expand Up @@ -111,7 +111,7 @@ async def create( # pylint: disable=too-many-arguments,too-many-locals
self,
*,
messages: List[Dict[str, Any]],
model: str,
model: Optional[str] = None,
frequency_penalty: float = 0.0,
presence_penalty: float = 0.0,
logprobs: bool = False,
Expand Down Expand Up @@ -160,7 +160,7 @@ async def create( # pylint: disable=too-many-arguments,too-many-locals
self,
*,
messages: List[Dict[str, Any]],
model: str,
model: Optional[str] = None,
frequency_penalty: float = 0.0,
presence_penalty: float = 0.0,
logprobs: bool = False,
Expand Down Expand Up @@ -238,8 +238,8 @@ def create( # pylint: disable=too-many-arguments,too-many-locals
self,
*,
messages: List[Dict[str, Any]],
model: str,
stream: Literal[True],
model: Optional[str] = None,
frequency_penalty: float = 0.0,
presence_penalty: float = 0.0,
logprobs: bool = False,
Expand Down Expand Up @@ -288,7 +288,7 @@ def create( # pylint: disable=too-many-arguments,too-many-locals
self,
*,
messages: List[Dict[str, Any]],
model: str,
model: Optional[str] = None,
frequency_penalty: float = 0.0,
presence_penalty: float = 0.0,
logprobs: bool = False,
Expand Down Expand Up @@ -335,7 +335,7 @@ def create( # pylint: disable=too-many-arguments,too-many-locals
self,
*,
messages: List[Dict[str, Any]],
model: str,
model: Optional[str] = None,
frequency_penalty: float = 0.0,
presence_penalty: float = 0.0,
logprobs: bool = False,
Expand Down Expand Up @@ -412,9 +412,9 @@ def __init__(self, engine: weakref.ReferenceType) -> None:
async def create( # pylint: disable=too-many-arguments,too-many-locals
self,
*,
model: str,
prompt: Union[str, List[int]],
stream: Literal[True],
model: Optional[str] = None,
best_of: int = 1,
echo: bool = False,
frequency_penalty: float = 0.0,
Expand Down Expand Up @@ -463,8 +463,8 @@ async def create( # pylint: disable=too-many-arguments,too-many-locals
async def create( # pylint: disable=too-many-arguments,too-many-locals
self,
*,
model: str,
prompt: Union[str, List[int]],
model: Optional[str] = None,
best_of: int = 1,
echo: bool = False,
frequency_penalty: float = 0.0,
Expand Down Expand Up @@ -511,8 +511,8 @@ async def create( # pylint: disable=too-many-arguments,too-many-locals
async def create( # pylint: disable=too-many-arguments,too-many-locals
self,
*,
model: str,
prompt: Union[str, List[int]],
model: Optional[str] = None,
best_of: int = 1,
echo: bool = False,
frequency_penalty: float = 0.0,
Expand Down Expand Up @@ -591,9 +591,9 @@ def __init__(self, engine: weakref.ReferenceType) -> None:
def create( # pylint: disable=too-many-arguments,too-many-locals
self,
*,
model: str,
prompt: Union[str, List[int]],
stream: Literal[True],
model: Optional[str] = None,
best_of: int = 1,
echo: bool = False,
frequency_penalty: float = 0.0,
Expand Down Expand Up @@ -642,8 +642,8 @@ def create( # pylint: disable=too-many-arguments,too-many-locals
def create( # pylint: disable=too-many-arguments,too-many-locals
self,
*,
model: str,
prompt: Union[str, List[int]],
model: Optional[str] = None,
best_of: int = 1,
echo: bool = False,
frequency_penalty: float = 0.0,
Expand Down Expand Up @@ -690,8 +690,8 @@ def create( # pylint: disable=too-many-arguments,too-many-locals
def create( # pylint: disable=too-many-arguments,too-many-locals
self,
*,
model: str,
prompt: Union[str, List[int]],
model: Optional[str] = None,
best_of: int = 1,
echo: bool = False,
frequency_penalty: float = 0.0,
Expand Down Expand Up @@ -883,7 +883,7 @@ async def _chat_completion( # pylint: disable=too-many-arguments,too-many-local
self,
*,
messages: List[Dict[str, Any]],
model: str,
model: Optional[str] = None,
frequency_penalty: float = 0.0,
presence_penalty: float = 0.0,
logprobs: bool = False,
Expand Down Expand Up @@ -1003,8 +1003,8 @@ async def _chat_completion( # pylint: disable=too-many-arguments,too-many-local
async def _completion( # pylint: disable=too-many-arguments,too-many-locals
self,
*,
model: str,
prompt: Union[str, List[int]],
model: Optional[str] = None,
best_of: int = 1,
echo: bool = False,
frequency_penalty: float = 0.0,
Expand Down Expand Up @@ -1429,7 +1429,7 @@ def _chat_completion( # pylint: disable=too-many-arguments,too-many-locals
self,
*,
messages: List[Dict[str, Any]],
model: str,
model: Optional[str] = None,
frequency_penalty: float = 0.0,
presence_penalty: float = 0.0,
logprobs: bool = False,
Expand Down Expand Up @@ -1549,8 +1549,8 @@ def _chat_completion( # pylint: disable=too-many-arguments,too-many-locals
def _completion( # pylint: disable=too-many-arguments,too-many-locals
self,
*,
model: str,
prompt: Union[str, List[int]],
model: Optional[str] = None,
best_of: int = 1,
echo: bool = False,
frequency_penalty: float = 0.0,
Expand Down
7 changes: 5 additions & 2 deletions python/mlc_llm/serve/server/server_context.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,8 +37,11 @@ def add_model(self, hosted_model: str, engine: AsyncLLMEngine) -> None:
raise RuntimeError(f"Model {hosted_model} already running.")
self._models[hosted_model] = engine

def get_engine(self, model: str) -> Optional[AsyncLLMEngine]:
"""Get the async engine of the requested model."""
def get_engine(self, model: Optional[str]) -> Optional[AsyncLLMEngine]:
"""Get the async engine of the requested model, or the unique async engine
if only one engine is served."""
if len(self._models) == 1:
return next(iter(self._models.values()))
return self._models.get(model, None)

def get_model_list(self) -> List[str]:
Expand Down
18 changes: 0 additions & 18 deletions tests/python/serve/server/test_server.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -329,23 +329,6 @@ def test_openai_v1_completions_openai_package(
)


def test_openai_v1_completions_invalid_requested_model(
launch_server, # pylint: disable=unused-argument
):
# `launch_server` is a pytest fixture defined in conftest.py.

model = "unserved_model"
payload = {
"model": model,
"prompt": "What is the meaning of life?",
"max_tokens": 10,
}
response = requests.post(OPENAI_V1_COMPLETION_URL, json=payload, timeout=180)
expect_error(
response_str=response.json(), msg_prefix=f'The requested model "{model}" is not served.'
)


@pytest.mark.parametrize("stream", [False, True])
def test_openai_v1_completions_echo(
served_model: Tuple[str, str],
Expand Down Expand Up @@ -1319,7 +1302,6 @@ def test_debug_dump_event_trace(
test_openai_v1_completions(MODEL, None, stream=True)
test_openai_v1_completions_openai_package(MODEL, None, stream=False)
test_openai_v1_completions_openai_package(MODEL, None, stream=True)
test_openai_v1_completions_invalid_requested_model(None)
test_openai_v1_completions_echo(MODEL, None, stream=False)
test_openai_v1_completions_echo(MODEL, None, stream=True)
test_openai_v1_completions_suffix(MODEL, None, stream=False)
Expand Down
Loading