Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix hanging when prompt exceeds limit #1029

Merged
merged 1 commit into from
Sep 17, 2023
Merged

Fix hanging when prompt exceeds limit #1029

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Fix hanging when prompt exceeds limit
  • Loading branch information
@chenxu2048
chenxu2048 committed Sep 13, 2023
commit 1c63f2413a4fc2d543017b45e8782ee196329ca2
2 changes: 1 addition & 1 deletion vllm/core/scheduler.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -175,7 +175,7 @@ def _schedule(self) -> SchedulerOutputs:
num_curr_seqs += num_new_seqs
scheduled.append(seq_group)

if scheduled:
if scheduled or ignored_seq_groups:
scheduler_outputs = SchedulerOutputs(
scheduled_seq_groups=scheduled,
prompt_run=True,
Expand Down
21 changes: 9 additions & 12 deletions vllm/engine/llm_engine.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -291,14 +291,12 @@ def has_unfinished_requests(self) -> bool:
def _schedule(
self
) -> Tuple[List[SequenceGroupMetadata], SchedulerOutputs,
Optional[List[RequestOutput]]]:
List[RequestOutput]]:
seq_group_metadata_list, scheduler_outputs = self.scheduler.schedule()
if scheduler_outputs.is_empty():
return seq_group_metadata_list, scheduler_outputs, [
RequestOutput.from_seq_group(seq_group)
for seq_group in scheduler_outputs.ignored_seq_groups
]
return seq_group_metadata_list, scheduler_outputs, None
return seq_group_metadata_list, scheduler_outputs, [
RequestOutput.from_seq_group(seq_group)
for seq_group in scheduler_outputs.ignored_seq_groups
]

def _check_beam_search_early_stopping(
self,
Expand Down Expand Up @@ -542,10 +540,9 @@ def step(self) -> List[RequestOutput]:
and updates the scheduler with the model outputs. Finally, it decodes
the sequences and returns the newly generated results.
"""
(seq_group_metadata_list, scheduler_outputs,
early_return) = self._schedule()
if early_return is not None:
return early_return
seq_group_metadata_list, scheduler_outputs, ignored = self._schedule()
if scheduler_outputs.is_empty():
return ignored

# Execute the model.
output = self._run_workers(
Expand All @@ -556,7 +553,7 @@ def step(self) -> List[RequestOutput]:
blocks_to_copy=scheduler_outputs.blocks_to_copy,
)

return self._process_model_outputs(output, scheduler_outputs)
return self._process_model_outputs(output, scheduler_outputs) + ignored

def _log_system_stats(
self,
Expand Down