Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[Core][Frontend] Support Passing Multimodal Processor Kwargs #8657

Merged
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
36 commits
Select commit Hold shift + click to select a range
550378b
Allow for processor kwarg overrides
alex-jw-brooks Sep 16, 2024
190606f
Pass processor through to partial
alex-jw-brooks Sep 17, 2024
b1ca041
Add default & processor kwarg override tests
alex-jw-brooks Sep 17, 2024
195e31c
Don't allow ctx or inputs as kwargs
alex-jw-brooks Sep 17, 2024
1472d04
Add kwarg override for processor to dummy data factories
alex-jw-brooks Sep 17, 2024
f10601f
Add kwarg override forr processor to max token calc
alex-jw-brooks Sep 19, 2024
429097a
Move kwarg only override func to utils
alex-jw-brooks Sep 19, 2024
159cfc2
Force processor kwargs to be keyword-only
alex-jw-brooks Sep 19, 2024
af91930
Pass unfiltered processor kwargs to default mapper
alex-jw-brooks Sep 19, 2024
9adad10
Add hack for mapper preprocessor kwargs
alex-jw-brooks Sep 19, 2024
9f7aed8
Simplify dummy data processor kwarg & add tests
alex-jw-brooks Sep 19, 2024
ff59e44
Add tests for max multimodal token kwarg overrides
alex-jw-brooks Sep 19, 2024
6b26454
Format registry
alex-jw-brooks Sep 20, 2024
0e2d53d
Fix default mapper comparison
alex-jw-brooks Sep 20, 2024
5a3341b
Move kwarg filtering into hf processor getter
alex-jw-brooks Sep 20, 2024
3e1fe54
Enable processor_kwargs in video processor
alex-jw-brooks Sep 20, 2024
feccfd7
Add tests for mapper processor_kwargs
alex-jw-brooks Sep 20, 2024
3ada64d
Update mapper not on multimodal processor kwargs
alex-jw-brooks Sep 20, 2024
58dcc63
processor kwarg test cleanup
alex-jw-brooks Sep 20, 2024
1cee215
Move context builder to test utils
alex-jw-brooks Sep 19, 2024
d5f9efa
Use common context builder in processor kwarg tests
alex-jw-brooks Sep 20, 2024
b5d434b
Update vllm/entrypoints/llm.py
alex-jw-brooks Sep 22, 2024
a096301
Update vllm/inputs/registry.py
alex-jw-brooks Sep 22, 2024
79962e0
Update vllm/inputs/registry.py
alex-jw-brooks Sep 22, 2024
2cb1f72
Update vllm/inputs/registry.py
alex-jw-brooks Sep 22, 2024
37eb532
Update vllm/inputs/registry.py
alex-jw-brooks Sep 22, 2024
a4c7c3d
Update vllm/inputs/registry.py
alex-jw-brooks Sep 22, 2024
36dd2cb
Fix formatting
alex-jw-brooks Sep 22, 2024
f95c86f
Rename processor kwargs to mm processor kwargs
alex-jw-brooks Sep 22, 2024
229604f
Update docstring
DarkLight1337 Sep 22, 2024
2a48452
Merge branch 'main' into support_processor_kwargs
DarkLight1337 Sep 22, 2024
b732d72
Try to fix CUDA reinitialization error
DarkLight1337 Sep 22, 2024
844524a
Consolidate processor loading
DarkLight1337 Sep 23, 2024
2dd742b
Fix CUDA reinitialization error
DarkLight1337 Sep 23, 2024
ebc1c02
Simplify code
DarkLight1337 Sep 23, 2024
a7f32f5
Fix tests
DarkLight1337 Sep 23, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Simplify code
  • Loading branch information
DarkLight1337 committed Sep 23, 2024
commit ebc1c0267614fef840d319e15d5005908ea0449a
4 changes: 2 additions & 2 deletions vllm/inputs/registry.py
Original file line number Diff line number Diff line change
Expand Up @@ -185,7 +185,7 @@ def dummy_data_for_profiling(
mm_counts = mm_registry.get_mm_limits_per_prompt(model_config)

mm_processor_kwargs = get_allowed_kwarg_only_overrides(
callable=dummy_factory, overrides=model_config.mm_processor_kwargs)
dummy_factory, overrides=model_config.mm_processor_kwargs)

seq_data, mm_data = dummy_factory(InputContext(model_config), seq_len,
_MultiModalCounts(mm_counts),
Expand Down Expand Up @@ -255,7 +255,7 @@ def process_input(self, model_config: "ModelConfig",
.get(model_cls, self._default_input_processor)

mm_processor_kwargs = get_allowed_kwarg_only_overrides(
callable=processor, overrides=model_config.mm_processor_kwargs)
processor, overrides=model_config.mm_processor_kwargs)

return processor(InputContext(model_config), inputs,
**mm_processor_kwargs)
Expand Down
7 changes: 3 additions & 4 deletions vllm/multimodal/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -260,9 +260,9 @@ def map_input(self, model_config: ModelConfig,
# Only get processor kwargs at mapping time if we are not using the
# input mapper; no overrides are used on the default here because they
# should be passed to the huggingface resource at initialization time.
if mapper != self._default_input_mapper:
if mapper is not None and mapper != self._default_input_mapper:
mm_processor_kwargs = get_allowed_kwarg_only_overrides(
callable=mapper, overrides=model_config.mm_processor_kwargs)
mapper, overrides=model_config.mm_processor_kwargs)
else:
mm_processor_kwargs = {}

Expand Down Expand Up @@ -344,8 +344,7 @@ def get_max_multimodal_tokens(self, model_config: ModelConfig) -> int:

if callable(max_mm_tokens):
mm_processor_kwargs = get_allowed_kwarg_only_overrides(
callable=max_mm_tokens,
overrides=model_config.mm_processor_kwargs)
max_mm_tokens, overrides=model_config.mm_processor_kwargs)
max_mm_tokens = max_mm_tokens(InputContext(model_config),
**mm_processor_kwargs)

Expand Down
6 changes: 3 additions & 3 deletions vllm/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -1239,7 +1239,7 @@ async def _run_task_with_lock(task: Callable, lock: asyncio.Lock, *args,


def get_allowed_kwarg_only_overrides(
callable: Optional[Callable],
callable: Callable[..., object],
overrides: Optional[Dict[str, Any]],
) -> Dict[str, Any]:
"""
Expand All @@ -1259,7 +1259,7 @@ def get_allowed_kwarg_only_overrides(
to overwrite one or more keyword only arguments when invoking the
callable.
"""
if not overrides or not callable:
if not overrides:
return {}

allowed_override_names = [
Expand All @@ -1276,7 +1276,7 @@ def get_allowed_kwarg_only_overrides(
}

# If anything is dropped, log a warning
dropped_keys = set(overrides) - set(filtered_overrides)
dropped_keys = overrides.keys() - filtered_overrides.keys()
if dropped_keys:
logger.warning(
"The following intended overrides are not keyword-only args "
Expand Down
Loading