Skip to content

Commit ab52c21

Browse files
DarkLight1337lk-chen
DarkLight1337
authored and
lk-chen
committed
[V1] Remove legacy input registry (vllm-project#15673)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
1 parent ab4fc51 commit ab52c21

File tree

17 files changed

+132
-153
lines changed

17 files changed

+132
-153
lines changed

‎tests/models/multimodal/processing/test_h2ovl.py‎

Lines changed: 1 addition & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,6 @@
1010
from vllm.multimodal import MULTIMODAL_REGISTRY
1111
from vllm.multimodal.image import rescale_image_size
1212
from vllm.multimodal.processing import BaseMultiModalProcessor
13-
from vllm.transformers_utils.tokenizer import cached_tokenizer_from_config
1413

1514
from ....conftest import _ImageAssets
1615
from ...utils import build_model_context
@@ -156,11 +155,7 @@ def test_processor_override(
156155
mm_processor_kwargs=mm_processor_kwargs if kwargs_on_init else None,
157156
limit_mm_per_prompt={"image": len(size_factors)},
158157
)
159-
tokenizer = cached_tokenizer_from_config(ctx.model_config)
160-
processor = MULTIMODAL_REGISTRY.create_processor(
161-
ctx.model_config,
162-
tokenizer=tokenizer,
163-
)
158+
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
164159
hf_processor_mm_kwargs = {} if kwargs_on_init else mm_processor_kwargs
165160

166161
min_num = min_dynamic_patch if dynamic_image_size else 1

‎tests/models/multimodal/processing/test_idefics3.py‎

Lines changed: 1 addition & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,6 @@
44
from transformers import Idefics3Config
55

66
from vllm.multimodal import MULTIMODAL_REGISTRY
7-
from vllm.transformers_utils.tokenizer import cached_tokenizer_from_config
87

98
from ....conftest import _ImageAssets
109
from ...utils import build_model_context
@@ -38,11 +37,7 @@ def test_processor_override(
3837
mm_processor_kwargs=mm_processor_kwargs if kwargs_on_init else None,
3938
limit_mm_per_prompt={"image": num_imgs},
4039
)
41-
tokenizer = cached_tokenizer_from_config(ctx.model_config)
42-
processor = MULTIMODAL_REGISTRY.create_processor(
43-
ctx.model_config,
44-
tokenizer=tokenizer,
45-
)
40+
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
4641
hf_processor_mm_kwargs = {} if kwargs_on_init else mm_processor_kwargs
4742

4843
# Build the image str / prompt based on the number of images we pass

‎tests/models/multimodal/processing/test_internvl.py‎

Lines changed: 1 addition & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,6 @@
1010
from vllm.multimodal import MULTIMODAL_REGISTRY
1111
from vllm.multimodal.image import rescale_image_size
1212
from vllm.multimodal.processing import BaseMultiModalProcessor
13-
from vllm.transformers_utils.tokenizer import cached_tokenizer_from_config
1413

1514
from ....conftest import _ImageAssets
1615
from ...utils import build_model_context
@@ -113,11 +112,7 @@ def test_processor_override(
113112
mm_processor_kwargs=mm_processor_kwargs if kwargs_on_init else None,
114113
limit_mm_per_prompt={"image": len(size_factors)},
115114
)
116-
tokenizer = cached_tokenizer_from_config(ctx.model_config)
117-
processor = MULTIMODAL_REGISTRY.create_processor(
118-
ctx.model_config,
119-
tokenizer=tokenizer,
120-
)
115+
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
121116
hf_processor_mm_kwargs = {} if kwargs_on_init else mm_processor_kwargs
122117

123118
min_num = min_dynamic_patch if dynamic_image_size else 1

‎tests/models/multimodal/processing/test_llava_next.py‎

Lines changed: 3 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,6 @@
1010
from vllm.multimodal import MULTIMODAL_REGISTRY
1111
from vllm.multimodal.parse import ImageSize
1212
from vllm.multimodal.processing import BaseMultiModalProcessor
13-
from vllm.transformers_utils.tokenizer import cached_tokenizer_from_config
1413

1514
from ...utils import build_model_context
1615

@@ -40,10 +39,7 @@ def test_processor_max_tokens(model_id):
4039
mm_processor_kwargs=None,
4140
limit_mm_per_prompt={"image": 1},
4241
)
43-
processor = MULTIMODAL_REGISTRY.create_processor(
44-
ctx.model_config,
45-
tokenizer=cached_tokenizer_from_config(ctx.model_config),
46-
)
42+
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
4743
info = processor.info
4844

4945
seen_aspect_ratios = set[float]()
@@ -139,10 +135,7 @@ def test_processor_prompt_replacements_regression(model_id, num_imgs):
139135
mm_processor_kwargs=None,
140136
limit_mm_per_prompt={"image": num_imgs},
141137
)
142-
processor = MULTIMODAL_REGISTRY.create_processor(
143-
ctx.model_config,
144-
tokenizer=cached_tokenizer_from_config(ctx.model_config),
145-
)
138+
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
146139

147140
image_ratios = [(171, 152), (184, 161), (198, 176), (333, 296), (369, 328),
148141
(488, 183), (2560, 1669)]
@@ -168,10 +161,7 @@ def test_processor_prompt_replacements_all(model_id, num_imgs):
168161
mm_processor_kwargs=None,
169162
limit_mm_per_prompt={"image": num_imgs},
170163
)
171-
processor = MULTIMODAL_REGISTRY.create_processor(
172-
ctx.model_config,
173-
tokenizer=cached_tokenizer_from_config(ctx.model_config),
174-
)
164+
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
175165

176166
seen_aspect_ratios = set[float]()
177167
image_sizes = list[ImageSize]()

‎tests/models/multimodal/processing/test_llava_onevision.py‎

Lines changed: 3 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,6 @@
1010
from vllm.multimodal import MULTIMODAL_REGISTRY
1111
from vllm.multimodal.parse import ImageSize
1212
from vllm.multimodal.processing import BaseMultiModalProcessor
13-
from vllm.transformers_utils.tokenizer import cached_tokenizer_from_config
1413

1514
from ...utils import build_model_context
1615

@@ -41,10 +40,7 @@ def test_processor_max_tokens(model_id):
4140
mm_processor_kwargs=None,
4241
limit_mm_per_prompt={"image": 1},
4342
)
44-
processor = MULTIMODAL_REGISTRY.create_processor(
45-
ctx.model_config,
46-
tokenizer=cached_tokenizer_from_config(ctx.model_config),
47-
)
43+
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
4844
info = processor.info
4945

5046
seen_aspect_ratios = set[float]()
@@ -139,10 +135,7 @@ def test_processor_prompt_replacements_regression(model_id, num_imgs):
139135
mm_processor_kwargs=None,
140136
limit_mm_per_prompt={"image": num_imgs},
141137
)
142-
processor = MULTIMODAL_REGISTRY.create_processor(
143-
ctx.model_config,
144-
tokenizer=cached_tokenizer_from_config(ctx.model_config),
145-
)
138+
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
146139

147140
image_ratios = [(171, 152), (184, 161), (198, 176), (333, 296), (369, 328),
148141
(488, 183), (2560, 1669)]
@@ -169,10 +162,7 @@ def test_processor_prompt_replacements_all(model_id, num_imgs):
169162
mm_processor_kwargs=None,
170163
limit_mm_per_prompt={"image": num_imgs},
171164
)
172-
processor = MULTIMODAL_REGISTRY.create_processor(
173-
ctx.model_config,
174-
tokenizer=cached_tokenizer_from_config(ctx.model_config),
175-
)
165+
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
176166

177167
seen_aspect_ratios = set[float]()
178168
image_sizes = list[ImageSize]()

‎tests/models/multimodal/processing/test_phi3v.py‎

Lines changed: 1 addition & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,6 @@
33
import pytest
44

55
from vllm.multimodal import MULTIMODAL_REGISTRY
6-
from vllm.transformers_utils.tokenizer import cached_tokenizer_from_config
76

87
from ....conftest import _ImageAssets
98
from ...utils import build_model_context
@@ -39,11 +38,7 @@ def test_processor_override(
3938
mm_processor_kwargs=mm_processor_kwargs if kwargs_on_init else None,
4039
limit_mm_per_prompt={"image": num_imgs},
4140
)
42-
tokenizer = cached_tokenizer_from_config(ctx.model_config)
43-
processor = MULTIMODAL_REGISTRY.create_processor(
44-
ctx.model_config,
45-
tokenizer=tokenizer,
46-
)
41+
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
4742
hf_processor_mm_kwargs = {} if kwargs_on_init else mm_processor_kwargs
4843

4944
# Build the image str / prompt based on the number of images we pass

‎tests/models/multimodal/processing/test_qwen2_vl.py‎

Lines changed: 2 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,6 @@
33
import pytest
44

55
from vllm.multimodal import MULTIMODAL_REGISTRY
6-
from vllm.transformers_utils.tokenizer import cached_tokenizer_from_config
76

87
from ....conftest import _ImageAssets
98
from ...utils import build_model_context
@@ -34,11 +33,8 @@ def test_processor_override(
3433
mm_processor_kwargs=mm_processor_kwargs if kwargs_on_init else None,
3534
limit_mm_per_prompt={"image": num_imgs},
3635
)
37-
tokenizer = cached_tokenizer_from_config(ctx.model_config)
38-
processor = MULTIMODAL_REGISTRY.create_processor(
39-
ctx.model_config,
40-
tokenizer=tokenizer,
41-
)
36+
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
37+
tokenizer = processor.info.get_tokenizer()
4238
hf_processor_mm_kwargs = {} if kwargs_on_init else mm_processor_kwargs
4339

4440
# Build the image str / prompt based on the number of images we pass

‎tests/multimodal/test_processing.py‎

Lines changed: 4 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -28,8 +28,7 @@
2828
replace_token_matches)
2929
# yapf: enable
3030
from vllm.multimodal.profiling import MultiModalProfiler
31-
from vllm.transformers_utils.tokenizer import (AnyTokenizer,
32-
cached_tokenizer_from_config)
31+
from vllm.transformers_utils.tokenizer import AnyTokenizer
3332
from vllm.utils import full_groupby
3433

3534
from .utils import random_image
@@ -955,10 +954,7 @@ def test_limit_mm_per_prompt_dummy(model_id, limit, num_supported, is_valid):
955954
limit_mm_per_prompt=limit_mm_per_prompt,
956955
)
957956

958-
processor = MULTIMODAL_REGISTRY.create_processor(
959-
model_config,
960-
tokenizer=cached_tokenizer_from_config(model_config),
961-
)
957+
processor = MULTIMODAL_REGISTRY.create_processor(model_config)
962958
profiler = MultiModalProfiler(processor)
963959

964960
mock_supported_mm_limits = MagicMock(return_value={"image": num_supported})
@@ -994,10 +990,7 @@ def test_limit_mm_per_prompt_apply(model_id, num_images, limit, is_valid):
994990
limit_mm_per_prompt=limit_mm_per_prompt,
995991
)
996992

997-
processor = MULTIMODAL_REGISTRY.create_processor(
998-
model_config,
999-
tokenizer=cached_tokenizer_from_config(model_config),
1000-
)
993+
processor = MULTIMODAL_REGISTRY.create_processor(model_config)
1001994

1002995
rng = np.random.RandomState(0)
1003996
image = random_image(rng, min_wh=128, max_wh=256)
@@ -1066,10 +1059,7 @@ def test_hf_processor_kwargs(model_id, call_kwargs, expected_kwargs):
10661059
revision=None,
10671060
)
10681061

1069-
processor = MULTIMODAL_REGISTRY.create_processor(
1070-
model_config,
1071-
tokenizer=cached_tokenizer_from_config(model_config),
1072-
)
1062+
processor = MULTIMODAL_REGISTRY.create_processor(model_config)
10731063
orig_get_hf_processor = processor.info.get_hf_processor
10741064

10751065
def get_hf_processor(self, **kwargs):

‎vllm/inputs/preprocess.py‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -261,13 +261,13 @@ def _process_multimodal(
261261
# initialized without a tokenizer while using also multi-modal
262262
# input.
263263
if not self.tokenizer:
264-
tokenizer = None
264+
tokenizer = object() # Dummy
265265
else:
266266
tokenizer_group = self.get_tokenizer_group()
267267
tokenizer = tokenizer_group.get_lora_tokenizer(lora_request)
268268

269-
mm_processor = self.mm_registry.create_processor(
270-
self.model_config, tokenizer)
269+
mm_processor = self.mm_registry.create_processor(self.model_config,
270+
tokenizer=tokenizer)
271271

272272
if mm_processor_kwargs is None:
273273
mm_processor_kwargs = {}
@@ -288,14 +288,14 @@ async def _process_multimodal_async(
288288
# initialized without a tokenizer while using also multi-modal
289289
# input.
290290
if not self.tokenizer:
291-
tokenizer = None
291+
tokenizer = object() # Dummy
292292
else:
293293
tokenizer_group = self.get_tokenizer_group()
294294
tokenizer = await tokenizer_group.get_lora_tokenizer_async(
295295
lora_request)
296296

297-
mm_processor = self.mm_registry.create_processor(
298-
self.model_config, tokenizer)
297+
mm_processor = self.mm_registry.create_processor(self.model_config,
298+
tokenizer=tokenizer)
299299
if mm_processor_kwargs is None:
300300
mm_processor_kwargs = {}
301301

‎vllm/inputs/registry.py‎

Lines changed: 17 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -13,8 +13,7 @@
1313

1414
from vllm.logger import init_logger
1515
from vllm.transformers_utils.processor import cached_processor_from_config
16-
from vllm.transformers_utils.tokenizer import (AnyTokenizer,
17-
cached_tokenizer_from_config)
16+
from vllm.transformers_utils.tokenizer import AnyTokenizer
1817
from vllm.utils import (ClassRegistry, get_allowed_kwarg_only_overrides,
1918
resolve_mm_processor_kwargs)
2019

@@ -329,17 +328,27 @@ def dummy_data_for_profiling(
329328
from vllm.model_executor.model_loader import get_model_architecture
330329
from vllm.multimodal import MultiModalKwargs
331330
from vllm.multimodal.profiling import MultiModalProfiler
331+
from vllm.sequence import SequenceData
332332

333333
if mm_registry.has_processor(model_config):
334-
tokenizer = cached_tokenizer_from_config(model_config)
335334
processor = mm_registry.create_processor(model_config,
336-
tokenizer,
337335
disable_cache=True)
338336
profiler = MultiModalProfiler(processor)
339-
dummy_data_factory = (profiler.get_encoder_dummy_data
340-
if is_encoder_data else
341-
profiler.get_decoder_dummy_data)
342-
dummy_data = dummy_data_factory(seq_len)
337+
338+
dummy_data_v1 = (profiler.get_encoder_dummy_data(seq_len)
339+
if is_encoder_data else
340+
profiler.get_decoder_dummy_data(seq_len))
341+
_seq_data = SequenceData.from_seqs(
342+
dummy_data_v1.prompt_token_ids) # type: ignore[attr-defined]
343+
344+
dummy_data = DummyData(
345+
seq_data=_seq_data,
346+
multi_modal_data=getattr(dummy_data_v1, "multi_modal_data",
347+
None),
348+
multi_modal_placeholders=getattr(dummy_data_v1,
349+
"multi_modal_placeholders",
350+
None),
351+
)
343352
else:
344353
model_cls, _ = get_model_architecture(model_config)
345354
if is_encoder_data:

0 commit comments

Comments
 (0)