File tree Expand file tree Collapse file tree 1 file changed +6
-6
lines changed Expand file tree Collapse file tree 1 file changed +6
-6
lines changed Original file line number Diff line number Diff line change @@ -423,14 +423,13 @@ def get_tokenizer(
423423 ** kwargs ,
424424) -> Union [PreTrainedTokenizer , PreTrainedTokenizerFast ]:
425425 if pretrained_model_name_or_path is not None and not os .path .exists (
426- pretrained_model_name_or_path
427- ):
428- pretrained_model_name_or_path = get_model ( pretrained_model_name_or_path )
426+ pretrained_model_name_or_path ):
427+ pretrained_model_name_or_path = get_model (
428+ pretrained_model_name_or_path )
429429 if tokenizer_mode == "slow" :
430430 if kwargs .get ("use_fast" , False ):
431431 raise ValueError (
432- "Cannot use the fast tokenizer in slow tokenizer mode."
433- )
432+ "Cannot use the fast tokenizer in slow tokenizer mode." )
434433 kwargs ["use_fast" ] = False
435434 if tokenizer_mode == "mistral" :
436435 vllm_dir = os .path .dirname (os .path .dirname (os .path .abspath (__file__ )))
@@ -439,7 +438,8 @@ def get_tokenizer(
439438 MistralTokenizer
440439 )
441440
442- return MistralTokenizer .from_pretrained (str (pretrained_model_name_or_path ))
441+ return MistralTokenizer .from_pretrained (
442+ str (pretrained_model_name_or_path ))
443443 else :
444444 return AutoTokenizer .from_pretrained (
445445 pretrained_model_name_or_path ,
You can’t perform that action at this time.
0 commit comments