[bug] Use PreTrainedTokenizerFast.from_pretrained instead of `PreTr…

…ainedTokenizerFast.__init__` (ServiceNow#75)
chrish42 · Dec 2, 2024 · 7d6c2cc · 7d6c2cc
1 parent 484e2f6
commit 7d6c2cc
Showing 1 changed file with 3 additions and 1 deletion.
diff --git a/fast_llm/data/tokenizer.py b/fast_llm/data/tokenizer.py
@@ -11,7 +11,9 @@ class Tokenizer:
 
     def __init__(self, config: TokenizerConfig):
         log_main_rank(f"> loading tokenizer from {config.path} ...")
-        self.tokenizer = PreTrainedTokenizerFast(tokenizer_file=config.path, errors="replace", max_len=None)
+        self.tokenizer = PreTrainedTokenizerFast.from_pretrained(
+            pretrained_model_name_or_path=config.path, errors="replace", max_len=None
+        )
         if self.tokenizer.eos_token_id is None:
             raise ValueError("Tokenizer does not have an EOS token.")
         self.eod_id = self.tokenizer.eos_token_id