AutoTokenizer - add from_model_name method

src/transformers/models/auto/tokenization_auto.py

@@ -348,9 +348,57 @@ class AutoTokenizer:
     def __init__(self):
         raise EnvironmentError(
             "AutoTokenizer is designed to be instantiated "
-            "using the `AutoTokenizer.from_pretrained(pretrained_model_name_or_path)` method."
+            "using the `AutoTokenizer.from_pretrained(pretrained_model_name_or_path)` or `AutoTokenizer.from_model_name(config)` method."
         )
 
+    @classmethod
+    @replace_list_option_in_docstrings(TOKENIZER_MAPPING_NAMES)
+    def from_model_name(cls, model_name, *args, **kwargs):
+        r"""
+        Instantiate one of the tokenizer classes of the library by passing the required vocabulary file.
+
+        The tokenizer class to instantiate is selected based on the :obj:`model_name` which is passed as an argument.
+
+        List options
+
+        Params:
+            model_name (:obj:`str`):
+                The :obj:`model_name` associated to the tokenizer class that should be instantiated. Should be one of
+                the keys shown in bold above.
+            use_fast (:obj:`bool`, `optional`, defaults to :obj:`True`):
+                Whether or not to try to load the fast version of the tokenizer.
+            args (additional positional arguments, `optional`):
+                Will be passed to the Tokenizer ``__init__()`` method. Can be used to pass the required vocabulary
+                files such as ``vocab_file`` or ``merges_file``.
+            kwargs (additional keyword arguments, `optional`):
+                Will be passed to the Tokenizer ``__init__()`` method. Can be used to pass the required vocabulary
+                files such as ``vocab_file=/path/tol/vocab_file.json`` and/or ``merges_file=/path/to/merges_file.txt``
+                as well as to set special tokens like ``bos_token``, ``eos_token``, ``unk_token``, ``sep_token``,
+                ``pad_token``, ``cls_token``, ``mask_token``, ``additional_special_tokens``. See parameters in the
+                ``__init__()`` for more details.
+
+        Examples::
+
+            >>> from transformers import AutoTokenizer
+
+            >>> # Instantiate BERT-like tokenizer
+            >>> tokenizer = AutoTokenizer.from_model_name("bert", vocab_file="./vocab.txt")
+
+            >>> # Instantiate GPT2-like tokenizer
+            >>> tokenizer = AutoTokenizer.from_model_name("gpt2", vocab_file="./vocab.json", merges_file="./merges.txt")
+        """
+
+        use_fast = kwargs.pop("use_fast", True)
+
+        tokenizer_class_name, tokenizer_class_name_fast = TOKENIZER_MAPPING_NAMES[model_name]
+
+        if use_fast and tokenizer_class_name_fast is not None:
+            tokenizer_cls_fast = tokenizer_class_from_name(tokenizer_class_name_fast)
+            return tokenizer_cls_fast(*args, **kwargs)
+
+        tokenizer_cls = tokenizer_class_from_name(tokenizer_class_name)
+        return tokenizer_cls(*args, **kwargs)
+
     @classmethod
     @replace_list_option_in_docstrings(TOKENIZER_MAPPING_NAMES)
     def from_pretrained(cls, pretrained_model_name_or_path, *inputs, **kwargs):

tests/fixtures/merges.txt

@@ -0,0 +1,5 @@
+#version: 0.2
+Ġ l
+Ġl o
+Ġlo w
+e r

tests/fixtures/vocab.json

@@ -0,0 +1 @@
+{"l": 0, "o": 1, "w": 2, "e": 3, "r": 4, "s": 5, "t": 6, "i": 7, "d": 8, "n": 9, "Ġ": 10, "Ġl": 11, "Ġn": 12, "Ġlo": 13, "Ġlow": 14, "er": 15, "Ġlowest": 16, "Ġnewer": 17, "Ġwider": 18, "<unk>": 19, "<|endoftext|>": 20}

tests/fixtures/vocab.txt

@@ -0,0 +1,10 @@
+[PAD]
+[SEP]
+[MASK]
+[CLS]
+[unused3]
+[unused4]
+[unused5]
+[unused6]
+[unused7]
+[unused8]

tests/test_tokenization_auto.py

@@ -78,6 +78,35 @@ def test_tokenizer_from_tokenizer_class(self):
         self.assertIsInstance(tokenizer, (BertTokenizer, BertTokenizerFast))
         self.assertEqual(tokenizer.vocab_size, 12)
 
+    def test_tokenizer_from_name(self):
+        name = "bert"
+        vocab_file = "./tests/fixtures/vocab.txt"
+        tokenizer = AutoTokenizer.from_model_name(name, vocab_file=vocab_file, use_fast=False)
+        self.assertIsInstance(tokenizer, BertTokenizer)
+        self.assertEqual(tokenizer.vocab_size, 10)
+
+        name = "gpt2"
+        vocab_file = "./tests/fixtures/vocab.json"
+        merges_file = "./tests/fixtures/merges.txt"
+        tokenizer = AutoTokenizer.from_model_name(name, vocab_file=vocab_file, merges_file=merges_file, use_fast=False)
+        self.assertIsInstance(tokenizer, GPT2Tokenizer)
+        self.assertEqual(tokenizer.vocab_size, 21)
+
+    @require_tokenizers
+    def test_tokenizer_from_name_fast(self):
+        name = "bert"
+        vocab_file = "./tests/fixtures/vocab.txt"
+        tokenizer = AutoTokenizer.from_model_name(name, vocab_file=vocab_file)
+        self.assertIsInstance(tokenizer, BertTokenizerFast)
+        self.assertEqual(tokenizer.vocab_size, 10)
+
+        name = "gpt2"
+        vocab_file = "./tests/fixtures/vocab.json"
+        merges_file = "./tests/fixtures/merges.txt"
+        tokenizer = AutoTokenizer.from_model_name(name, vocab_file=vocab_file, merges_file=merges_file)
+        self.assertIsInstance(tokenizer, GPT2TokenizerFast)
+        self.assertEqual(tokenizer.vocab_size, 21)
+
     @require_tokenizers
     def test_tokenizer_identifier_with_correct_config(self):
         for tokenizer_class in [BertTokenizer, BertTokenizerFast, AutoTokenizer]: