use Qwen2Tokenizer instead of Qwen2MoeTokenizer

docs/source/en/model_doc/qwen2_moe.md

@@ -61,15 +61,6 @@ In the following, we demonstrate how to use `Qwen1.5-MoE-A2.7B-Chat` for the inf
 
 [[autodoc]] Qwen2MoeConfig
 
-## Qwen2MoeTokenizer
-
-[[autodoc]] Qwen2MoeTokenizer
-    - save_vocabulary
-
-## Qwen2MoeTokenizerFast
-
-[[autodoc]] Qwen2MoeTokenizerFast
-
 ## Qwen2MoeModel
 
 [[autodoc]] Qwen2MoeModel

src/transformers/__init__.py

@@ -721,7 +721,6 @@
     "models.qwen2_moe": [
         "QWEN2MOE_PRETRAINED_CONFIG_ARCHIVE_MAP",
         "Qwen2MoeConfig",
-        "Qwen2MoeTokenizer",
     ],
     "models.rag": ["RagConfig", "RagRetriever", "RagTokenizer"],
     "models.realm": [
@@ -1210,7 +1209,6 @@
     _import_structure["models.openai"].append("OpenAIGPTTokenizerFast")
     _import_structure["models.pegasus"].append("PegasusTokenizerFast")
     _import_structure["models.qwen2"].append("Qwen2TokenizerFast")
-    _import_structure["models.qwen2_moe"].append("Qwen2MoeTokenizerFast")
     _import_structure["models.realm"].append("RealmTokenizerFast")
     _import_structure["models.reformer"].append("ReformerTokenizerFast")
     _import_structure["models.rembert"].append("RemBertTokenizerFast")
@@ -5516,7 +5514,7 @@
     from .models.pvt import PVT_PRETRAINED_CONFIG_ARCHIVE_MAP, PvtConfig
     from .models.qdqbert import QDQBERT_PRETRAINED_CONFIG_ARCHIVE_MAP, QDQBertConfig
     from .models.qwen2 import QWEN2_PRETRAINED_CONFIG_ARCHIVE_MAP, Qwen2Config, Qwen2Tokenizer
-    from .models.qwen2_moe import QWEN2MOE_PRETRAINED_CONFIG_ARCHIVE_MAP, Qwen2MoeConfig, Qwen2MoeTokenizer
+    from .models.qwen2_moe import QWEN2MOE_PRETRAINED_CONFIG_ARCHIVE_MAP, Qwen2MoeConfig
     from .models.rag import RagConfig, RagRetriever, RagTokenizer
     from .models.realm import (
         REALM_PRETRAINED_CONFIG_ARCHIVE_MAP,
@@ -5993,7 +5991,6 @@
         from .models.openai import OpenAIGPTTokenizerFast
         from .models.pegasus import PegasusTokenizerFast
         from .models.qwen2 import Qwen2TokenizerFast
-        from .models.qwen2_moe import Qwen2MoeTokenizerFast
         from .models.realm import RealmTokenizerFast
         from .models.reformer import ReformerTokenizerFast
         from .models.rembert import RemBertTokenizerFast

src/transformers/convert_slow_tokenizer.py

@@ -432,48 +432,6 @@ def converted(self) -> Tokenizer:
         return tokenizer
 
 
-class Qwen2MoeConverter(Converter):
-    def converted(self) -> Tokenizer:
-        vocab = self.original_tokenizer.encoder
-        merges = list(self.original_tokenizer.bpe_ranks.keys())
-
-        tokenizer = Tokenizer(
-            BPE(
-                vocab=vocab,
-                merges=merges,
-                dropout=None,
-                unk_token=None,
-                continuing_subword_prefix="",
-                end_of_word_suffix="",
-                fuse_unk=False,
-                byte_fallback=False,
-            )
-        )
-
-        tokenizer.normalizer = normalizers.NFC()
-
-        tokenizer.pre_tokenizer = pre_tokenizers.Sequence(
-            [
-                pre_tokenizers.Split(
-                    Regex(
-                        r"""(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\r\n\p{L}\p{N}]?\p{L}+|\p{N}| ?[^\s\p{L}\p{N}]+[\r\n]*|\s*[\r\n]+|\s+(?!\S)|\s+"""
-                    ),
-                    behavior="isolated",
-                    invert=False,
-                ),
-                pre_tokenizers.ByteLevel(
-                    add_prefix_space=getattr(self.original_tokenizer, "add_prefix_space", False),
-                    use_regex=False,
-                ),
-            ]
-        )
-
-        tokenizer.decoder = decoders.ByteLevel()
-        tokenizer.post_processor = processors.ByteLevel(trim_offsets=False)
-
-        return tokenizer
-
-
 class RobertaConverter(Converter):
     def converted(self) -> Tokenizer:
         ot = self.original_tokenizer
@@ -1504,7 +1462,6 @@ def converted(self) -> Tokenizer:
     "OpenAIGPTTokenizer": OpenAIGPTConverter,
     "PegasusTokenizer": PegasusConverter,
     "Qwen2Tokenizer": Qwen2Converter,
-    "Qwen2MoeTokenizer": Qwen2MoeConverter,
     "RealmTokenizer": BertConverter,
     "ReformerTokenizer": ReformerConverter,
     "RemBertTokenizer": RemBertConverter,

src/transformers/models/auto/tokenization_auto.py

@@ -353,8 +353,8 @@
             (
                 "qwen2_moe",
                 (
-                    "Qwen2MoeTokenizer",
-                    "Qwen2MoeTokenizerFast" if is_tokenizers_available() else None,
+                    "Qwen2Tokenizer",
+                    "Qwen2TokenizerFast" if is_tokenizers_available() else None,
                 ),
             ),
             ("rag", ("RagTokenizer", None)),

src/transformers/models/qwen2_moe/__init__.py

@@ -16,23 +16,14 @@
 from ...utils import (
     OptionalDependencyNotAvailable,
     _LazyModule,
-    is_tokenizers_available,
     is_torch_available,
 )
 
 
 _import_structure = {
     "configuration_qwen2_moe": ["QWEN2MOE_PRETRAINED_CONFIG_ARCHIVE_MAP", "Qwen2MoeConfig"],
-    "tokenization_qwen2_moe": ["Qwen2MoeTokenizer"],
 }
 
-try:
-    if not is_tokenizers_available():
-        raise OptionalDependencyNotAvailable()
-except OptionalDependencyNotAvailable:
-    pass
-else:
-    _import_structure["tokenization_qwen2_moe_fast"] = ["Qwen2MoeTokenizerFast"]
 
 try:
     if not is_torch_available():
@@ -50,15 +41,6 @@
 
 if TYPE_CHECKING:
     from .configuration_qwen2_moe import QWEN2MOE_PRETRAINED_CONFIG_ARCHIVE_MAP, Qwen2MoeConfig
-    from .tokenization_qwen2_moe import Qwen2MoeTokenizer
-
-    try:
-        if not is_tokenizers_available():
-            raise OptionalDependencyNotAvailable()
-    except OptionalDependencyNotAvailable:
-        pass
-    else:
-        from .tokenization_qwen2_moe_fast import Qwen2MoeTokenizerFast
 
     try:
         if not is_torch_available():