Avoid a top-level import of tokenizers.

Author: mark

axlearn/experiments/text/gpt/c4_trainer.py

@@ -40,14 +40,12 @@
 ```
 """
 
-
 from axlearn.common.config import InstantiableConfig, config_for_class, config_for_function
 from axlearn.common.input_lm import lm_text_preprocessor
 from axlearn.common.utils import get_data_dir
 from axlearn.experiments.text.common import DataMixtureComponent, vocab
 from axlearn.experiments.text.gpt import fuji, gspmd
 from axlearn.experiments.text.gpt.common import mixture_train_input_source, tfds_input
-from axlearn.experiments.text.gpt.vocabulary_fuji_v3 import FujiV3Vocabulary
 from axlearn.experiments.trainer_config_utils import TrainerConfigFn
 
 
@@ -59,6 +57,10 @@ def _vocab_cfg(vocab_size: int):
     if vocab_size == 128 * 1024:
         return config_for_function(vocab).set(sentencepiece_model_name="bpe_128k_c4.model")
     if vocab_size == 128256:
+        # Avoid a global tokenizers dependency.
+        # pylint: disable-next=import-outside-toplevel
+        from axlearn.experiments.text.gpt.vocabulary_fuji_v3 import FujiV3Vocabulary
+
         # TikToken.
         return config_for_class(FujiV3Vocabulary).set(filename="Llama-3-tokenizer.json")
     raise ValueError(f"Tokenizer with vocab size {vocab_size} does not exist.")