Description
Traceback (most recent call last):
File "/mnt/d/ai/RLHF/test.py", line 3, in
tokenizer = AutoTokenizer.from_pretrained("/mnt/d/ai/pretrain_models/pangu", trust_remote_code=True)
File "/home/liu/anaconda3/envs/rlhf/lib/python3.10/site-packages/transformers/models/auto/tokenization_auto.py", line 774, in from_pretrained
return tokenizer_class.from_pretrained(pretrained_model_name_or_path, *inputs, **kwargs)
File "/home/liu/anaconda3/envs/rlhf/lib/python3.10/site-packages/transformers/tokenization_utils_base.py", line 2028, in from_pretrained
return cls._from_pretrained(
File "/home/liu/anaconda3/envs/rlhf/lib/python3.10/site-packages/transformers/tokenization_utils_base.py", line 2260, in _from_pretrained
tokenizer = cls(*init_inputs, **init_kwargs)
File "/home/liu/.cache/huggingface/modules/transformers_modules/pangu/tokenization_gptpangu.py", line 28, in init
super().init(**kwargs)
File "/home/liu/anaconda3/envs/rlhf/lib/python3.10/site-packages/transformers/tokenization_utils.py", line 367, in init
self._add_tokens(
File "/home/liu/anaconda3/envs/rlhf/lib/python3.10/site-packages/transformers/tokenization_utils.py", line 467, in _add_tokens
current_vocab = self.get_vocab().copy()
File "/home/liu/anaconda3/envs/rlhf/lib/python3.10/site-packages/transformers/tokenization_utils_base.py", line 1675, in get_vocab
raise NotImplementedError()
NotImplementedError