File tree Expand file tree Collapse file tree 1 file changed +1
-1
lines changed
examples/qualcomm/oss_scripts/llama Expand file tree Collapse file tree 1 file changed +1
-1
lines changed Original file line number Diff line number Diff line change 5757from executorch .examples .models .llama .source_transformation .quantize import (
5858 get_quant_embedding_transform ,
5959)
60- from executorch .examples .models .llama .tokenizer .tiktoken import Tokenizer as Tiktoken
6160from executorch .examples .qualcomm .oss_scripts .llama .model .static_llama import (
6261 LlamaModel ,
6362 ModelArgs ,
7776from executorch .extension .llm .export .builder import DType
7877from pytorch_tokenizers import get_tokenizer
7978from pytorch_tokenizers .llama2c import Llama2cTokenizer as SentencePieceTokenizer
79+ from pytorch_tokenizers import TiktokenTokenizer as Tiktoken
8080
8181from torch .ao .quantization .observer import MinMaxObserver
8282from torch .ao .quantization .quantize_pt2e import convert_pt2e , prepare_pt2e
You can’t perform that action at this time.
0 commit comments