small fix to remove unused eos in processor when it's not used. (hugg…

…ingface#23408)
sheonhan · Jun 1, 2023 · a76d30e · a76d30e
1 parent 0b69d43
commit a76d30e
Showing 1 changed file with 5 additions and 1 deletion.
diff --git a/src/transformers/convert_slow_tokenizer.py b/src/transformers/convert_slow_tokenizer.py
@@ -1175,7 +1175,11 @@ def post_processor(self):
             single = f"{(bos+':0 ') * add_bos}$A:0{(' '+eos+':0') * add_eos}"
             pair = f"{single}{(' '+bos+':1') * add_bos} $B:1{(' '+eos+':1') * add_eos}"
 
-            special_tokens = [(bos, bos_token_id), (eos, eos_token_id)]
+            special_tokens = []
+            if add_bos:
+                special_tokens.append((bos, bos_token_id))
+            if add_eos:
+                special_tokens.append((eos, eos_token_id))
             return processors.TemplateProcessing(single=single, pair=pair, special_tokens=special_tokens)
 
         else: