Fix for issue #147 for transformers>4.17.0 (#148)

Fix for issue #147 for transformers>4.17.0

Author: shirley-wu

bert_score/utils.py

@@ -10,7 +10,7 @@
 from packaging import version
 from torch.nn.utils.rnn import pad_sequence
 from tqdm.auto import tqdm
-from transformers import (AutoModel, AutoTokenizer, BertConfig, GPT2Tokenizer,
+from transformers import (AutoModel, AutoTokenizer, BertConfig, GPT2Tokenizer, RobertaTokenizer,
                           RobertaConfig, XLMConfig, XLNetConfig)
 from transformers import __version__ as trans_version
 
@@ -190,7 +190,7 @@ def sent_encode(tokenizer, sent):
     sent = sent.strip()
     if sent == "":
         return tokenizer.build_inputs_with_special_tokens([])
-    elif isinstance(tokenizer, GPT2Tokenizer):
+    elif isinstance(tokenizer, GPT2Tokenizer) or isinstance(tokenizer, RobertaTokenizer):
         # for RoBERTa and GPT-2
         if version.parse(trans_version) >= version.parse("4.0.0"):
             return tokenizer.encode(