fix more tests

junrae6454 · Sep 4, 2023 · d9829cd · d9829cd
1 parent 39bd27e
commit d9829cd
Show file tree

Hide file tree

Showing 2 changed files with 2 additions and 2 deletions.
diff --git a/bindings/python/src/trainers.rs b/bindings/python/src/trainers.rs
@@ -226,7 +226,7 @@ impl PyBpeTrainer {
                     if let Ok(content) = token.extract::<String>() {
                         Ok(tk::tokenizer::AddedToken::from(content, true))
                     } else if let Ok(mut token) = token.extract::<PyRefMut<PyAddedToken>>() {
-                        token.special = true;
+                        token.special = false;
                         Ok(token.get_token())
                     } else {
                         Err(exceptions::PyTypeError::new_err(

diff --git a/bindings/python/tests/bindings/test_tokenizer.py b/bindings/python/tests/bindings/test_tokenizer.py
@@ -19,7 +19,7 @@ def test_instantiate_with_content_only(self):
         assert type(added_token) == AddedToken
         assert str(added_token) == "<mask>"
         assert (
-            repr(added_token) == 'AddedToken("<mask>", rstrip=False, lstrip=False, single_word=False, normalized=True)'
+            repr(added_token) == 'AddedToken("<mask>", rstrip=False, lstrip=False, single_word=False, normalized=True, special=False)'
         )
         assert added_token.rstrip == False
         assert added_token.lstrip == False