diff --git a/bindings/python/src/tokenizer.rs b/bindings/python/src/tokenizer.rs index ac3eab2fb..1cd431861 100644 --- a/bindings/python/src/tokenizer.rs +++ b/bindings/python/src/tokenizer.rs @@ -55,6 +55,8 @@ use crate::utils::{MaybeSizedIterator, PyBufferedIterator}; /// text. For example, with the added token ``"yesterday"``, and a normalizer in charge of /// lowercasing the text, the token could be extract from the input ``"I saw a lion /// Yesterday"``. +/// special (:obj:`bool`, defaults to :obj:`False` with :meth:`~tokenizers.Tokenizer.add_tokens` and :obj:`False` with :meth:`~tokenizers.Tokenizer.add_special_tokens`): +/// Defines whether this token should be skipped when decoding. /// #[pyclass(dict, module = "tokenizers", name = "AddedToken")] pub struct PyAddedToken { @@ -179,6 +181,12 @@ impl PyAddedToken { &self.content } + /// Set the content of this :obj:`AddedToken` + #[setter] + fn set_content(&self, content: String){ + self.get_token().content = content + } + /// Get the value of the :obj:`rstrip` option #[getter] fn get_rstrip(&self) -> bool { diff --git a/bindings/python/tests/bindings/test_tokenizer.py b/bindings/python/tests/bindings/test_tokenizer.py index ae58b157c..f8e2a0b1d 100644 --- a/bindings/python/tests/bindings/test_tokenizer.py +++ b/bindings/python/tests/bindings/test_tokenizer.py @@ -16,6 +16,7 @@ class TestAddedToken: def test_instantiate_with_content_only(self): added_token = AddedToken("") + added_token.content = "" assert type(added_token) == AddedToken assert str(added_token) == "" assert (