Merge pull request #9 from lilt/unk_hack

UNK hack
lilt · Dec 1, 2022 · e68969f · e68969f
2 parents cea83a2 + 8b69a77
commit e68969f
Showing 1 changed file with 3 additions and 0 deletions.
diff --git a/src/model_interface.cc b/src/model_interface.cc
@@ -130,6 +130,9 @@ void ModelInterface::InitializePieces() {
     }
   }
 
+  // Allow sentencepiece to encode literate "<unk>" tokens in the raw input.
+  user_defined_symbols.insert("<unk>");
+
   matcher_ = absl::make_unique<normalizer::PrefixMatcher>(user_defined_symbols);
 }