Remove compound word boundary markers from the gold lemmas

predict.py

@@ -100,7 +100,7 @@ def predict_lemma_and_pos(model, testset, outputdir):
 
 
 def remove_compund_word_boundary_markers(word):
-    return re.sub(r'(?<=\w)#(?=\w)', '', word)
+    return re.sub(r'(?<=[-–\w])#(?=\w)', '', word)
 
 
 def write_results_conllu(f, predicted):

preprocess_data.py

@@ -9,6 +9,7 @@
 import logging
 import os
 import os.path
+import re
 
 
 def main():
@@ -138,6 +139,7 @@ def preprocess(filename, inputdir, destdir, tag_map, aux_from_deprel):
                     num_non_continuous_index += 1
                     cols[0] = str(prev_index + 1)
 
+                cols[2] = remove_compound_word_boundaries(cols[2])
                 cols[3] = normalize_pos(cols[3], tag_map)
 
                 if is_invalid_pos_tag(cols[3]):
@@ -218,6 +220,10 @@ def split_into_sentences(lines):
         yield sentence
 
 
+def remove_compound_word_boundaries(lemma):
+    return re.sub(r'(?<=[-–\w])#(?=\w)', '', lemma)
+
+
 def normalize_pos(pos, tag_map):
     if '|' in pos:
         # FTB2 has some erroneous tags like "N|Sg|Ine"