PoC using polib and how to silent padpo on false positives

padpo/checkers/__init__.py

@@ -3,14 +3,12 @@
 from padpo.checkers.fuzzy import FuzzyChecker
 from padpo.checkers.glossary import GlossaryChecker
 from padpo.checkers.grammalecte import GrammalecteChecker
-from padpo.checkers.linelength import LineLengthChecker
 from padpo.checkers.nbsp import NonBreakableSpaceChecker
 
 checkers = [
     EmptyChecker(),
     FuzzyChecker(),
     GrammalecteChecker(),
     GlossaryChecker(),
-    LineLengthChecker(),
     NonBreakableSpaceChecker(),
 ]

padpo/checkers/baseclass.py

@@ -26,6 +26,23 @@ def check_item(self, item: PoItem):
         """Check an item in a `*.po` file."""
         return NotImplementedError
 
+    def match_silent(self, regex, item: PoItem, match: str):
+        """ Generic function to silent a warning directly raised by the checker in the *match* field """
+        list_ok = regex.findall(item.entry.tcomment)
+        return any([match in term for term in list_ok])
+
+
+    def add_warning(self, level: str, item: PoItem, text: str, prefix:str=None, match:str=None, suffix:str=None):
+        silent_method_names = [self.__getattribute__(name) for name in self.__dir__() if name.startswith('silent_')]
+        if any([m(item, text, prefix, match, suffix) for m in silent_method_names if callable(m)]):
+                return
+        if level == 'warning':
+            item.add_warning(self.name, text)
+        else:
+            item.add_error(self.name, text)
+
+
+
 
 def replace_quotes(match):
     """Replace match with « xxxxxxx »."""

padpo/checkers/empty.py

@@ -12,5 +12,5 @@ class EmptyChecker(Checker):
 
     def check_item(self, item: PoItem):
         """Check an item in a `*.po` file."""
-        if item.msgid_full_content and not item.msgstr_full_content:
+        if item.entry.msgid and not item.entry.msgstr:
             item.add_warning(self.name, "This entry is not translated yet.")

padpo/checkers/fuzzy.py

@@ -11,5 +11,5 @@ class FuzzyChecker(Checker):
 
     def check_item(self, item: PoItem):
         """Check an item in a `*.po` file."""
-        if item.fuzzy:
+        if item.entry.fuzzy:
             item.add_warning(self.name, "This entry is tagged as fuzzy.")

padpo/checkers/glossary.py

@@ -10,14 +10,20 @@ class GlossaryChecker(Checker):
     """Checker for glossary usage."""
 
     name = "Glossary"
+    re_silent = re.compile('silent-glossary:`([^`]*)`')
+
+    def silent_glossary(self, item: PoItem, text: str, prefix, match, suffix):
+        for term in self.re_silent.findall(item.entry.tcomment):
+            if term in text[:text.find("that is not translated")]:
+                return True
 
     def check_item(self, item: PoItem):
         """Check an item in a `*.po` file."""
-        if not item.msgstr_full_content:
+        if not item.entry.msgstr:
             return  # no warning
         original_content = item.msgid_rst2txt.lower()
         original_content = re.sub(r"« .*? »", "", original_content)
-        translated_content = item.msgstr_full_content.lower()
+        translated_content = item.entry.msgstr.lower()
         for word, translations in glossary.items():
             if re.match(fr"\b{word.lower()}\b", original_content):
                 for translated_word in translations:
@@ -30,10 +36,9 @@ def check_item(self, item: PoItem):
                         possibilities += '" or "'
                     possibilities += translations[-1]
                     possibilities += '"'
-                    item.add_warning(
-                        self.name,
+                    self.add_warning('warning', item,
                         f'Found "{word}" that is not translated in '
-                        f"{possibilities} in ###{item.msgstr_full_content}"
+                        f"{possibilities} in ###{item.entry.msgstr}"
                         "###.",
                     )
 

padpo/checkers/grammalecte.py

@@ -23,6 +23,16 @@ class GrammalecteChecker(Checker):
     """Checker for grammar errors."""
 
     name = "Grammalecte"
+    capital_silent_re = re.compile('silent-capital:`([^`]*)`')
+    unknown_word_silent_re = re.compile('silent-unknown:`([^`]*)`')
+
+    def silent_capital(self, item: PoItem, text: str, prefix: str, match: str, suffix:str):
+        return (text.startswith("Majuscule en début de phrase") and
+                self.match_silent(self.capital_silent_re, item, match))
+
+    def silent_unknown_word(self, item: PoItem, text: str, prefix: str, match: str, suffix: str):
+        return (text.startswith("Mot inconnu") and
+                self.match_silent(self.unknown_word_silent_re, item, match))
 
     def __init__(self):
         """Initialiser."""
@@ -54,9 +64,11 @@ def manage_warnings(self, warnings: GrammalecteMessage, pofile: PoFile) -> None:
             item = pofile.content[item_index]
             start = max(0, warning.start - 40)
             end = warning.end + 10
-            item.add_warning(
-                self.name,
+            self.add_warning("warning", item,
                 f"{warning.message} => " f"###{item.msgstr_rst2txt[start:end]}###",
+                             item.msgstr_rst2txt[start:warning.start],
+                             item.msgstr_rst2txt[warning.start:warning.end],
+                             item.msgstr_rst2txt[warning.end:end]
             )
 
     def filter_out_grammar_error(self, warning: GrammalecteMessage) -> bool:

padpo/checkers/nbsp.py

@@ -10,6 +10,15 @@ class NonBreakableSpaceChecker(Checker):
     """Checker for missing non breakable spaces."""
 
     name = "NBSP"
+    after_silent_re=re.compile('silent-nbsp-after:`([^`]*)`')
+
+
+    def silent_nbsp_after(self, item: PoItem, text: str, prefix: str, match: str, suffix: str):
+        list_ok = self.after_silent_re.findall(item.entry.tcomment)
+        for term in list_ok:
+            if term in prefix:
+                return True
+
 
     def check_item(self, item: PoItem):
         """Check an item in a `*.po` file."""
@@ -31,16 +40,14 @@ def check_item(self, item: PoItem):
                     self.__add_message_space_before(item, prefix, match, suffix)
 
     def __add_message(self, item, prefix, match, suffix):
-        item.add_error(
-            self.name,
+        self.add_warning('error', item,
             "Space should be replaced with a non-breakable space in "
-            f'"{match}": between ###{prefix}### and ###{suffix}###',
+            f'"{match}": between ###{prefix}### and ###{suffix}###', prefix, match, suffix,
         )
 
     def __add_message_space_before(self, item, prefix, match, suffix):
-        item.add_error(
-            self.name,
+        self.add_warning('error', item,
             f"There should be a non-breakable space before "
             f'"{match[1:]}": between ###{prefix}{match[0]}### and '
-            f"###{match[1:]}{suffix}###",
+            f"###{match[1:]}{suffix}###", prefix, match, suffix,
         )

padpo/pofile.py

@@ -2,6 +2,7 @@
 
 import re
 from typing import List
+from polib import pofile
 
 import simplelogging
 
@@ -11,62 +12,31 @@
 class PoItem:
     """Translation item."""
 
-    def __init__(self, path, lineno):
+    def __init__(self, entry):
         """Initializer."""
-        self.path = path[3:]
-        self.lineno_start = lineno
-        self.lineno_end = lineno
-        self.parsing_msgid = None
-        self.msgid = []
-        self.msgstr = []
-        self.fuzzy = False
         self.warnings = []
         self.inside_pull_request = False
+        self.entry = entry
 
-    def append_line(self, line):
-        """Append a line of a `*.po` file to the item."""
-        self.lineno_end += 1
-        if line.startswith("msgid"):
-            self.parsing_msgid = True
-            self.msgid.append(line[7:-2])
-        elif line.startswith("msgstr"):
-            self.parsing_msgid = False
-            self.msgstr.append(line[8:-2])
-        elif line.startswith("#, fuzzy"):
-            self.fuzzy = True
-        elif line.startswith('"'):
-            if self.parsing_msgid:
-                self.msgid.append(line[1:-2])
-            elif not self.parsing_msgid is None:
-                self.msgstr.append(line[1:-2])
 
     def __str__(self):
         """Return string representation."""
         return (
-            f"    - {self.msgid_full_content}\n"
-            f"        => {self.msgstr_full_content}\n"
+            f"    - {self.entry.msgid}\n"
+            f"        => {self.entry.msgstr}\n"
             f"        => {self.msgstr_rst2txt}\n"
         )
 
-    @property
-    def msgid_full_content(self):
-        """Full content of the msgid."""
-        return "".join(self.msgid)
-
-    @property
-    def msgstr_full_content(self):
-        """Full content of the msgstr."""
-        return "".join(self.msgstr)
 
     @property
     def msgid_rst2txt(self):
         """Full content of the msgid (reStructuredText escaped)."""
-        return self.rst2txt(self.msgid_full_content)
+        return self.rst2txt(self.entry.msgid)
 
     @property
     def msgstr_rst2txt(self):
         """Full content of the msgstr (reStructuredText escaped)."""
-        return self.rst2txt(self.msgstr_full_content)
+        return self.rst2txt(self.entry.msgstr)
 
     @staticmethod
     def rst2txt(text):
@@ -112,18 +82,19 @@ def __init__(self, path=None):
     def parse_file(self, path):
         """Parse a `*.po` file according to its path."""
         # TODO assert path is a file, not a dir
-        item = None
-        with open(path, encoding="utf8") as f:
-            for lineno, line in enumerate(f):
-                if line.startswith("#: "):
-                    if item:
-                        self.content.append(item)
-                    item = PoItem(line, lineno + 1)
-                elif item:
-                    item.append_line(line)
-        if item:
+        self.pofile = pofile(path)
+        for entry in self.pofile:
+            item = PoItem(entry)
             self.content.append(item)
 
+        # lineno_end only needed for github patch. May be removed ?
+        import sys
+        lineno_end = sys.maxsize
+        for item in sorted(self.content, key=lambda x: x.entry.linenum, reverse=True):
+            item.lineno_end = lineno_end
+            lineno_end = item.entry.linenum - 1
+
+
     def __str__(self):
         """Return string representation."""
         ret = f"Po file: {self.path}\n"
@@ -148,7 +119,7 @@ def display_warnings(self, pull_request_info=None):
                         message.text,
                         extra={
                             "pofile": self.path,
-                            "poline": item.lineno_start,
+                            "poline": item.entry.linenum,
                             "checker": message.checker_name,
                             "leveldesc": "error",
                         },
@@ -159,7 +130,7 @@ def display_warnings(self, pull_request_info=None):
                         message.text,
                         extra={
                             "pofile": self.path,
-                            "poline": item.lineno_start,
+                            "poline": item.entry.linenum,
                             "checker": message.checker_name,
                             "leveldesc": "warning",
                         },
@@ -178,7 +149,7 @@ def tag_in_pull_request(self, pull_request_info):
                 item.inside_pull_request = False
             for lineno_diff in self.lines_in_diff(diff):
                 for item in self.content:
-                    if item.lineno_start <= lineno_diff <= item.lineno_end:
+                    if item.entry.linenum <= lineno_diff <= item.lineno_end:
                         item.inside_pull_request = True
 
     @staticmethod