Pass homoglyph variants of names to trademark checks and NARC (#23842)

* Pass homoglyph variants of names to trademark checks and NARC

Author: diox

requirements/prod.txt

@@ -799,6 +799,9 @@ cachetools==5.5.2 \
 sqlparse==0.5.3 \
     --hash=sha256:09f67787f56a0b16ecdbde1bfc7f5d9c3371ca683cfeaa8e6ff60b4807ec9272 \
     --hash=sha256:cf2196ed3418f3ba5de6af7e82c694a9fbdbfecccdfc72e281548517081f16ca
+homoglyphs-fork==2.1.1 \
+    --hash=sha256:42b8e77e82fff9eaa63422e3ff3149daa0c8ecc03fb2381e6aae5a961c3104c2 \
+    --hash=sha256:d0994a1f5fa74b6bd0a25e714eb527a43335c80d0e86abf6f099d0e108dc94bd
 yara-python==4.5.4 \
     --hash=sha256:0b9de86fbe8a646c0644df9e1396d6941dc6ed0f89be2807e6c52ab39161fd9f \
     --hash=sha256:0e762e6c5b47ddf30b0128ba723da46fcc2aa7959a252748497492cb452d1c84 \

src/olympia/addons/tests/test_utils_.py

@@ -37,6 +37,11 @@
         ('Firefox add-on for Firefox', True, True),
         ('Foobarfor Firefox', False, False),
         ('F i r e f o x', False, False),
+        ('F î r \u0435fox', False, False),
+        ('FiRѐF0 x', False, False),
+        ('F i r \u0435 f o x', False, False),
+        ('F\u2800i r e f o x', False, False),
+        ('F\u2800 i r \u0435 f o x', False, False),
         ('Fïrefox is great', False, False),
         ('Foobarfor Firefox!', False, False),
         ('Better Privacy for Firefox!', True, False),

src/olympia/addons/utils.py

@@ -9,6 +9,7 @@
 from olympia import amo, core
 from olympia.access.acl import action_allowed_for
 from olympia.amo.utils import (
+    generate_lowercase_homoglyphs_variants_for_string,
     normalize_string_for_name_checks,
     verify_condition_with_locales,
 )
@@ -30,24 +31,27 @@ def verify_mozilla_trademark(name, user, *, form=None):
     )
 
     def _check(name):
-        fully_normalized_name = normalize_string_for_name_checks(name).lower()
         name_without_punctuation = normalize_string_for_name_checks(
             name, categories_to_strip=('P')
         ).lower()
 
-        for symbol in amo.MOZILLA_TRADEMARK_SYMBOLS:
-            symbol_count = fully_normalized_name.count(symbol)
-            violates_trademark = symbol_count > 1 or (
-                symbol_count >= 1
-                and not name_without_punctuation.endswith(f' for {symbol}')
-            )
+        variants = generate_lowercase_homoglyphs_variants_for_string(
+            normalize_string_for_name_checks(name)
+        )
+
+        for variant in variants:
+            for symbol in amo.MOZILLA_TRADEMARK_SYMBOLS:
+                symbol_count = variant.count(symbol)
+                violates_trademark = symbol_count > 1 or (
+                    symbol_count >= 1
+                    and not name_without_punctuation.endswith(f' for {symbol}')
+                )
 
-            if violates_trademark:
-                raise forms.ValidationError(
-                    gettext(
+                if violates_trademark:
+                    msg = gettext(
                         'Add-on names cannot contain the Mozilla or Firefox trademarks.'
                     )
-                )
+                    raise forms.ValidationError(msg)
 
     if not skip_trademark_check:
         verify_condition_with_locales(

src/olympia/amo/tests/test_utils.py

@@ -29,6 +29,7 @@
     create_signed_url_for_file_backup,
     download_file_contents_from_backup_storage,
     extract_colors_from_image,
+    generate_lowercase_homoglyphs_variants_for_string,
     get_locale_from_lang,
     id_to_path,
     is_safe_url,
@@ -453,6 +454,24 @@ def test_normalize_string_for_name_checks_with_specific_category(value, expected
     )
 
 
+@pytest.mark.parametrize(
+    'value, expected',
+    [
+        ('aBc', {'abc'}),
+        ('\u0430bc', {'abc'}),
+        ('l\u04300', {'iao', 'lao'}),
+        ('𝐪1lt', {'qiit', 'qilt', 'qlit', 'qllt'}),
+        ('bеta', {'beta'}),
+        ('қѺʍѕ', {'koms'}),
+        ('Zoom', {'zoom'}),
+        ('ТЕСТ0n1𝓀', {'tectonik', 'tectonlk'}),
+        ('ТЄСТ0n1к', {'tectonik', 'tectonlk'}),
+    ],
+)
+def test_generate_lowercase_homoglyphs_variants_for_string(value, expected):
+    assert generate_lowercase_homoglyphs_variants_for_string(value) == expected
+
+
 @pytest.mark.parametrize(
     'value, expected',
     [

src/olympia/amo/utils.py

@@ -46,6 +46,7 @@
 
 import basket
 import colorgram
+import homoglyphs_fork
 import html5lib
 import markupsafe
 import pytz
@@ -494,6 +495,37 @@ def normalize_string_for_name_checks(
     return value
 
 
+def generate_lowercase_homoglyphs_variants_for_string(value):
+    """Generates a set of lowercase homoglyph variants for a given string.
+
+    Value passed as argument is a string that is expected to have gone through
+    normalization to remove characters we don't want first, see
+    normalize_string_for_name_checks().
+    """
+    # These are not normally considered confusables, but we think they should.
+    additional_replacements = {
+        'e': ('Э', '℈', 'Є', '€', 'Ꞓ'),
+        'k': ('ĸ', 'κ', 'к', 'қ', 'ҝ', 'ҟ', 'ҡ', 'ᴋ'),
+        'm': ('ʍ', 'м', 'ᴍ'),
+        'o': ('Ѻ', 'ѻ'),
+    }
+    additional_replacement_table = dict(
+        itertools.chain(
+            *(
+                list(zip(map(ord, letters), itertools.repeat(ord(replacement))))
+                for replacement, letters in additional_replacements.items()
+            )
+        )
+    )
+    value = value.translate(additional_replacement_table)
+    homoglyphs = homoglyphs_fork.Homoglyphs(
+        languages={'en'},
+        strategy=homoglyphs_fork.STRATEGY_LOAD,
+        ascii_range=range(ord('A'), ord('z') + 1),
+    )
+    return {variant.lower() for variant in homoglyphs.to_ascii(value)}
+
+
 def slug_validator(slug, message=validate_slug.message):
     """
     Raise an error if the string has any punctuation characters.

src/olympia/scanners/tasks.py

@@ -18,7 +18,11 @@
 from olympia.addons.models import Addon
 from olympia.amo.celery import create_chunked_tasks_signatures, task
 from olympia.amo.decorators import use_primary_db
-from olympia.amo.utils import attach_trans_dict, normalize_string_for_name_checks
+from olympia.amo.utils import (
+    attach_trans_dict,
+    generate_lowercase_homoglyphs_variants_for_string,
+    normalize_string_for_name_checks,
+)
 from olympia.constants.scanners import (
     ABORTED,
     ABORTING,
@@ -228,10 +232,21 @@ def _run_narc(*, version, run_action_on_match):
             ),
         ):
             value = str(value)
-            variants = [(value, False)]
+            variants = [(value, None)]
             if (normalized_value := normalize_string_for_name_checks(value)) != value:
-                variants.append((normalized_value, True))
-            for variant, is_normalized in variants:
+                variants.append((normalized_value, 'normalized'))
+            homoglyph_variants = generate_lowercase_homoglyphs_variants_for_string(
+                normalized_value
+            )
+            if homoglyph_variants:
+                variants.extend(
+                    (homoglyph_variant, 'homoglyph')
+                    for homoglyph_variant in homoglyph_variants
+                    if homoglyph_variant != value.lower()
+                    and homoglyph_variant != normalized_value.lower()
+                )
+
+            for variant, variant_type in variants:
                 if match := definition.search(variant):
                     result = {
                         'rule': rule.name,
@@ -243,8 +258,8 @@ def _run_narc(*, version, run_action_on_match):
                             'span': match.span(),
                         },
                     }
-                    if is_normalized:
-                        result['meta']['is_normalized'] = True
+                    if variant_type is not None:
+                        result['meta']['variant'] = variant_type
                         result['meta']['original_string'] = value
                     results.add(json.dumps(result, sort_keys=True))
 

src/olympia/scanners/tests/test_tasks.py

@@ -292,7 +292,7 @@ def test_run(self, incr_mock):
         assert narc_result.results == [
             {
                 'meta': {
-                    'is_normalized': True,
+                    'variant': 'normalized',
                     'locale': 'en-us',
                     'pattern': '.*',
                     'source': 'db_addon',
@@ -307,44 +307,44 @@ def test_run(self, incr_mock):
             },
             {
                 'meta': {
-                    'is_normalized': True,
-                    'locale': None,
+                    'locale': 'en-us',
                     'pattern': '.*',
-                    'source': 'author',
+                    'source': 'db_addon',
                     'span': [
                         0,
-                        3,
+                        27,
                     ],
-                    'string': 'Foo',
-                    'original_string': 'Fôo',
+                    'string': 'My Fancy WebExtension Addon',
                 },
                 'rule': 'always_match_rule',
             },
             {
                 'meta': {
-                    'is_normalized': True,
+                    'variant': 'normalized',
                     'locale': None,
                     'pattern': '.*',
-                    'source': 'xpi',
+                    'source': 'author',
                     'span': [
                         0,
-                        19,
+                        3,
                     ],
-                    'string': 'MyWebExtensionAddon',
-                    'original_string': 'My WebExtension Addon',
+                    'string': 'Foo',
+                    'original_string': 'Fôo',
                 },
                 'rule': 'always_match_rule',
             },
             {
                 'meta': {
-                    'locale': 'en-us',
+                    'variant': 'normalized',
+                    'locale': None,
                     'pattern': '.*',
-                    'source': 'db_addon',
+                    'source': 'xpi',
                     'span': [
                         0,
-                        27,
+                        19,
                     ],
-                    'string': 'My Fancy WebExtension Addon',
+                    'string': 'MyWebExtensionAddon',
+                    'original_string': 'My WebExtension Addon',
                 },
                 'rule': 'always_match_rule',
             },
@@ -379,6 +379,104 @@ def test_run(self, incr_mock):
             ]
         )
 
+    @mock.patch('olympia.scanners.tasks.statsd.incr')
+    def test_run_normalized_match(self, incr_mock):
+        self.addon.name = 'My\u2800 Fäncy WebExtension 𝕒ddon'
+        self.addon.save()
+
+        rule = ScannerRule.objects.create(
+            name='always_match_rule',
+            scanner=NARC,
+            definition='MyFancyWebExtensionAddon',
+        )
+
+        run_narc_on_version(self.version.pk)
+
+        scanner_results = ScannerResult.objects.all()
+        assert len(scanner_results) == 1
+        narc_result = scanner_results[0]
+        assert narc_result.scanner == NARC
+        assert narc_result.upload is None
+        assert narc_result.version == self.version
+        assert narc_result.has_matches
+        assert list(narc_result.matched_rules.all()) == [rule]
+        assert len(narc_result.results) == 1
+        assert narc_result.results == [
+            {
+                'meta': {
+                    'locale': 'en-us',
+                    'original_string': 'My⠀ Fäncy WebExtension 𝕒ddon',
+                    'pattern': 'MyFancyWebExtensionAddon',
+                    'source': 'db_addon',
+                    'span': [
+                        0,
+                        24,
+                    ],
+                    'string': 'MyFancyWebExtensionaddon',
+                    'variant': 'normalized',
+                },
+                'rule': 'always_match_rule',
+            },
+        ]
+        assert incr_mock.called
+        assert incr_mock.call_count == 3
+        incr_mock.assert_has_calls(
+            [
+                mock.call('devhub.narc.has_matches'),
+                mock.call(f'devhub.narc.rule.{rule.id}.match'),
+                mock.call('devhub.narc.success'),
+            ]
+        )
+
+    @mock.patch('olympia.scanners.tasks.statsd.incr')
+    def test_run_homoglyph_match(self, incr_mock):
+        self.addon.name = 'My\u2800 Fäncy W\u0435bExtѐnsion addon'
+        self.addon.save()
+
+        rule = ScannerRule.objects.create(
+            name='always_match_rule',
+            scanner=NARC,
+            definition='MyFancyWebExtensionAddon',
+        )
+
+        run_narc_on_version(self.version.pk)
+
+        scanner_results = ScannerResult.objects.all()
+        assert len(scanner_results) == 1
+        narc_result = scanner_results[0]
+        assert narc_result.scanner == NARC
+        assert narc_result.upload is None
+        assert narc_result.version == self.version
+        assert narc_result.has_matches
+        assert list(narc_result.matched_rules.all()) == [rule]
+        assert len(narc_result.results) == 1
+        assert narc_result.results == [
+            {
+                'meta': {
+                    'locale': 'en-us',
+                    'original_string': 'My⠀ Fäncy WеbExtѐnsion addon',
+                    'pattern': 'MyFancyWebExtensionAddon',
+                    'source': 'db_addon',
+                    'span': [
+                        0,
+                        24,
+                    ],
+                    'string': 'myfancywebextensionaddon',
+                    'variant': 'homoglyph',
+                },
+                'rule': 'always_match_rule',
+            },
+        ]
+        assert incr_mock.called
+        assert incr_mock.call_count == 3
+        incr_mock.assert_has_calls(
+            [
+                mock.call('devhub.narc.has_matches'),
+                mock.call(f'devhub.narc.rule.{rule.id}.match'),
+                mock.call('devhub.narc.success'),
+            ]
+        )
+
     @mock.patch('olympia.scanners.tasks.statsd.timer')
     def test_calls_statsd_timer(self, timer_mock):
         run_narc_on_version(self.version.pk)
@@ -515,7 +613,7 @@ def test_run_multiple_authors_match(self, incr_mock):
         assert narc_result.results == [
             {
                 'meta': {
-                    'is_normalized': True,
+                    'variant': 'normalized',
                     'span': [0, 3],
                     'locale': None,
                     'source': 'author',