feat(pipeline): IE errors translation

Author: kamilogorek

src/sentry/lang/javascript/errorlocale.py

@@ -0,0 +1,98 @@
+from __future__ import absolute_import, print_function
+
+import six
+import os
+import io
+import re
+
+LOCALES_DIR = os.path.join(os.path.dirname(__file__), '../../data/error-locale')
+TARGET_LOCALE = 'en-US'
+
+translation_lookup_table = set()
+target_locale_lookup_table = {}
+
+for locale in os.listdir(LOCALES_DIR):
+    fn = os.path.join(LOCALES_DIR, locale)
+    if not os.path.isfile(fn):
+        continue
+
+    with io.open(fn, encoding='utf-8') as f:
+        for line in f:
+            key, translation = line.split(',', 1)
+            translation = translation.strip()
+
+            if TARGET_LOCALE in locale:
+                target_locale_lookup_table[key] = translation
+            else:
+                translation_regexp = re.escape(translation)
+                translation_regexp = translation_regexp.replace(
+                    '\%s', r'(?P<format_string_data>[a-zA-Z0-9-_\$]+)')
+                # Some errors are substrings of more detailed ones, so we need exact match
+                translation_regexp = re.compile('^' + translation_regexp + '$')
+                translation_lookup_table.add((translation_regexp, key))
+
+
+def find_translation(message):
+    for translation in translation_lookup_table:
+        translation_regexp, key = translation
+        match = translation_regexp.search(message)
+
+        if match is not None:
+            format_string_data = match.groupdict().get('format_string_data')
+
+            if format_string_data is None:
+                return [key, None]
+            else:
+                return [key, format_string_data]
+
+    return [None, None]
+
+
+def format_message(message, data):
+    return message.replace('%s', data)
+
+message_type_regexp = re.compile('^(?P<type>[a-zA-Z]*Error): (?P<message>.*)')
+
+
+def translate_message(original_message):
+    if not isinstance(original_message, six.string_types):
+        return original_message
+
+    type = None
+    message = original_message.strip()
+
+    # Handle both cases. Just a message and message preceeded with error type
+    # eg. `ReferenceError: foo`, `TypeError: bar`
+    match = message_type_regexp.search(message)
+
+    if match is not None:
+        type = match.groupdict().get('type')
+        message = match.groupdict().get('message')
+
+    translation, format_string_data = find_translation(message)
+
+    if translation is None:
+        return original_message
+    else:
+        translated_message = target_locale_lookup_table.get(translation, original_message)
+
+        if type is not None:
+            translated_message = type + ': ' + translated_message
+
+        if format_string_data is None:
+            return translated_message
+        else:
+            return format_message(translated_message, format_string_data)
+
+
+def translate_exception(data):
+    if 'sentry.interfaces.Message' in data:
+        data['sentry.interfaces.Message']['message'] = translate_message(
+            data['sentry.interfaces.Message']['message'])
+
+    if 'sentry.interfaces.Exception' in data:
+        for entry in data['sentry.interfaces.Exception']['values']:
+            if 'value' in entry:
+                entry['value'] = translate_message(entry['value'])
+
+    return data

src/sentry/lang/javascript/plugin.py

@@ -7,10 +7,12 @@
 
 from .processor import JavaScriptStacktraceProcessor
 from .errormapping import rewrite_exception
+from .errorlocale import translate_exception
 
 
 def preprocess_event(data):
     rewrite_exception(data)
+    translate_exception(data)
     fix_culprit(data)
     inject_device_data(data)
     generate_modules(data)

tests/sentry/lang/javascript/test_errorlocale.py

@@ -0,0 +1,65 @@
+from __future__ import absolute_import, unicode_literals
+
+from sentry.testutils import TestCase
+from sentry.lang.javascript.errorlocale import translate_message
+
+
+class ErrorLocaleTest(TestCase):
+    def test_basic_translation(self):
+        actual = 'Type mismatch'
+        expected = translate_message('Typenkonflikt')
+        assert actual == expected
+
+    def test_unicode_translation(self):
+        expected = 'Division by zero'
+        actual = translate_message(u'Divisi\xf3n por cero')
+        assert actual == expected
+
+    def test_same_translation(self):
+        expected = 'Out of memory'
+        actual = translate_message('Out of memory')
+        assert actual == expected
+
+    def test_unknown_translation(self):
+        expected = 'Some unknown message'
+        actual = translate_message('Some unknown message')
+        assert actual == expected
+
+    def test_translation_with_type(self):
+        expected = 'RangeError: Subscript out of range'
+        actual = translate_message('RangeError: Indeks poza zakresem')
+        assert actual == expected
+
+    def test_translation_with_type_and_colon(self):
+        expected = 'RangeError: Cannot define property: object is not extensible'
+        actual = translate_message(
+            u'RangeError: Nie mo\u017cna zdefiniowa\u0107 w\u0142a\u015bciwo\u015bci: obiekt nie jest rozszerzalny')
+        assert actual == expected
+
+    def test_interpolated_translation(self):
+        expected = 'Type \'foo\' not found'
+        actual = translate_message(u'Nie odnaleziono typu \u201efoo\u201d')
+        assert actual == expected
+
+    def test_interpolated_translation_with_colon(self):
+        expected = '\'this\' is not of expected type: foo'
+        actual = translate_message(
+            u'Typ obiektu \u201ethis\u201d jest inny ni\u017c oczekiwany: foo')
+        assert actual == expected
+
+    def test_interpolated_translation_with_colon_in_front(self):
+        expected = 'foo: an unexpected failure occurred while trying to obtain metadata information'
+        actual = translate_message(
+            u'foo: wyst\u0105pi\u0142 nieoczekiwany b\u0142\u0105d podczas pr\xf3by uzyskania informacji o metadanych')
+        assert actual == expected
+
+    def test_interpolated_translation_with_type(self):
+        expected = 'TypeError: Type \'foo\' not found'
+        actual = translate_message(u'TypeError: Nie odnaleziono typu \u201efoo\u201d')
+        assert actual == expected
+
+    def test_interpolated_translation_with_type_and_colon(self):
+        expected = 'ReferenceError: Cannot modify property \'foo\': \'length\' is not writable'
+        actual = translate_message(
+            u'ReferenceError: Nie mo\u017cna zmodyfikowa\u0107 w\u0142a\u015bciwo\u015bci \u201efoo\u201d: warto\u015b\u0107 \u201elength\u201d jest niezapisywalna')
+        assert actual == expected

tests/sentry/lang/javascript/test_plugin.py

@@ -239,6 +239,40 @@ def test_inlined_sources(self, mock_discover_sourcemap, mock_fetch_file):
         assert not frame.post_context
         assert frame.data['sourcemap'] == 'http://example.com/test.min.js'
 
+    @responses.activate
+    def test_error_message_translations(self):
+        data = {
+            'message': 'hello',
+            'platform': 'javascript',
+            'sentry.interfaces.Message': {
+                'message': u'ReferenceError: Impossible de d\xe9finir une propri\xe9t\xe9 \xab foo \xbb : objet non extensible'
+            },
+            'sentry.interfaces.Exception': {
+                'values': [
+                    {
+                        'type': 'Error',
+                        'value': u'P\u0159\xedli\u0161 mnoho soubor\u016f'
+                    },
+                    {
+                        'type': 'Error',
+                        'value': u'foo: wyst\u0105pi\u0142 nieoczekiwany b\u0142\u0105d podczas pr\xf3by uzyskania informacji o metadanych'
+                    }
+                ],
+            }
+        }
+
+        resp = self._postWithHeader(data)
+        assert resp.status_code, 200
+
+        event = Event.objects.get()
+
+        message = event.interfaces['sentry.interfaces.Message']
+        assert message.message == 'ReferenceError: Cannot define property \'foo\': object is not extensible'
+
+        exception = event.interfaces['sentry.interfaces.Exception']
+        assert exception.values[0].value == 'Too many files'
+        assert exception.values[1].value == 'foo: an unexpected failure occurred while trying to obtain metadata information'
+
     @responses.activate
     def test_sourcemap_source_expansion(self):
         responses.add(