fix(symbolicator): Tests are no longer allowed failures

.travis.yml

@@ -239,12 +239,6 @@ matrix:
     - language: node_js
       env: STORYBOOK_BUILD=1
 
-    # XXX(ja): Should be removed once symbolicator's interface is more stable or
-    # tests have been refactored to be more lenient. At the moment, snapshots
-    # break every time when a new attribute is added or some secondary attribute
-    # is changed.
-    - env: TEST_SUITE=symbolicator
-
 notifications:
   webhooks:
     urls:

tests/symbolicator/__init__.py

@@ -2,6 +2,8 @@
 
 import os
 
+from sentry.utils.safe import get_path
+
 
 def get_fixture_path(name):
     return os.path.join(
@@ -13,11 +15,49 @@ def get_fixture_path(name):
     )
 
 
+def strip_frame(frame):
+    if frame:
+        frame = {
+            'data': {'symbolicator_status': get_path(frame, 'data', 'symbolicator_status')},
+            'function': frame.get('function'),
+            'instruction_addr': frame.get('instruction_addr'),
+            'symbol': frame.get('symbol'),
+            'package': frame.get('package'),
+            'lineno': frame.get('lineno'),
+            'in_app': frame.get('in_app'),
+            'trust': frame.get('trust'),
+        }
+
+    return frame
+
+
+def strip_stacktrace(stacktrace):
+    if stacktrace:
+        stacktrace = dict(stacktrace)
+        stacktrace['frames'] = [strip_frame(x) for x in stacktrace.get('frames') or ()]
+
+    return stacktrace
+
+
+def strip_stacktrace_container(container):
+    if container:
+        container = dict(container)
+        container['stacktrace'] = strip_stacktrace(container.get('stacktrace'))
+        container['raw_stacktrace'] = strip_stacktrace(container.get('raw_stacktrace'))
+
+    return container
+
+
 def insta_snapshot_stacktrace_data(self, event):
+    # limit amount of data going into a snapshot so that they don't break all
+    # the time due to unrelated changes.
     self.insta_snapshot({
-        "stacktrace": event.get('stacktrace'),
-        "exception": event.get('exception'),
-        "threads": event.get('threads'),
+        "stacktrace": strip_stacktrace(event.get('stacktrace')),
+        "exception": {'values': [strip_stacktrace_container(x) for x in get_path(event, 'exception', 'values') or ()]},
+        "threads": {'values': [
+            strip_stacktrace_container(x)
+            for x in get_path(event, 'threads', 'values') or ()
+        ]},
         "debug_meta": event.get('debug_meta'),
         "contexts": event.get('contexts'),
         "errors": event.get('errors'),

tests/symbolicator/snapshots/SymbolicatorMinidumpIntegrationTest/test_full_minidump.pysnap

@@ -1,5 +1,5 @@
 ---
-created: '2019-06-03T14:50:05.902803Z'
+created: '2019-06-11T21:25:58.992806Z'
 creator: sentry
 source: tests/symbolicator/test_minidump_full.py
 ---
@@ -144,41 +144,47 @@ exception:
       handled: false
       synthetic: true
       type: minidump
+    raw_stacktrace: null
     stacktrace:
       frames:
       - data:
           symbolicator_status: missing
+        function: null
         in_app: false
         instruction_addr: '0x771d0f44'
+        lineno: null
         package: C:\Windows\System32\ntdll.dll
+        symbol: null
         trust: fp
       - data:
           symbolicator_status: missing
+        function: null
         in_app: false
         instruction_addr: '0x771d0f79'
+        lineno: null
         package: C:\Windows\System32\ntdll.dll
+        symbol: null
         trust: fp
       - data:
           symbolicator_status: missing
+        function: null
         in_app: false
         instruction_addr: '0x750662c4'
+        lineno: null
         package: C:\Windows\System32\kernel32.dll
+        symbol: null
         trust: cfi
-      - abs_path: f:\dd\vctools\crt\vcstartup\src\startup\exe_common.inl
-        data:
+      - data:
           symbolicator_status: symbolicated
-        filename: exe_common.inl
         function: __scrt_common_main_seh
         in_app: false
         instruction_addr: '0x2a2d96'
         lineno: 283
         package: C:\projects\breakpad-tools\windows\Release\crash.exe
         symbol: __scrt_common_main_seh
         trust: cfi
-      - abs_path: c:\projects\breakpad-tools\windows\crash\main.cpp
-        data:
+      - data:
           symbolicator_status: symbolicated
-        filename: main.cpp
         function: main
         in_app: false
         instruction_addr: '0x2a2a3d'
@@ -205,33 +211,48 @@ threads:
   values:
   - crashed: true
     id: 1636
+    raw_stacktrace: null
+    stacktrace: null
   - crashed: false
     id: 3580
+    raw_stacktrace: null
     stacktrace:
       frames:
       - data:
           symbolicator_status: missing
+        function: null
         in_app: false
         instruction_addr: '0x771d0f44'
+        lineno: null
         package: C:\Windows\System32\ntdll.dll
+        symbol: null
         trust: fp
       - data:
           symbolicator_status: missing
+        function: null
         in_app: false
         instruction_addr: '0x771d0f79'
+        lineno: null
         package: C:\Windows\System32\ntdll.dll
+        symbol: null
         trust: fp
       - data:
           symbolicator_status: missing
+        function: null
         in_app: false
         instruction_addr: '0x750662c4'
+        lineno: null
         package: C:\Windows\System32\kernel32.dll
+        symbol: null
         trust: fp
       - data:
           symbolicator_status: missing
+        function: null
         in_app: false
         instruction_addr: '0x771e016c'
+        lineno: null
         package: C:\Windows\System32\ntdll.dll
+        symbol: null
         trust: context
       registers:
         eax: '0x0'
@@ -246,31 +267,44 @@ threads:
         esp: '0x159f900'
   - crashed: false
     id: 2600
+    raw_stacktrace: null
     stacktrace:
       frames:
       - data:
           symbolicator_status: missing
+        function: null
         in_app: false
         instruction_addr: '0x771d0f44'
+        lineno: null
         package: C:\Windows\System32\ntdll.dll
+        symbol: null
         trust: fp
       - data:
           symbolicator_status: missing
+        function: null
         in_app: false
         instruction_addr: '0x771d0f79'
+        lineno: null
         package: C:\Windows\System32\ntdll.dll
+        symbol: null
         trust: fp
       - data:
           symbolicator_status: missing
+        function: null
         in_app: false
         instruction_addr: '0x750662c4'
+        lineno: null
         package: C:\Windows\System32\kernel32.dll
+        symbol: null
         trust: fp
       - data:
           symbolicator_status: missing
+        function: null
         in_app: false
         instruction_addr: '0x771e016c'
+        lineno: null
         package: C:\Windows\System32\ntdll.dll
+        symbol: null
         trust: context
       registers:
         eax: '0x0'
@@ -285,13 +319,17 @@ threads:
         esp: '0x169f9f4'
   - crashed: false
     id: 2920
+    raw_stacktrace: null
     stacktrace:
       frames:
       - data:
           symbolicator_status: missing
+        function: null
         in_app: false
         instruction_addr: '0x771df3dc'
+        lineno: null
         package: C:\Windows\System32\ntdll.dll
+        symbol: null
         trust: context
       registers:
         eax: '0x0'