stubtest: catch all for errors from the runtime

mypy/stubtest.py

@@ -13,6 +13,7 @@
 import pkgutil
 import re
 import sys
+import traceback
 import types
 import typing
 import typing_extensions
@@ -201,7 +202,28 @@ def test_module(module_name: str) -> Iterator[Error]:
 
     with warnings.catch_warnings():
         warnings.simplefilter("ignore")
-        yield from verify(stub, runtime, [module_name])
+        try:
+            yield from verify(stub, runtime, [module_name])
+        except Exception as e:
+            bottom_frame = list(traceback.walk_tb(e.__traceback__))[-1][0]
+            bottom_module = bottom_frame.f_globals.get("__name__", "")
+            # Pass on any errors originating from stubtest or mypy
+            # These can occur expectedly, e.g. StubtestFailure
+            if bottom_module == "__main__" or bottom_module.split(".")[0] == "mypy":
+                raise
+            yield Error(
+                [module_name],
+                f"encountered unexpected error, {type(e).__name__}: {e}",
+                stub,
+                runtime,
+                stub_desc="N/A",
+                runtime_desc=(
+                    "This is most likely the fault of something very dynamic in your library. "
+                    "It's also possible this is a bug in stubtest.\nIf in doubt, please "
+                    "open an issue at https://github.com/python/mypy\n\n"
+                    + traceback.format_exc().strip()
+                ),
+            )
 
 
 @singledispatch