Skip to content

Commit

Permalink
Fixes to stubtest's new check for missing stdlib modules (#15960)
Browse files Browse the repository at this point in the history
- It's not easy to predict where stdlib modules are going to be located.
(It varies between platforms, and between venvs and conda envs; on some
platforms it's in a completely different directory to the Python
executable.)
- Some modules appear to raise `SystemExit` when stubtest tries to
import them in CI, leading stubtest to instantly exit without logging a
message to the terminal.
- Importing some `test.*` submodules leads to unraisable exceptions
being printed to the terminal at the end of the stubtest run, which is
somewhat annoying.
  • Loading branch information
AlexWaygood authored Aug 27, 2023
1 parent efecd59 commit d7b2451
Showing 1 changed file with 30 additions and 11 deletions.
41 changes: 30 additions & 11 deletions mypy/stubtest.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
import typing
import typing_extensions
import warnings
from collections import defaultdict
from contextlib import redirect_stderr, redirect_stdout
from functools import singledispatch
from pathlib import Path
Expand Down Expand Up @@ -1679,16 +1680,22 @@ def get_importable_stdlib_modules() -> set[str]:
all_stdlib_modules = sys.stdlib_module_names
else:
all_stdlib_modules = set(sys.builtin_module_names)
python_exe_dir = Path(sys.executable).parent
modules_by_finder: defaultdict[importlib.machinery.FileFinder, set[str]] = defaultdict(set)
for m in pkgutil.iter_modules():
finder = m.module_finder
if isinstance(finder, importlib.machinery.FileFinder):
finder_path = Path(finder.path)
if (
python_exe_dir in finder_path.parents
and "site-packages" not in finder_path.parts
):
all_stdlib_modules.add(m.name)
if isinstance(m.module_finder, importlib.machinery.FileFinder):
modules_by_finder[m.module_finder].add(m.name)
for finder, module_group in modules_by_finder.items():
if (
"site-packages" not in Path(finder.path).parents
# if "_queue" is present, it's most likely the module finder
# for stdlib extension modules;
# if "queue" is present, it's most likely the module finder
# for pure-Python stdlib modules.
# In either case, we'll want to add all the modules that the finder has to offer us.
# This is a bit hacky, but seems to work well in a cross-platform way.
and {"_queue", "queue"} & module_group
):
all_stdlib_modules.update(module_group)

importable_stdlib_modules: set[str] = set()
for module_name in all_stdlib_modules:
Expand Down Expand Up @@ -1719,13 +1726,25 @@ def get_importable_stdlib_modules() -> set[str]:
# The idlelib.* submodules are similarly annoying in opening random tkinter windows,
# and we're unlikely to ever add stubs for idlelib in typeshed
# (see discussion in https://github.com/python/typeshed/pull/9193)
if submodule_name.endswith(".__main__") or submodule_name.startswith("idlelib."):
#
# test.* modules do weird things like raising exceptions in __del__ methods,
# leading to unraisable exceptions being logged to the terminal
# as a warning at the end of the stubtest run
if (
submodule_name.endswith(".__main__")
or submodule_name.startswith("idlelib.")
or submodule_name.startswith("test.")
):
continue

try:
silent_import_module(submodule_name)
except KeyboardInterrupt:
raise
# importing multiprocessing.popen_forkserver on Windows raises AttributeError...
except Exception:
# some submodules also appear to raise SystemExit as well on some Python versions
# (not sure exactly which)
except BaseException:
continue
else:
importable_stdlib_modules.add(submodule_name)
Expand Down

0 comments on commit d7b2451

Please sign in to comment.