Implement PEP 561 searching

docs/source/command_line.rst

@@ -8,17 +8,21 @@ summary of command line flags can always be printed using the ``-h``
 flag (or its long form ``--help``)::
 
   $ mypy -h
-  usage: mypy [-h] [-v] [-V] [--python-version x.y] [--platform PLATFORM] [-2]
+  usage: mypy [-h] [-v] [-V] [--python-version x.y]
+              [--python-executable PYTHON_EXECUTABLE] [--platform PLATFORM] [-2]
               [--ignore-missing-imports]
               [--follow-imports {normal,silent,skip,error}]
-              [--disallow-any-{unimported,expr,decorated,explicit,generics}]
-              [--disallow-untyped-calls] [--disallow-untyped-defs]
+              [--disallow-any-unimported] [--disallow-any-expr]
+              [--disallow-any-decorated] [--disallow-any-explicit]
+              [--disallow-any-generics] [--disallow-untyped-calls]
+              [--disallow-untyped-defs] [--disallow-incomplete-defs]
               [--check-untyped-defs] [--disallow-subclassing-any]
-              [--warn-incomplete-stub] [--warn-redundant-casts]
-              [--no-warn-no-return] [--warn-return-any] [--warn-unused-ignores]
+              [--warn-incomplete-stub] [--disallow-untyped-decorators]
+              [--warn-redundant-casts] [--no-warn-no-return] [--warn-return-any]
+              [--warn-unused-ignores] [--warn-unused-configs]
               [--show-error-context] [--no-implicit-optional] [-i]
-              [--quick-and-dirty] [--cache-dir DIR] [--skip-version-check]
-              [--strict-optional]
+              [--quick-and-dirty] [--cache-dir DIR] [--cache-fine-grained]
+              [--skip-version-check] [--strict-optional]
               [--strict-optional-whitelist [GLOB [GLOB ...]]]
               [--junit-xml JUNIT_XML] [--pdb] [--show-traceback] [--stats]
               [--inferstats] [--custom-typing MODULE]
@@ -28,9 +32,9 @@ flag (or its long form ``--help``)::
               [--shadow-file SOURCE_FILE SHADOW_FILE] [--any-exprs-report DIR]
               [--cobertura-xml-report DIR] [--html-report DIR]
               [--linecount-report DIR] [--linecoverage-report DIR]
-              [--memory-xml-report DIR]
-              [--txt-report DIR] [--xml-report DIR] [--xslt-html-report DIR]
-              [--xslt-txt-report DIR] [-m MODULE] [-c PROGRAM_TEXT] [-p PACKAGE]
+              [--memory-xml-report DIR] [--txt-report DIR] [--xml-report DIR]
+              [--xslt-html-report DIR] [--xslt-txt-report DIR] [-m MODULE]
+              [-c PROGRAM_TEXT] [-p PACKAGE]
               [files [files ...]]
 
   (etc., too long to show everything here)
@@ -366,6 +370,12 @@ Here are some more useful flags:
   updates the cache, but regular incremental mode ignores cache files
   written by quick mode.
 
+- ``--python-executable EXECUTABLE`` will have mypy collect type information
+  from PEP 561 compliant packages installed with the given Python executable.
+  By default, mypy will use PEP 561 compliant packages installed for the Python
+  executable running mypy. See :ref:`installed_packages` for more on making
+  PEP 561 compliant packages.
+
 - ``--python-version X.Y`` will make mypy typecheck your code as if it were
   run under Python version X.Y. Without this option, mypy will default to using
   whatever version of Python is running mypy. Note that the ``-2`` and

docs/source/index.rst

@@ -30,6 +30,7 @@ Mypy is a static type checker for Python.
    command_line
    config_file
    python36
+   installed_packages
    faq
    cheat_sheet
    cheat_sheet_py3

docs/source/installed_packages.rst

@@ -0,0 +1,73 @@
+.. _installed-packages:
+
+Using Installed Packages
+========================
+
+`PEP 561 <https://www.python.org/dev/peps/pep-0561/>`_ specifies how to mark
+a package as supporting type checking. Below is a summary of how to create
+PEP 561 compatible packages and have mypy use them in type checking.
+
+Making PEP 561 compatible packages
+**********************************
+
+Packages that must be imported at runtime that supply type information should
+put a ``py.typed`` in their package directory. For example, with a directory
+structure as follows:
+
+.. code-block:: text
+
+    setup.py
+    package_a/
+        __init__.py
+        lib.py
+        py.typed
+
+the setup.py might look like:
+
+.. code-block:: python
+
+    from distutils.core import setup
+
+    setup(
+        name="SuperPackageA",
+        author="Me",
+        version="0.1",
+        package_data={"package_a": ["py.typed"]},
+        packages=["package_a"]
+    )
+
+If the package is entirely made up of stub (``*.pyi``) files, the package
+should have a suffix of ``-stubs``. For example, if we had stubs for
+``package_b``, we might do the following:
+
+.. code-block:: text
+
+    setup.py
+    package_b-stubs/
+        __init__.pyi
+        lib.pyi
+
+    the setup.py might look like:
+
+    .. code-block:: python
+
+    from distutils.core import setup
+
+    setup(
+        name="SuperPackageB",
+        author="Me",
+        version="0.1",
+        package_data={"package_b-stubs": ["__init__.pyi", "lib.pyi"]},
+        packages=["package_b-stubs"]
+    )
+
+Using PEP 561 compatible packages with mypy
+*******************************************
+
+Generally, you do not need to do anything to use installed packages for the
+Python executable used to run mypy. They should be automatically picked up by
+mypy and used for type checking.
+
+If you use mypy to type check a Python other than the version running mypy, you
+can use the ``--python-executable`` flag to point to the executable, and mypy
+will find packages installed for that python executable.

mypy/build.py

@@ -21,6 +21,7 @@
 import re
 import site
 import stat
+from subprocess import check_output, STDOUT
 import sys
 import time
 from os.path import dirname, basename
@@ -683,7 +684,7 @@ def correct_rel_imp(imp: Union[ImportFrom, ImportAll]) -> str:
 
     def is_module(self, id: str) -> bool:
         """Is there a file in the file system corresponding to module id?"""
-        return find_module(id, self.lib_path) is not None
+        return find_module(id, self.lib_path, self.options.python_executable) is not None
 
     def parse_file(self, id: str, path: str, source: str, ignore_errors: bool) -> MypyFile:
         """Parse the source of a file with the given name.
@@ -796,14 +797,14 @@ def remove_cwd_prefix_from_path(p: str) -> str:
     return p
 
 
-# Cache find_module: (id, lib_path) -> result.
-find_module_cache = {}  # type: Dict[Tuple[str, Tuple[str, ...]], Optional[str]]
+# Cache find_module: (id) -> result.
+find_module_cache = {}  # type: Dict[str, Optional[str]]
 
 # Cache some repeated work within distinct find_module calls: finding which
 # elements of lib_path have even the subdirectory they'd need for the module
 # to exist.  This is shared among different module ids when they differ only
 # in the last component.
-find_module_dir_cache = {}  # type: Dict[Tuple[str, Tuple[str, ...]], List[str]]
+find_module_dir_cache = {}  # type: Dict[str, List[str]]
 
 # Cache directory listings.  We assume that while one os.listdir()
 # call may be more expensive than one os.stat() call, a small number
@@ -818,6 +819,9 @@ def remove_cwd_prefix_from_path(p: str) -> str:
 # Cache for isdir(join(head, tail))
 find_module_isdir_cache = {}  # type: Dict[Tuple[str, str], bool]
 
+# Cache packages for Python executable
+package_dirs_cache = []  # type: List[str]
+
 
 def find_module_clear_caches() -> None:
     find_module_cache.clear()
@@ -861,19 +865,70 @@ def is_file(path: str) -> bool:
     return res
 
 
-def find_module(id: str, lib_path_arg: Iterable[str]) -> Optional[str]:
+USER_SITE_PACKAGES = \
+    '"import site; print(site.getusersitepackages()); print(*site.getsitepackages(), sep=\'\\n\')"'
+VIRTUALENV_SITE_PACKAGES = \
+    '"from distutils.sysconfig import get_python_lib; print(get_python_lib())"'
+
+
+def call_python(python: str, command: str) -> str:
+    return check_output([python, '-c', command]).decode(sys.stdout.encoding)
+
+
+def get_package_dirs(python: Optional[str]) -> List[str]:
+    """Find package directories for given python
+
+     This defaults to the Python running mypy."""
+    if package_dirs_cache:
+        return package_dirs_cache
+    package_dirs = []  # type: List[str]
+    if python:
+        # Use subprocess to get the package directory of given Python
+        # executable
+        check = check_output([python, '-V'], stderr=STDOUT).decode('UTF-8')
+        assert check.startswith('Python'), \
+            "Mypy could not use the Python executable: {}".format(python)
+        # If we have a working python executable, query information from it
+        output = call_python(python, USER_SITE_PACKAGES)
+        for line in output.splitlines():
+            if os.path.isdir(line):
+                package_dirs.append(line)
+        if not package_dirs:
+            # if no paths are found, we fall back on sysconfig
+            output = call_python(python, VIRTUALENV_SITE_PACKAGES)
+            for line in output.splitlines():
+                if os.path.isdir(line):
+                    package_dirs.append(line)
+    else:
+        # Use running Python's package dirs
+        try:
+            user_dir = site.getusersitepackages()
+            package_dirs = site.getsitepackages() + [user_dir]
+        except AttributeError:
+            package_dirs = [get_python_lib()]
+    package_dirs_cache[:] = package_dirs
+    return package_dirs
+
+
+def find_module(id: str, lib_path_arg: Iterable[str],
+                python: Optional[str] = None) -> Optional[str]:
     """Return the path of the module source file, or None if not found."""
     lib_path = tuple(lib_path_arg)
+    package_dirs = get_package_dirs(python)
+    if python:
+        assert package_dirs, "Could not find package directories for Python '{}'".format(python)
+    components = id.split('.')
+    dir_chain = os.sep.join(components[:-1])  # e.g., 'foo/bar'
 
     def find() -> Optional[str]:
         # If we're looking for a module like 'foo.bar.baz', it's likely that most of the
         # many elements of lib_path don't even have a subdirectory 'foo/bar'.  Discover
         # that only once and cache it for when we look for modules like 'foo.bar.blah'
         # that will require the same subdirectory.
-        components = id.split('.')
-        dir_chain = os.sep.join(components[:-1])  # e.g., 'foo/bar'
-        if (dir_chain, lib_path) not in find_module_dir_cache:
-            dirs = []
+
+        dirs = find_module_dir_cache.get(dir_chain, [])
+        if not dirs:
+            # Regular packages on the PATH
             for pathitem in lib_path:
                 # e.g., '/usr/lib/python3.4/foo/bar'
                 isdir = find_module_isdir_cache.get((pathitem, dir_chain))
@@ -883,8 +938,24 @@ def find() -> Optional[str]:
                     find_module_isdir_cache[pathitem, dir_chain] = isdir
                 if isdir:
                     dirs.append(dir)
-            find_module_dir_cache[dir_chain, lib_path] = dirs
-        candidate_base_dirs = find_module_dir_cache[dir_chain, lib_path]
+
+        # Third-party stub/typed packages
+        for pkg_dir in package_dirs:
+            stub_name = components[0] + '-stubs'
+            typed_file = os.path.join(pkg_dir, components[0], 'py.typed')
+            stub_typed_file = os.path.join(pkg_dir, stub_name, 'py.typed')
+            if os.path.isfile(stub_typed_file):
+                components[0] = stub_name
+                rest = components[:-1]
+                path = os.path.join(pkg_dir, *rest)
+                if os.path.isdir(path):
+                    dirs.append(path)
+            elif os.path.isfile(typed_file):
+                path = os.path.join(pkg_dir, dir_chain)
+                dirs.append(path)
+
+        find_module_dir_cache[dir_chain] = dirs
+        candidate_base_dirs = find_module_dir_cache[dir_chain]
 
         # If we're looking for a module like 'foo.bar.baz', then candidate_base_dirs now
         # contains just the subdirectories 'foo/bar' that actually exist under the
@@ -897,23 +968,33 @@ def find() -> Optional[str]:
             # Prefer package over module, i.e. baz/__init__.py* over baz.py*.
             for extension in PYTHON_EXTENSIONS:
                 path = base_path + sepinit + extension
+                path_stubs = base_path + '_stubs' + sepinit + extension
                 if is_file(path) and verify_module(id, path):
                     return path
+                elif is_file(path_stubs) and verify_module(id, path_stubs):
+                    return path_stubs
             # No package, look for module.
             for extension in PYTHON_EXTENSIONS:
                 path = base_path + extension
                 if is_file(path) and verify_module(id, path):
                     return path
         return None
 
-    key = (id, lib_path)
-    if key not in find_module_cache:
-        find_module_cache[key] = find()
-    return find_module_cache[key]
+    if id not in find_module_cache:
+        find_module_cache[id] = find()
+
+    #  If we searched for items with a base directory of site-packages/ we need to
+    # remove it to avoid searching it for non-typed ids.
+    for dir in package_dirs:
+        if dir + os.sep in find_module_dir_cache[dir_chain]:
+            find_module_dir_cache[dir_chain].remove(dir + os.sep)
+
+    return find_module_cache[id]
 
 
-def find_modules_recursive(module: str, lib_path: List[str]) -> List[BuildSource]:
-    module_path = find_module(module, lib_path)
+def find_modules_recursive(module: str, lib_path: List[str],
+                           python: Optional[str]) -> List[BuildSource]:
+    module_path = find_module(module, lib_path, python)
     if not module_path:
         return []
     result = [BuildSource(module_path, module, None)]
@@ -933,14 +1014,14 @@ def find_modules_recursive(module: str, lib_path: List[str]) -> List[BuildSource
                     (os.path.isfile(os.path.join(abs_path, '__init__.py')) or
                     os.path.isfile(os.path.join(abs_path, '__init__.pyi'))):
                 hits.add(item)
-                result += find_modules_recursive(module + '.' + item, lib_path)
+                result += find_modules_recursive(module + '.' + item, lib_path, python)
             elif item != '__init__.py' and item != '__init__.pyi' and \
                     item.endswith(('.py', '.pyi')):
                 mod = item.split('.')[0]
                 if mod not in hits:
                     hits.add(mod)
-                    result += find_modules_recursive(
-                        module + '.' + mod, lib_path)
+                    result += find_modules_recursive(module + '.' + mod,
+                                                     lib_path, python)
     return result
 
 
@@ -1606,8 +1687,15 @@ def __init__(self,
                 # difference and just assume 'builtins' everywhere,
                 # which simplifies code.
                 file_id = '__builtin__'
-            path = find_module(file_id, manager.lib_path)
+            path = find_module(file_id, manager.lib_path, manager.options.python_executable)
             if path:
+                # Installed package modules should be silenced. They are all under absolute
+                # paths. When 3.4 is dropped, this should just use os.path.commonpath.
+                if os.path.isabs(path):
+                    # Silence errors from module if it is in a package directory
+                    for dir in package_dirs_cache:
+                        if path.startswith(dir + os.sep):
+                            self.ignore_all = True
                 # For non-stubs, look at options.follow_imports:
                 # - normal (default) -> fully analyze
                 # - silent -> analyze but silence errors

mypy/main.py

@@ -256,6 +256,9 @@ def add_invertible_flag(flag: str,
                         version='%(prog)s ' + __version__)
     parser.add_argument('--python-version', type=parse_version, metavar='x.y',
                         help='use Python x.y')
+    parser.add_argument('--python-executable', action='store',
+                        help="Python executable whose installed packages will be"
+                             " used in typechecking.")
     parser.add_argument('--platform', action='store', metavar='PLATFORM',
                         help="typecheck special-cased code for the given OS platform "
                         "(defaults to sys.platform).")
@@ -524,7 +527,8 @@ def add_invertible_flag(flag: str,
                  .format(special_opts.package))
         options.build_type = BuildType.MODULE
         lib_path = [os.getcwd()] + build.mypy_path()
-        targets = build.find_modules_recursive(special_opts.package, lib_path)
+        targets = build.find_modules_recursive(special_opts.package, lib_path,
+                                               options.python_executable)
         if not targets:
             fail("Can't find package '{}'".format(special_opts.package))
         return targets, options

mypy/options.py

@@ -53,6 +53,7 @@ def __init__(self) -> None:
         # -- build options --
         self.build_type = BuildType.STANDARD
         self.python_version = defaults.PYTHON3_VERSION
+        self.python_executable = None  # type: Optional[str]
         self.platform = sys.platform
         self.custom_typing_module = None  # type: Optional[str]
         self.custom_typeshed_dir = None  # type: Optional[str]

mypy/stubgen.py

@@ -156,7 +156,8 @@ def find_module_path_and_all(module: str, pyversion: Tuple[int, int],
             module_all = getattr(mod, '__all__', None)
     else:
         # Find module by going through search path.
-        module_path = mypy.build.find_module(module, ['.'] + search_path)
+        module_path = mypy.build.find_module(module, ['.'] + search_path,
+                                             interpreter)
         if not module_path:
             raise SystemExit(
                 "Can't find module '{}' (consider using --search-path)".format(module))