Add several undocumented flags (--bazel, --package-root and --cache-map) in support of Bazel

mypy/build.py

@@ -66,13 +66,38 @@
 PYTHON_EXTENSIONS = ['.pyi', '.py']
 
 
+if os.altsep:
+    SEPARATORS = frozenset([os.sep, os.altsep])
+else:
+    SEPARATORS = frozenset([os.sep])
+
+
 Graph = Dict[str, 'State']
 
 
 def getmtime(name: str) -> int:
     return int(os.path.getmtime(name))
 
 
+def relpath(path: str) -> str:
+    if not path or path[0] not in SEPARATORS:
+        # Relative path, or has drive letter prefix (C: etc.) on Windows.
+        assert not os.path.isabs(path), "%s is neither rel nor abs" % (path,)
+        return path
+    if os.altsep and os.altsep in path:
+        # So the rest of this code doesn't need to handle altsep on Windows.
+        path = path.replace(os.altsep, os.sep)
+    cwd = os.getcwd()
+    for i in range(10):
+        if path.startswith(cwd + os.sep):
+            # The .lstrip(os.sep) call strips duplicate leading slashes.
+            return '../' * i + path[len(cwd) + 1:].lstrip(os.sep)
+        cwd = os.path.dirname(cwd)
+        if all(c == os.sep for c in cwd):
+            break
+    return path
+
+
 # TODO: Get rid of BuildResult.  We might as well return a BuildManager.
 class BuildResult:
     """The result of a successful build.
@@ -228,7 +253,12 @@ def _build(sources: List[BuildSource],
         # to the lib_path
         # TODO: Don't do this in some cases; for motivation see see
         # https://github.com/python/mypy/issues/4195#issuecomment-341915031
-        lib_path.insert(0, os.getcwd())
+        if options.bazel:
+            dir = '.'
+        else:
+            dir = os.getcwd()
+        if dir not in lib_path:
+            lib_path.insert(0, dir)
 
     # Prepend a config-defined mypy path.
     lib_path[:0] = options.mypy_path
@@ -931,14 +961,17 @@ def get_cache_names(id: str, path: str, manager: BuildManager) -> Tuple[str, str
 
     Args:
       id: module ID
-      path: module path (used to recognize packages)
+      path: module path
       cache_dir: cache directory
       pyversion: Python version (major, minor)
 
     Returns:
       A tuple with the file names to be used for the meta JSON and the
       data JSON, respectively.
     """
+    pair = manager.options.cache_map.get(path)
+    if pair is not None:
+        return pair
     cache_dir = manager.options.cache_dir
     pyversion = manager.options.python_version
     prefix = os.path.join(cache_dir, '%d.%d' % pyversion, *id.split('.'))
@@ -1055,15 +1088,16 @@ def validate_meta(meta: Optional[CacheMeta], id: str, path: Optional[str],
         manager.log('Metadata abandoned for {}: errors were previously ignored'.format(id))
         return None
 
+    bazel = manager.options.bazel
     assert path is not None, "Internal error: meta was provided without a path"
     # Check data_json; assume if its mtime matches it's good.
     # TODO: stat() errors
-    data_mtime = getmtime(meta.data_json)
+    data_mtime = 0 if bazel else getmtime(meta.data_json)
     if data_mtime != meta.data_mtime:
         manager.log('Metadata abandoned for {}: data cache is modified'.format(id))
         return None
 
-    path = os.path.abspath(path)
+    path = relpath(path) if bazel else os.path.abspath(path)
     try:
         st = manager.get_stat(path)
     except OSError:
@@ -1083,11 +1117,11 @@ def validate_meta(meta: Optional[CacheMeta], id: str, path: Optional[str],
         return meta
 
     size = st.st_size
-    if size != meta.size:
+    if not bazel and size != meta.size:
         manager.log('Metadata abandoned for {}: file {} has different size'.format(id, path))
         return None
 
-    mtime = int(st.st_mtime)
+    mtime = 0 if bazel else int(st.st_mtime)
     if mtime != meta.mtime or path != meta.path:
         try:
             source_hash = manager.fscache.md5(path)
@@ -1171,8 +1205,18 @@ def write_cache(id: str, path: str, tree: MypyFile,
       corresponding to the metadata that was written (the latter may
       be None if the cache could not be written).
     """
+    # For Bazel we use relative paths and zero mtimes.
+    bazel = manager.options.bazel
+
     # Obtain file paths
-    path = os.path.abspath(path)
+    if not bazel:
+        # Normally, make all paths absolute.
+        path = os.path.abspath(path)
+    else:
+        # For Bazel, make all paths relative (else Bazel caching is thwarted).
+        # And also patch tree.path, which might have been made absolute earlier.
+        tree.path = path = relpath(path)
+
     meta_json, data_json = get_cache_names(id, path, manager)
     manager.log('Writing {} {} {} {}'.format(id, path, meta_json, data_json))
 
@@ -1192,7 +1236,8 @@ def write_cache(id: str, path: str, tree: MypyFile,
 
     # Obtain and set up metadata
     try:
-        os.makedirs(parent, exist_ok=True)
+        if parent:
+            os.makedirs(parent, exist_ok=True)
         st = manager.get_stat(path)
     except OSError as err:
         manager.log("Cannot get stat for {}: {}".format(path, err))
@@ -1207,10 +1252,11 @@ def write_cache(id: str, path: str, tree: MypyFile,
         return interface_hash, None
 
     # Write data cache file, if applicable
+    # Note that for Bazel we don't record the data file's mtime.
     if old_interface_hash == interface_hash:
         # If the interface is unchanged, the cached data is guaranteed
         # to be equivalent, and we only need to update the metadata.
-        data_mtime = getmtime(data_json)
+        data_mtime = 0 if bazel else getmtime(data_json)
         manager.trace("Interface for {} is unchanged".format(id))
     else:
         manager.trace("Interface for {} has changed".format(id))
@@ -1227,9 +1273,9 @@ def write_cache(id: str, path: str, tree: MypyFile,
             # Both have the effect of slowing down the next run a
             # little bit due to an out-of-date cache file.
             return interface_hash, None
-        data_mtime = getmtime(data_json)
+        data_mtime = 0 if bazel else getmtime(data_json)
 
-    mtime = int(st.st_mtime)
+    mtime = 0 if bazel else int(st.st_mtime)
     size = st.st_size
     options = manager.options.clone_for_module(id)
     assert source_hash is not None
@@ -1271,7 +1317,7 @@ def delete_cache(id: str, path: str, manager: BuildManager) -> None:
     This avoids inconsistent states with cache files from different mypy runs,
     see #4043 for an example.
     """
-    path = os.path.abspath(path)
+    path = relpath(path) if manager.options.bazel else os.path.abspath(path)
     meta_json, data_json = get_cache_names(id, path, manager)
     manager.log('Deleting {} {} {} {}'.format(id, path, meta_json, data_json))
 

mypy/main.py

@@ -367,6 +367,7 @@ def add_invertible_flag(flag: str,
     parser.add_argument('--shadow-file', nargs=2, metavar=('SOURCE_FILE', 'SHADOW_FILE'),
                         dest='shadow_file',
                         help='Typecheck SHADOW_FILE in place of SOURCE_FILE.')
+
     # hidden options
     # --debug-cache will disable any cache-related compressions/optimizations,
     # which will make the cache writing process output pretty-printed JSON (which
@@ -381,6 +382,15 @@ def add_invertible_flag(flag: str,
     # --local-partial-types disallows partial types spanning module top level and a function
     # (implicitly defined in fine-grained incremental mode)
     parser.add_argument('--local-partial-types', action='store_true', help=argparse.SUPPRESS)
+    # --bazel changes some behaviors for use with Bazel (https://bazel.build).
+    parser.add_argument('--bazel', action='store_true', help=argparse.SUPPRESS)
+    # --cache-map FILE ... gives a mapping from source files to cache files.
+    # Each triple of arguments is a source file, a cache meta file, and a cache data file.
+    # Modules not mentioned in the file will go through cache_dir.
+    # Must be followed by another flag or by '--' (and then only file args may follow).
+    parser.add_argument('--cache-map', nargs='+', dest='special-opts:cache_map',
+                        help=argparse.SUPPRESS)
+
     # deprecated options
     parser.add_argument('--disallow-any', dest='special-opts:disallow_any',
                         help=argparse.SUPPRESS)
@@ -520,6 +530,18 @@ def add_invertible_flag(flag: str,
             report_dir = val
             options.report_dirs[report_type] = report_dir
 
+    # Parse cache map.  Uses assertions for checking.
+    if special_opts.cache_map:
+        n = len(special_opts.cache_map)
+        assert n % 3 == 0, "--cache-map requires one or more triples (see source)"
+        for i in range(0, n, 3):
+            source, meta_file, data_file = special_opts.cache_map[i:i + 3]
+            assert source not in options.cache_map
+            assert source.endswith('.py') or source.endswith('.pyi'), source
+            assert meta_file.endswith('.meta.json'), meta_file
+            assert data_file.endswith('.data.json'), data_file
+            options.cache_map[source] = (meta_file, data_file)
+
     # Let quick_and_dirty imply incremental.
     if options.quick_and_dirty:
         options.incremental = True

mypy/options.py

@@ -43,7 +43,7 @@ class Options:
     }
 
     OPTIONS_AFFECTING_CACHE = ((PER_MODULE_OPTIONS |
-                                {"quick_and_dirty", "platform", "cache_fine_grained"})
+                                {"quick_and_dirty", "platform", "cache_fine_grained", "bazel"})
                                - {"debug_cache"})
 
     def __init__(self) -> None:
@@ -176,6 +176,9 @@ def __init__(self) -> None:
         self.dump_deps = False
         # If True, partial types can't span a module top level and a function
         self.local_partial_types = False
+        # Some behaviors are changed when using Bazel (https://bazel.build).
+        self.bazel = False
+        self.cache_map = {}  # type: Dict[str, Tuple[str, str]]
 
     def __eq__(self, other: object) -> bool:
         return self.__class__ == other.__class__ and self.__dict__ == other.__dict__