Quick and dirty variant of incremental mode (#2972)

* "Quick and dirty" variant of incremental mode.

This uses the cache whenever the cache file exists and is newer than the source.

The difference with regular incremental mode is that regular
incremental mode forces a re-check whenever any dependency is stale,
and it forces re-checks for all modules in an SCC.  In quick-and-dirty
mode, by contrast, if some modules in an SCC have a new-enough cache
file, they are loaded from the cache even if some other module in the
same SCC or one of its dependencies was stale.

This is much quicker but obviously not always correct.  The hope is
that it speeds up checking single files when there are large SCCs (or
large dependency graphs) around.

A typical workflow would be:

1. mypy -i (slow, but warms the cache)
2. edit some file (or a few)
3. mypy --quick-and-dirty (re-checks only the file(s) you edited)
4. repeat steps 2-4 any number of times (maybe for different files)
5. mypy -i (re-check everything -- this could be part of CI)
6. if any errors left, go to step 2

Author: gvanrossum

mypy/build.py

@@ -787,6 +787,9 @@ def find_cache_meta(id: str, path: str, manager: BuildManager) -> Optional[Cache
     # Ignore cache if (relevant) options aren't the same.
     cached_options = m.options
     current_options = manager.options.clone_for_module(id).select_options_affecting_cache()
+    if manager.options.quick_and_dirty:
+        # In quick_and_dirty mode allow non-quick_and_dirty cache files.
+        cached_options['quick_and_dirty'] = True
     if cached_options != current_options:
         manager.trace('Metadata abandoned for {}: options differ'.format(id))
         return None
@@ -1318,10 +1321,12 @@ def load_tree(self) -> None:
         self.manager.modules[self.id] = self.tree
 
     def fix_cross_refs(self) -> None:
-        fixup_module_pass_one(self.tree, self.manager.modules)
+        fixup_module_pass_one(self.tree, self.manager.modules,
+                              self.manager.options.quick_and_dirty)
 
     def calculate_mros(self) -> None:
-        fixup_module_pass_two(self.tree, self.manager.modules)
+        fixup_module_pass_two(self.tree, self.manager.modules,
+                              self.manager.options.quick_and_dirty)
 
     def fix_suppressed_dependencies(self, graph: Graph) -> None:
         """Corrects whether dependencies are considered stale in silent mode.
@@ -1513,7 +1518,14 @@ def valid_references(self) -> Set[str]:
         return valid_refs
 
     def write_cache(self) -> None:
-        if self.path and self.options.incremental and not self.manager.errors.is_errors():
+        ok = self.path and self.options.incremental
+        if ok:
+            if self.manager.options.quick_and_dirty:
+                is_errors = self.manager.errors.is_errors_for_file(self.path)
+            else:
+                is_errors = self.manager.errors.is_errors()
+            ok = not is_errors
+        if ok:
             dep_prios = [self.priorities.get(dep, PRI_HIGH) for dep in self.dependencies]
             new_interface_hash = write_cache(
                 self.id, self.path, self.tree,
@@ -1702,7 +1714,8 @@ def process_graph(graph: Graph, manager: BuildManager) -> None:
             deps.update(graph[id].dependencies)
         deps -= ascc
         stale_deps = {id for id in deps if not graph[id].is_interface_fresh()}
-        fresh = fresh and not stale_deps
+        if not manager.options.quick_and_dirty:
+            fresh = fresh and not stale_deps
         undeps = set()
         if fresh:
             # Check if any dependencies that were suppressed according
@@ -1717,7 +1730,7 @@ def process_graph(graph: Graph, manager: BuildManager) -> None:
             # All cache files are fresh.  Check that no dependency's
             # cache file is newer than any scc node's cache file.
             oldest_in_scc = min(graph[id].meta.data_mtime for id in scc)
-            viable = {id for id in deps if not graph[id].is_interface_fresh()}
+            viable = {id for id in stale_deps if graph[id].meta is not None}
             newest_in_deps = 0 if not viable else max(graph[dep].meta.data_mtime for dep in viable)
             if manager.options.verbosity >= 3:  # Dump all mtimes for extreme debugging.
                 all_ids = sorted(ascc | viable, key=lambda id: graph[id].meta.data_mtime)
@@ -1735,7 +1748,9 @@ def process_graph(graph: Graph, manager: BuildManager) -> None:
                     manager.trace(" %5s %.0f %s" % (key, graph[id].meta.data_mtime, id))
             # If equal, give the benefit of the doubt, due to 1-sec time granularity
             # (on some platforms).
-            if oldest_in_scc < newest_in_deps:
+            if manager.options.quick_and_dirty and stale_deps:
+                fresh_msg = "fresh(ish)"
+            elif oldest_in_scc < newest_in_deps:
                 fresh = False
                 fresh_msg = "out of date by %.0f seconds" % (newest_in_deps - oldest_in_scc)
             else:
@@ -1753,7 +1768,7 @@ def process_graph(graph: Graph, manager: BuildManager) -> None:
 
         scc_str = " ".join(scc)
         if fresh:
-            manager.log("Queuing fresh SCC (%s)" % scc_str)
+            manager.log("Queuing %s SCC (%s)" % (fresh_msg, scc_str))
             fresh_scc_queue.append(scc)
         else:
             if len(fresh_scc_queue) > 0:
@@ -1775,7 +1790,7 @@ def process_graph(graph: Graph, manager: BuildManager) -> None:
                 manager.log("Processing SCC singleton (%s) as %s" % (scc_str, fresh_msg))
             else:
                 manager.log("Processing SCC of size %d (%s) as %s" % (size, scc_str, fresh_msg))
-            process_stale_scc(graph, scc)
+            process_stale_scc(graph, scc, manager)
 
     sccs_left = len(fresh_scc_queue)
     if sccs_left:
@@ -1842,26 +1857,46 @@ def process_fresh_scc(graph: Graph, scc: List[str]) -> None:
         graph[id].calculate_mros()
 
 
-def process_stale_scc(graph: Graph, scc: List[str]) -> None:
-    """Process the modules in one SCC from source code."""
-    for id in scc:
+def process_stale_scc(graph: Graph, scc: List[str], manager: BuildManager) -> None:
+    """Process the modules in one SCC from source code.
+
+    Exception: If quick_and_dirty is set, use the cache for fresh modules.
+    """
+    if manager.options.quick_and_dirty:
+        fresh = [id for id in scc if graph[id].is_fresh()]
+        fresh_set = set(fresh)  # To avoid running into O(N**2)
+        stale = [id for id in scc if id not in fresh_set]
+        if fresh:
+            manager.log("  Fresh ids: %s" % (", ".join(fresh)))
+        if stale:
+            manager.log("  Stale ids: %s" % (", ".join(stale)))
+    else:
+        fresh = []
+        stale = scc
+    for id in fresh:
+        graph[id].load_tree()
+    for id in stale:
         # We may already have parsed the module, or not.
         # If the former, parse_file() is a no-op.
         graph[id].parse_file()
         graph[id].fix_suppressed_dependencies(graph)
-    for id in scc:
+    for id in fresh:
+        graph[id].fix_cross_refs()
+    for id in stale:
         graph[id].semantic_analysis()
-    for id in scc:
+    for id in stale:
         graph[id].semantic_analysis_pass_three()
-    for id in scc:
+    for id in fresh:
+        graph[id].calculate_mros()
+    for id in stale:
         graph[id].type_check_first_pass()
     more = True
     while more:
         more = False
-        for id in scc:
+        for id in stale:
             if graph[id].type_check_second_pass():
                 more = True
-    for id in scc:
+    for id in stale:
         graph[id].finish_passes()
         graph[id].write_cache()
         graph[id].mark_as_rechecked()

mypy/errors.py

@@ -76,6 +76,9 @@ class Errors:
     # Current error context: nested import context/stack, as a list of (path, line) pairs.
     import_ctx = None  # type: List[Tuple[str, int]]
 
+    # Set of files with errors.
+    error_files = None  # type: Set[str]
+
     # Path name prefix that is removed from all paths, if set.
     ignore_prefix = None  # type: str
 
@@ -110,6 +113,7 @@ def __init__(self, show_error_context: bool = False,
                  show_column_numbers: bool = False) -> None:
         self.error_info = []
         self.import_ctx = []
+        self.error_files = set()
         self.type_name = [None]
         self.function_or_member = [None]
         self.ignored_lines = OrderedDict()
@@ -230,6 +234,7 @@ def add_error_info(self, info: ErrorInfo) -> None:
                 return
             self.only_once_messages.add(info.message)
         self.error_info.append(info)
+        self.error_files.add(file)
 
     def generate_unused_ignore_notes(self) -> None:
         for file, ignored_lines in self.ignored_lines.items():
@@ -257,6 +262,10 @@ def is_blockers(self) -> bool:
         """Are the any errors that are blockers?"""
         return any(err for err in self.error_info if err.blocker)
 
+    def is_errors_for_file(self, file: str) -> bool:
+        """Are there any errors for the given file?"""
+        return file in self.error_files
+
     def raise_error(self) -> None:
         """Raise a CompileError with the generated messages.
 

mypy/fixup.py

@@ -16,12 +16,14 @@
 from mypy.visitor import NodeVisitor
 
 
-def fixup_module_pass_one(tree: MypyFile, modules: Dict[str, MypyFile]) -> None:
-    node_fixer = NodeFixer(modules)
+def fixup_module_pass_one(tree: MypyFile, modules: Dict[str, MypyFile],
+                          quick_and_dirty: bool) -> None:
+    node_fixer = NodeFixer(modules, quick_and_dirty)
     node_fixer.visit_symbol_table(tree.names)
 
 
-def fixup_module_pass_two(tree: MypyFile, modules: Dict[str, MypyFile]) -> None:
+def fixup_module_pass_two(tree: MypyFile, modules: Dict[str, MypyFile],
+                          quick_and_dirty: bool) -> None:
     compute_all_mros(tree.names, modules)
 
 
@@ -38,11 +40,10 @@ def compute_all_mros(symtab: SymbolTable, modules: Dict[str, MypyFile]) -> None:
 class NodeFixer(NodeVisitor[None]):
     current_info = None  # type: Optional[TypeInfo]
 
-    def __init__(self, modules: Dict[str, MypyFile], type_fixer: 'TypeFixer' = None) -> None:
+    def __init__(self, modules: Dict[str, MypyFile], quick_and_dirty: bool) -> None:
         self.modules = modules
-        if type_fixer is None:
-            type_fixer = TypeFixer(self.modules)
-        self.type_fixer = type_fixer
+        self.quick_and_dirty = quick_and_dirty
+        self.type_fixer = TypeFixer(self.modules, quick_and_dirty)
 
     # NOTE: This method isn't (yet) part of the NodeVisitor API.
     def visit_type_info(self, info: TypeInfo) -> None:
@@ -76,10 +77,13 @@ def visit_symbol_table(self, symtab: SymbolTable) -> None:
                 if cross_ref in self.modules:
                     value.node = self.modules[cross_ref]
                 else:
-                    stnode = lookup_qualified_stnode(self.modules, cross_ref)
-                    assert stnode is not None, "Could not find cross-ref %s" % (cross_ref,)
-                    value.node = stnode.node
-                    value.type_override = stnode.type_override
+                    stnode = lookup_qualified_stnode(self.modules, cross_ref,
+                                                     self.quick_and_dirty)
+                    if stnode is not None:
+                        value.node = stnode.node
+                        value.type_override = stnode.type_override
+                    elif not self.quick_and_dirty:
+                        assert stnode is not None, "Could not find cross-ref %s" % (cross_ref,)
             else:
                 if isinstance(value.node, TypeInfo):
                     # TypeInfo has no accept().  TODO: Add it?
@@ -132,16 +136,17 @@ def visit_var(self, v: Var) -> None:
 
 
 class TypeFixer(TypeVisitor[None]):
-    def __init__(self, modules: Dict[str, MypyFile]) -> None:
+    def __init__(self, modules: Dict[str, MypyFile], quick_and_dirty: bool) -> None:
         self.modules = modules
+        self.quick_and_dirty = quick_and_dirty
 
     def visit_instance(self, inst: Instance) -> None:
         # TODO: Combine Instances that are exactly the same?
         type_ref = inst.type_ref
         if type_ref is None:
             return  # We've already been here.
         del inst.type_ref
-        node = lookup_qualified(self.modules, type_ref)
+        node = lookup_qualified(self.modules, type_ref, self.quick_and_dirty)
         if isinstance(node, TypeInfo):
             inst.type = node
             # TODO: Is this needed or redundant?
@@ -230,29 +235,40 @@ def visit_type_type(self, t: TypeType) -> None:
         t.item.accept(self)
 
 
-def lookup_qualified(modules: Dict[str, MypyFile], name: str) -> SymbolNode:
-    stnode = lookup_qualified_stnode(modules, name)
+def lookup_qualified(modules: Dict[str, MypyFile], name: str,
+                     quick_and_dirty: bool) -> SymbolNode:
+    stnode = lookup_qualified_stnode(modules, name, quick_and_dirty)
     if stnode is None:
         return None
     else:
         return stnode.node
 
 
-def lookup_qualified_stnode(modules: Dict[str, MypyFile], name: str) -> SymbolTableNode:
+def lookup_qualified_stnode(modules: Dict[str, MypyFile], name: str,
+                            quick_and_dirty: bool) -> Optional[SymbolTableNode]:
     head = name
     rest = []
     while True:
-        assert '.' in head, "Cannot find %s" % (name,)
+        if '.' not in head:
+            if not quick_and_dirty:
+                assert '.' in head, "Cannot find %s" % (name,)
+            return None
         head, tail = head.rsplit('.', 1)
         rest.append(tail)
         mod = modules.get(head)
         if mod is not None:
             break
     names = mod.names
     while True:
-        assert rest, "Cannot find %s" % (name,)
+        if not rest:
+            if not quick_and_dirty:
+                assert rest, "Cannot find %s" % (name,)
+            return None
         key = rest.pop()
-        assert key in names, "Cannot find %s for %s" % (key, name)
+        if key not in names:
+            return None
+        elif not quick_and_dirty:
+            assert key in names, "Cannot find %s for %s" % (key, name)
         stnode = names[key]
         if not rest:
             return stnode

mypy/main.py

@@ -239,7 +239,10 @@ def add_invertible_flag(flag: str,
     add_invertible_flag('--no-fast-parser', default=True, dest='fast_parser',
                         help="disable the fast parser (not recommended)")
     parser.add_argument('-i', '--incremental', action='store_true',
-                        help="enable experimental module cache")
+                        help="enable module cache")
+    parser.add_argument('--quick-and-dirty', action='store_true',
+                        help="use cache even if dependencies out of date "
+                        "(implies --incremental)")
     parser.add_argument('--cache-dir', action='store', metavar='DIR',
                         help="store module cache info in the given folder in incremental mode "
                         "(defaults to '{}')".format(defaults.CACHE_DIR))
@@ -405,6 +408,10 @@ def add_invertible_flag(flag: str,
             report_dir = val
             options.report_dirs[report_type] = report_dir
 
+    # Let quick_and_dirty imply incremental.
+    if options.quick_and_dirty:
+        options.incremental = True
+
     # Set target.
     if special_opts.modules:
         options.build_type = BuildType.MODULE

mypy/options.py

@@ -31,7 +31,7 @@ class Options:
         "strict_boolean",
     }
 
-    OPTIONS_AFFECTING_CACHE = PER_MODULE_OPTIONS | {"strict_optional"}
+    OPTIONS_AFFECTING_CACHE = PER_MODULE_OPTIONS | {"strict_optional", "quick_and_dirty"}
 
     def __init__(self) -> None:
         # -- build options --
@@ -101,6 +101,15 @@ def __init__(self) -> None:
         # Write junit.xml to given file
         self.junit_xml = None  # type: Optional[str]
 
+        # Fast parser is on by default
+        self.fast_parser = True
+
+        # Caching options
+        self.incremental = False
+        self.cache_dir = defaults.CACHE_DIR
+        self.debug_cache = False
+        self.quick_and_dirty = False
+
         # Per-module options (raw)
         self.per_module_options = {}  # type: Dict[Pattern[str], Dict[str, object]]
 
@@ -119,10 +128,6 @@ def __init__(self) -> None:
         self.use_builtins_fixtures = False
 
         # -- experimental options --
-        self.fast_parser = True
-        self.incremental = False
-        self.cache_dir = defaults.CACHE_DIR
-        self.debug_cache = False
         self.shadow_file = None  # type: Optional[Tuple[str, str]]
         self.show_column_numbers = False  # type: bool
         self.dump_graph = False

mypy/test/testcheck.py

@@ -149,7 +149,7 @@ def run_case_once(self, testcase: DataDrivenTestCase, incremental: int = 0) -> N
                             os.utime(target, times=(new_time, new_time))
 
         # Parse options after moving files (in case mypy.ini is being moved).
-        options = self.parse_options(original_program_text, testcase)
+        options = self.parse_options(original_program_text, testcase, incremental)
         options.use_builtins_fixtures = True
         options.show_traceback = True
         if 'optional' in testcase.file:
@@ -305,9 +305,14 @@ def parse_module(self, program_text: str, incremental: int = 0) -> List[Tuple[st
         else:
             return [('__main__', 'main', program_text)]
 
-    def parse_options(self, program_text: str, testcase: DataDrivenTestCase) -> Options:
+    def parse_options(self, program_text: str, testcase: DataDrivenTestCase,
+                      incremental: int) -> Options:
         options = Options()
         flags = re.search('# flags: (.*)$', program_text, flags=re.MULTILINE)
+        if incremental == 2:
+            flags2 = re.search('# flags2: (.*)$', program_text, flags=re.MULTILINE)
+            if flags2:
+                flags = flags2
 
         flag_list = None
         if flags: