From abab1cd84f002894022da513075ab6d4cf82b065 Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Sun, 20 Mar 2022 18:04:32 -0700 Subject: [PATCH] Fix caching of PEP 561 namespace packages with missing submodules (#12250) Fixes #12232 Another "fail every 2nd run" bug `if not self.is_module(sub_id):`, the parent can come before another child, and we fail to resolve the parent. Co-authored-by: hauntsaninja <> --- mypy/build.py | 30 +++++++++++++++++----------- test-data/unit/check-errorcodes.test | 6 +++--- test-data/unit/cmdline.test | 4 ++-- test-data/unit/pep561.test | 10 ++++++++++ test-data/unit/semanal-errors.test | 2 +- 5 files changed, 34 insertions(+), 18 deletions(-) diff --git a/mypy/build.py b/mypy/build.py index 6513414dd298..694261411d80 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -730,20 +730,19 @@ def correct_rel_imp(imp: Union[ImportFrom, ImportAll]) -> str: return new_id res: List[Tuple[int, str, int]] = [] + delayed_res: List[Tuple[int, str, int]] = [] for imp in file.imports: if not imp.is_unreachable: if isinstance(imp, Import): pri = import_priority(imp, PRI_MED) ancestor_pri = import_priority(imp, PRI_LOW) for id, _ in imp.ids: - # We append the target (e.g. foo.bar.baz) - # before the ancestors (e.g. foo and foo.bar) - # so that, if FindModuleCache finds the target - # module in a package marked with py.typed - # underneath a namespace package installed in - # site-packages, (gasp), that cache's - # knowledge of the ancestors can be primed - # when it is asked to find the target. + # We append the target (e.g. foo.bar.baz) before the ancestors (e.g. foo + # and foo.bar) so that, if FindModuleCache finds the target module in a + # package marked with py.typed underneath a namespace package installed in + # site-packages, (gasp), that cache's knowledge of the ancestors + # (aka FindModuleCache.ns_ancestors) can be primed when it is asked to find + # the parent. res.append((pri, id, imp.line)) ancestor_parts = id.split(".")[:-1] ancestors = [] @@ -752,6 +751,7 @@ def correct_rel_imp(imp: Union[ImportFrom, ImportAll]) -> str: res.append((ancestor_pri, ".".join(ancestors), imp.line)) elif isinstance(imp, ImportFrom): cur_id = correct_rel_imp(imp) + any_are_submodules = False all_are_submodules = True # Also add any imported names that are submodules. pri = import_priority(imp, PRI_MED) @@ -759,6 +759,7 @@ def correct_rel_imp(imp: Union[ImportFrom, ImportAll]) -> str: sub_id = cur_id + '.' + name if self.is_module(sub_id): res.append((pri, sub_id, imp.line)) + any_are_submodules = True else: all_are_submodules = False # Add cur_id as a dependency, even if all of the @@ -768,14 +769,19 @@ def correct_rel_imp(imp: Union[ImportFrom, ImportAll]) -> str: # if all of the imports are submodules, do the import at a lower # priority. pri = import_priority(imp, PRI_HIGH if not all_are_submodules else PRI_LOW) - # The imported module goes in after the - # submodules, for the same namespace related - # reasons discussed in the Import case. - res.append((pri, cur_id, imp.line)) + # The imported module goes in after the submodules, for the same namespace + # related reasons discussed in the Import case. + # There is an additional twist: if none of the submodules exist, + # we delay the import in case other imports of other submodules succeed. + if any_are_submodules: + res.append((pri, cur_id, imp.line)) + else: + delayed_res.append((pri, cur_id, imp.line)) elif isinstance(imp, ImportAll): pri = import_priority(imp, PRI_HIGH) res.append((pri, correct_rel_imp(imp), imp.line)) + res.extend(delayed_res) return res def is_module(self, id: str) -> bool: diff --git a/test-data/unit/check-errorcodes.test b/test-data/unit/check-errorcodes.test index a74771108ca2..9fde5ce0d0cc 100644 --- a/test-data/unit/check-errorcodes.test +++ b/test-data/unit/check-errorcodes.test @@ -596,10 +596,10 @@ if int() is str(): # E: Non-overlapping identity check (left operand type: "int [builtins fixtures/primitives.pyi] [case testErrorCodeMissingModule] -from defusedxml import xyz # E: Cannot find implementation or library stub for module named "defusedxml" [import] \ - # N: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports +from defusedxml import xyz # E: Cannot find implementation or library stub for module named "defusedxml" [import] from nonexistent import foobar # E: Cannot find implementation or library stub for module named "nonexistent" [import] -import nonexistent2 # E: Cannot find implementation or library stub for module named "nonexistent2" [import] +import nonexistent2 # E: Cannot find implementation or library stub for module named "nonexistent2" [import] \ + # N: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports from nonexistent3 import * # E: Cannot find implementation or library stub for module named "nonexistent3" [import] from pkg import bad # E: Module "pkg" has no attribute "bad" [attr-defined] from pkg.bad2 import bad3 # E: Cannot find implementation or library stub for module named "pkg.bad2" [import] diff --git a/test-data/unit/cmdline.test b/test-data/unit/cmdline.test index f29c183f20ba..918493b8df48 100644 --- a/test-data/unit/cmdline.test +++ b/test-data/unit/cmdline.test @@ -415,9 +415,9 @@ follow_imports = error [file a.py] / # No error reported [out] -main.py:1: error: Import of "a" ignored -main.py:1: note: (Using --follow-imports=error, module not passed on command line) main.py:2: note: Revealed type is "Any" +main.py:3: error: Import of "a" ignored +main.py:3: note: (Using --follow-imports=error, module not passed on command line) main.py:4: note: Revealed type is "Any" [case testConfigFollowImportsSelective] diff --git a/test-data/unit/pep561.test b/test-data/unit/pep561.test index 364414b202e1..290cf73b7829 100644 --- a/test-data/unit/pep561.test +++ b/test-data/unit/pep561.test @@ -222,3 +222,13 @@ b.bf(1) [out] testNamespacePkgWStubsWithNamespacePackagesFlag.py:7: error: Argument 1 to "bf" has incompatible type "int"; expected "bool" testNamespacePkgWStubsWithNamespacePackagesFlag.py:8: error: Argument 1 to "bf" has incompatible type "int"; expected "bool" + + +[case testTypedPkgNamespaceRegFromImportTwiceMissing] +# pkgs: typedpkg_ns_a +from typedpkg_ns import b # type: ignore +from typedpkg_ns import a +-- dummy should trigger a second iteration +[file dummy.py.2] +[out] +[out2] diff --git a/test-data/unit/semanal-errors.test b/test-data/unit/semanal-errors.test index 1eea55fca799..a0efccef5913 100644 --- a/test-data/unit/semanal-errors.test +++ b/test-data/unit/semanal-errors.test @@ -293,8 +293,8 @@ from m.n import x from a.b import * [out] main:2: error: Cannot find implementation or library stub for module named "m.n" -main:2: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports main:3: error: Cannot find implementation or library stub for module named "a.b" +main:3: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports [case testErrorInImportedModule] import m