diff --git a/changelog/11475.feature.rst b/changelog/11475.feature.rst new file mode 100644 index 00000000000..42550235d80 --- /dev/null +++ b/changelog/11475.feature.rst @@ -0,0 +1,3 @@ +Added the new :confval:`consider_namespace_packages` configuration option, defaulting to ``False``. + +If set to ``True``, pytest will attempt to identify modules that are part of `namespace packages `__ when importing modules. diff --git a/changelog/11475.improvement.rst b/changelog/11475.improvement.rst new file mode 100644 index 00000000000..4f6a4bffaad --- /dev/null +++ b/changelog/11475.improvement.rst @@ -0,0 +1,3 @@ +:ref:`--import-mode=importlib ` now tries to import modules using the standard import mechanism (but still without changing :py:data:`sys.path`), falling back to importing modules directly only if that fails. + +This means that installed packages will be imported under their canonical name if possible first, for example ``app.core.models``, instead of having the module name always be derived from their path (for example ``.env310.lib.site_packages.app.core.models``). diff --git a/doc/en/explanation/goodpractices.rst b/doc/en/explanation/goodpractices.rst index efde420cd8f..1390ba4e8fe 100644 --- a/doc/en/explanation/goodpractices.rst +++ b/doc/en/explanation/goodpractices.rst @@ -60,8 +60,10 @@ Within Python modules, ``pytest`` also discovers tests using the standard :ref:`unittest.TestCase ` subclassing technique. -Choosing a test layout / import rules -------------------------------------- +.. _`test layout`: + +Choosing a test layout +---------------------- ``pytest`` supports two common test layouts: diff --git a/doc/en/explanation/pythonpath.rst b/doc/en/explanation/pythonpath.rst index 5b533f47fdc..33eba86b57a 100644 --- a/doc/en/explanation/pythonpath.rst +++ b/doc/en/explanation/pythonpath.rst @@ -10,19 +10,27 @@ Import modes pytest as a testing framework needs to import test modules and ``conftest.py`` files for execution. -Importing files in Python (at least until recently) is a non-trivial processes, often requiring -changing :data:`sys.path`. Some aspects of the +Importing files in Python is a non-trivial processes, so aspects of the import process can be controlled through the ``--import-mode`` command-line flag, which can assume these values: +.. _`import-mode-prepend`: + * ``prepend`` (default): the directory path containing each module will be inserted into the *beginning* - of :py:data:`sys.path` if not already there, and then imported with the :func:`importlib.import_module ` function. + of :py:data:`sys.path` if not already there, and then imported with + the :func:`importlib.import_module ` function. + + It is highly recommended to arrange your test modules as packages by adding ``__init__.py`` files to your directories + containing tests. This will make the tests part of a proper Python package, allowing pytest to resolve their full + name (for example ``tests.core.test_core`` for ``test_core.py`` inside the ``tests.core`` package). - This requires test module names to be unique when the test directory tree is not arranged in - packages, because the modules will put in :py:data:`sys.modules` after importing. + If the test directory tree is not arranged as packages, then each test file needs to have a unique name + compared to the other test files, otherwise pytest will raise an error if it finds two tests with the same name. This is the classic mechanism, dating back from the time Python 2 was still supported. +.. _`import-mode-append`: + * ``append``: the directory containing each module is appended to the end of :py:data:`sys.path` if not already there, and imported with :func:`importlib.import_module `. @@ -38,32 +46,78 @@ these values: the tests will run against the installed version of ``pkg_under_test`` when ``--import-mode=append`` is used whereas with ``prepend`` they would pick up the local version. This kind of confusion is why - we advocate for using :ref:`src ` layouts. + we advocate for using :ref:`src-layouts `. Same as ``prepend``, requires test module names to be unique when the test directory tree is not arranged in packages, because the modules will put in :py:data:`sys.modules` after importing. -* ``importlib``: new in pytest-6.0, this mode uses more fine control mechanisms provided by :mod:`importlib` to import test modules. This gives full control over the import process, and doesn't require changing :py:data:`sys.path`. +.. _`import-mode-importlib`: + +* ``importlib``: this mode uses more fine control mechanisms provided by :mod:`importlib` to import test modules, without changing :py:data:`sys.path`. + + Advantages of this mode: + + * pytest will not change :py:data:`sys.path` at all. + * Test module names do not need to be unique -- pytest will generate a unique name automatically based on the ``rootdir``. + + Disadvantages: + + * Test modules can't import each other. + * Testing utility modules in the tests directories (for example a ``tests.helpers`` module containing test-related functions/classes) + are not importable. The recommendation in this case it to place testing utility modules together with the application/library + code, for example ``app.testing.helpers``. + + Important: by "test utility modules" we mean functions/classes which are imported by + other tests directly; this does not include fixtures, which should be placed in ``conftest.py`` files, along + with the test modules, and are discovered automatically by pytest. + + It works like this: + + 1. Given a certain module path, for example ``tests/core/test_models.py``, derives a canonical name + like ``tests.core.test_models`` and tries to import it. - For this reason this doesn't require test module names to be unique. + For non-test modules this will work if they are accessible via :py:data:`sys.path`, so + for example ``.env/lib/site-packages/app/core.py`` will be importable as ``app.core``. + This is happens when plugins import non-test modules (for example doctesting). - One drawback however is that test modules are non-importable by each other. Also, utility - modules in the tests directories are not automatically importable because the tests directory is no longer - added to :py:data:`sys.path`. + If this step succeeds, the module is returned. - Initially we intended to make ``importlib`` the default in future releases, however it is clear now that - it has its own set of drawbacks so the default will remain ``prepend`` for the foreseeable future. + For test modules, unless they are reachable from :py:data:`sys.path`, this step will fail. + + 2. If the previous step fails, we import the module directly using ``importlib`` facilities, which lets us import it without + changing :py:data:`sys.path`. + + Because Python requires the module to also be available in :py:data:`sys.modules`, pytest derives a unique name for it based + on its relative location from the ``rootdir``, and adds the module to :py:data:`sys.modules`. + + For example, ``tests/core/test_models.py`` will end up being imported as the module ``tests.core.test_models``. + + .. versionadded:: 6.0 + +.. note:: + + Initially we intended to make ``importlib`` the default in future releases, however it is clear now that + it has its own set of drawbacks so the default will remain ``prepend`` for the foreseeable future. + +.. note:: + + By default, pytest will not attempt to resolve namespace packages automatically, but that can + be changed via the :confval:`consider_namespace_packages` configuration variable. .. seealso:: The :confval:`pythonpath` configuration variable. + The :confval:`consider_namespace_packages` configuration variable. + + :ref:`test layout`. + ``prepend`` and ``append`` import modes scenarios ------------------------------------------------- Here's a list of scenarios when using ``prepend`` or ``append`` import modes where pytest needs to -change ``sys.path`` in order to import test modules or ``conftest.py`` files, and the issues users +change :py:data:`sys.path` in order to import test modules or ``conftest.py`` files, and the issues users might encounter because of that. Test modules / ``conftest.py`` files inside packages @@ -92,7 +146,7 @@ pytest will find ``foo/bar/tests/test_foo.py`` and realize it is part of a packa there's an ``__init__.py`` file in the same folder. It will then search upwards until it can find the last folder which still contains an ``__init__.py`` file in order to find the package *root* (in this case ``foo/``). To load the module, it will insert ``root/`` to the front of -``sys.path`` (if not there already) in order to load +:py:data:`sys.path` (if not there already) in order to load ``test_foo.py`` as the *module* ``foo.bar.tests.test_foo``. The same logic applies to the ``conftest.py`` file: it will be imported as ``foo.conftest`` module. @@ -122,8 +176,8 @@ When executing: pytest will find ``foo/bar/tests/test_foo.py`` and realize it is NOT part of a package given that there's no ``__init__.py`` file in the same folder. It will then add ``root/foo/bar/tests`` to -``sys.path`` in order to import ``test_foo.py`` as the *module* ``test_foo``. The same is done -with the ``conftest.py`` file by adding ``root/foo`` to ``sys.path`` to import it as ``conftest``. +:py:data:`sys.path` in order to import ``test_foo.py`` as the *module* ``test_foo``. The same is done +with the ``conftest.py`` file by adding ``root/foo`` to :py:data:`sys.path` to import it as ``conftest``. For this reason this layout cannot have test modules with the same name, as they all will be imported in the global import namespace. @@ -136,7 +190,7 @@ Invoking ``pytest`` versus ``python -m pytest`` ----------------------------------------------- Running pytest with ``pytest [...]`` instead of ``python -m pytest [...]`` yields nearly -equivalent behaviour, except that the latter will add the current directory to ``sys.path``, which +equivalent behaviour, except that the latter will add the current directory to :py:data:`sys.path`, which is standard ``python`` behavior. See also :ref:`invoke-python`. diff --git a/doc/en/reference/reference.rst b/doc/en/reference/reference.rst index bba4a399c72..f84b7ea4847 100644 --- a/doc/en/reference/reference.rst +++ b/doc/en/reference/reference.rst @@ -1274,6 +1274,19 @@ passed multiple times. The expected format is ``name=value``. For example:: variables, that will be expanded. For more information about cache plugin please refer to :ref:`cache_provider`. +.. confval:: consider_namespace_packages + + Controls if pytest should attempt to identify `namespace packages `__ + when collecting Python modules. Default is ``False``. + + Set to ``True`` if you are testing namespace packages installed into a virtual environment and it is important for + your packages to be imported using their full namespace package name. + + Only `native namespace packages `__ + are supported, with no plans to support `legacy namespace packages `__. + + .. versionadded:: 8.1 + .. confval:: console_output_style Sets the console output style while running tests: diff --git a/src/_pytest/config/__init__.py b/src/_pytest/config/__init__.py index 069e2196d25..7ed79483c4e 100644 --- a/src/_pytest/config/__init__.py +++ b/src/_pytest/config/__init__.py @@ -547,6 +547,8 @@ def _set_initial_conftests( confcutdir: Optional[Path], invocation_dir: Path, importmode: Union[ImportMode, str], + *, + consider_namespace_packages: bool, ) -> None: """Load initial conftest files given a preparsed "namespace". @@ -572,10 +574,20 @@ def _set_initial_conftests( # Ensure we do not break if what appears to be an anchor # is in fact a very long option (#10169, #11394). if safe_exists(anchor): - self._try_load_conftest(anchor, importmode, rootpath) + self._try_load_conftest( + anchor, + importmode, + rootpath, + consider_namespace_packages=consider_namespace_packages, + ) foundanchor = True if not foundanchor: - self._try_load_conftest(invocation_dir, importmode, rootpath) + self._try_load_conftest( + invocation_dir, + importmode, + rootpath, + consider_namespace_packages=consider_namespace_packages, + ) def _is_in_confcutdir(self, path: Path) -> bool: """Whether to consider the given path to load conftests from.""" @@ -593,17 +605,37 @@ def _is_in_confcutdir(self, path: Path) -> bool: return path not in self._confcutdir.parents def _try_load_conftest( - self, anchor: Path, importmode: Union[str, ImportMode], rootpath: Path + self, + anchor: Path, + importmode: Union[str, ImportMode], + rootpath: Path, + *, + consider_namespace_packages: bool, ) -> None: - self._loadconftestmodules(anchor, importmode, rootpath) + self._loadconftestmodules( + anchor, + importmode, + rootpath, + consider_namespace_packages=consider_namespace_packages, + ) # let's also consider test* subdirs if anchor.is_dir(): for x in anchor.glob("test*"): if x.is_dir(): - self._loadconftestmodules(x, importmode, rootpath) + self._loadconftestmodules( + x, + importmode, + rootpath, + consider_namespace_packages=consider_namespace_packages, + ) def _loadconftestmodules( - self, path: Path, importmode: Union[str, ImportMode], rootpath: Path + self, + path: Path, + importmode: Union[str, ImportMode], + rootpath: Path, + *, + consider_namespace_packages: bool, ) -> None: if self._noconftest: return @@ -620,7 +652,12 @@ def _loadconftestmodules( if self._is_in_confcutdir(parent): conftestpath = parent / "conftest.py" if conftestpath.is_file(): - mod = self._importconftest(conftestpath, importmode, rootpath) + mod = self._importconftest( + conftestpath, + importmode, + rootpath, + consider_namespace_packages=consider_namespace_packages, + ) clist.append(mod) self._dirpath2confmods[directory] = clist @@ -642,7 +679,12 @@ def _rget_with_confmod( raise KeyError(name) def _importconftest( - self, conftestpath: Path, importmode: Union[str, ImportMode], rootpath: Path + self, + conftestpath: Path, + importmode: Union[str, ImportMode], + rootpath: Path, + *, + consider_namespace_packages: bool, ) -> types.ModuleType: conftestpath_plugin_name = str(conftestpath) existing = self.get_plugin(conftestpath_plugin_name) @@ -661,7 +703,12 @@ def _importconftest( pass try: - mod = import_path(conftestpath, mode=importmode, root=rootpath) + mod = import_path( + conftestpath, + mode=importmode, + root=rootpath, + consider_namespace_packages=consider_namespace_packages, + ) except Exception as e: assert e.__traceback__ is not None raise ConftestImportFailure(conftestpath, cause=e) from e @@ -1177,6 +1224,9 @@ def pytest_load_initial_conftests(self, early_config: "Config") -> None: confcutdir=early_config.known_args_namespace.confcutdir, invocation_dir=early_config.invocation_params.dir, importmode=early_config.known_args_namespace.importmode, + consider_namespace_packages=early_config.getini( + "consider_namespace_packages" + ), ) def _initini(self, args: Sequence[str]) -> None: diff --git a/src/_pytest/main.py b/src/_pytest/main.py index d8cd023cc6e..1de86be868c 100644 --- a/src/_pytest/main.py +++ b/src/_pytest/main.py @@ -222,6 +222,12 @@ def pytest_addoption(parser: Parser) -> None: help="Prepend/append to sys.path when importing test modules and conftest " "files. Default: prepend.", ) + parser.addini( + "consider_namespace_packages", + type="bool", + default=False, + help="Consider namespace packages when resolving module names during import", + ) group = parser.getgroup("debugconfig", "test session debugging and configuration") group.addoption( diff --git a/src/_pytest/pathlib.py b/src/_pytest/pathlib.py index 1e0891153e5..a19e89aa116 100644 --- a/src/_pytest/pathlib.py +++ b/src/_pytest/pathlib.py @@ -484,73 +484,86 @@ class ImportPathMismatchError(ImportError): def import_path( - p: Union[str, "os.PathLike[str]"], + path: Union[str, "os.PathLike[str]"], *, mode: Union[str, ImportMode] = ImportMode.prepend, root: Path, + consider_namespace_packages: bool, ) -> ModuleType: - """Import and return a module from the given path, which can be a file (a module) or + """ + Import and return a module from the given path, which can be a file (a module) or a directory (a package). - The import mechanism used is controlled by the `mode` parameter: + :param path: + Path to the file to import. - * `mode == ImportMode.prepend`: the directory containing the module (or package, taking - `__init__.py` files into account) will be put at the *start* of `sys.path` before - being imported with `importlib.import_module`. + :param mode: + Controls the underlying import mechanism that will be used: - * `mode == ImportMode.append`: same as `prepend`, but the directory will be appended - to the end of `sys.path`, if not already in `sys.path`. + * ImportMode.prepend: the directory containing the module (or package, taking + `__init__.py` files into account) will be put at the *start* of `sys.path` before + being imported with `importlib.import_module`. - * `mode == ImportMode.importlib`: uses more fine control mechanisms provided by `importlib` - to import the module, which avoids having to muck with `sys.path` at all. It effectively - allows having same-named test modules in different places. + * ImportMode.append: same as `prepend`, but the directory will be appended + to the end of `sys.path`, if not already in `sys.path`. + + * ImportMode.importlib: uses more fine control mechanisms provided by `importlib` + to import the module, which avoids having to muck with `sys.path` at all. It effectively + allows having same-named test modules in different places. :param root: Used as an anchor when mode == ImportMode.importlib to obtain a unique name for the module being imported so it can safely be stored into ``sys.modules``. + :param consider_namespace_packages: + If True, consider namespace packages when resolving module names. + :raises ImportPathMismatchError: If after importing the given `path` and the module `__file__` are different. Only raised in `prepend` and `append` modes. """ + path = Path(path) mode = ImportMode(mode) - path = Path(p) - if not path.exists(): raise ImportError(path) if mode is ImportMode.importlib: + # Try to import this module using the standard import mechanisms, but + # without touching sys.path. + try: + pkg_root, module_name = resolve_pkg_root_and_module_name( + path, consider_namespace_packages=consider_namespace_packages + ) + except CouldNotResolvePathError: + pass + else: + mod = _import_module_using_spec( + module_name, path, pkg_root, insert_modules=False + ) + if mod is not None: + return mod + + # Could not import the module with the current sys.path, so we fall back + # to importing the file as a single module, not being a part of a package. module_name = module_name_from_path(path, root) with contextlib.suppress(KeyError): return sys.modules[module_name] - for meta_importer in sys.meta_path: - spec = meta_importer.find_spec(module_name, [str(path.parent)]) - if spec is not None: - break - else: - spec = importlib.util.spec_from_file_location(module_name, str(path)) - - if spec is None: + mod = _import_module_using_spec( + module_name, path, path.parent, insert_modules=True + ) + if mod is None: raise ImportError(f"Can't find module {module_name} at location {path}") - mod = importlib.util.module_from_spec(spec) - sys.modules[module_name] = mod - spec.loader.exec_module(mod) # type: ignore[union-attr] - insert_missing_modules(sys.modules, module_name) return mod - pkg_path = resolve_package_path(path) - if pkg_path is not None: - pkg_root = pkg_path.parent - names = list(path.with_suffix("").relative_to(pkg_root).parts) - if names[-1] == "__init__": - names.pop() - module_name = ".".join(names) - else: - pkg_root = path.parent - module_name = path.stem + try: + pkg_root, module_name = resolve_pkg_root_and_module_name( + path, consider_namespace_packages=consider_namespace_packages + ) + except CouldNotResolvePathError: + pkg_root, module_name = path.parent, path.stem # Change sys.path permanently: restoring it at the end of this function would cause surprising # problems because of delayed imports: for example, a conftest.py file imported by this function @@ -592,6 +605,40 @@ def import_path( return mod +def _import_module_using_spec( + module_name: str, module_path: Path, module_location: Path, *, insert_modules: bool +) -> Optional[ModuleType]: + """ + Tries to import a module by its canonical name, path to the .py file, and its + parent location. + + :param insert_modules: + If True, will call insert_missing_modules to create empty intermediate modules + for made-up module names (when importing test files not reachable from sys.path). + Note: we can probably drop insert_missing_modules altogether: instead of + generating module names such as "src.tests.test_foo", which require intermediate + empty modules, we might just as well generate unique module names like + "src_tests_test_foo". + """ + # Checking with sys.meta_path first in case one of its hooks can import this module, + # such as our own assertion-rewrite hook. + for meta_importer in sys.meta_path: + spec = meta_importer.find_spec(module_name, [str(module_location)]) + if spec is not None: + break + else: + spec = importlib.util.spec_from_file_location(module_name, str(module_path)) + if spec is not None: + mod = importlib.util.module_from_spec(spec) + sys.modules[module_name] = mod + spec.loader.exec_module(mod) # type: ignore[union-attr] + if insert_modules: + insert_missing_modules(sys.modules, module_name) + return mod + + return None + + # Implement a special _is_same function on Windows which returns True if the two filenames # compare equal, to circumvent os.path.samefile returning False for mounts in UNC (#7678). if sys.platform.startswith("win"): @@ -628,6 +675,11 @@ def module_name_from_path(path: Path, root: Path) -> str: if len(path_parts) >= 2 and path_parts[-1] == "__init__": path_parts = path_parts[:-1] + # Module names cannot contain ".", normalize them to "_". This prevents + # a directory having a "." in the name (".env.310" for example) causing extra intermediate modules. + # Also, important to replace "." at the start of paths, as those are considered relative imports. + path_parts = tuple(x.replace(".", "_") for x in path_parts) + return ".".join(path_parts) @@ -689,6 +741,60 @@ def resolve_package_path(path: Path) -> Optional[Path]: return result +def resolve_pkg_root_and_module_name( + path: Path, *, consider_namespace_packages: bool = False +) -> Tuple[Path, str]: + """ + Return the path to the directory of the root package that contains the + given Python file, and its module name: + + src/ + app/ + __init__.py + core/ + __init__.py + models.py + + Passing the full path to `models.py` will yield Path("src") and "app.core.models". + + If consider_namespace_packages is True, then we additionally check upwards in the hierarchy + until we find a directory that is reachable from sys.path, which marks it as a namespace package: + + https://packaging.python.org/en/latest/guides/packaging-namespace-packages + + Raises CouldNotResolvePathError if the given path does not belong to a package (missing any __init__.py files). + """ + pkg_path = resolve_package_path(path) + if pkg_path is not None: + pkg_root = pkg_path.parent + # https://packaging.python.org/en/latest/guides/packaging-namespace-packages/ + if consider_namespace_packages: + # Go upwards in the hierarchy, if we find a parent path included + # in sys.path, it means the package found by resolve_package_path() + # actually belongs to a namespace package. + for parent in pkg_root.parents: + # If any of the parent paths has a __init__.py, it means it is not + # a namespace package (see the docs linked above). + if (parent / "__init__.py").is_file(): + break + if str(parent) in sys.path: + # Point the pkg_root to the root of the namespace package. + pkg_root = parent + break + + names = list(path.with_suffix("").relative_to(pkg_root).parts) + if names[-1] == "__init__": + names.pop() + module_name = ".".join(names) + return pkg_root, module_name + + raise CouldNotResolvePathError(f"Could not resolve for {path}") + + +class CouldNotResolvePathError(Exception): + """Custom exception raised by resolve_pkg_root_and_module_name.""" + + def scandir( path: Union[str, "os.PathLike[str]"], sort_key: Callable[["os.DirEntry[str]"], object] = lambda entry: entry.name, diff --git a/src/_pytest/python.py b/src/_pytest/python.py index ca64a877d42..e1730b1a7e0 100644 --- a/src/_pytest/python.py +++ b/src/_pytest/python.py @@ -516,7 +516,12 @@ def importtestmodule( # We assume we are only called once per module. importmode = config.getoption("--import-mode") try: - mod = import_path(path, mode=importmode, root=config.rootpath) + mod = import_path( + path, + mode=importmode, + root=config.rootpath, + consider_namespace_packages=config.getini("consider_namespace_packages"), + ) except SyntaxError as e: raise nodes.Collector.CollectError( ExceptionInfo.from_current().getrepr(style="short") diff --git a/src/_pytest/runner.py b/src/_pytest/runner.py index b60af9dd3fb..16abb895d58 100644 --- a/src/_pytest/runner.py +++ b/src/_pytest/runner.py @@ -380,6 +380,9 @@ def collect() -> List[Union[Item, Collector]]: collector.path, collector.config.getoption("importmode"), rootpath=collector.config.rootpath, + consider_namespace_packages=collector.config.getini( + "consider_namespace_packages" + ), ) return list(collector.collect()) diff --git a/testing/code/test_excinfo.py b/testing/code/test_excinfo.py index cce23bf87d4..49c5dd3715b 100644 --- a/testing/code/test_excinfo.py +++ b/testing/code/test_excinfo.py @@ -180,7 +180,7 @@ def test_traceback_cut(self) -> None: def test_traceback_cut_excludepath(self, pytester: Pytester) -> None: p = pytester.makepyfile("def f(): raise ValueError") with pytest.raises(ValueError) as excinfo: - import_path(p, root=pytester.path).f() # type: ignore[attr-defined] + import_path(p, root=pytester.path, consider_namespace_packages=False).f() # type: ignore[attr-defined] basedir = Path(pytest.__file__).parent newtraceback = excinfo.traceback.cut(excludepath=basedir) for x in newtraceback: @@ -543,7 +543,9 @@ def importasmod(source): tmp_path.joinpath("__init__.py").touch() modpath.write_text(source, encoding="utf-8") importlib.invalidate_caches() - return import_path(modpath, root=tmp_path) + return import_path( + modpath, root=tmp_path, consider_namespace_packages=False + ) return importasmod diff --git a/testing/code/test_source.py b/testing/code/test_source.py index 9d0565380e4..12ea27b3517 100644 --- a/testing/code/test_source.py +++ b/testing/code/test_source.py @@ -296,7 +296,7 @@ def method(self): ) path = tmp_path.joinpath("a.py") path.write_text(str(source), encoding="utf-8") - mod: Any = import_path(path, root=tmp_path) + mod: Any = import_path(path, root=tmp_path, consider_namespace_packages=False) s2 = Source(mod.A) assert str(source).strip() == str(s2).strip() diff --git a/testing/test_conftest.py b/testing/test_conftest.py index bb74fa75d61..3116dfe2584 100644 --- a/testing/test_conftest.py +++ b/testing/test_conftest.py @@ -38,6 +38,7 @@ def conftest_setinitial( confcutdir=confcutdir, invocation_dir=Path.cwd(), importmode="prepend", + consider_namespace_packages=False, ) @@ -64,7 +65,9 @@ def basedir( def test_basic_init(self, basedir: Path) -> None: conftest = PytestPluginManager() p = basedir / "adir" - conftest._loadconftestmodules(p, importmode="prepend", rootpath=basedir) + conftest._loadconftestmodules( + p, importmode="prepend", rootpath=basedir, consider_namespace_packages=False + ) assert conftest._rget_with_confmod("a", p)[1] == 1 def test_immediate_initialiation_and_incremental_are_the_same( @@ -72,15 +75,26 @@ def test_immediate_initialiation_and_incremental_are_the_same( ) -> None: conftest = PytestPluginManager() assert not len(conftest._dirpath2confmods) - conftest._loadconftestmodules(basedir, importmode="prepend", rootpath=basedir) + conftest._loadconftestmodules( + basedir, + importmode="prepend", + rootpath=basedir, + consider_namespace_packages=False, + ) snap1 = len(conftest._dirpath2confmods) assert snap1 == 1 conftest._loadconftestmodules( - basedir / "adir", importmode="prepend", rootpath=basedir + basedir / "adir", + importmode="prepend", + rootpath=basedir, + consider_namespace_packages=False, ) assert len(conftest._dirpath2confmods) == snap1 + 1 conftest._loadconftestmodules( - basedir / "b", importmode="prepend", rootpath=basedir + basedir / "b", + importmode="prepend", + rootpath=basedir, + consider_namespace_packages=False, ) assert len(conftest._dirpath2confmods) == snap1 + 2 @@ -92,10 +106,18 @@ def test_value_access_not_existing(self, basedir: Path) -> None: def test_value_access_by_path(self, basedir: Path) -> None: conftest = ConftestWithSetinitial(basedir) adir = basedir / "adir" - conftest._loadconftestmodules(adir, importmode="prepend", rootpath=basedir) + conftest._loadconftestmodules( + adir, + importmode="prepend", + rootpath=basedir, + consider_namespace_packages=False, + ) assert conftest._rget_with_confmod("a", adir)[1] == 1 conftest._loadconftestmodules( - adir / "b", importmode="prepend", rootpath=basedir + adir / "b", + importmode="prepend", + rootpath=basedir, + consider_namespace_packages=False, ) assert conftest._rget_with_confmod("a", adir / "b")[1] == 1.5 @@ -152,7 +174,12 @@ def test_conftest_global_import(pytester: Pytester) -> None: import pytest from _pytest.config import PytestPluginManager conf = PytestPluginManager() - mod = conf._importconftest(Path("conftest.py"), importmode="prepend", rootpath=Path.cwd()) + mod = conf._importconftest( + Path("conftest.py"), + importmode="prepend", + rootpath=Path.cwd(), + consider_namespace_packages=False, + ) assert mod.x == 3 import conftest assert conftest is mod, (conftest, mod) @@ -160,7 +187,12 @@ def test_conftest_global_import(pytester: Pytester) -> None: sub.mkdir() subconf = sub / "conftest.py" subconf.write_text("y=4", encoding="utf-8") - mod2 = conf._importconftest(subconf, importmode="prepend", rootpath=Path.cwd()) + mod2 = conf._importconftest( + subconf, + importmode="prepend", + rootpath=Path.cwd(), + consider_namespace_packages=False, + ) assert mod != mod2 assert mod2.y == 4 import conftest @@ -176,17 +208,30 @@ def test_conftestcutdir(pytester: Pytester) -> None: p = pytester.mkdir("x") conftest = PytestPluginManager() conftest_setinitial(conftest, [pytester.path], confcutdir=p) - conftest._loadconftestmodules(p, importmode="prepend", rootpath=pytester.path) + conftest._loadconftestmodules( + p, + importmode="prepend", + rootpath=pytester.path, + consider_namespace_packages=False, + ) values = conftest._getconftestmodules(p) assert len(values) == 0 conftest._loadconftestmodules( - conf.parent, importmode="prepend", rootpath=pytester.path + conf.parent, + importmode="prepend", + rootpath=pytester.path, + consider_namespace_packages=False, ) values = conftest._getconftestmodules(conf.parent) assert len(values) == 0 assert not conftest.has_plugin(str(conf)) # but we can still import a conftest directly - conftest._importconftest(conf, importmode="prepend", rootpath=pytester.path) + conftest._importconftest( + conf, + importmode="prepend", + rootpath=pytester.path, + consider_namespace_packages=False, + ) values = conftest._getconftestmodules(conf.parent) assert values[0].__file__ is not None assert values[0].__file__.startswith(str(conf)) @@ -405,13 +450,18 @@ def test_conftest_import_order(pytester: Pytester, monkeypatch: MonkeyPatch) -> ct2 = sub / "conftest.py" ct2.write_text("", encoding="utf-8") - def impct(p, importmode, root): + def impct(p, importmode, root, consider_namespace_packages): return p conftest = PytestPluginManager() conftest._confcutdir = pytester.path monkeypatch.setattr(conftest, "_importconftest", impct) - conftest._loadconftestmodules(sub, importmode="prepend", rootpath=pytester.path) + conftest._loadconftestmodules( + sub, + importmode="prepend", + rootpath=pytester.path, + consider_namespace_packages=False, + ) mods = cast(List[Path], conftest._getconftestmodules(sub)) expected = [ct1, ct2] assert mods == expected diff --git a/testing/test_doctest.py b/testing/test_doctest.py index 32897a916fe..58fce244f45 100644 --- a/testing/test_doctest.py +++ b/testing/test_doctest.py @@ -117,12 +117,12 @@ def test_simple_doctestfile(self, pytester: Pytester): def test_importmode(self, pytester: Pytester): pytester.makepyfile( **{ - "namespacepkg/innerpkg/__init__.py": "", - "namespacepkg/innerpkg/a.py": """ + "src/namespacepkg/innerpkg/__init__.py": "", + "src/namespacepkg/innerpkg/a.py": """ def some_func(): return 42 """, - "namespacepkg/innerpkg/b.py": """ + "src/namespacepkg/innerpkg/b.py": """ from namespacepkg.innerpkg.a import some_func def my_func(): ''' @@ -133,6 +133,10 @@ def my_func(): """, } ) + # For 'namespacepkg' to be considered a namespace package, its containing directory + # needs to be reachable from sys.path: + # https://packaging.python.org/en/latest/guides/packaging-namespace-packages + pytester.syspathinsert(pytester.path / "src") reprec = pytester.inline_run("--doctest-modules", "--import-mode=importlib") reprec.assertoutcome(passed=1) diff --git a/testing/test_pathlib.py b/testing/test_pathlib.py index 075259009de..357860563a8 100644 --- a/testing/test_pathlib.py +++ b/testing/test_pathlib.py @@ -3,17 +3,20 @@ import os.path from pathlib import Path import pickle +import shutil import sys from textwrap import dedent from types import ModuleType from typing import Any from typing import Generator from typing import Iterator +from typing import Tuple import unittest.mock from _pytest.monkeypatch import MonkeyPatch from _pytest.pathlib import bestrelpath from _pytest.pathlib import commonpath +from _pytest.pathlib import CouldNotResolvePathError from _pytest.pathlib import ensure_deletable from _pytest.pathlib import fnmatch_ex from _pytest.pathlib import get_extended_length_path_str @@ -25,6 +28,7 @@ from _pytest.pathlib import maybe_delete_a_numbered_dir from _pytest.pathlib import module_name_from_path from _pytest.pathlib import resolve_package_path +from _pytest.pathlib import resolve_pkg_root_and_module_name from _pytest.pathlib import safe_exists from _pytest.pathlib import symlink_or_skip from _pytest.pathlib import visit @@ -33,6 +37,20 @@ import pytest +@pytest.fixture(autouse=True) +def autouse_pytester(pytester: Pytester) -> None: + """ + Fixture to make pytester() being autouse for all tests in this module. + + pytester makes sure to restore sys.path to its previous state, and many tests in this module + import modules and change sys.path because of that, so common module names such as "test" or "test.conftest" + end up leaking to tests in other modules. + + Note: we might consider extracting the sys.path restoration aspect into its own fixture, and apply it + to the entire test suite always. + """ + + class TestFNMatcherPort: """Test our port of py.common.FNMatcher (fnmatch_ex).""" @@ -82,6 +100,15 @@ def test_not_matching(self, pattern: str, path: str) -> None: assert not fnmatch_ex(pattern, path) +@pytest.fixture(params=[True, False]) +def ns_param(request: pytest.FixtureRequest) -> bool: + """ + Simple parametrized fixture for tests which call import_path() with consider_namespace_packages + using True and False. + """ + return bool(request.param) + + class TestImportPath: """ @@ -152,87 +179,113 @@ def setuptestfs(self, path: Path) -> None: encoding="utf-8", ) - def test_smoke_test(self, path1: Path) -> None: - obj = import_path(path1 / "execfile.py", root=path1) + def test_smoke_test(self, path1: Path, ns_param: bool) -> None: + obj = import_path( + path1 / "execfile.py", root=path1, consider_namespace_packages=ns_param + ) assert obj.x == 42 # type: ignore[attr-defined] assert obj.__name__ == "execfile" - def test_import_path_missing_file(self, path1: Path) -> None: + def test_import_path_missing_file(self, path1: Path, ns_param: bool) -> None: with pytest.raises(ImportPathMismatchError): - import_path(path1 / "sampledir", root=path1) + import_path( + path1 / "sampledir", root=path1, consider_namespace_packages=ns_param + ) def test_renamed_dir_creates_mismatch( - self, tmp_path: Path, monkeypatch: MonkeyPatch + self, tmp_path: Path, monkeypatch: MonkeyPatch, ns_param: bool ) -> None: tmp_path.joinpath("a").mkdir() p = tmp_path.joinpath("a", "test_x123.py") p.touch() - import_path(p, root=tmp_path) + import_path(p, root=tmp_path, consider_namespace_packages=ns_param) tmp_path.joinpath("a").rename(tmp_path.joinpath("b")) with pytest.raises(ImportPathMismatchError): - import_path(tmp_path.joinpath("b", "test_x123.py"), root=tmp_path) + import_path( + tmp_path.joinpath("b", "test_x123.py"), + root=tmp_path, + consider_namespace_packages=ns_param, + ) # Errors can be ignored. monkeypatch.setenv("PY_IGNORE_IMPORTMISMATCH", "1") - import_path(tmp_path.joinpath("b", "test_x123.py"), root=tmp_path) + import_path( + tmp_path.joinpath("b", "test_x123.py"), + root=tmp_path, + consider_namespace_packages=ns_param, + ) # PY_IGNORE_IMPORTMISMATCH=0 does not ignore error. monkeypatch.setenv("PY_IGNORE_IMPORTMISMATCH", "0") with pytest.raises(ImportPathMismatchError): - import_path(tmp_path.joinpath("b", "test_x123.py"), root=tmp_path) + import_path( + tmp_path.joinpath("b", "test_x123.py"), + root=tmp_path, + consider_namespace_packages=ns_param, + ) - def test_messy_name(self, tmp_path: Path) -> None: + def test_messy_name(self, tmp_path: Path, ns_param: bool) -> None: # https://bitbucket.org/hpk42/py-trunk/issue/129 path = tmp_path / "foo__init__.py" path.touch() - module = import_path(path, root=tmp_path) + module = import_path(path, root=tmp_path, consider_namespace_packages=ns_param) assert module.__name__ == "foo__init__" - def test_dir(self, tmp_path: Path) -> None: + def test_dir(self, tmp_path: Path, ns_param: bool) -> None: p = tmp_path / "hello_123" p.mkdir() p_init = p / "__init__.py" p_init.touch() - m = import_path(p, root=tmp_path) + m = import_path(p, root=tmp_path, consider_namespace_packages=ns_param) assert m.__name__ == "hello_123" - m = import_path(p_init, root=tmp_path) + m = import_path(p_init, root=tmp_path, consider_namespace_packages=ns_param) assert m.__name__ == "hello_123" - def test_a(self, path1: Path) -> None: + def test_a(self, path1: Path, ns_param: bool) -> None: otherdir = path1 / "otherdir" - mod = import_path(otherdir / "a.py", root=path1) + mod = import_path( + otherdir / "a.py", root=path1, consider_namespace_packages=ns_param + ) assert mod.result == "got it" # type: ignore[attr-defined] assert mod.__name__ == "otherdir.a" - def test_b(self, path1: Path) -> None: + def test_b(self, path1: Path, ns_param: bool) -> None: otherdir = path1 / "otherdir" - mod = import_path(otherdir / "b.py", root=path1) + mod = import_path( + otherdir / "b.py", root=path1, consider_namespace_packages=ns_param + ) assert mod.stuff == "got it" # type: ignore[attr-defined] assert mod.__name__ == "otherdir.b" - def test_c(self, path1: Path) -> None: + def test_c(self, path1: Path, ns_param: bool) -> None: otherdir = path1 / "otherdir" - mod = import_path(otherdir / "c.py", root=path1) + mod = import_path( + otherdir / "c.py", root=path1, consider_namespace_packages=ns_param + ) assert mod.value == "got it" # type: ignore[attr-defined] - def test_d(self, path1: Path) -> None: + def test_d(self, path1: Path, ns_param: bool) -> None: otherdir = path1 / "otherdir" - mod = import_path(otherdir / "d.py", root=path1) + mod = import_path( + otherdir / "d.py", root=path1, consider_namespace_packages=ns_param + ) assert mod.value2 == "got it" # type: ignore[attr-defined] - def test_import_after(self, tmp_path: Path) -> None: + def test_import_after(self, tmp_path: Path, ns_param: bool) -> None: tmp_path.joinpath("xxxpackage").mkdir() tmp_path.joinpath("xxxpackage", "__init__.py").touch() mod1path = tmp_path.joinpath("xxxpackage", "module1.py") mod1path.touch() - mod1 = import_path(mod1path, root=tmp_path) + mod1 = import_path( + mod1path, root=tmp_path, consider_namespace_packages=ns_param + ) assert mod1.__name__ == "xxxpackage.module1" from xxxpackage import module1 assert module1 is mod1 def test_check_filepath_consistency( - self, monkeypatch: MonkeyPatch, tmp_path: Path + self, monkeypatch: MonkeyPatch, tmp_path: Path, ns_param: bool ) -> None: name = "pointsback123" p = tmp_path.joinpath(name + ".py") @@ -244,7 +297,9 @@ def test_check_filepath_consistency( pseudopath.touch() mod.__file__ = str(pseudopath) mp.setitem(sys.modules, name, mod) - newmod = import_path(p, root=tmp_path) + newmod = import_path( + p, root=tmp_path, consider_namespace_packages=ns_param + ) assert mod == newmod mod = ModuleType(name) pseudopath = tmp_path.joinpath(name + "123.py") @@ -252,40 +307,32 @@ def test_check_filepath_consistency( mod.__file__ = str(pseudopath) monkeypatch.setitem(sys.modules, name, mod) with pytest.raises(ImportPathMismatchError) as excinfo: - import_path(p, root=tmp_path) + import_path(p, root=tmp_path, consider_namespace_packages=ns_param) modname, modfile, orig = excinfo.value.args assert modname == name assert modfile == str(pseudopath) assert orig == p assert issubclass(ImportPathMismatchError, ImportError) - def test_issue131_on__init__(self, tmp_path: Path) -> None: - # __init__.py files may be namespace packages, and thus the - # __file__ of an imported module may not be ourselves - # see issue - tmp_path.joinpath("proja").mkdir() - p1 = tmp_path.joinpath("proja", "__init__.py") - p1.touch() - tmp_path.joinpath("sub", "proja").mkdir(parents=True) - p2 = tmp_path.joinpath("sub", "proja", "__init__.py") - p2.touch() - m1 = import_path(p1, root=tmp_path) - m2 = import_path(p2, root=tmp_path) - assert m1 == m2 - - def test_ensuresyspath_append(self, tmp_path: Path) -> None: + def test_ensuresyspath_append(self, tmp_path: Path, ns_param: bool) -> None: root1 = tmp_path / "root1" root1.mkdir() file1 = root1 / "x123.py" file1.touch() assert str(root1) not in sys.path - import_path(file1, mode="append", root=tmp_path) + import_path( + file1, mode="append", root=tmp_path, consider_namespace_packages=ns_param + ) assert str(root1) == sys.path[-1] assert str(root1) not in sys.path[:-1] - def test_invalid_path(self, tmp_path: Path) -> None: + def test_invalid_path(self, tmp_path: Path, ns_param: bool) -> None: with pytest.raises(ImportError): - import_path(tmp_path / "invalid.py", root=tmp_path) + import_path( + tmp_path / "invalid.py", + root=tmp_path, + consider_namespace_packages=ns_param, + ) @pytest.fixture def simple_module( @@ -300,10 +347,19 @@ def simple_module( sys.modules.pop(module_name, None) def test_importmode_importlib( - self, simple_module: Path, tmp_path: Path, request: pytest.FixtureRequest + self, + simple_module: Path, + tmp_path: Path, + request: pytest.FixtureRequest, + ns_param: bool, ) -> None: """`importlib` mode does not change sys.path.""" - module = import_path(simple_module, mode="importlib", root=tmp_path) + module = import_path( + simple_module, + mode="importlib", + root=tmp_path, + consider_namespace_packages=ns_param, + ) assert module.foo(2) == 42 # type: ignore[attr-defined] assert str(simple_module.parent) not in sys.path assert module.__name__ in sys.modules @@ -312,19 +368,38 @@ def test_importmode_importlib( assert "_src.tests" in sys.modules def test_remembers_previous_imports( - self, simple_module: Path, tmp_path: Path + self, simple_module: Path, tmp_path: Path, ns_param: bool ) -> None: """`importlib` mode called remembers previous module (#10341, #10811).""" - module1 = import_path(simple_module, mode="importlib", root=tmp_path) - module2 = import_path(simple_module, mode="importlib", root=tmp_path) + module1 = import_path( + simple_module, + mode="importlib", + root=tmp_path, + consider_namespace_packages=ns_param, + ) + module2 = import_path( + simple_module, + mode="importlib", + root=tmp_path, + consider_namespace_packages=ns_param, + ) assert module1 is module2 def test_no_meta_path_found( - self, simple_module: Path, monkeypatch: MonkeyPatch, tmp_path: Path + self, + simple_module: Path, + monkeypatch: MonkeyPatch, + tmp_path: Path, + ns_param: bool, ) -> None: """Even without any meta_path should still import module.""" monkeypatch.setattr(sys, "meta_path", []) - module = import_path(simple_module, mode="importlib", root=tmp_path) + module = import_path( + simple_module, + mode="importlib", + root=tmp_path, + consider_namespace_packages=ns_param, + ) assert module.foo(2) == 42 # type: ignore[attr-defined] # mode='importlib' fails if no spec is found to load the module @@ -337,7 +412,12 @@ def test_no_meta_path_found( importlib.util, "spec_from_file_location", lambda *args: None ) with pytest.raises(ImportError): - import_path(simple_module, mode="importlib", root=tmp_path) + import_path( + simple_module, + mode="importlib", + root=tmp_path, + consider_namespace_packages=False, + ) def test_resolve_package_path(tmp_path: Path) -> None: @@ -473,12 +553,16 @@ def test_samefile_false_negatives(tmp_path: Path, monkeypatch: MonkeyPatch) -> N # the paths too. Using a context to narrow the patch as much as possible given # this is an important system function. mp.setattr(os.path, "samefile", lambda x, y: False) - module = import_path(module_path, root=tmp_path) + module = import_path( + module_path, root=tmp_path, consider_namespace_packages=False + ) assert getattr(module, "foo")() == 42 class TestImportLibMode: - def test_importmode_importlib_with_dataclass(self, tmp_path: Path) -> None: + def test_importmode_importlib_with_dataclass( + self, tmp_path: Path, ns_param: bool + ) -> None: """Ensure that importlib mode works with a module containing dataclasses (#7856).""" fn = tmp_path.joinpath("_src/tests/test_dataclass.py") fn.parent.mkdir(parents=True) @@ -495,13 +579,17 @@ class Data: encoding="utf-8", ) - module = import_path(fn, mode="importlib", root=tmp_path) + module = import_path( + fn, mode="importlib", root=tmp_path, consider_namespace_packages=ns_param + ) Data: Any = getattr(module, "Data") data = Data(value="foo") assert data.value == "foo" assert data.__module__ == "_src.tests.test_dataclass" - def test_importmode_importlib_with_pickle(self, tmp_path: Path) -> None: + def test_importmode_importlib_with_pickle( + self, tmp_path: Path, ns_param: bool + ) -> None: """Ensure that importlib mode works with pickle (#7859).""" fn = tmp_path.joinpath("_src/tests/test_pickle.py") fn.parent.mkdir(parents=True) @@ -521,13 +609,15 @@ def round_trip(): encoding="utf-8", ) - module = import_path(fn, mode="importlib", root=tmp_path) + module = import_path( + fn, mode="importlib", root=tmp_path, consider_namespace_packages=ns_param + ) round_trip = getattr(module, "round_trip") action = round_trip() assert action() == 42 def test_importmode_importlib_with_pickle_separate_modules( - self, tmp_path: Path + self, tmp_path: Path, ns_param: bool ) -> None: """ Ensure that importlib mode works can load pickles that look similar but are @@ -571,10 +661,14 @@ def round_trip(obj): s = pickle.dumps(obj) return pickle.loads(s) - module = import_path(fn1, mode="importlib", root=tmp_path) + module = import_path( + fn1, mode="importlib", root=tmp_path, consider_namespace_packages=ns_param + ) Data1 = getattr(module, "Data") - module = import_path(fn2, mode="importlib", root=tmp_path) + module = import_path( + fn2, mode="importlib", root=tmp_path, consider_namespace_packages=ns_param + ) Data2 = getattr(module, "Data") assert round_trip(Data1(20)) == Data1(20) @@ -598,6 +692,53 @@ def test_module_name_from_path(self, tmp_path: Path) -> None: result = module_name_from_path(tmp_path / "__init__.py", tmp_path) assert result == "__init__" + # Modules which start with "." are considered relative and will not be imported + # unless part of a package, so we replace it with a "_" when generating the fake module name. + result = module_name_from_path(tmp_path / ".env/tests/test_foo.py", tmp_path) + assert result == "_env.tests.test_foo" + + # We want to avoid generating extra intermediate modules if some directory just happens + # to contain a "." in the name. + result = module_name_from_path( + tmp_path / ".env.310/tests/test_foo.py", tmp_path + ) + assert result == "_env_310.tests.test_foo" + + def test_resolve_pkg_root_and_module_name( + self, tmp_path: Path, monkeypatch: MonkeyPatch + ) -> None: + # Create a directory structure first without __init__.py files. + (tmp_path / "src/app/core").mkdir(parents=True) + models_py = tmp_path / "src/app/core/models.py" + models_py.touch() + with pytest.raises(CouldNotResolvePathError): + _ = resolve_pkg_root_and_module_name(models_py) + + # Create the __init__.py files, it should now resolve to a proper module name. + (tmp_path / "src/app/__init__.py").touch() + (tmp_path / "src/app/core/__init__.py").touch() + assert resolve_pkg_root_and_module_name( + models_py, consider_namespace_packages=True + ) == ( + tmp_path / "src", + "app.core.models", + ) + + # If we add tmp_path to sys.path, src becomes a namespace package. + monkeypatch.syspath_prepend(tmp_path) + assert resolve_pkg_root_and_module_name( + models_py, consider_namespace_packages=True + ) == ( + tmp_path, + "src.app.core.models", + ) + assert resolve_pkg_root_and_module_name( + models_py, consider_namespace_packages=False + ) == ( + tmp_path / "src", + "app.core.models", + ) + def test_insert_missing_modules( self, monkeypatch: MonkeyPatch, tmp_path: Path ) -> None: @@ -629,7 +770,9 @@ def test_parent_contains_child_module_attribute( assert modules["xxx"].tests is modules["xxx.tests"] assert modules["xxx.tests"].foo is modules["xxx.tests.foo"] - def test_importlib_package(self, monkeypatch: MonkeyPatch, tmp_path: Path): + def test_importlib_package( + self, monkeypatch: MonkeyPatch, tmp_path: Path, ns_param: bool + ): """ Importing a package using --importmode=importlib should not import the package's __init__.py file more than once (#11306). @@ -666,7 +809,12 @@ def __init__(self) -> None: encoding="ascii", ) - mod = import_path(init, root=tmp_path, mode=ImportMode.importlib) + mod = import_path( + init, + root=tmp_path, + mode=ImportMode.importlib, + consider_namespace_packages=ns_param, + ) assert len(mod.instance.INSTANCES) == 1 def test_importlib_root_is_package(self, pytester: Pytester) -> None: @@ -685,6 +833,203 @@ def test_my_test(): result = pytester.runpytest("--import-mode=importlib") result.stdout.fnmatch_lines("* 1 passed *") + def create_installed_doctests_and_tests_dir( + self, path: Path, monkeypatch: MonkeyPatch + ) -> Tuple[Path, Path, Path]: + """ + Create a directory structure where the application code is installed in a virtual environment, + and the tests are in an outside ".tests" directory. + + Return the paths to the core module (installed in the virtualenv), and the test modules. + """ + app = path / "src/app" + app.mkdir(parents=True) + (app / "__init__.py").touch() + core_py = app / "core.py" + core_py.write_text( + dedent( + """ + def foo(): + ''' + >>> 1 + 1 + 2 + ''' + """ + ), + encoding="ascii", + ) + + # Install it into a site-packages directory, and add it to sys.path, mimicking what + # happens when installing into a virtualenv. + site_packages = path / ".env/lib/site-packages" + site_packages.mkdir(parents=True) + shutil.copytree(app, site_packages / "app") + assert (site_packages / "app/core.py").is_file() + + monkeypatch.syspath_prepend(site_packages) + + # Create the tests files, outside 'src' and the virtualenv. + # We use the same test name on purpose, but in different directories, to ensure + # this works as advertised. + conftest_path1 = path / ".tests/a/conftest.py" + conftest_path1.parent.mkdir(parents=True) + conftest_path1.write_text( + dedent( + """ + import pytest + @pytest.fixture + def a_fix(): return "a" + """ + ), + encoding="ascii", + ) + test_path1 = path / ".tests/a/test_core.py" + test_path1.write_text( + dedent( + """ + import app.core + def test(a_fix): + assert a_fix == "a" + """, + ), + encoding="ascii", + ) + + conftest_path2 = path / ".tests/b/conftest.py" + conftest_path2.parent.mkdir(parents=True) + conftest_path2.write_text( + dedent( + """ + import pytest + @pytest.fixture + def b_fix(): return "b" + """ + ), + encoding="ascii", + ) + + test_path2 = path / ".tests/b/test_core.py" + test_path2.write_text( + dedent( + """ + import app.core + def test(b_fix): + assert b_fix == "b" + """, + ), + encoding="ascii", + ) + return (site_packages / "app/core.py"), test_path1, test_path2 + + def test_import_using_normal_mechanism_first( + self, monkeypatch: MonkeyPatch, pytester: Pytester, ns_param: bool + ) -> None: + """ + Test import_path imports from the canonical location when possible first, only + falling back to its normal flow when the module being imported is not reachable via sys.path (#11475). + """ + core_py, test_path1, test_path2 = self.create_installed_doctests_and_tests_dir( + pytester.path, monkeypatch + ) + + # core_py is reached from sys.path, so should be imported normally. + mod = import_path( + core_py, + mode="importlib", + root=pytester.path, + consider_namespace_packages=ns_param, + ) + assert mod.__name__ == "app.core" + assert mod.__file__ and Path(mod.__file__) == core_py + + # tests are not reachable from sys.path, so they are imported as a standalone modules. + # Instead of '.tests.a.test_core', we import as "_tests.a.test_core" because + # importlib considers module names starting with '.' to be local imports. + mod = import_path( + test_path1, + mode="importlib", + root=pytester.path, + consider_namespace_packages=ns_param, + ) + assert mod.__name__ == "_tests.a.test_core" + mod = import_path( + test_path2, + mode="importlib", + root=pytester.path, + consider_namespace_packages=ns_param, + ) + assert mod.__name__ == "_tests.b.test_core" + + def test_import_using_normal_mechanism_first_integration( + self, monkeypatch: MonkeyPatch, pytester: Pytester, ns_param: bool + ) -> None: + """ + Same test as above, but verify the behavior calling pytest. + + We should not make this call in the same test as above, as the modules have already + been imported by separate import_path() calls. + """ + core_py, test_path1, test_path2 = self.create_installed_doctests_and_tests_dir( + pytester.path, monkeypatch + ) + result = pytester.runpytest( + "--import-mode=importlib", + "-o", + f"consider_namespace_packages={ns_param}", + "--doctest-modules", + "--pyargs", + "app", + "./.tests", + ) + result.stdout.fnmatch_lines( + [ + f"{core_py.relative_to(pytester.path)} . *", + f"{test_path1.relative_to(pytester.path)} . *", + f"{test_path2.relative_to(pytester.path)} . *", + "* 3 passed*", + ] + ) + + def test_import_path_imports_correct_file( + self, pytester: Pytester, ns_param: bool + ) -> None: + """ + Import the module by the given path, even if other module with the same name + is reachable from sys.path. + """ + pytester.syspathinsert() + # Create a 'x.py' module reachable from sys.path that raises AssertionError + # if imported. + x_at_root = pytester.path / "x.py" + x_at_root.write_text("raise AssertionError('x at root')", encoding="ascii") + + # Create another x.py module, but in some subdirectories to ensure it is not + # accessible from sys.path. + x_in_sub_folder = pytester.path / "a/b/x.py" + x_in_sub_folder.parent.mkdir(parents=True) + x_in_sub_folder.write_text("X = 'a/b/x'", encoding="ascii") + + # Import our x.py module from the subdirectories. + # The 'x.py' module from sys.path was not imported for sure because + # otherwise we would get an AssertionError. + mod = import_path( + x_in_sub_folder, + mode=ImportMode.importlib, + root=pytester.path, + consider_namespace_packages=ns_param, + ) + assert mod.__file__ and Path(mod.__file__) == x_in_sub_folder + assert mod.X == "a/b/x" + + # Attempt to import root 'x.py'. + with pytest.raises(AssertionError, match="x at root"): + _ = import_path( + x_at_root, + mode=ImportMode.importlib, + root=pytester.path, + consider_namespace_packages=ns_param, + ) + def test_safe_exists(tmp_path: Path) -> None: d = tmp_path.joinpath("some_dir") @@ -713,3 +1058,109 @@ def test_safe_exists(tmp_path: Path) -> None: side_effect=ValueError("name too long"), ): assert safe_exists(p) is False + + +class TestNamespacePackages: + """Test import_path support when importing from properly namespace packages.""" + + def setup_directories( + self, tmp_path: Path, monkeypatch: MonkeyPatch, pytester: Pytester + ) -> Tuple[Path, Path]: + # Set up a namespace package "com.company", containing + # two subpackages, "app" and "calc". + (tmp_path / "src/dist1/com/company/app/core").mkdir(parents=True) + (tmp_path / "src/dist1/com/company/app/__init__.py").touch() + (tmp_path / "src/dist1/com/company/app/core/__init__.py").touch() + models_py = tmp_path / "src/dist1/com/company/app/core/models.py" + models_py.touch() + + (tmp_path / "src/dist2/com/company/calc/algo").mkdir(parents=True) + (tmp_path / "src/dist2/com/company/calc/__init__.py").touch() + (tmp_path / "src/dist2/com/company/calc/algo/__init__.py").touch() + algorithms_py = tmp_path / "src/dist2/com/company/calc/algo/algorithms.py" + algorithms_py.touch() + + # Validate the namespace package by importing it in a Python subprocess. + r = pytester.runpython_c( + dedent( + f""" + import sys + sys.path.append(r{str(tmp_path / "src/dist1")!r}) + sys.path.append(r{str(tmp_path / "src/dist2")!r}) + import com.company.app.core.models + import com.company.calc.algo.algorithms + """ + ) + ) + assert r.ret == 0 + + monkeypatch.syspath_prepend(tmp_path / "src/dist1") + monkeypatch.syspath_prepend(tmp_path / "src/dist2") + return models_py, algorithms_py + + @pytest.mark.parametrize("import_mode", ["prepend", "append", "importlib"]) + def test_resolve_pkg_root_and_module_name_ns_multiple_levels( + self, + tmp_path: Path, + monkeypatch: MonkeyPatch, + pytester: Pytester, + import_mode: str, + ) -> None: + models_py, algorithms_py = self.setup_directories( + tmp_path, monkeypatch, pytester + ) + + pkg_root, module_name = resolve_pkg_root_and_module_name( + models_py, consider_namespace_packages=True + ) + assert (pkg_root, module_name) == ( + tmp_path / "src/dist1", + "com.company.app.core.models", + ) + + mod = import_path( + models_py, mode=import_mode, root=tmp_path, consider_namespace_packages=True + ) + assert mod.__name__ == "com.company.app.core.models" + assert mod.__file__ == str(models_py) + + pkg_root, module_name = resolve_pkg_root_and_module_name( + algorithms_py, consider_namespace_packages=True + ) + assert (pkg_root, module_name) == ( + tmp_path / "src/dist2", + "com.company.calc.algo.algorithms", + ) + + mod = import_path( + algorithms_py, + mode=import_mode, + root=tmp_path, + consider_namespace_packages=True, + ) + assert mod.__name__ == "com.company.calc.algo.algorithms" + assert mod.__file__ == str(algorithms_py) + + @pytest.mark.parametrize("import_mode", ["prepend", "append", "importlib"]) + def test_incorrect_namespace_package( + self, + tmp_path: Path, + monkeypatch: MonkeyPatch, + pytester: Pytester, + import_mode: str, + ) -> None: + models_py, algorithms_py = self.setup_directories( + tmp_path, monkeypatch, pytester + ) + # Namespace packages must not have an __init__.py at any of its + # directories; if it does, we then fall back to importing just the + # part of the package containing the __init__.py files. + (tmp_path / "src/dist1/com/__init__.py").touch() + + pkg_root, module_name = resolve_pkg_root_and_module_name( + models_py, consider_namespace_packages=True + ) + assert (pkg_root, module_name) == ( + tmp_path / "src/dist1/com/company", + "app.core.models", + ) diff --git a/testing/test_pluginmanager.py b/testing/test_pluginmanager.py index f68f143f433..da43364f643 100644 --- a/testing/test_pluginmanager.py +++ b/testing/test_pluginmanager.py @@ -46,7 +46,10 @@ def pytest_myhook(xyz): kwargs=dict(pluginmanager=config.pluginmanager) ) config.pluginmanager._importconftest( - conf, importmode="prepend", rootpath=pytester.path + conf, + importmode="prepend", + rootpath=pytester.path, + consider_namespace_packages=False, ) # print(config.pluginmanager.get_plugins()) res = config.hook.pytest_myhook(xyz=10) @@ -75,7 +78,10 @@ def pytest_addoption(parser): """ ) config.pluginmanager._importconftest( - p, importmode="prepend", rootpath=pytester.path + p, + importmode="prepend", + rootpath=pytester.path, + consider_namespace_packages=False, ) assert config.option.test123 @@ -115,6 +121,7 @@ def test_conftestpath_case_sensitivity(self, pytester: Pytester) -> None: conftest, importmode="prepend", rootpath=pytester.path, + consider_namespace_packages=False, ) plugin = config.pluginmanager.get_plugin(str(conftest)) assert plugin is mod @@ -123,6 +130,7 @@ def test_conftestpath_case_sensitivity(self, pytester: Pytester) -> None: conftest_upper_case, importmode="prepend", rootpath=pytester.path, + consider_namespace_packages=False, ) plugin_uppercase = config.pluginmanager.get_plugin(str(conftest_upper_case)) assert plugin_uppercase is mod_uppercase @@ -174,12 +182,18 @@ def test_hook_proxy(self, pytester: Pytester) -> None: conftest2 = pytester.path.joinpath("tests/subdir/conftest.py") config.pluginmanager._importconftest( - conftest1, importmode="prepend", rootpath=pytester.path + conftest1, + importmode="prepend", + rootpath=pytester.path, + consider_namespace_packages=False, ) ihook_a = session.gethookproxy(pytester.path / "tests") assert ihook_a is not None config.pluginmanager._importconftest( - conftest2, importmode="prepend", rootpath=pytester.path + conftest2, + importmode="prepend", + rootpath=pytester.path, + consider_namespace_packages=False, ) ihook_b = session.gethookproxy(pytester.path / "tests") assert ihook_a is not ihook_b @@ -398,7 +412,9 @@ def test_consider_conftest_deps( pytestpm: PytestPluginManager, ) -> None: mod = import_path( - pytester.makepyfile("pytest_plugins='xyz'"), root=pytester.path + pytester.makepyfile("pytest_plugins='xyz'"), + root=pytester.path, + consider_namespace_packages=False, ) with pytest.raises(ImportError): pytestpm.consider_conftest(mod, registration_name="unused")