Add initial implementation of editable strategy based on a MetaPathFinder for top-level packages

setuptools/command/editable_wheel.py

@@ -11,17 +11,22 @@
 """
 
 import os
+import re
 import shutil
 import sys
+import logging
+from itertools import chain
+from inspect import cleandoc
 from pathlib import Path
 from tempfile import TemporaryDirectory
-from typing import Dict, Iterable, Iterator, List, Union
+from typing import Dict, Iterable, Iterator, List, Mapping, Set, Union
 
 from setuptools import Command, namespaces
 from setuptools.discovery import find_package_path
 from setuptools.dist import Distribution
 
 _Path = Union[str, Path]
+_logger = logging.getLogger(__name__)
 
 
 class editable_wheel(Command):
@@ -120,20 +125,30 @@ def _populate_strategy(self, name, tag):
             # any OS, even if that means using hardlinks instead of symlinks
             auxiliar_build_dir = _empty_dir(auxiliar_build_dir)
             # TODO: return _LinkTree(dist, name, auxiliar_build_dir)
+            msg = """
+            Strict editable install will be performed using a link tree.
+            New files will not be automatically picked up without a new installation.
+            """
+            _logger.info(cleandoc(msg))
             raise NotImplementedError
 
         packages = _find_packages(dist)
         has_simple_layout = _simple_layout(packages, self.package_dir, project_dir)
         if set(self.package_dir) == {""} and has_simple_layout:
             # src-layout(ish) package detected. These kind of packages are relatively
             # safe so we can simply add the src directory to the pth file.
-            return _StaticPth(dist, name, [Path(project_dir, self.package_dir[""])])
+            src_dir = self.package_dir[""]
+            msg = f"Editable install will be performed using .pth file to {src_dir}."
+            _logger.info(msg)
+            return _StaticPth(dist, name, [Path(project_dir, src_dir)])
 
-        # >>> msg = "TODO: Explain limitations with meta path finder"
-        # >>> warnings.warn(msg)
-        paths = [Path(project_dir, p) for p in (".", self.package_dir.get("")) if p]
-        # TODO: return _TopLevelFinder(dist, name, auxiliar_build_dir)
-        return _StaticPth(dist, name, paths)
+        msg = """
+        Editable install will be performed using a meta path finder.
+        If you add any top-level packages or modules, they might not be automatically
+        picked up without a new installation.
+        """
+        _logger.info(cleandoc(msg))
+        return _TopLevelFinder(dist, name)
 
 
 class _StaticPth:
@@ -148,11 +163,38 @@ def __call__(self, unpacked_wheel_dir: Path):
         pth.write_text(f"{entries}\n", encoding="utf-8")
 
 
+class _TopLevelFinder:
+    def __init__(self, dist: Distribution, name: str):
+        self.dist = dist
+        self.name = name
+
+    def __call__(self, unpacked_wheel_dir: Path):
+        src_root = self.dist.src_root or os.curdir
+        packages = chain(_find_packages(self.dist), _find_top_level_modules(self.dist))
+        package_dir = self.dist.package_dir or {}
+        pkg_roots = _find_pkg_roots(packages, package_dir, src_root)
+        namespaces_ = set(_find_mapped_namespaces(pkg_roots))
+
+        finder = _make_identifier(f"__editable__.{self.name}.finder")
+        content = _finder_template(pkg_roots, namespaces_)
+        Path(unpacked_wheel_dir, f"{finder}.py").write_text(content, encoding="utf-8")
+
+        pth = f"__editable__.{self.name}.pth"
+        content = f"import {finder}; {finder}.install()"
+        Path(unpacked_wheel_dir, pth).write_text(content, encoding="utf-8")
+
+
 def _simple_layout(
     packages: Iterable[str], package_dir: Dict[str, str], project_dir: Path
 ) -> bool:
     """Make sure all packages are contained by the same parent directory.
 
+    >>> _simple_layout(['a'], {"": "src"}, "/tmp/myproj")
+    True
+    >>> _simple_layout(['a', 'a.b'], {"": "src"}, "/tmp/myproj")
+    True
+    >>> _simple_layout(['a', 'a.b'], {}, "/tmp/myproj")
+    True
     >>> _simple_layout(['a', 'a.a1', 'a.a1.a2', 'b'], {"": "src"}, "/tmp/myproj")
     True
     >>> _simple_layout(['a', 'a.a1', 'a.a1.a2', 'b'], {"a": "a", "b": "b"}, ".")
@@ -172,13 +214,27 @@ def _simple_layout(
         pkg: find_package_path(pkg, package_dir, project_dir)
         for pkg in packages
     }
-    parent = os.path.commonpath(list(layout.values()))
+    if not layout:
+        return False
+    parent = os.path.commonpath([_parent_path(k, v) for k, v in layout.items()])
     return all(
         _normalize_path(Path(parent, *key.split('.'))) == _normalize_path(value)
         for key, value in layout.items()
     )
 
 
+def _parent_path(pkg, pkg_path):
+    """Infer the parent path for a package if possible. When the pkg is directly mapped
+    into a directory with a different name, return its own path.
+    >>> _parent_path("a", "src/a")
+    'src'
+    >>> _parent_path("b", "src/c")
+    'src/c'
+    """
+    parent = pkg_path[:-len(pkg)] if pkg_path.endswith(pkg) else pkg_path
+    return parent.rstrip("/" + os.sep)
+
+
 def _find_packages(dist: Distribution) -> Iterator[str]:
     yield from iter(dist.packages or [])
 
@@ -195,6 +251,85 @@ def _find_packages(dist: Distribution) -> Iterator[str]:
         yield package
 
 
+def _find_top_level_modules(dist: Distribution) -> Iterator[str]:
+    py_modules = dist.py_modules or []
+    yield from (mod for mod in py_modules if "." not in mod)
+
+    if not dist.ext_package:
+        ext_modules = dist.ext_modules or []
+        yield from (x.name for x in ext_modules if "." not in x.name)
+
+
+def _find_pkg_roots(
+    packages: Iterable[str],
+    package_dir: Mapping[str, str],
+    src_root: _Path,
+) -> Dict[str, str]:
+    pkg_roots: Dict[str, str] = {
+        pkg: _absolute_root(find_package_path(pkg, package_dir, src_root))
+        for pkg in sorted(packages)
+    }
+
+    return _remove_nested(pkg_roots)
+
+
+def _absolute_root(path: _Path) -> str:
+    """Works for packages and top-level modules"""
+    path_ = Path(path)
+    parent = path_.parent
+
+    if path_.exists():
+        return str(path_.resolve())
+    else:
+        return str(parent.resolve() / path_.name)
+
+
+def _find_mapped_namespaces(pkg_roots: Dict[str, str]) -> Iterator[str]:
+    """By carefully designing ``package_dir``, it is possible to implement
+    PEP 420 compatible namespaces without creating extra folders.
+    This function will try to find this kind of namespaces.
+    """
+    for pkg in pkg_roots:
+        if "." not in pkg:
+            continue
+        parts = pkg.split(".")
+        for i in range(len(parts) - 1, 0, -1):
+            partial_name = ".".join(parts[:i])
+            path = find_package_path(partial_name, pkg_roots, "")
+            if not Path(path, "__init__.py").exists():
+                yield partial_name
+
+
+def _remove_nested(pkg_roots: Dict[str, str]) -> Dict[str, str]:
+    output = dict(pkg_roots.copy())
+
+    for pkg, path in reversed(list(pkg_roots.items())):
+        if any(
+            pkg != other and _is_nested(pkg, path, other, other_path)
+            for other, other_path in pkg_roots.items()
+        ):
+            output.pop(pkg)
+
+    return output
+
+
+def _is_nested(pkg: str, pkg_path: str, parent: str, parent_path: str) -> bool:
+    """
+    >>> _is_nested("a.b", "path/a/b", "a", "path/a")
+    True
+    >>> _is_nested("a.b", "path/a/b", "a", "otherpath/a")
+    False
+    >>> _is_nested("a.b", "path/a/b", "c", "path/c")
+    False
+    """
+    norm_pkg_path = _normalize_path(pkg_path)
+    rest = pkg.replace(parent, "").strip(".").split(".")
+    return (
+        pkg.startswith(parent)
+        and norm_pkg_path == _normalize_path(Path(parent_path, *rest))
+    )
+
+
 def _normalize_path(filename: _Path) -> str:
     """Normalize a file/dir name for comparison purposes"""
     # See pkg_resources.normalize_path
@@ -208,6 +343,18 @@ def _empty_dir(dir_: Path) -> Path:
     return dir_
 
 
+def _make_identifier(name: str) -> str:
+    """Make a string safe to be used as Python identifier.
+    >>> _make_identifier("12abc")
+    '_12abc'
+    >>> _make_identifier("__editable__.myns.pkg-78.9.3_local")
+    '__editable___myns_pkg_78_9_3_local'
+    """
+    safe = re.sub(r'\W|^(?=\d)', '_', name)
+    assert safe.isidentifier()
+    return safe
+
+
 class _NamespaceInstaller(namespaces.Installer):
     def __init__(self, distribution, installation_dir, editable_name, src_root):
         self.distribution = distribution
@@ -223,3 +370,62 @@ def _get_target(self):
     def _get_root(self):
         """Where the modules/packages should be loaded from."""
         return repr(str(self.src_root))
+
+
+_FINDER_TEMPLATE = """\
+import sys
+from importlib.machinery import all_suffixes as module_suffixes
+from importlib.machinery import ModuleSpec
+from importlib.util import spec_from_file_location
+from itertools import chain
+from pathlib import Path
+
+class __EditableFinder:
+    MAPPING = {mapping!r}
+    NAMESPACES = {namespaces!r}
+
+    @classmethod
+    def find_spec(cls, fullname, path, target=None):
+        if fullname in cls.NAMESPACES:
+            return cls._namespace_spec(fullname)
+
+        for pkg, pkg_path in reversed(list(cls.MAPPING.items())):
+            if fullname.startswith(pkg):
+                return cls._find_spec(fullname, pkg, pkg_path)
+
+        return None
+
+    @classmethod
+    def _namespace_spec(cls, name):
+        # Since `cls` is appended to the path, this will only trigger
+        # when no other package is installed in the same namespace.
+        return ModuleSpec(name, None, is_package=True)
+        # ^-- PEP 451 mentions setting loader to None for namespaces.
+
+    @classmethod
+    def _find_spec(cls, fullname, parent, parent_path):
+        rest = fullname.replace(parent, "").strip(".").split(".")
+        candidate_path = Path(parent_path, *rest)
+
+        init = candidate_path / "__init__.py"
+        candidates = (candidate_path.with_suffix(x) for x in module_suffixes())
+        for candidate in chain([init], candidates):
+            if candidate.exists():
+                spec = spec_from_file_location(fullname, candidate)
+                return spec
+
+        if candidate_path.exists():
+            return cls._namespace_spec(fullname)
+
+        return None
+
+
+def install():
+    if not any(finder == __EditableFinder for finder in sys.meta_path):
+        sys.meta_path.append(__EditableFinder)
+"""
+
+
+def _finder_template(mapping: Mapping[str, str], namespaces: Set[str]):
+    mapping = dict(sorted(mapping.items(), key=lambda p: p[0]))
+    return _FINDER_TEMPLATE.format(mapping=mapping, namespaces=namespaces)

setuptools/discovery.py

@@ -42,8 +42,18 @@
 from fnmatch import fnmatchcase
 from glob import glob
 from pathlib import Path
-from typing import TYPE_CHECKING
-from typing import Callable, Dict, Iterator, Iterable, List, Optional, Tuple, Union
+from typing import (
+    TYPE_CHECKING,
+    Callable,
+    Dict,
+    Iterable,
+    Iterator,
+    List,
+    Mapping,
+    Optional,
+    Tuple,
+    Union
+)
 
 import _distutils_hack.override  # noqa: F401
 
@@ -435,6 +445,7 @@ def _analyse_flat_modules(self) -> bool:
     def _ensure_no_accidental_inclusion(self, detected: List[str], kind: str):
         if len(detected) > 1:
             from inspect import cleandoc
+
             from setuptools.errors import PackageDiscoveryError
 
             msg = f"""Multiple top-level {kind} discovered in a flat-layout: {detected}.
@@ -527,7 +538,7 @@ def remove_stubs(packages: List[str]) -> List[str]:
 
 
 def find_parent_package(
-    packages: List[str], package_dir: Dict[str, str], root_dir: _Path
+    packages: List[str], package_dir: Mapping[str, str], root_dir: _Path
 ) -> Optional[str]:
     """Find the parent package that is not a namespace."""
     packages = sorted(packages, key=len)
@@ -550,7 +561,9 @@ def find_parent_package(
     return None
 
 
-def find_package_path(name: str, package_dir: Dict[str, str], root_dir: _Path) -> str:
+def find_package_path(
+    name: str, package_dir: Mapping[str, str], root_dir: _Path
+) -> str:
     """Given a package name, return the path where it should be found on
     disk, considering the ``package_dir`` option.
 

setuptools/tests/contexts.py

@@ -123,3 +123,26 @@ def session_locked_tmp_dir(request, tmp_path_factory, name):
         # ^-- prevent multiple workers to access the directory at once
         locked_dir.mkdir(exist_ok=True, parents=True)
         yield locked_dir
+
+
+@contextlib.contextmanager
+def save_paths():
+    """Make sure initial ``sys.path`` and ``sys.meta_path`` are preserved"""
+    prev_paths = sys.path[:], sys.meta_path[:]
+
+    try:
+        yield
+    finally:
+        sys.path, sys.meta_path = prev_paths
+
+
+@contextlib.contextmanager
+def save_sys_modules():
+    """Make sure initial ``sys.modules`` is preserved"""
+    prev_modules = sys.modules
+
+    try:
+        sys.modules = sys.modules.copy()
+        yield
+    finally:
+        sys.modules = prev_modules