Isolate EXTERNALLY-MANAGED logic better and improve build-time depend…

…ency constraints (#34)

* Isolate EXTERNALLY-MANAGED logic better

* fix imports in paths

* No need for this 2nd argument

* Adjust dependencies in pyproject.toml too

* Do enable conda-build tests

* do print

* fix recipe building

* Fix recipe test

* More macOS

* Do not assume Python is installed for potential paths

.github/workflows/test.yml

@@ -52,7 +52,26 @@ jobs:
           pixi run --environment ${{ env.PIXI_ENV_NAME }} conda info
       - name: Run tests
         run: pixi run --environment ${{ env.PIXI_ENV_NAME }} test --basetemp=${{ runner.os == 'Windows' && 'D:\\temp' || runner.temp }}
-      - name: Build recipe (${{ env.PIXI_ENV_NAME }})
-        if: matrix.python-version == '3.10'
-        run: pixi run build
+
+  build-conda:
+    name: Build conda package (${{ matrix.os }})
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, windows-latest, macos-13, macos-14]
+    env:
+      PYTHONUNBUFFERED: "1"
+    steps:
+      - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
+        with:
+          fetch-depth: 0
+      - uses: prefix-dev/setup-pixi@ba3bb36eb2066252b2363392b7739741bb777659 # v0.8.1
+        with:
+          environments: build
+      - name: Setup project
+        run: |
+          pixi run --environment build dev
+      - name: Build recipe
+        run: pixi run --environment build build
 

conda_pypi/cli/install.py

@@ -1,16 +1,14 @@
 from __future__ import annotations
 
-import sys
 from logging import getLogger
-from pathlib import Path
 from typing import TYPE_CHECKING
 
 from conda.base.context import context
 from conda.common.io import Spinner
 from conda.exceptions import CondaVerificationError, CondaFileIOError
 
 from ..main import run_pip_install, compute_record_sum, PyPIDistribution
-from ..utils import get_env_site_packages
+from ..python_paths import get_env_site_packages
 
 if TYPE_CHECKING:
     from typing import Iterable, Literal

conda_pypi/main.py

@@ -13,12 +13,6 @@
 from tempfile import NamedTemporaryFile
 from typing import Any, Iterable, Literal
 
-try:
-    from importlib.resources import files as importlib_files
-except ImportError:
-    from importlib_resources import files as importlib_files
-
-
 from conda.base.context import context
 from conda.common.pkg_formats.python import PythonDistribution
 from conda.core.prefix_data import PrefixData
@@ -33,12 +27,13 @@
 from packaging.requirements import Requirement
 from packaging.tags import parse_tag
 
-from .utils import (
+from .python_paths import (
+    ensure_externally_managed,
     get_env_python,
     get_env_site_packages,
-    get_externally_managed_path,
-    pypi_spec_variants,
+    get_externally_managed_paths,
 )
+from .utils import pypi_spec_variants
 
 logger = getLogger(f"conda.{__name__}")
 HERE = Path(__file__).parent.resolve()
@@ -154,21 +149,6 @@ def run_pip_install(
     return process
 
 
-def ensure_externally_managed(prefix: os.PathLike = None) -> Path:
-    """
-    conda-pypi places its own EXTERNALLY-MANAGED file when it is installed in an environment.
-    We also need to place it in _new_ environments created by conda. We do this by implementing
-    some extra plugin hooks.
-    """
-    target_path = next(get_externally_managed_path(prefix))
-    if target_path.exists():
-        return target_path
-    logger.info("Placing EXTERNALLY-MANAGED in %s", target_path.parent)
-    resource = importlib_files("conda_pypi") / "data" / "EXTERNALLY-MANAGED"
-    target_path.write_text(resource.read_text())
-    return target_path
-
-
 def ensure_target_env_has_externally_managed(command: str):
     """
     post-command hook to ensure that the target env has the EXTERNALLY-MANAGED file
@@ -178,7 +158,7 @@ def ensure_target_env_has_externally_managed(command: str):
         return
     base_prefix = Path(context.conda_prefix)
     target_prefix = Path(context.target_prefix)
-    if base_prefix == target_prefix:
+    if base_prefix == target_prefix or base_prefix.resolve() == target_prefix.resolve():
         return
     # ensure conda-pypi was explicitly installed in base env (and not as a dependency)
     requested_specs_map = History(base_prefix).get_requested_specs_map()
@@ -191,15 +171,15 @@ def ensure_target_env_has_externally_managed(command: str):
             return
         # Check if there are some leftover EXTERNALLY-MANAGED files from other Python versions
         if command != "create" and os.name != "nt":
-            for path in get_externally_managed_path(target_prefix):
+            for path in get_externally_managed_paths(target_prefix):
                 if path.exists():
                     path.unlink()
         ensure_externally_managed(target_prefix)
     elif command == "remove":
         if list(prefix_data.query("pip")):
             # leave in place if pip is still installed
             return
-        for path in get_externally_managed_path(target_prefix):
+        for path in get_externally_managed_paths(target_prefix):
             if path.exists():
                 path.unlink()
     else:
@@ -387,7 +367,7 @@ def from_conda_record(
     def from_lockfile_line(cls, line: str | Iterable[str]):
         if isinstance(line, str):
             if line.startswith(cls._line_prefix):
-                line = line[len(cls._line_prefix):]
+                line = line[len(cls._line_prefix) :]
             line = shlex.split(line.strip())
         if cls._arg_parser is None:
             cls._arg_parser = cls._build_arg_parser()
@@ -437,7 +417,7 @@ def to_lockfile_line(self) -> list[str]:
             line += f" --abi {abi}"
         for platform in self.python_platform_tags:
             line += f" --platform {platform}"
-        for algo, checksum in  self.record_checksums.items():
+        for algo, checksum in self.record_checksums.items():
             line += f" --record-checksum={algo}:{checksum}"
 
         # Here we could invoke self.find_wheel_url() to get the resolved URL but I'm not sure it's
@@ -503,7 +483,8 @@ def _build_arg_parser() -> argparse.ArgumentParser:
         parser.add_argument("--platform", action="append", default=[])
         parser.add_argument("--record-checksum", action="append", default=[])
         return parser
-
+
+
 def compute_record_sum(manifest: str, algos: Iterable[str] = ("sha256",)) -> dict[str, str]:
     """
     Given a `RECORD` file, compute hashes out of a subset of its sorted contents.

conda_pypi/python_paths.py

@@ -0,0 +1,93 @@
+"""
+Logic to place and find Python paths and EXTERNALLY-MANAGED in target (conda) environments.
+
+Since functions in this module might be called to facilitate installation of the package,
+this module MUST only use the Python stdlib. No 3rd party allowed (except for importlib-resources).
+"""
+
+import os
+import sys
+import sysconfig
+from logging import getLogger
+from pathlib import Path
+from subprocess import check_output
+from typing import Iterator
+
+try:
+    from importlib.resources import files as importlib_files
+except ImportError:  # python <3.9
+    from importlib_resources import files as importlib_files
+
+
+logger = getLogger(f"conda.{__name__}")
+
+
+def get_env_python(prefix: os.PathLike = None) -> Path:
+    prefix = Path(prefix or sys.prefix)
+    if os.name == "nt":
+        return prefix / "python.exe"
+    return prefix / "bin" / "python"
+
+
+def _get_env_sysconfig_path(key: str, prefix: os.PathLike = None) -> Path:
+    prefix = Path(prefix or sys.prefix)
+    if str(prefix) == sys.prefix or prefix.resolve() == Path(sys.prefix).resolve():
+        return Path(sysconfig.get_path(key))
+    path = check_output(
+        [get_env_python(prefix), "-c", f"import sysconfig as s; print(s.get_path('{key}'))"],
+        text=True,
+    ).strip()
+    if not path:
+        raise RuntimeError(f"Could not identify sysconfig path for '{key}' at '{prefix}'")
+    return Path(path)
+
+
+def get_env_stdlib(prefix: os.PathLike = None) -> Path:
+    return _get_env_sysconfig_path("stdlib", prefix)
+
+
+def get_env_site_packages(prefix: os.PathLike = None) -> Path:
+    return _get_env_sysconfig_path("purelib", prefix)
+
+
+def get_current_externally_managed_path(prefix: os.PathLike = None) -> Path:
+    """
+    Returns the path for EXTERNALLY-MANAGED for the given Python installation in 'prefix'.
+    Not guaranteed to exist. There might be more EXTERNALLY-MANAGED files in 'prefix' for
+    older Python versions. These are not returned.
+
+    It assumes Python is installed in 'prefix' and will call it with a subprocess if needed.
+    """
+    prefix = Path(prefix or sys.prefix)
+    return get_env_stdlib(prefix) / "EXTERNALLY-MANAGED"
+
+
+def get_externally_managed_paths(prefix: os.PathLike = None) -> Iterator[Path]:
+    """
+    Returns all the possible EXTERNALLY-MANAGED paths in 'prefix', for all found
+    Python (former) installations. The paths themselves are not guaranteed to exist.
+
+    This does NOT invoke python's sysconfig because Python  might not be installed (anymore).
+    """
+    prefix = Path(prefix or sys.prefix)
+    if os.name == "nt":
+        yield prefix / "Lib" / "EXTERNALLY-MANAGED"
+    else:
+        for python_dir in sorted(Path(prefix, "lib").glob("python*")):
+            if python_dir.is_dir():
+                yield Path(python_dir, "EXTERNALLY-MANAGED")
+
+
+def ensure_externally_managed(prefix: os.PathLike = None) -> Path:
+    """
+    conda-pypi places its own EXTERNALLY-MANAGED file when it is installed in an environment.
+    We also need to place it in _new_ environments created by conda. We do this by implementing
+    some extra plugin hooks.
+    """
+    target_path = get_current_externally_managed_path(prefix)
+    if target_path.exists():
+        return target_path
+    logger.info("Placing EXTERNALLY-MANAGED in %s", target_path.parent)
+    resource = importlib_files("conda_pypi") / "data" / "EXTERNALLY-MANAGED"
+    target_path.write_text(resource.read_text())
+    return target_path

conda_pypi/utils.py

@@ -1,11 +1,8 @@
 from __future__ import annotations
 
 import os
-import sys
-import sysconfig
 from logging import getLogger
 from pathlib import Path
-from subprocess import check_output
 from typing import Iterator
 
 from conda.base.context import context, locate_prefix_by_name
@@ -24,51 +21,6 @@ def get_prefix(prefix: os.PathLike = None, name: str = None) -> Path:
         return Path(context.target_prefix)
 
 
-def get_env_python(prefix: os.PathLike = None) -> Path:
-    prefix = Path(prefix or sys.prefix)
-    if os.name == "nt":
-        return prefix / "python.exe"
-    return prefix / "bin" / "python"
-
-
-def _get_env_sysconfig_path(key: str, prefix: os.PathLike = None) -> Path:
-    prefix = Path(prefix or sys.prefix)
-    if str(prefix) == sys.prefix:
-        return Path(sysconfig.get_path(key))
-    return Path(
-        check_output(
-            [
-                get_env_python(prefix),
-                "-c",
-                f"import sysconfig; print(sysconfig.get_paths()['{key}'])",
-            ],
-            text=True,
-        ).strip()
-    )
-
-
-def get_env_stdlib(prefix: os.PathLike = None) -> Path:
-    return _get_env_sysconfig_path("stdlib", prefix)
-
-
-def get_env_site_packages(prefix: os.PathLike = None) -> Path:
-    return _get_env_sysconfig_path("purelib", prefix)
-
-
-def get_externally_managed_path(prefix: os.PathLike = None) -> Iterator[Path]:
-    prefix = Path(prefix or sys.prefix)
-    if os.name == "nt":
-        yield Path(prefix, "Lib", "EXTERNALLY-MANAGED")
-    else:
-        found = False
-        for python_dir in sorted(Path(prefix, "lib").glob("python*")):
-            if python_dir.is_dir():
-                found = True
-                yield Path(python_dir, "EXTERNALLY-MANAGED")
-        if not found:
-            raise ValueError("Could not locate EXTERNALLY-MANAGED file")
-
-
 def pypi_spec_variants(spec_str: str) -> Iterator[str]:
     yield spec_str
     spec = MatchSpec(spec_str)

pyproject.toml

@@ -26,7 +26,8 @@ dependencies = [
   # "conda >=23.9.0",
   "pip",
   "grayskull",
-  "importlib_resources",
+  "importlib_resources; python_version < '3.9'",
+  "packaging",
 ]
 dynamic = [
   "version"

recipe/meta.yaml

@@ -17,31 +17,30 @@ build:
     - set -x  # [unix]
     - "@ECHO ON"  # [win]
     - {{ PYTHON }} -m pip install . --no-deps --no-build-isolation -vv
-    - {{ PYTHON }} -c "from conda_pypi.main import ensure_externally_managed; ensure_externally_managed()"
+    - {{ PYTHON }} -c "from conda_pypi.python_paths import ensure_externally_managed; ensure_externally_managed()"
 
 requirements:
   host:
     - python
     - pip
     - hatchling >=1.12.2
     - hatch-vcs >=0.2.0
-    - conda >=23.7.3
-    - importlib_resources
+    - importlib_resources  # [py<39]
   run:
     - python
-    - conda >=23.7.3
+    - conda >=23.9.0
     - pip >=23.0.1
     - grayskull
-    - importlib_resources
-    - conda-libmamba-solver
+    - importlib_resources  # [py<39]
+    - packaging
 
 test:
   imports:
     - conda_pypi
     - conda_pypi.main
   commands:
     - conda pip --help
-    - python -c "from conda_pypi.utils import get_env_stdlib; assert (get_env_stdlib() / 'EXTERNALLY-MANAGED').exists()"
+    - python -c "from conda_pypi.python_paths import get_env_stdlib; assert (get_env_stdlib() / 'EXTERNALLY-MANAGED').exists()"
     - pip install requests && exit 1 || exit 0
 
 about:

tests/test_install.py

@@ -11,7 +11,7 @@
 from conda.testing import CondaCLIFixture, TmpEnvFixture
 
 from conda_pypi.dependencies import NAME_MAPPINGS, BACKENDS, _pypi_spec_to_conda_spec
-from conda_pypi.utils import get_env_python
+from conda_pypi.python_paths import get_env_python
 
 
 @pytest.mark.parametrize("source", NAME_MAPPINGS.keys())

tests/test_validate.py

@@ -9,7 +9,7 @@
 from conda.testing.integration import package_is_installed
 from pytest_mock import MockerFixture
 
-from conda_pypi.utils import get_env_python, get_env_stdlib
+from conda_pypi.python_paths import get_env_python, get_env_stdlib
 
 
 def test_pip_required_in_target_env(