diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 753c1b0..3995912 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -52,7 +52,26 @@ jobs: pixi run --environment ${{ env.PIXI_ENV_NAME }} conda info - name: Run tests run: pixi run --environment ${{ env.PIXI_ENV_NAME }} test --basetemp=${{ runner.os == 'Windows' && 'D:\\temp' || runner.temp }} - - name: Build recipe (${{ env.PIXI_ENV_NAME }}) - if: matrix.python-version == '3.10' - run: pixi run build + + build-conda: + name: Build conda package (${{ matrix.os }}) + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, windows-latest, macos-13, macos-14] + env: + PYTHONUNBUFFERED: "1" + steps: + - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 + with: + fetch-depth: 0 + - uses: prefix-dev/setup-pixi@ba3bb36eb2066252b2363392b7739741bb777659 # v0.8.1 + with: + environments: build + - name: Setup project + run: | + pixi run --environment build dev + - name: Build recipe + run: pixi run --environment build build \ No newline at end of file diff --git a/conda_pypi/cli/install.py b/conda_pypi/cli/install.py index 7036b9b..0a44670 100644 --- a/conda_pypi/cli/install.py +++ b/conda_pypi/cli/install.py @@ -1,8 +1,6 @@ from __future__ import annotations -import sys from logging import getLogger -from pathlib import Path from typing import TYPE_CHECKING from conda.base.context import context @@ -10,7 +8,7 @@ from conda.exceptions import CondaVerificationError, CondaFileIOError from ..main import run_pip_install, compute_record_sum, PyPIDistribution -from ..utils import get_env_site_packages +from ..python_paths import get_env_site_packages if TYPE_CHECKING: from typing import Iterable, Literal diff --git a/conda_pypi/main.py b/conda_pypi/main.py index 53545ac..2aae29b 100644 --- a/conda_pypi/main.py +++ b/conda_pypi/main.py @@ -13,12 +13,6 @@ from tempfile import NamedTemporaryFile from typing import Any, Iterable, Literal -try: - from importlib.resources import files as importlib_files -except ImportError: - from importlib_resources import files as importlib_files - - from conda.base.context import context from conda.common.pkg_formats.python import PythonDistribution from conda.core.prefix_data import PrefixData @@ -33,12 +27,13 @@ from packaging.requirements import Requirement from packaging.tags import parse_tag -from .utils import ( +from .python_paths import ( + ensure_externally_managed, get_env_python, get_env_site_packages, - get_externally_managed_path, - pypi_spec_variants, + get_externally_managed_paths, ) +from .utils import pypi_spec_variants logger = getLogger(f"conda.{__name__}") HERE = Path(__file__).parent.resolve() @@ -154,21 +149,6 @@ def run_pip_install( return process -def ensure_externally_managed(prefix: os.PathLike = None) -> Path: - """ - conda-pypi places its own EXTERNALLY-MANAGED file when it is installed in an environment. - We also need to place it in _new_ environments created by conda. We do this by implementing - some extra plugin hooks. - """ - target_path = next(get_externally_managed_path(prefix)) - if target_path.exists(): - return target_path - logger.info("Placing EXTERNALLY-MANAGED in %s", target_path.parent) - resource = importlib_files("conda_pypi") / "data" / "EXTERNALLY-MANAGED" - target_path.write_text(resource.read_text()) - return target_path - - def ensure_target_env_has_externally_managed(command: str): """ post-command hook to ensure that the target env has the EXTERNALLY-MANAGED file @@ -178,7 +158,7 @@ def ensure_target_env_has_externally_managed(command: str): return base_prefix = Path(context.conda_prefix) target_prefix = Path(context.target_prefix) - if base_prefix == target_prefix: + if base_prefix == target_prefix or base_prefix.resolve() == target_prefix.resolve(): return # ensure conda-pypi was explicitly installed in base env (and not as a dependency) requested_specs_map = History(base_prefix).get_requested_specs_map() @@ -191,7 +171,7 @@ def ensure_target_env_has_externally_managed(command: str): return # Check if there are some leftover EXTERNALLY-MANAGED files from other Python versions if command != "create" and os.name != "nt": - for path in get_externally_managed_path(target_prefix): + for path in get_externally_managed_paths(target_prefix): if path.exists(): path.unlink() ensure_externally_managed(target_prefix) @@ -199,7 +179,7 @@ def ensure_target_env_has_externally_managed(command: str): if list(prefix_data.query("pip")): # leave in place if pip is still installed return - for path in get_externally_managed_path(target_prefix): + for path in get_externally_managed_paths(target_prefix): if path.exists(): path.unlink() else: @@ -387,7 +367,7 @@ def from_conda_record( def from_lockfile_line(cls, line: str | Iterable[str]): if isinstance(line, str): if line.startswith(cls._line_prefix): - line = line[len(cls._line_prefix):] + line = line[len(cls._line_prefix) :] line = shlex.split(line.strip()) if cls._arg_parser is None: cls._arg_parser = cls._build_arg_parser() @@ -437,7 +417,7 @@ def to_lockfile_line(self) -> list[str]: line += f" --abi {abi}" for platform in self.python_platform_tags: line += f" --platform {platform}" - for algo, checksum in self.record_checksums.items(): + for algo, checksum in self.record_checksums.items(): line += f" --record-checksum={algo}:{checksum}" # Here we could invoke self.find_wheel_url() to get the resolved URL but I'm not sure it's @@ -503,7 +483,8 @@ def _build_arg_parser() -> argparse.ArgumentParser: parser.add_argument("--platform", action="append", default=[]) parser.add_argument("--record-checksum", action="append", default=[]) return parser - + + def compute_record_sum(manifest: str, algos: Iterable[str] = ("sha256",)) -> dict[str, str]: """ Given a `RECORD` file, compute hashes out of a subset of its sorted contents. diff --git a/conda_pypi/python_paths.py b/conda_pypi/python_paths.py new file mode 100644 index 0000000..a88bef2 --- /dev/null +++ b/conda_pypi/python_paths.py @@ -0,0 +1,93 @@ +""" +Logic to place and find Python paths and EXTERNALLY-MANAGED in target (conda) environments. + +Since functions in this module might be called to facilitate installation of the package, +this module MUST only use the Python stdlib. No 3rd party allowed (except for importlib-resources). +""" + +import os +import sys +import sysconfig +from logging import getLogger +from pathlib import Path +from subprocess import check_output +from typing import Iterator + +try: + from importlib.resources import files as importlib_files +except ImportError: # python <3.9 + from importlib_resources import files as importlib_files + + +logger = getLogger(f"conda.{__name__}") + + +def get_env_python(prefix: os.PathLike = None) -> Path: + prefix = Path(prefix or sys.prefix) + if os.name == "nt": + return prefix / "python.exe" + return prefix / "bin" / "python" + + +def _get_env_sysconfig_path(key: str, prefix: os.PathLike = None) -> Path: + prefix = Path(prefix or sys.prefix) + if str(prefix) == sys.prefix or prefix.resolve() == Path(sys.prefix).resolve(): + return Path(sysconfig.get_path(key)) + path = check_output( + [get_env_python(prefix), "-c", f"import sysconfig as s; print(s.get_path('{key}'))"], + text=True, + ).strip() + if not path: + raise RuntimeError(f"Could not identify sysconfig path for '{key}' at '{prefix}'") + return Path(path) + + +def get_env_stdlib(prefix: os.PathLike = None) -> Path: + return _get_env_sysconfig_path("stdlib", prefix) + + +def get_env_site_packages(prefix: os.PathLike = None) -> Path: + return _get_env_sysconfig_path("purelib", prefix) + + +def get_current_externally_managed_path(prefix: os.PathLike = None) -> Path: + """ + Returns the path for EXTERNALLY-MANAGED for the given Python installation in 'prefix'. + Not guaranteed to exist. There might be more EXTERNALLY-MANAGED files in 'prefix' for + older Python versions. These are not returned. + + It assumes Python is installed in 'prefix' and will call it with a subprocess if needed. + """ + prefix = Path(prefix or sys.prefix) + return get_env_stdlib(prefix) / "EXTERNALLY-MANAGED" + + +def get_externally_managed_paths(prefix: os.PathLike = None) -> Iterator[Path]: + """ + Returns all the possible EXTERNALLY-MANAGED paths in 'prefix', for all found + Python (former) installations. The paths themselves are not guaranteed to exist. + + This does NOT invoke python's sysconfig because Python might not be installed (anymore). + """ + prefix = Path(prefix or sys.prefix) + if os.name == "nt": + yield prefix / "Lib" / "EXTERNALLY-MANAGED" + else: + for python_dir in sorted(Path(prefix, "lib").glob("python*")): + if python_dir.is_dir(): + yield Path(python_dir, "EXTERNALLY-MANAGED") + + +def ensure_externally_managed(prefix: os.PathLike = None) -> Path: + """ + conda-pypi places its own EXTERNALLY-MANAGED file when it is installed in an environment. + We also need to place it in _new_ environments created by conda. We do this by implementing + some extra plugin hooks. + """ + target_path = get_current_externally_managed_path(prefix) + if target_path.exists(): + return target_path + logger.info("Placing EXTERNALLY-MANAGED in %s", target_path.parent) + resource = importlib_files("conda_pypi") / "data" / "EXTERNALLY-MANAGED" + target_path.write_text(resource.read_text()) + return target_path diff --git a/conda_pypi/utils.py b/conda_pypi/utils.py index ed025aa..6c098f4 100644 --- a/conda_pypi/utils.py +++ b/conda_pypi/utils.py @@ -1,11 +1,8 @@ from __future__ import annotations import os -import sys -import sysconfig from logging import getLogger from pathlib import Path -from subprocess import check_output from typing import Iterator from conda.base.context import context, locate_prefix_by_name @@ -24,51 +21,6 @@ def get_prefix(prefix: os.PathLike = None, name: str = None) -> Path: return Path(context.target_prefix) -def get_env_python(prefix: os.PathLike = None) -> Path: - prefix = Path(prefix or sys.prefix) - if os.name == "nt": - return prefix / "python.exe" - return prefix / "bin" / "python" - - -def _get_env_sysconfig_path(key: str, prefix: os.PathLike = None) -> Path: - prefix = Path(prefix or sys.prefix) - if str(prefix) == sys.prefix: - return Path(sysconfig.get_path(key)) - return Path( - check_output( - [ - get_env_python(prefix), - "-c", - f"import sysconfig; print(sysconfig.get_paths()['{key}'])", - ], - text=True, - ).strip() - ) - - -def get_env_stdlib(prefix: os.PathLike = None) -> Path: - return _get_env_sysconfig_path("stdlib", prefix) - - -def get_env_site_packages(prefix: os.PathLike = None) -> Path: - return _get_env_sysconfig_path("purelib", prefix) - - -def get_externally_managed_path(prefix: os.PathLike = None) -> Iterator[Path]: - prefix = Path(prefix or sys.prefix) - if os.name == "nt": - yield Path(prefix, "Lib", "EXTERNALLY-MANAGED") - else: - found = False - for python_dir in sorted(Path(prefix, "lib").glob("python*")): - if python_dir.is_dir(): - found = True - yield Path(python_dir, "EXTERNALLY-MANAGED") - if not found: - raise ValueError("Could not locate EXTERNALLY-MANAGED file") - - def pypi_spec_variants(spec_str: str) -> Iterator[str]: yield spec_str spec = MatchSpec(spec_str) diff --git a/pyproject.toml b/pyproject.toml index 832a8fc..a985026 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,7 +26,8 @@ dependencies = [ # "conda >=23.9.0", "pip", "grayskull", - "importlib_resources", + "importlib_resources; python_version < '3.9'", + "packaging", ] dynamic = [ "version" diff --git a/recipe/meta.yaml b/recipe/meta.yaml index c1b320e..23ae135 100644 --- a/recipe/meta.yaml +++ b/recipe/meta.yaml @@ -17,7 +17,7 @@ build: - set -x # [unix] - "@ECHO ON" # [win] - {{ PYTHON }} -m pip install . --no-deps --no-build-isolation -vv - - {{ PYTHON }} -c "from conda_pypi.main import ensure_externally_managed; ensure_externally_managed()" + - {{ PYTHON }} -c "from conda_pypi.python_paths import ensure_externally_managed; ensure_externally_managed()" requirements: host: @@ -25,15 +25,14 @@ requirements: - pip - hatchling >=1.12.2 - hatch-vcs >=0.2.0 - - conda >=23.7.3 - - importlib_resources + - importlib_resources # [py<39] run: - python - - conda >=23.7.3 + - conda >=23.9.0 - pip >=23.0.1 - grayskull - - importlib_resources - - conda-libmamba-solver + - importlib_resources # [py<39] + - packaging test: imports: @@ -41,7 +40,7 @@ test: - conda_pypi.main commands: - conda pip --help - - python -c "from conda_pypi.utils import get_env_stdlib; assert (get_env_stdlib() / 'EXTERNALLY-MANAGED').exists()" + - python -c "from conda_pypi.python_paths import get_env_stdlib; assert (get_env_stdlib() / 'EXTERNALLY-MANAGED').exists()" - pip install requests && exit 1 || exit 0 about: diff --git a/tests/test_install.py b/tests/test_install.py index f1d3cda..e60cfce 100644 --- a/tests/test_install.py +++ b/tests/test_install.py @@ -11,7 +11,7 @@ from conda.testing import CondaCLIFixture, TmpEnvFixture from conda_pypi.dependencies import NAME_MAPPINGS, BACKENDS, _pypi_spec_to_conda_spec -from conda_pypi.utils import get_env_python +from conda_pypi.python_paths import get_env_python @pytest.mark.parametrize("source", NAME_MAPPINGS.keys()) diff --git a/tests/test_validate.py b/tests/test_validate.py index 293bb08..c734548 100644 --- a/tests/test_validate.py +++ b/tests/test_validate.py @@ -9,7 +9,7 @@ from conda.testing.integration import package_is_installed from pytest_mock import MockerFixture -from conda_pypi.utils import get_env_python, get_env_stdlib +from conda_pypi.python_paths import get_env_python, get_env_stdlib def test_pip_required_in_target_env(