Windows embedable support (#2353)

* Bump pip and setuptools (#2348)

Signed-off-by: Bernát Gábor <bgabor8@bloomberg.net>

* Use shlex.quote instead of deprecated pipes.quote (#2351)

* Embeds the "python<VERSION>.zip" for Windows.

For example, for Python 3.10 the embeddable file name would be
"python310.zip". If this file would be found in `sys.path`, the
virtualenv should copy it into the "<venv>\Scripts\python310.zip".

* For Windows CPython3: *.dll/*.pyd -> to_bin

* Fixture for a Python interpreter info.

Helps to test virtualenv creator classes.

* Creators tests: path_mock as separate module.

* Clarifies tests, separates testing tools.

* Tests for CPython3Windows sources.

* Tests for the embedded Python std lib for Windows.

* Add news entry.

* Replaces `yield from` for backward compability.

* FIX: Path mocking in pypy tests.

* Wrap `sys` `Path` with `str` for importlib.

The importlib accepts a Path-like objects from Python 3.6

* Makes PathMock ABC compatible with Python 2

* Does not collect tests for Python3 under Python 2

It is possible to make pass CPython3 tests under Python 2,
but it's better to disable it instead of decreasing the
readability and performance of Python 3 style.

* Allows empty `Path()` in Windows with Python 2

* Allows to load fixture files with PY2 Windows Path

* Skips one PY3 POSIX test in PY2 Windows

Co-authored-by: Bernát Gábor <gaborjbernat@gmail.com>
Co-authored-by: Lumír 'Frenzy' Balhar <lbalhar@redhat.com>

docs/changelog/1774.feature.rst

@@ -0,0 +1,2 @@
+Support for Windows embeddable Python package: includes ``python<VERSION>.zip``
+in the creator sources.

src/virtualenv/create/via_global_ref/builtin/cpython/cpython3.py

@@ -1,6 +1,9 @@
 from __future__ import absolute_import, unicode_literals
 
 import abc
+import fnmatch
+from itertools import chain
+from operator import methodcaller as method
 from textwrap import dedent
 
 from six import add_metaclass
@@ -53,11 +56,20 @@ def setup_meta(cls, interpreter):
 
     @classmethod
     def sources(cls, interpreter):
-        for src in super(CPython3Windows, cls).sources(interpreter):
-            yield src
-        if not cls.has_shim(interpreter):
-            for src in cls.include_dll_and_pyd(interpreter):
-                yield src
+        if cls.has_shim(interpreter):
+            refs = cls.executables(interpreter)
+        else:
+            refs = chain(
+                cls.executables(interpreter),
+                cls.dll_and_pyd(interpreter),
+                cls.python_zip(interpreter),
+            )
+        for ref in refs:
+            yield ref
+
+    @classmethod
+    def executables(cls, interpreter):
+        return super(CPython3Windows, cls).sources(interpreter)
 
     @classmethod
     def has_shim(cls, interpreter):
@@ -79,13 +91,32 @@ def host_python(cls, interpreter):
         return super(CPython3Windows, cls).host_python(interpreter)
 
     @classmethod
-    def include_dll_and_pyd(cls, interpreter):
+    def dll_and_pyd(cls, interpreter):
         dll_folder = Path(interpreter.system_prefix) / "DLLs"
         host_exe_folder = Path(interpreter.system_executable).parent
         for folder in [host_exe_folder, dll_folder]:
             for file in folder.iterdir():
                 if file.suffix in (".pyd", ".dll"):
-                    yield PathRefToDest(file, dest=cls.to_dll_and_pyd)
+                    yield PathRefToDest(file, cls.to_bin)
 
-    def to_dll_and_pyd(self, src):
-        return self.bin_dir / src.name
+    @classmethod
+    def python_zip(cls, interpreter):
+        """
+        "python{VERSION}.zip" contains compiled *.pyc std lib packages, where
+        "VERSION" is `py_version_nodot` var from the `sysconfig` module.
+        :see: https://docs.python.org/3/using/windows.html#the-embeddable-package
+        :see: `discovery.py_info.PythonInfo` class (interpreter).
+        :see: `python -m sysconfig` output.
+
+        :note: The embeddable Python distribution for Windows includes
+        "python{VERSION}.zip" and "python{VERSION}._pth" files. User can
+        move/rename *zip* file and edit `sys.path` by editing *_pth* file.
+        Here the `pattern` is used only for the default *zip* file name!
+        """
+        pattern = "*python{}.zip".format(interpreter.version_nodot)
+        matches = fnmatch.filter(interpreter.path, pattern)
+        matched_paths = map(Path, matches)
+        existing_paths = filter(method("exists"), matched_paths)
+        path = next(existing_paths, None)
+        if path is not None:
+            yield PathRefToDest(path, cls.to_bin)

src/virtualenv/discovery/py_info.py

@@ -47,6 +47,10 @@ def abs_path(v):
         self.version_info = VersionInfo(*list(u(i) for i in sys.version_info))
         self.architecture = 64 if sys.maxsize > 2**32 else 32
 
+        # Used to determine some file names.
+        # See `CPython3Windows.python_zip()`.
+        self.version_nodot = sysconfig.get_config_var("py_version_nodot")
+
         self.version = u(sys.version)
         self.os = u(os.name)
 

src/virtualenv/util/path/_pathlib/via_os_path.py

@@ -1,5 +1,6 @@
 from __future__ import absolute_import, unicode_literals
 
+import fnmatch
 import os
 import platform
 from contextlib import contextmanager
@@ -10,7 +11,7 @@
 
 
 class Path(object):
-    def __init__(self, path):
+    def __init__(self, path=""):
         if isinstance(path, Path):
             _path = path._path
         else:
@@ -147,5 +148,13 @@ def chmod(self, mode):
     def absolute(self):
         return Path(os.path.abspath(self._path))
 
+    def rglob(self, pattern):
+        """
+        Rough emulation of the origin method. Just for searching fixture files.
+        """
+        for root, _dirs, files in os.walk(self._path):
+            for filename in fnmatch.filter(files, pattern):
+                yield Path(os.path.join(root, filename))
+
 
 __all__ = ("Path",)

tests/conftest.py

@@ -14,7 +14,7 @@
 from virtualenv.app_data import AppDataDiskFolder
 from virtualenv.discovery.builtin import get_interpreter
 from virtualenv.discovery.py_info import PythonInfo
-from virtualenv.info import IS_PYPY, IS_WIN, fs_supports_symlink
+from virtualenv.info import IS_PYPY, IS_WIN, PY2, fs_supports_symlink
 from virtualenv.report import LOGGER
 from virtualenv.util.path import Path
 from virtualenv.util.six import ensure_str, ensure_text
@@ -388,3 +388,11 @@ def skip_if_test_in_system(session_app_data):
     current = PythonInfo.current(session_app_data)
     if current.system_executable is not None:
         pytest.skip("test not valid if run under system")
+
+
+def pytest_ignore_collect(path):
+    """
+    We can't just skip these tests due to syntax errors that occurs during
+    collecting tests under a Python 2 host.
+    """
+    return PY2 and str(path).endswith("test_cpython3_win.py")

tests/unit/create/via_global_ref/builtin/conftest.py

@@ -0,0 +1,25 @@
+import sys
+
+import pytest
+from testing import path
+from testing.py_info import read_fixture
+
+from virtualenv.util.path import Path
+
+# Allows to import from `testing` into test submodules.
+sys.path.append(str(Path(__file__).parent))
+
+
+@pytest.fixture
+def py_info(py_info_name):
+    return read_fixture(py_info_name)
+
+
+@pytest.fixture
+def mock_files(mocker):
+    return lambda paths, files: path.mock_files(mocker, paths, files)
+
+
+@pytest.fixture
+def mock_pypy_libs(mocker):
+    return lambda pypy, libs: path.mock_pypy_libs(mocker, pypy, libs)

tests/unit/create/via_global_ref/builtin/cpython/cpython3_win_embed.json

@@ -0,0 +1,61 @@
+{
+  "platform": "win32",
+  "implementation": "CPython",
+  "version_info": {
+    "major": 3,
+    "minor": 10,
+    "micro": 4,
+    "releaselevel": "final",
+    "serial": 0
+  },
+  "architecture": 64,
+  "version_nodot": "310",
+  "version": "3.10.4 (tags/v3.10.4:9d38120, Mar 23 2022, 23:13:41) [MSC v.1929 64 bit (AMD64)]",
+  "os": "nt",
+  "prefix": "c:\\path\\to\\python",
+  "base_prefix": "c:\\path\\to\\python",
+  "real_prefix": null,
+  "base_exec_prefix": "c:\\path\\to\\python",
+  "exec_prefix": "c:\\path\\to\\python",
+  "executable": "c:\\path\\to\\python\\python.exe",
+  "original_executable": "c:\\path\\to\\python\\python.exe",
+  "system_executable": "c:\\path\\to\\python\\python.exe",
+  "has_venv": false,
+  "path": [
+    "c:\\path\\to\\python\\Scripts\\virtualenv.exe",
+    "c:\\path\\to\\python\\python310.zip",
+    "c:\\path\\to\\python",
+    "c:\\path\\to\\python\\Lib\\site-packages"
+  ],
+  "file_system_encoding": "utf-8",
+  "stdout_encoding": "utf-8",
+  "sysconfig_scheme": null,
+  "sysconfig_paths": {
+    "stdlib": "{installed_base}/Lib",
+    "platstdlib": "{base}/Lib",
+    "purelib": "{base}/Lib/site-packages",
+    "platlib": "{base}/Lib/site-packages",
+    "include": "{installed_base}/Include",
+    "scripts": "{base}/Scripts",
+    "data": "{base}"
+  },
+  "distutils_install": {
+    "purelib": "Lib\\site-packages",
+    "platlib": "Lib\\site-packages",
+    "headers": "Include\\UNKNOWN",
+    "scripts": "Scripts",
+    "data": ""
+  },
+  "sysconfig": {
+    "makefile_filename": "c:\\path\\to\\python\\Lib\\config\\Makefile"
+  },
+  "sysconfig_vars": {
+    "PYTHONFRAMEWORK": "",
+    "installed_base": "c:\\path\\to\\python",
+    "base": "c:\\path\\to\\python"
+  },
+  "system_stdlib": "c:\\path\\to\\python\\Lib",
+  "system_stdlib_platform": "c:\\path\\to\\python\\Lib",
+  "max_size": 9223372036854775807,
+  "_creators": null
+}

tests/unit/create/via_global_ref/builtin/cpython/test_cpython3_win.py

@@ -0,0 +1,99 @@
+import pytest
+from testing.helpers import contains_exe, contains_ref
+from testing.path import join as path
+
+from virtualenv.create.via_global_ref.builtin.cpython.cpython3 import CPython3Windows
+
+CPYTHON3_PATH = (
+    "virtualenv.create.via_global_ref.builtin.cpython.common.Path",
+    "virtualenv.create.via_global_ref.builtin.cpython.cpython3.Path",
+)
+
+
+@pytest.mark.parametrize("py_info_name", ["cpython3_win_embed"])
+def test_2_exe_on_default_py_host(py_info, mock_files):
+    mock_files(CPYTHON3_PATH, [py_info.system_executable])
+    sources = tuple(CPython3Windows.sources(interpreter=py_info))
+    # Default Python exe.
+    assert contains_exe(sources, py_info.system_executable)
+    # Should always exist.
+    assert contains_exe(sources, path(py_info.prefix, "pythonw.exe"))
+
+
+@pytest.mark.parametrize("py_info_name", ["cpython3_win_embed"])
+def test_3_exe_on_not_default_py_host(py_info, mock_files):
+    # Not default python host.
+    py_info.system_executable = path(py_info.prefix, "python666.exe")
+    mock_files(CPYTHON3_PATH, [py_info.system_executable])
+    sources = tuple(CPython3Windows.sources(interpreter=py_info))
+    # Not default Python exe linked to both the default name and origin.
+    assert contains_exe(sources, py_info.system_executable, "python.exe")
+    assert contains_exe(sources, py_info.system_executable, "python666.exe")
+    # Should always exist.
+    assert contains_exe(sources, path(py_info.prefix, "pythonw.exe"))
+
+
+@pytest.mark.parametrize("py_info_name", ["cpython3_win_embed"])
+def test_only_shim(py_info, mock_files):
+    shim = path(py_info.system_stdlib, "venv\\scripts\\nt\\python.exe")
+    py_files = (
+        path(py_info.prefix, "libcrypto-1_1.dll"),
+        path(py_info.prefix, "libffi-7.dll"),
+        path(py_info.prefix, "_asyncio.pyd"),
+        path(py_info.prefix, "_bz2.pyd"),
+    )
+    mock_files(CPYTHON3_PATH, [shim, *py_files])
+    sources = tuple(CPython3Windows.sources(interpreter=py_info))
+    assert CPython3Windows.has_shim(interpreter=py_info)
+    assert contains_exe(sources, shim)
+    assert not contains_exe(sources, py_info.system_executable)
+    for file in py_files:
+        assert not contains_ref(sources, file)
+
+
+@pytest.mark.parametrize("py_info_name", ["cpython3_win_embed"])
+def test_exe_dll_pyd_without_shim(py_info, mock_files):
+    py_files = (
+        path(py_info.prefix, "libcrypto-1_1.dll"),
+        path(py_info.prefix, "libffi-7.dll"),
+        path(py_info.prefix, "_asyncio.pyd"),
+        path(py_info.prefix, "_bz2.pyd"),
+    )
+    mock_files(CPYTHON3_PATH, py_files)
+    sources = tuple(CPython3Windows.sources(interpreter=py_info))
+    assert not CPython3Windows.has_shim(interpreter=py_info)
+    assert contains_exe(sources, py_info.system_executable)
+    for file in py_files:
+        assert contains_ref(sources, file)
+
+
+@pytest.mark.parametrize("py_info_name", ["cpython3_win_embed"])
+def test_python_zip_if_exists_and_set_in_path(py_info, mock_files):
+    python_zip_name = "python{}.zip".format(py_info.version_nodot)
+    python_zip = path(py_info.prefix, python_zip_name)
+    mock_files(CPYTHON3_PATH, [python_zip])
+    sources = tuple(CPython3Windows.sources(interpreter=py_info))
+    assert python_zip in py_info.path
+    assert contains_ref(sources, python_zip)
+
+
+@pytest.mark.parametrize("py_info_name", ["cpython3_win_embed"])
+def test_no_python_zip_if_exists_and_not_set_in_path(py_info, mock_files):
+    python_zip_name = "python{}.zip".format(py_info.version_nodot)
+    python_zip = path(py_info.prefix, python_zip_name)
+    py_info.path.remove(python_zip)
+    mock_files(CPYTHON3_PATH, [python_zip])
+    sources = tuple(CPython3Windows.sources(interpreter=py_info))
+    assert python_zip not in py_info.path
+    assert not contains_ref(sources, python_zip)
+
+
+@pytest.mark.parametrize("py_info_name", ["cpython3_win_embed"])
+def test_no_python_zip_if_not_exists(py_info, mock_files):
+    python_zip_name = "python{}.zip".format(py_info.version_nodot)
+    python_zip = path(py_info.prefix, python_zip_name)
+    # No `python_zip`, just python.exe file.
+    mock_files(CPYTHON3_PATH, [py_info.system_executable])
+    sources = tuple(CPython3Windows.sources(interpreter=py_info))
+    assert python_zip in py_info.path
+    assert not contains_ref(sources, python_zip)