bpo-42856: Add --with-wheel-pkg-dir=PATH configure option

Doc/library/ensurepip.rst

@@ -48,7 +48,7 @@ The simplest possible invocation is::
 
 This invocation will install ``pip`` if it is not already installed,
 but otherwise does nothing. To ensure the installed version of ``pip``
-is at least as recent as the one bundled with ``ensurepip``, pass the
+is at least as recent as the one available in ``ensurepip``, pass the
 ``--upgrade`` option::
 
     python -m ensurepip --upgrade
@@ -86,7 +86,7 @@ Module API
 
 .. function:: version()
 
-   Returns a string specifying the bundled version of pip that will be
+   Returns a string specifying the available version of pip that will be
    installed when bootstrapping an environment.
 
 .. function:: bootstrap(root=None, upgrade=False, user=False, \
@@ -100,7 +100,7 @@ Module API
    for the current environment.
 
    *upgrade* indicates whether or not to upgrade an existing installation
-   of an earlier version of ``pip`` to the bundled version.
+   of an earlier version of ``pip`` to the available version.
 
    *user* indicates whether to use the user scheme rather than installing
    globally.

Doc/whatsnew/3.10.rst

@@ -617,6 +617,18 @@ Build Changes
   don't build nor install test modules.
   (Contributed by Xavier de Gaye, Thomas Petazzoni and Peixing Xin in :issue:`27640`.)
 
+* Add ``--with-wheel-pkg-dir=PATH`` option to the ``./configure`` script. If
+  specified, the :mod:`ensurepip` module looks for ``setuptools`` and ``pip``
+  wheel packages in this directory: if both are present, these wheel packages
+  are used instead of ensurepip bundled wheel packages.
+
+  Some Linux distribution packaging policies recommend against bundling
+  dependencies. For example, Fedora installs wheel packages in the
+  ``/usr/share/python-wheels/`` directory and don't install the
+  ``ensurepip._bundled`` package.
+
+  (Contributed by Victor Stinner in :issue:`42856`.)
+
 
 C API Changes
 =============

Lib/ensurepip/__init__.py

@@ -1,27 +1,82 @@
+import collections
 import os
 import os.path
+import subprocess
 import sys
-import runpy
+import sysconfig
 import tempfile
-import subprocess
 from importlib import resources
 
-from . import _bundled
-
 
 
 __all__ = ["version", "bootstrap"]
-
-
+_PACKAGE_NAMES = ('setuptools', 'pip')
 _SETUPTOOLS_VERSION = "47.1.0"
-
 _PIP_VERSION = "20.2.3"
-
 _PROJECTS = [
     ("setuptools", _SETUPTOOLS_VERSION, "py3"),
     ("pip", _PIP_VERSION, "py2.py3"),
 ]
 
+# Packages bundled in ensurepip._bundled have wheel_name set.
+# Packages from WHEEL_PKG_DIR have wheel_path set.
+_Package = collections.namedtuple('Package',
+                                  ('version', 'wheel_name', 'wheel_path'))
+
+# Directory of system wheel packages. Some Linux distribution packaging
+# policies recommend against bundling dependencies. For example, Fedora
+# installs wheel packages in the /usr/share/python-wheels/ directory and don't
+# install the ensurepip._bundled package.
+_WHEEL_PKG_DIR = sysconfig.get_config_var('WHEEL_PKG_DIR')
+
+
+def _find_packages(path):
+    packages = {}
+    try:
+        filenames = os.listdir(path)
+    except OSError:
+        # Ignore: path doesn't exist or permission error
+        filenames = ()
+    # Make the code deterministic if a directory contains multiple wheel files
+    # of the same package, but don't attempt to implement correct version
+    # comparison since this case should not happen.
+    filenames = sorted(filenames)
+    for filename in filenames:
+        # filename is like 'pip-20.2.3-py2.py3-none-any.whl'
+        if not filename.endswith(".whl"):
+            continue
+        for name in _PACKAGE_NAMES:
+            prefix = name + '-'
+            if filename.startswith(prefix):
+                break
+        else:
+            continue
+
+        # Extract '20.2.2' from 'pip-20.2.2-py2.py3-none-any.whl'
+        version = filename.removeprefix(prefix).partition('-')[0]
+        wheel_path = os.path.join(path, filename)
+        packages[name] = _Package(version, None, wheel_path)
+    return packages
+
+
+def _get_packages():
+    global _PACKAGES, _WHEEL_PKG_DIR
+    if _PACKAGES is not None:
+        return _PACKAGES
+
+    packages = {}
+    for name, version, py_tag in _PROJECTS:
+        wheel_name = f"{name}-{version}-{py_tag}-none-any.whl"
+        packages[name] = _Package(version, wheel_name, None)
+    if _WHEEL_PKG_DIR:
+        dir_packages = _find_packages(_WHEEL_PKG_DIR)
+        # only used the wheel package directory if all packages are found there
+        if all(name in dir_packages for name in _PACKAGE_NAMES):
+            packages = dir_packages
+    _PACKAGES = packages
+    return packages
+_PACKAGES = None
+
 
 def _run_pip(args, additional_paths=None):
     # Run the bootstraping in a subprocess to avoid leaking any state that happens
@@ -42,7 +97,8 @@ def version():
     """
     Returns a string specifying the bundled version of pip.
     """
-    return _PIP_VERSION
+    return _get_packages()['pip'].version
+
 
 def _disable_pip_configuration_settings():
     # We deliberately ignore all pip environment variables
@@ -104,16 +160,23 @@ def _bootstrap(*, root=None, upgrade=False, user=False,
         # Put our bundled wheels into a temporary directory and construct the
         # additional paths that need added to sys.path
         additional_paths = []
-        for project, version, py_tag in _PROJECTS:
-            wheel_name = "{}-{}-{}-none-any.whl".format(project, version, py_tag)
-            whl = resources.read_binary(
-                _bundled,
-                wheel_name,
-            )
-            with open(os.path.join(tmpdir, wheel_name), "wb") as fp:
+        for name, package in _get_packages().items():
+            if package.wheel_name:
+                # Use bundled wheel package
+                from ensurepip import _bundled
+                wheel_name = package.wheel_name
+                whl = resources.read_binary(_bundled, wheel_name)
+            else:
+                # Use the wheel package directory
+                with open(package.wheel_path, "rb") as fp:
+                    whl = fp.read()
+                wheel_name = os.path.basename(package.wheel_path)
+
+            filename = os.path.join(tmpdir, wheel_name)
+            with open(filename, "wb") as fp:
                 fp.write(whl)
 
-            additional_paths.append(os.path.join(tmpdir, wheel_name))
+            additional_paths.append(filename)
 
         # Construct the arguments to be passed to the pip command
         args = ["install", "--no-cache-dir", "--no-index", "--find-links", tmpdir]
@@ -126,7 +189,7 @@ def _bootstrap(*, root=None, upgrade=False, user=False,
         if verbosity:
             args += ["-" + "v" * verbosity]
 
-        return _run_pip(args + [p[0] for p in _PROJECTS], additional_paths)
+        return _run_pip([*args, *_PACKAGE_NAMES], additional_paths)
 
 def _uninstall_helper(*, verbosity=0):
     """Helper to support a clean default uninstall process on Windows
@@ -139,11 +202,14 @@ def _uninstall_helper(*, verbosity=0):
     except ImportError:
         return
 
-    # If the pip version doesn't match the bundled one, leave it alone
-    if pip.__version__ != _PIP_VERSION:
-        msg = ("ensurepip will only uninstall a matching version "
-               "({!r} installed, {!r} bundled)")
-        print(msg.format(pip.__version__, _PIP_VERSION), file=sys.stderr)
+    # If the installed pip version doesn't match the available one,
+    # leave it alone
+    available_version = version()
+    if pip.__version__ != available_version:
+        print(f"ensurepip will only uninstall a matching version "
+              f"({pip.__version__!r} installed, "
+              f"{available_version!r} available)",
+              file=sys.stderr)
         return
 
     _disable_pip_configuration_settings()
@@ -153,7 +219,7 @@ def _uninstall_helper(*, verbosity=0):
     if verbosity:
         args += ["-" + "v" * verbosity]
 
-    return _run_pip(args + [p[0] for p in reversed(_PROJECTS)])
+    return _run_pip([*args, *reversed(_PACKAGE_NAMES)])
 
 
 def _main(argv=None):

Lib/test/test_ensurepip.py

@@ -1,19 +1,68 @@
-import unittest
-import unittest.mock
-import test.support
+import contextlib
 import os
 import os.path
-import contextlib
 import sys
+import tempfile
+import test.support
+import unittest
+import unittest.mock
 
 import ensurepip
 import ensurepip._uninstall
 
 
-class TestEnsurePipVersion(unittest.TestCase):
+class TestPackages(unittest.TestCase):
+    def touch(self, directory, filename):
+        fullname = os.path.join(directory, filename)
+        open(fullname, "wb").close()
+
+    def test_version(self):
+        # Test version()
+        with tempfile.TemporaryDirectory() as tmpdir:
+            self.touch(tmpdir, "pip-1.2.3b1-py2.py3-none-any.whl")
+            self.touch(tmpdir, "setuptools-49.1.3-py3-none-any.whl")
+            with (unittest.mock.patch.object(ensurepip, '_PACKAGES', None),
+                  unittest.mock.patch.object(ensurepip, '_WHEEL_PKG_DIR', tmpdir)):
+                self.assertEqual(ensurepip.version(), '1.2.3b1')
+
+    def test_get_packages_no_dir(self):
+        # Test _get_packages() without a wheel package directory
+        with (unittest.mock.patch.object(ensurepip, '_PACKAGES', None),
+              unittest.mock.patch.object(ensurepip, '_WHEEL_PKG_DIR', None)):
+            packages = ensurepip._get_packages()
+
+            # when bundled wheel packages are used, we get _PIP_VERSION
+            self.assertEqual(ensurepip._PIP_VERSION, ensurepip.version())
+
+        # use bundled wheel packages
+        self.assertIsNotNone(packages['pip'].wheel_name)
+        self.assertIsNotNone(packages['setuptools'].wheel_name)
+
+    def test_get_packages_with_dir(self):
+        # Test _get_packages() with a wheel package directory
+        setuptools_filename = "setuptools-49.1.3-py3-none-any.whl"
+        pip_filename = "pip-20.2.2-py2.py3-none-any.whl"
+
+        with tempfile.TemporaryDirectory() as tmpdir:
+            self.touch(tmpdir, setuptools_filename)
+            self.touch(tmpdir, pip_filename)
+            # not used, make sure that it's ignored
+            self.touch(tmpdir, "wheel-0.34.2-py2.py3-none-any.whl")
+
+            with (unittest.mock.patch.object(ensurepip, '_PACKAGES', None),
+                  unittest.mock.patch.object(ensurepip, '_WHEEL_PKG_DIR', tmpdir)):
+                packages = ensurepip._get_packages()
+
+            self.assertEqual(packages['setuptools'].version, '49.1.3')
+            self.assertEqual(packages['setuptools'].wheel_path,
+                             os.path.join(tmpdir, setuptools_filename))
+            self.assertEqual(packages['pip'].version, '20.2.2')
+            self.assertEqual(packages['pip'].wheel_path,
+                             os.path.join(tmpdir, pip_filename))
+
+            # wheel package is ignored
+            self.assertEqual(sorted(packages), ['pip', 'setuptools'])
 
-    def test_returns_version(self):
-        self.assertEqual(ensurepip._PIP_VERSION, ensurepip.version())
 
 class EnsurepipMixin:
 
@@ -27,6 +76,8 @@ def setUp(self):
         real_devnull = os.devnull
         os_patch = unittest.mock.patch("ensurepip.os")
         patched_os = os_patch.start()
+        # But expose os.listdir() used by _find_packages()
+        patched_os.listdir = os.listdir
         self.addCleanup(os_patch.stop)
         patched_os.devnull = real_devnull
         patched_os.path = os.path
@@ -147,7 +198,7 @@ def test_pip_config_file_disabled(self):
         self.assertEqual(self.os_environ["PIP_CONFIG_FILE"], os.devnull)
 
 @contextlib.contextmanager
-def fake_pip(version=ensurepip._PIP_VERSION):
+def fake_pip(version=ensurepip.version()):
     if version is None:
         pip = None
     else:
@@ -243,7 +294,7 @@ def test_pip_config_file_disabled(self):
 
 # Basic testing of the main functions and their argument parsing
 
-EXPECTED_VERSION_OUTPUT = "pip " + ensurepip._PIP_VERSION
+EXPECTED_VERSION_OUTPUT = "pip " + ensurepip.version()
 
 class TestBootstrappingMainFunction(EnsurepipMixin, unittest.TestCase):
 

Makefile.pre.in

@@ -146,6 +146,8 @@ CONFINCLUDEDIR=	$(exec_prefix)/include
 PLATLIBDIR=	@PLATLIBDIR@
 SCRIPTDIR=	$(prefix)/$(PLATLIBDIR)
 ABIFLAGS=	@ABIFLAGS@
+# Variable used by ensurepip
+WHEEL_PKG_DIR=	@WHEEL_PKG_DIR@
 
 # Detailed destination directories
 BINLIBDEST=	@BINLIBDEST@

Misc/NEWS.d/next/Build/2021-01-07-12-51-38.bpo-42856.n3cMHV.rst

@@ -0,0 +1,9 @@
+Add ``--with-wheel-pkg-dir=PATH`` option to the ``./configure`` script. If
+specified, the :mod:`ensurepip` module looks for ``setuptools`` and ``pip``
+wheel packages in this directory: if both are present, these wheel packages are
+used instead of ensurepip bundled wheel packages.
+
+Some Linux distribution packaging policies recommend against bundling
+dependencies. For example, Fedora installs wheel packages in the
+``/usr/share/python-wheels/`` directory and don't install the
+``ensurepip._bundled`` package.

configure

@@ -630,6 +630,7 @@ OPENSSL_INCLUDES
 ENSUREPIP
 SRCDIRS
 THREADHEADERS
+WHEEL_PKG_DIR
 LIBPL
 PY_ENABLE_SHARED
 PLATLIBDIR
@@ -847,6 +848,7 @@ with_libm
 with_libc
 enable_big_digits
 with_platlibdir
+with_wheel_pkg_dir
 with_computed_gotos
 with_ensurepip
 with_openssl
@@ -1576,6 +1578,9 @@ Optional Packages:
                           system-dependent)
   --with-platlibdir=DIRNAME
                           Python library directory name (default is "lib")
+  --with-wheel-pkg-dir=PATH
+                          Directory of wheel packages used by ensurepip
+                          (default: none)
   --with-computed-gotos   enable computed gotos in evaluation loop (enabled by
                           default on supported compilers)
   --with-ensurepip[=install|upgrade|no]
@@ -15493,6 +15498,29 @@ else
 fi
 
 
+# Check for --with-wheel-pkg-dir=PATH
+
+WHEEL_PKG_DIR=""
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for --with-wheel-pkg-dir" >&5
+$as_echo_n "checking for --with-wheel-pkg-dir... " >&6; }
+
+# Check whether --with-wheel-pkg-dir was given.
+if test "${with_wheel_pkg_dir+set}" = set; then :
+  withval=$with_wheel_pkg_dir;
+if test -n "$withval"; then
+  { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+  WHEEL_PKG_DIR="$withval"
+else
+  { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+fi
+else
+  { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+fi
+
+
 # Check whether right shifting a negative integer extends the sign bit
 # or fills with zeros (like the Cray J90, according to Tim Peters).
 { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether right shift extends the sign bit" >&5