Bump pip's src from 20.1.1 to 21.2.4

WARNING: fetchcode's vcs will not work on this commit.
This is result of separating pip's code from changes made in scope of
this repository.

This should make it easier to track potentially replicated issues from
pip when taking their vcs pkg.

It also made cleaning up easier, due to some maintenance activities done
in pip:
- dropping Python 2 & 3.5 support
  pypa/pip#9189
- modernized code after above - partially done, tracked in:
  pypa/pip#8802
- added py3.9 support
- updated vendored libraries (e.g. fixing CVE-2021-28363)
  multiple PRs

pip._internal.vcs (and related code) changes:
- Fetch resources that are missing locally:
  pypa/pip#8817
- Improve SVN version parser (Windows)
  pypa/pip#8665
- Always close stderr after subprocess completion:
  pypa/pip#9156
- Remove vcs export feature:
  pypa/pip#9713
- Remove support for git+ssh@ scheme in favour of git+ssh://
  pypa/pip#9436
- Security fix in git tags parsing (CVE-2021-3572):
  pypa/pip#9827
- Reimplement Git version parsing:
  pypa/pip#10117

In next commits, most of pip's internals will be removed from fetchcode,
leaving only vcs module with supporting code (like utils functions,
tests (which will be added & submitted with this change))

This will allow for changes such as ability to add return codes
(probably via futures) from long running downloads and other features.

Switching to having own vcs module might also be a good call due to
pip._internal.vcs close integration with pip's cli in vcs module (some
pip code has been commented out in commit mentioned below)

While generally copy-pasting code without history is bad idea, this
commit follows precedence set in this repo by:
8046215
with exception that all changes to pip's code will be submitted as separate
commits.

It has been agreed with @pombredanne & @TG1999 that history from pip
will be rebased on fetchcode by @pombredanne (thanks!). It will be done
only for the files that are of concern for fetchcode to limit noise in
git history.

I'm leaving this commit without SoB intentionally, as this is not my
work, but that of the many pip's authors:
https://github.com/pypa/pip/blob/21.2.4/AUTHORS.txt
License of pip: MIT (https://pypi.org/project/pip/)

src/fetchcode/vcs/pip/__init__.py

@@ -1,18 +1,13 @@
-from fetchcode.vcs.pip._internal.utils.typing import MYPY_CHECK_RUNNING
+from typing import List, Optional
 
-if MYPY_CHECK_RUNNING:
-    from typing import List, Optional
+__version__ = "21.2.4"
 
 
-__version__ = "20.1.1"
-
-
-def main(args=None):
-    # type: (Optional[List[str]]) -> int
+def main(args: Optional[List[str]] = None) -> int:
     """This is an internal API only meant for use by pip's own console scripts.
 
     For additional details, see https://github.com/pypa/pip/issues/7498.
     """
-    from fetchcode.vcs.pip._internal.utils.entrypoints import _wrapper
+    from pip._internal.utils.entrypoints import _wrapper
 
     return _wrapper(args)

src/fetchcode/vcs/pip/__main__.py

@@ -1,26 +1,31 @@
-from __future__ import absolute_import
-
 import os
 import sys
+import warnings
 
 # Remove '' and current working directory from the first entry
 # of sys.path, if present to avoid using current directory
 # in pip commands check, freeze, install, list and show,
 # when invoked as python -m pip <command>
-if sys.path[0] in ('', os.getcwd()):
+if sys.path[0] in ("", os.getcwd()):
     sys.path.pop(0)
 
 # If we are running from a wheel, add the wheel to sys.path
 # This allows the usage python pip-*.whl/pip install pip-*.whl
-if __package__ == '':
+if __package__ == "":
     # __file__ is pip-*.whl/pip/__main__.py
     # first dirname call strips of '/__main__.py', second strips off '/pip'
     # Resulting path is the name of the wheel itself
     # Add that to sys.path so we can import pip
     path = os.path.dirname(os.path.dirname(__file__))
     sys.path.insert(0, path)
 
-from fetchcode.vcs.pip._internal.cli.main import main as _main  # isort:skip # noqa
+if __name__ == "__main__":
+    # Work around the error reported in #9540, pending a proper fix.
+    # Note: It is essential the warning filter is set *before* importing
+    #       pip, as the deprecation happens at import time, not runtime.
+    warnings.filterwarnings(
+        "ignore", category=DeprecationWarning, module=".*packaging\\.version"
+    )
+    from pip._internal.cli.main import main as _main
 
-if __name__ == '__main__':
     sys.exit(_main())

src/fetchcode/vcs/pip/_internal/__init__.py

@@ -1,17 +1,19 @@
-import fetchcode.vcs.pip._internal.utils.inject_securetransport  # noqa
-from fetchcode.vcs.pip._internal.utils.typing import MYPY_CHECK_RUNNING
+from typing import List, Optional
 
-if MYPY_CHECK_RUNNING:
-    from typing import Optional, List
+import pip._internal.utils.inject_securetransport  # noqa
+from pip._internal.utils import _log
 
+# init_logging() must be called before any call to logging.getLogger()
+# which happens at import of most modules.
+_log.init_logging()
 
-def main(args=None):
-    # type: (Optional[List[str]]) -> int
+
+def main(args: (Optional[List[str]]) = None) -> int:
     """This is preserved for old console scripts that may still be referencing
     it.
 
     For additional details, see https://github.com/pypa/pip/issues/7498.
     """
-    from fetchcode.vcs.pip._internal.utils.entrypoints import _wrapper
+    from pip._internal.utils.entrypoints import _wrapper
 
     return _wrapper(args)

src/fetchcode/vcs/pip/_internal/build_env.py

@@ -1,29 +1,31 @@
 """Build Environment used for isolation during sdist building
 """
 
-# The following comment should be removed at some point in the future.
-# mypy: strict-optional=False
-# mypy: disallow-untyped-defs=False
-
+import contextlib
 import logging
 import os
+import pathlib
 import sys
 import textwrap
+import zipfile
 from collections import OrderedDict
-from distutils.sysconfig import get_python_lib
 from sysconfig import get_paths
+from types import TracebackType
+from typing import TYPE_CHECKING, Iterable, Iterator, List, Optional, Set, Tuple, Type
 
-from fetchcode.vcs.pip._vendor.pkg_resources import Requirement, VersionConflict, WorkingSet
+from pip._vendor.certifi import where
+from pip._vendor.packaging.requirements import Requirement
+from pip._vendor.packaging.version import Version
 
 from pip import __file__ as pip_location
-from fetchcode.vcs.pip._internal.cli.spinners import open_spinner
-from fetchcode.vcs.pip._internal.utils.subprocess import call_subprocess
-from fetchcode.vcs.pip._internal.utils.temp_dir import TempDirectory, tempdir_kinds
-from fetchcode.vcs.pip._internal.utils.typing import MYPY_CHECK_RUNNING
+from pip._internal.cli.spinners import open_spinner
+from pip._internal.locations import get_platlib, get_prefixed_libs, get_purelib
+from pip._internal.metadata import get_environment
+from pip._internal.utils.subprocess import call_subprocess
+from pip._internal.utils.temp_dir import TempDirectory, tempdir_kinds
 
-if MYPY_CHECK_RUNNING:
-    from typing import Tuple, Set, Iterable, Optional, List
-    from fetchcode.vcs.pip._internal.index.package_finder import PackageFinder
+if TYPE_CHECKING:
+    from pip._internal.index.package_finder import PackageFinder
 
 logger = logging.getLogger(__name__)
 
@@ -38,17 +40,36 @@ def __init__(self, path):
             'nt' if os.name == 'nt' else 'posix_prefix',
             vars={'base': path, 'platbase': path}
         )['scripts']
-        # Note: prefer distutils' sysconfig to get the
-        # library paths so PyPy is correctly supported.
-        purelib = get_python_lib(plat_specific=False, prefix=path)
-        platlib = get_python_lib(plat_specific=True, prefix=path)
-        if purelib == platlib:
-            self.lib_dirs = [purelib]
-        else:
-            self.lib_dirs = [purelib, platlib]
+        self.lib_dirs = get_prefixed_libs(path)
 
 
-class BuildEnvironment(object):
+@contextlib.contextmanager
+def _create_standalone_pip() -> Iterator[str]:
+    """Create a "standalone pip" zip file.
+
+    The zip file's content is identical to the currently-running pip.
+    It will be used to install requirements into the build environment.
+    """
+    source = pathlib.Path(pip_location).resolve().parent
+
+    # Return the current instance if `source` is not a directory. We can't build
+    # a zip from this, and it likely means the instance is already standalone.
+    if not source.is_dir():
+        yield str(source)
+        return
+
+    with TempDirectory(kind="standalone-pip") as tmp_dir:
+        pip_zip = os.path.join(tmp_dir.path, "__env_pip__.zip")
+        kwargs = {}
+        if sys.version_info >= (3, 8):
+            kwargs["strict_timestamps"] = False
+        with zipfile.ZipFile(pip_zip, "w", **kwargs) as zf:
+            for child in source.rglob("*"):
+                zf.write(child, child.relative_to(source.parent).as_posix())
+        yield os.path.join(pip_zip, "pip")
+
+
+class BuildEnvironment:
     """Creates and manages an isolated environment to install build deps
     """
 
@@ -58,10 +79,10 @@ def __init__(self):
             kind=tempdir_kinds.BUILD_ENV, globally_managed=True
         )
 
-        self._prefixes = OrderedDict((
+        self._prefixes = OrderedDict(
             (name, _Prefix(os.path.join(temp_dir.path, name)))
             for name in ('normal', 'overlay')
-        ))
+        )
 
         self._bin_dirs = []  # type: List[str]
         self._lib_dirs = []  # type: List[str]
@@ -73,10 +94,7 @@ def __init__(self):
         # - ensure .pth files are honored
         # - prevent access to system site packages
         system_sites = {
-            os.path.normcase(site) for site in (
-                get_python_lib(plat_specific=False),
-                get_python_lib(plat_specific=True),
-            )
+            os.path.normcase(site) for site in (get_purelib(), get_platlib())
         }
         self._site_dir = os.path.join(temp_dir.path, 'site')
         if not os.path.exists(self._site_dir):
@@ -110,6 +128,7 @@ def __init__(self):
             ).format(system_sites=system_sites, lib_dirs=self._lib_dirs))
 
     def __enter__(self):
+        # type: () -> None
         self._save_env = {
             name: os.environ.get(name, None)
             for name in ('PATH', 'PYTHONNOUSERSITE', 'PYTHONPATH')
@@ -128,7 +147,13 @@ def __enter__(self):
             'PYTHONPATH': os.pathsep.join(pythonpath),
         })
 
-    def __exit__(self, exc_type, exc_val, exc_tb):
+    def __exit__(
+        self,
+        exc_type,  # type: Optional[Type[BaseException]]
+        exc_val,  # type: Optional[BaseException]
+        exc_tb  # type: Optional[TracebackType]
+    ):
+        # type: (...) -> None
         for varname, old_value in self._save_env.items():
             if old_value is None:
                 os.environ.pop(varname, None)
@@ -144,31 +169,62 @@ def check_requirements(self, reqs):
         missing = set()
         conflicting = set()
         if reqs:
-            ws = WorkingSet(self._lib_dirs)
-            for req in reqs:
-                try:
-                    if ws.find(Requirement.parse(req)) is None:
-                        missing.add(req)
-                except VersionConflict as e:
-                    conflicting.add((str(e.args[0].as_requirement()),
-                                     str(e.args[1])))
+            env = get_environment(self._lib_dirs)
+            for req_str in reqs:
+                req = Requirement(req_str)
+                dist = env.get_distribution(req.name)
+                if not dist:
+                    missing.add(req_str)
+                    continue
+                if isinstance(dist.version, Version):
+                    installed_req_str = f"{req.name}=={dist.version}"
+                else:
+                    installed_req_str = f"{req.name}==={dist.version}"
+                if dist.version not in req.specifier:
+                    conflicting.add((installed_req_str, req_str))
+                # FIXME: Consider direct URL?
         return conflicting, missing
 
     def install_requirements(
         self,
         finder,  # type: PackageFinder
         requirements,  # type: Iterable[str]
         prefix_as_string,  # type: str
-        message  # type: Optional[str]
+        message  # type: str
     ):
         # type: (...) -> None
         prefix = self._prefixes[prefix_as_string]
         assert not prefix.setup
         prefix.setup = True
         if not requirements:
             return
+        with contextlib.ExitStack() as ctx:
+            # TODO: Remove this block when dropping 3.6 support. Python 3.6
+            # lacks importlib.resources and pep517 has issues loading files in
+            # a zip, so we fallback to the "old" method by adding the current
+            # pip directory to the child process's sys.path.
+            if sys.version_info < (3, 7):
+                pip_runnable = os.path.dirname(pip_location)
+            else:
+                pip_runnable = ctx.enter_context(_create_standalone_pip())
+            self._install_requirements(
+                pip_runnable,
+                finder,
+                requirements,
+                prefix,
+                message,
+            )
+
+    @staticmethod
+    def _install_requirements(
+        pip_runnable: str,
+        finder: "PackageFinder",
+        requirements: Iterable[str],
+        prefix: _Prefix,
+        message: str,
+    ) -> None:
         args = [
-            sys.executable, os.path.dirname(pip_location), 'install',
+            sys.executable, pip_runnable, 'install',
             '--ignore-installed', '--no-user', '--prefix', prefix.path,
             '--no-warn-script-location',
         ]  # type: List[str]
@@ -193,27 +249,46 @@ def install_requirements(
             args.extend(['--trusted-host', host])
         if finder.allow_all_prereleases:
             args.append('--pre')
+        if finder.prefer_binary:
+            args.append('--prefer-binary')
         args.append('--')
         args.extend(requirements)
+        extra_environ = {"_PIP_STANDALONE_CERT": where()}
         with open_spinner(message) as spinner:
-            call_subprocess(args, spinner=spinner)
+            call_subprocess(args, spinner=spinner, extra_environ=extra_environ)
 
 
 class NoOpBuildEnvironment(BuildEnvironment):
     """A no-op drop-in replacement for BuildEnvironment
     """
 
     def __init__(self):
+        # type: () -> None
         pass
 
     def __enter__(self):
+        # type: () -> None
         pass
 
-    def __exit__(self, exc_type, exc_val, exc_tb):
+    def __exit__(
+        self,
+        exc_type,  # type: Optional[Type[BaseException]]
+        exc_val,  # type: Optional[BaseException]
+        exc_tb  # type: Optional[TracebackType]
+    ):
+        # type: (...) -> None
         pass
 
     def cleanup(self):
+        # type: () -> None
         pass
 
-    def install_requirements(self, finder, requirements, prefix, message):
+    def install_requirements(
+        self,
+        finder,  # type: PackageFinder
+        requirements,  # type: Iterable[str]
+        prefix_as_string,  # type: str
+        message  # type: str
+    ):
+        # type: (...) -> None
         raise NotImplementedError()