Merge pull request #9673 from mwchase/url-constraints-final-2

Support URL constraints in the new resolver

Author: sbidoul

docs/html/user_guide.rst

@@ -254,9 +254,11 @@ Constraints Files
 
 Constraints files are requirements files that only control which version of a
 requirement is installed, not whether it is installed or not. Their syntax and
-contents is nearly identical to :ref:`Requirements Files`. There is one key
-difference: Including a package in a constraints file does not trigger
-installation of the package.
+contents is a subset of :ref:`Requirements Files`, with several kinds of syntax
+not allowed: constraints must have a name, they cannot be editable, and they
+cannot specify extras. In terms of semantics, there is one key difference:
+Including a package in a constraints file does not trigger installation of the
+package.
 
 Use a constraints file like so:
 

news/8253.feature.rst

@@ -0,0 +1 @@
+Add the ability for the new resolver to process URL constraints.

src/pip/_internal/models/link.py

@@ -240,3 +240,9 @@ def is_hash_allowed(self, hashes):
         assert self.hash is not None
 
         return hashes.is_hash_allowed(self.hash_name, hex_digest=self.hash)
+
+
+# TODO: Relax this comparison logic to ignore, for example, fragments.
+def links_equivalent(link1, link2):
+    # type: (Link, Link) -> bool
+    return link1 == link2

src/pip/_internal/req/constructors.py

@@ -468,3 +468,19 @@ def install_req_from_parsed_requirement(
             user_supplied=user_supplied,
         )
     return req
+
+
+def install_req_from_link_and_ireq(link, ireq):
+    # type: (Link, InstallRequirement) -> InstallRequirement
+    return InstallRequirement(
+        req=ireq.req,
+        comes_from=ireq.comes_from,
+        editable=ireq.editable,
+        link=link,
+        markers=ireq.markers,
+        use_pep517=ireq.use_pep517,
+        isolated=ireq.isolated,
+        install_options=ireq.install_options,
+        global_options=ireq.global_options,
+        hash_options=ireq.hash_options,
+    )

src/pip/_internal/req/req_install.py

@@ -848,8 +848,8 @@ def check_invalid_constraint_type(req):
     problem = ""
     if not req.name:
         problem = "Unnamed requirements are not allowed as constraints"
-    elif req.link:
-        problem = "Links are not allowed as constraints"
+    elif req.editable:
+        problem = "Editable requirements are not allowed as constraints"
     elif req.extras:
         problem = "Constraints cannot have extras"
 

src/pip/_internal/resolution/resolvelib/base.py

@@ -4,7 +4,7 @@
 from pip._vendor.packaging.utils import NormalizedName, canonicalize_name
 from pip._vendor.packaging.version import LegacyVersion, Version
 
-from pip._internal.models.link import Link
+from pip._internal.models.link import Link, links_equivalent
 from pip._internal.req.req_install import InstallRequirement
 from pip._internal.utils.hashes import Hashes
 
@@ -21,24 +21,26 @@ def format_name(project, extras):
 
 
 class Constraint:
-    def __init__(self, specifier, hashes):
-        # type: (SpecifierSet, Hashes) -> None
+    def __init__(self, specifier, hashes, links):
+        # type: (SpecifierSet, Hashes, FrozenSet[Link]) -> None
         self.specifier = specifier
         self.hashes = hashes
+        self.links = links
 
     @classmethod
     def empty(cls):
         # type: () -> Constraint
-        return Constraint(SpecifierSet(), Hashes())
+        return Constraint(SpecifierSet(), Hashes(), frozenset())
 
     @classmethod
     def from_ireq(cls, ireq):
         # type: (InstallRequirement) -> Constraint
-        return Constraint(ireq.specifier, ireq.hashes(trust_internet=False))
+        links = frozenset([ireq.link]) if ireq.link else frozenset()
+        return Constraint(ireq.specifier, ireq.hashes(trust_internet=False), links)
 
     def __nonzero__(self):
         # type: () -> bool
-        return bool(self.specifier) or bool(self.hashes)
+        return bool(self.specifier) or bool(self.hashes) or bool(self.links)
 
     def __bool__(self):
         # type: () -> bool
@@ -50,10 +52,16 @@ def __and__(self, other):
             return NotImplemented
         specifier = self.specifier & other.specifier
         hashes = self.hashes & other.hashes(trust_internet=False)
-        return Constraint(specifier, hashes)
+        links = self.links
+        if other.link:
+            links = links.union([other.link])
+        return Constraint(specifier, hashes, links)
 
     def is_satisfied_by(self, candidate):
         # type: (Candidate) -> bool
+        # Reject if there are any mismatched URL constraints on this package.
+        if self.links and not all(_match_link(link, candidate) for link in self.links):
+            return False
         # We can safely always allow prereleases here since PackageFinder
         # already implements the prerelease logic, and would have filtered out
         # prerelease candidates if the user does not expect them.
@@ -95,6 +103,13 @@ def format_for_error(self):
         raise NotImplementedError("Subclass should override")
 
 
+def _match_link(link, candidate):
+    # type: (Link, Candidate) -> bool
+    if candidate.source_link:
+        return links_equivalent(link, candidate.source_link)
+    return False
+
+
 class Candidate:
     @property
     def project_name(self):

src/pip/_internal/resolution/resolvelib/candidates.py

@@ -9,7 +9,7 @@
 from pip._vendor.pkg_resources import Distribution
 
 from pip._internal.exceptions import HashError, MetadataInconsistent
-from pip._internal.models.link import Link
+from pip._internal.models.link import Link, links_equivalent
 from pip._internal.models.wheel import Wheel
 from pip._internal.req.constructors import (
     install_req_from_editable,
@@ -156,7 +156,7 @@ def __hash__(self):
     def __eq__(self, other):
         # type: (Any) -> bool
         if isinstance(other, self.__class__):
-            return self._link == other._link
+            return links_equivalent(self._link, other._link)
         return False
 
     @property

src/pip/_internal/resolution/resolvelib/factory.py

@@ -34,6 +34,7 @@
 from pip._internal.models.link import Link
 from pip._internal.models.wheel import Wheel
 from pip._internal.operations.prepare import RequirementPreparer
+from pip._internal.req.constructors import install_req_from_link_and_ireq
 from pip._internal.req.req_install import InstallRequirement
 from pip._internal.resolution.base import InstallRequirementProvider
 from pip._internal.utils.compatibility_tags import get_supported
@@ -296,6 +297,46 @@ def find_candidates(
             if ireq is not None:
                 ireqs.append(ireq)
 
+        for link in constraint.links:
+            if not ireqs:
+                # If we hit this condition, then we cannot construct a candidate.
+                # However, if we hit this condition, then none of the requirements
+                # provided an ireq, so they must have provided an explicit candidate.
+                # In that case, either the candidate matches, in which case this loop
+                # doesn't need to do anything, or it doesn't, in which case there's
+                # nothing this loop can do to recover.
+                break
+            if link.is_wheel:
+                wheel = Wheel(link.filename)
+                # Check whether the provided wheel is compatible with the target
+                # platform.
+                if not wheel.supported(self._finder.target_python.get_tags()):
+                    # We are constrained to install a wheel that is incompatible with
+                    # the target architecture, so there are no valid candidates.
+                    # Return early, with no candidates.
+                    return ()
+            # Create a "fake" InstallRequirement that's basically a clone of
+            # what "should" be the template, but with original_link set to link.
+            # Using the given requirement is necessary for preserving hash
+            # requirements, but without the original_link, direct_url.json
+            # won't be created.
+            ireq = install_req_from_link_and_ireq(link, ireqs[0])
+            candidate = self._make_candidate_from_link(
+                link,
+                extras=frozenset(),
+                template=ireq,
+                name=canonicalize_name(ireq.name) if ireq.name else None,
+                version=None,
+            )
+            if candidate is None:
+                # _make_candidate_from_link returns None if the wheel fails to build.
+                # We are constrained to install this wheel, so there are no valid
+                # candidates.
+                # Return early, with no candidates.
+                return ()
+
+            explicit_candidates.add(candidate)
+
         # If none of the requirements want an explicit candidate, we can ask
         # the finder for candidates.
         if not explicit_candidates:

tests/functional/test_install_direct_url.py

@@ -1,5 +1,7 @@
 import re
 
+import pytest
+
 from pip._internal.models.direct_url import DIRECT_URL_METADATA_NAME, DirectUrl
 from tests.lib import _create_test_package, path_to_url
 
@@ -46,3 +48,24 @@ def test_install_archive_direct_url(script, data, with_wheel):
     assert req.startswith("simple @ file://")
     result = script.pip("install", req)
     assert _get_created_direct_url(result, "simple")
+
+
+@pytest.mark.network
+def test_install_vcs_constraint_direct_url(script, with_wheel):
+    constraints_file = script.scratch_path / "constraints.txt"
+    constraints_file.write_text(
+        "git+https://github.com/pypa/pip-test-package"
+        "@5547fa909e83df8bd743d3978d6667497983a4b7"
+        "#egg=pip-test-package"
+    )
+    result = script.pip("install", "pip-test-package", "-c", constraints_file)
+    assert _get_created_direct_url(result, "pip_test_package")
+
+
+def test_install_vcs_constraint_direct_file_url(script, with_wheel):
+    pkg_path = _create_test_package(script, name="testpkg")
+    url = path_to_url(pkg_path)
+    constraints_file = script.scratch_path / "constraints.txt"
+    constraints_file.write_text(f"git+{url}#egg=testpkg")
+    result = script.pip("install", "testpkg", "-c", constraints_file)
+    assert _get_created_direct_url(result, "testpkg")

tests/functional/test_install_reqs.py

@@ -405,7 +405,7 @@ def test_constraints_constrain_to_local_editable(
         expect_error=(resolver_variant == "2020-resolver"),
     )
     if resolver_variant == "2020-resolver":
-        assert 'Links are not allowed as constraints' in result.stderr
+        assert 'Editable requirements are not allowed as constraints' in result.stderr
     else:
         assert 'Running setup.py develop for singlemodule' in result.stdout
 
@@ -419,12 +419,8 @@ def test_constraints_constrain_to_local(script, data, resolver_variant):
         'install', '--no-index', '-f', data.find_links, '-c',
         script.scratch_path / 'constraints.txt', 'singlemodule',
         allow_stderr_warning=True,
-        expect_error=(resolver_variant == "2020-resolver"),
     )
-    if resolver_variant == "2020-resolver":
-        assert 'Links are not allowed as constraints' in result.stderr
-    else:
-        assert 'Running setup.py install for singlemodule' in result.stdout
+    assert 'Running setup.py install for singlemodule' in result.stdout
 
 
 def test_constrained_to_url_install_same_url(script, data, resolver_variant):
@@ -438,7 +434,11 @@ def test_constrained_to_url_install_same_url(script, data, resolver_variant):
         expect_error=(resolver_variant == "2020-resolver"),
     )
     if resolver_variant == "2020-resolver":
-        assert 'Links are not allowed as constraints' in result.stderr
+        assert 'Cannot install singlemodule 0.0.1' in result.stderr, str(result)
+        assert (
+            'because these package versions have conflicting dependencies.'
+            in result.stderr
+        ), str(result)
     else:
         assert ('Running setup.py install for singlemodule'
                 in result.stdout), str(result)
@@ -489,7 +489,7 @@ def test_install_with_extras_from_constraints(script, data, resolver_variant):
         expect_error=(resolver_variant == "2020-resolver"),
     )
     if resolver_variant == "2020-resolver":
-        assert 'Links are not allowed as constraints' in result.stderr
+        assert 'Constraints cannot have extras' in result.stderr
     else:
         result.did_create(script.site_packages / 'simple')
 
@@ -521,7 +521,7 @@ def test_install_with_extras_joined(script, data, resolver_variant):
         expect_error=(resolver_variant == "2020-resolver"),
     )
     if resolver_variant == "2020-resolver":
-        assert 'Links are not allowed as constraints' in result.stderr
+        assert 'Constraints cannot have extras' in result.stderr
     else:
         result.did_create(script.site_packages / 'simple')
         result.did_create(script.site_packages / 'singlemodule.py')
@@ -538,7 +538,7 @@ def test_install_with_extras_editable_joined(script, data, resolver_variant):
         expect_error=(resolver_variant == "2020-resolver"),
     )
     if resolver_variant == "2020-resolver":
-        assert 'Links are not allowed as constraints' in result.stderr
+        assert 'Editable requirements are not allowed as constraints' in result.stderr
     else:
         result.did_create(script.site_packages / 'simple')
         result.did_create(script.site_packages / 'singlemodule.py')