diff --git a/src/pip/_internal/build_env.py b/src/pip/_internal/build_env.py index 95187f49f0c..a060ceea2ca 100644 --- a/src/pip/_internal/build_env.py +++ b/src/pip/_internal/build_env.py @@ -177,14 +177,17 @@ def install_requirements( formats = getattr(finder.format_control, format_control) args.extend(('--' + format_control.replace('_', '-'), ','.join(sorted(formats or {':none:'})))) - if finder.index_urls: - args.extend(['-i', finder.index_urls[0]]) - for extra_index in finder.index_urls[1:]: + + index_urls = finder.index_urls + if index_urls: + args.extend(['-i', index_urls[0]]) + for extra_index in index_urls[1:]: args.extend(['--extra-index-url', extra_index]) else: args.append('--no-index') for link in finder.find_links: args.extend(['--find-links', link]) + for host in finder.trusted_hosts: args.extend(['--trusted-host', host]) if finder.allow_all_prereleases: diff --git a/src/pip/_internal/index.py b/src/pip/_internal/index.py index 1c8f1ecb1cc..7bc20e758ef 100644 --- a/src/pip/_internal/index.py +++ b/src/pip/_internal/index.py @@ -6,7 +6,6 @@ import logging import mimetypes import os -import posixpath import re from collections import namedtuple @@ -19,21 +18,21 @@ from pip._vendor.six.moves.urllib import parse as urllib_parse from pip._vendor.six.moves.urllib import request as urllib_request -from pip._internal.download import HAS_TLS, is_url, url_to_path +from pip._internal.download import is_url, url_to_path from pip._internal.exceptions import ( BestVersionAlreadyInstalled, DistributionNotFound, InvalidWheelFilename, UnsupportedWheel, ) from pip._internal.models.candidate import InstallationCandidate from pip._internal.models.format_control import FormatControl -from pip._internal.models.index import PyPI from pip._internal.models.link import Link +from pip._internal.models.search_scope import SearchScope from pip._internal.models.target_python import TargetPython from pip._internal.utils.compat import ipaddress from pip._internal.utils.logging import indent_log from pip._internal.utils.misc import ( - ARCHIVE_EXTENSIONS, SUPPORTED_EXTENSIONS, WHEEL_EXTENSION, normalize_path, - path_to_url, redact_password_from_url, + ARCHIVE_EXTENSIONS, SUPPORTED_EXTENSIONS, WHEEL_EXTENSION, path_to_url, + redact_password_from_url, ) from pip._internal.utils.packaging import check_requires_python from pip._internal.utils.typing import MYPY_CHECK_RUNNING @@ -560,8 +559,7 @@ class PackageFinder(object): def __init__( self, candidate_evaluator, # type: CandidateEvaluator - find_links, # type: List[str] - index_urls, # type: List[str] + search_scope, # type: SearchScope session, # type: PipSession format_control=None, # type: Optional[FormatControl] trusted_hosts=None, # type: Optional[List[str]] @@ -583,8 +581,7 @@ def __init__( format_control = format_control or FormatControl(set(), set()) self.candidate_evaluator = candidate_evaluator - self.find_links = find_links - self.index_urls = index_urls + self.search_scope = search_scope self.session = session self.format_control = format_control self.trusted_hosts = trusted_hosts @@ -626,47 +623,36 @@ def create( "'session'" ) - # Build find_links. If an argument starts with ~, it may be - # a local file relative to a home directory. So try normalizing - # it and if it exists, use the normalized version. - # This is deliberately conservative - it might be fine just to - # blindly normalize anything starting with a ~... - built_find_links = [] # type: List[str] - for link in find_links: - if link.startswith('~'): - new_link = normalize_path(link) - if os.path.exists(new_link): - link = new_link - built_find_links.append(link) + search_scope = SearchScope.create( + find_links=find_links, + index_urls=index_urls, + ) candidate_evaluator = CandidateEvaluator( - target_python=target_python, prefer_binary=prefer_binary, + target_python=target_python, + prefer_binary=prefer_binary, allow_all_prereleases=allow_all_prereleases, ignore_requires_python=ignore_requires_python, ) - # If we don't have TLS enabled, then WARN if anyplace we're looking - # relies on TLS. - if not HAS_TLS: - for link in itertools.chain(index_urls, built_find_links): - parsed = urllib_parse.urlparse(link) - if parsed.scheme == "https": - logger.warning( - "pip is configured with locations that require " - "TLS/SSL, however the ssl module in Python is not " - "available." - ) - break - return cls( candidate_evaluator=candidate_evaluator, - find_links=built_find_links, - index_urls=index_urls, + search_scope=search_scope, session=session, format_control=format_control, trusted_hosts=trusted_hosts, ) + @property + def find_links(self): + # type: () -> List[str] + return self.search_scope.find_links + + @property + def index_urls(self): + # type: () -> List[str] + return self.search_scope.index_urls + @property def allow_all_prereleases(self): # type: () -> bool @@ -701,21 +687,6 @@ def iter_secure_origins(self): for host in self.trusted_hosts: yield ('*', host, '*') - def get_formatted_locations(self): - # type: () -> str - lines = [] - if self.index_urls and self.index_urls != [PyPI.simple_url]: - lines.append( - "Looking in indexes: {}".format(", ".join( - redact_password_from_url(url) for url in self.index_urls)) - ) - if self.find_links: - lines.append( - "Looking in links: {}".format(", ".join( - redact_password_from_url(url) for url in self.find_links)) - ) - return "\n".join(lines) - @staticmethod def _sort_locations(locations, expand_dir=False): # type: (Sequence[str], bool) -> Tuple[List[str], List[str]] @@ -848,29 +819,6 @@ def _validate_secure_origin(self, logger, location): return False - def _get_index_urls_locations(self, project_name): - # type: (str) -> List[str] - """Returns the locations found via self.index_urls - - Checks the url_name on the main (first in the list) index and - use this url_name to produce all locations - """ - - def mkurl_pypi_url(url): - loc = posixpath.join( - url, - urllib_parse.quote(canonicalize_name(project_name))) - # For maximum compatibility with easy_install, ensure the path - # ends in a trailing slash. Although this isn't in the spec - # (and PyPI can handle it without the slash) some other index - # implementations might break if they relied on easy_install's - # behavior. - if not loc.endswith('/'): - loc = loc + '/' - return loc - - return [mkurl_pypi_url(url) for url in self.index_urls] - def find_all_candidates(self, project_name): # type: (str) -> List[InstallationCandidate] """Find all available InstallationCandidate for project_name @@ -881,7 +829,8 @@ def find_all_candidates(self, project_name): See CandidateEvaluator.evaluate_link() for details on which files are accepted. """ - index_locations = self._get_index_urls_locations(project_name) + search_scope = self.search_scope + index_locations = search_scope.get_index_urls_locations(project_name) index_file_loc, index_url_loc = self._sort_locations(index_locations) fl_file_loc, fl_url_loc = self._sort_locations( self.find_links, expand_dir=True, diff --git a/src/pip/_internal/legacy_resolve.py b/src/pip/_internal/legacy_resolve.py index c95e388a44c..1d9229cb621 100644 --- a/src/pip/_internal/legacy_resolve.py +++ b/src/pip/_internal/legacy_resolve.py @@ -179,7 +179,8 @@ def resolve(self, requirement_set): ) # Display where finder is looking for packages - locations = self.finder.get_formatted_locations() + search_scope = self.finder.search_scope + locations = search_scope.get_formatted_locations() if locations: logger.info(locations) diff --git a/src/pip/_internal/models/search_scope.py b/src/pip/_internal/models/search_scope.py new file mode 100644 index 00000000000..c570586d2d7 --- /dev/null +++ b/src/pip/_internal/models/search_scope.py @@ -0,0 +1,113 @@ +import itertools +import logging +import os +import posixpath + +from pip._vendor.packaging.utils import canonicalize_name +from pip._vendor.six.moves.urllib import parse as urllib_parse + +from pip._internal.download import HAS_TLS +from pip._internal.models.index import PyPI +from pip._internal.utils.misc import normalize_path, redact_password_from_url +from pip._internal.utils.typing import MYPY_CHECK_RUNNING + +if MYPY_CHECK_RUNNING: + from typing import List + + +logger = logging.getLogger(__name__) + + +class SearchScope(object): + + """ + Encapsulates the locations that pip is configured to search. + """ + + @classmethod + def create( + cls, + find_links, # type: List[str] + index_urls, # type: List[str] + ): + # type: (...) -> SearchScope + """ + Create a SearchScope object after normalizing the `find_links`. + """ + # Build find_links. If an argument starts with ~, it may be + # a local file relative to a home directory. So try normalizing + # it and if it exists, use the normalized version. + # This is deliberately conservative - it might be fine just to + # blindly normalize anything starting with a ~... + built_find_links = [] # type: List[str] + for link in find_links: + if link.startswith('~'): + new_link = normalize_path(link) + if os.path.exists(new_link): + link = new_link + built_find_links.append(link) + + # If we don't have TLS enabled, then WARN if anyplace we're looking + # relies on TLS. + if not HAS_TLS: + for link in itertools.chain(index_urls, built_find_links): + parsed = urllib_parse.urlparse(link) + if parsed.scheme == 'https': + logger.warning( + 'pip is configured with locations that require ' + 'TLS/SSL, however the ssl module in Python is not ' + 'available.' + ) + break + + return cls( + find_links=built_find_links, + index_urls=index_urls, + ) + + def __init__( + self, + find_links, # type: List[str] + index_urls, # type: List[str] + ): + # type: (...) -> None + self.find_links = find_links + self.index_urls = index_urls + + def get_formatted_locations(self): + # type: () -> str + lines = [] + if self.index_urls and self.index_urls != [PyPI.simple_url]: + lines.append( + 'Looking in indexes: {}'.format(', '.join( + redact_password_from_url(url) for url in self.index_urls)) + ) + if self.find_links: + lines.append( + 'Looking in links: {}'.format(', '.join( + redact_password_from_url(url) for url in self.find_links)) + ) + return '\n'.join(lines) + + def get_index_urls_locations(self, project_name): + # type: (str) -> List[str] + """Returns the locations found via self.index_urls + + Checks the url_name on the main (first in the list) index and + use this url_name to produce all locations + """ + + def mkurl_pypi_url(url): + loc = posixpath.join( + url, + urllib_parse.quote(canonicalize_name(project_name))) + # For maximum compatibility with easy_install, ensure the path + # ends in a trailing slash. Although this isn't in the spec + # (and PyPI can handle it without the slash) some other index + # implementations might break if they relied on easy_install's + # behavior. + if not loc.endswith('/'): + loc = loc + '/' + return loc + + return [mkurl_pypi_url(url) for url in self.index_urls] diff --git a/src/pip/_internal/req/req_file.py b/src/pip/_internal/req/req_file.py index bfad9d16180..5a9920fe963 100644 --- a/src/pip/_internal/req/req_file.py +++ b/src/pip/_internal/req/req_file.py @@ -16,6 +16,7 @@ from pip._internal.cli import cmdoptions from pip._internal.download import get_file_content from pip._internal.exceptions import RequirementsFileParseError +from pip._internal.models.search_scope import SearchScope from pip._internal.req.constructors import ( install_req_from_editable, install_req_from_line, ) @@ -238,12 +239,14 @@ def process_line( # set finder options elif finder: + find_links = finder.find_links + index_urls = finder.index_urls if opts.index_url: - finder.index_urls = [opts.index_url] + index_urls = [opts.index_url] if opts.no_index is True: - finder.index_urls = [] + index_urls = [] if opts.extra_index_urls: - finder.index_urls.extend(opts.extra_index_urls) + index_urls.extend(opts.extra_index_urls) if opts.find_links: # FIXME: it would be nice to keep track of the source # of the find_links: support a find-links local path @@ -253,7 +256,14 @@ def process_line( relative_to_reqs_file = os.path.join(req_dir, value) if os.path.exists(relative_to_reqs_file): value = relative_to_reqs_file - finder.find_links.append(value) + find_links.append(value) + + search_scope = SearchScope( + find_links=find_links, + index_urls=index_urls, + ) + finder.search_scope = search_scope + if opts.pre: finder.set_allow_all_prereleases() for host in opts.trusted_hosts or []: diff --git a/tests/unit/test_finder.py b/tests/unit/test_finder.py index a87e30d7082..e5c0f29d499 100644 --- a/tests/unit/test_finder.py +++ b/tests/unit/test_finder.py @@ -463,15 +463,6 @@ def test_evaluate_link__substring_fails(self, url, expected_msg): assert actual == (False, expected_msg) -def test_get_index_urls_locations(): - """Check that the canonical name is on all indexes""" - finder = make_test_finder(index_urls=['file://index1/', 'file://index2']) - locations = finder._get_index_urls_locations( - install_req_from_line('Complex_Name').name) - assert locations == ['file://index1/complex-name/', - 'file://index2/complex-name/'] - - def test_find_all_candidates_nothing(): """Find nothing without anything""" finder = make_test_finder() diff --git a/tests/unit/test_index.py b/tests/unit/test_index.py index 4ffc6475e33..4ebcaf1b735 100644 --- a/tests/unit/test_index.py +++ b/tests/unit/test_index.py @@ -385,27 +385,6 @@ def test_secure_origin(location, trusted, expected): assert logger.called == expected -def test_get_formatted_locations_basic_auth(): - """ - Test that basic authentication credentials defined in URL - is not included in formatted output. - """ - index_urls = [ - 'https://pypi.org/simple', - 'https://repo-user:repo-pass@repo.domain.com', - ] - find_links = [ - 'https://links-user:links-pass@page.domain.com' - ] - finder = make_test_finder(find_links=find_links, index_urls=index_urls) - - result = finder.get_formatted_locations() - assert 'repo-user:****@repo.domain.com' in result - assert 'repo-pass' not in result - assert 'links-user:****@page.domain.com' in result - assert 'links-pass' not in result - - @pytest.mark.parametrize( ("egg_info", "canonical_name", "expected"), [ diff --git a/tests/unit/test_search_scope.py b/tests/unit/test_search_scope.py new file mode 100644 index 00000000000..e7f4e3f16b5 --- /dev/null +++ b/tests/unit/test_search_scope.py @@ -0,0 +1,41 @@ +from pip._internal.models.search_scope import SearchScope +from pip._internal.req.constructors import install_req_from_line + + +class TestSearchScope: + + def test_get_formatted_locations_basic_auth(self): + """ + Test that basic authentication credentials defined in URL + is not included in formatted output. + """ + index_urls = [ + 'https://pypi.org/simple', + 'https://repo-user:repo-pass@repo.domain.com', + ] + find_links = [ + 'https://links-user:links-pass@page.domain.com' + ] + search_scope = SearchScope( + find_links=find_links, index_urls=index_urls, + ) + + result = search_scope.get_formatted_locations() + assert 'repo-user:****@repo.domain.com' in result + assert 'repo-pass' not in result + assert 'links-user:****@page.domain.com' in result + assert 'links-pass' not in result + + def test_get_index_urls_locations(self): + """Check that the canonical name is on all indexes""" + search_scope = SearchScope( + find_links=[], + index_urls=['file://index1/', 'file://index2'], + ) + actual = search_scope.get_index_urls_locations( + install_req_from_line('Complex_Name').name + ) + assert actual == [ + 'file://index1/complex-name/', + 'file://index2/complex-name/', + ]