Add --insecure option #62

Reference: #62
Signed-off-by: Tushar Goel <tushar.goel.dav@gmail.com>

Author: TG1999

CHANGELOG.rst

@@ -1,6 +1,10 @@
 Changelog
 =========
 
+v0.6.6
+------
+- Add --insecure option to compute arguments.
+
 v0.6.5
 ------
 

src/python_inspector/resolution.py

@@ -26,6 +26,7 @@
 from packaging.version import LegacyVersion
 from packaging.version import Version
 from packaging.version import parse as parse_version
+from requirements_builder.requirements_builder import iter_requirements
 from resolvelib import AbstractProvider
 from resolvelib import Resolver
 from resolvelib.reporters import BaseReporter
@@ -134,6 +135,17 @@ def is_requirements_file_in_setup_files(setup_files: List[str]) -> bool:
     return False
 
 
+def parse_setup_py_insecurely(setup_py):
+    """
+    Yield requirements from the setup.py file at ``setup_py``.
+    """
+    if not os.path.exists(setup_py):
+        return []
+    unparsed_requirements = iter_requirements(level="", extras=[], setup_file=setup_py)
+    for requirement in unparsed_requirements:
+        yield get_requirements_from_string(requirement)
+
+
 def is_valid_version(
     parsed_version: Union[LegacyVersion, Version],
     requirements: Dict,
@@ -222,7 +234,14 @@ def get_requirements_from_dependencies(
         # and other pip options for now
         # https://github.com/nexB/python-inspector/issues/41
         if can_process_dependent_package(dep):
-            yield Requirement(str(dep.extracted_requirement))
+            yield get_requirements_from_string(str(dep.extracted_requirement))
+
+
+def get_requirements_from_string(string: str) -> List[Requirement]:
+    """
+    Generate parsed requirements for the given ``string``.
+    """
+    return Requirement(string)
 
 
 def remove_extras(identifier: str) -> str:
@@ -235,13 +254,14 @@ def remove_extras(identifier: str) -> str:
 
 
 class PythonInputProvider(AbstractProvider):
-    def __init__(self, environment=None, repos=tuple()):
+    def __init__(self, environment=None, repos=tuple(), insecure=False):
         self.environment = environment
         self.environment_marker = get_environment_marker_from_environment(environment)
         self.repos = repos or []
         self.versions_by_package = {}
         self.dependencies_by_purl = {}
         self.wheel_or_sdist_by_package = {}
+        self.insecure = insecure
 
     def identify(self, requirement_or_candidate: Union[Candidate, Requirement]) -> str:
         """Given a requirement, return an identifier for it. Overridden."""
@@ -357,12 +377,10 @@ def get_requirements_for_package_from_pypi_simple(
             if deps:
                 has_wheels = True
                 yield from deps
-
         if not has_wheels:
             sdist_location = fetch_and_extract_sdist(
                 repos=self.repos, candidate=candidate, python_version=python_version
             )
-
             if sdist_location:
                 setup_py_location = os.path.join(
                     sdist_location,
@@ -393,6 +411,8 @@ def get_requirements_for_package_from_pypi_simple(
                     sdist_location,
                     "requirements.txt",
                 )
+
+                has_deps_yielded = False
                 if not deps_in_setup and is_requirements_file_in_setup_files(
                     setup_files=[setup_py_location, setup_cfg_location]
                 ):
@@ -401,8 +421,12 @@ def get_requirements_for_package_from_pypi_simple(
                         location=requirement_location,
                     )
                     if deps:
+                        has_deps_yielded = True
                         yield from deps
 
+                if not has_deps_yielded and self.insecure:
+                    yield from parse_setup_py_insecurely(setup_py=setup_py_location)
+
     def get_requirements_for_package_from_pypi_json_api(
         self, purl: PackageURL
     ) -> List[Requirement]:
@@ -686,6 +710,7 @@ def get_resolved_dependencies(
     max_rounds: int = 200000,
     verbose: bool = False,
     pdt_output: bool = False,
+    insecure: bool = False,
 ):
     """
     Return resolved dependencies of a ``requirements`` list of Requirement for
@@ -697,7 +722,7 @@ def get_resolved_dependencies(
     """
     try:
         resolver = Resolver(
-            provider=PythonInputProvider(environment=environment, repos=repos),
+            provider=PythonInputProvider(environment=environment, repos=repos, insecure=insecure),
             reporter=BaseReporter(),
         )
         resolver_results = resolver.resolve(requirements=requirements, max_rounds=max_rounds)

src/python_inspector/resolve_cli.py

@@ -156,6 +156,11 @@ def print_version(ctx, param, value):
     help="Use PyPI JSON API to fetch dependency data. Faster but not always correct. "
     "--index-url are ignored when this option is active.",
 )
+@click.option(
+    "--insecure",
+    is_flag=True,
+    help="Resolve insecurely",
+)
 @click.option(
     "--verbose",
     is_flag=True,
@@ -186,6 +191,7 @@ def resolve_dependencies(
     max_rounds,
     use_cached_index=False,
     use_pypi_json_api=False,
+    insecure=False,
     verbose=TRACE,
 ):
     """
@@ -329,6 +335,7 @@ def resolve_dependencies(
         max_rounds=max_rounds,
         verbose=verbose,
         pdt_output=pdt_output,
+        insecure=insecure,
     )
 
     cli_options = [f"--requirement {rf}" for rf in requirement_files]
@@ -383,6 +390,7 @@ def resolve(
     max_rounds=200000,
     verbose=False,
     pdt_output=False,
+    insecure=False,
 ):
     """
     Resolve dependencies given a ``direct_dependencies`` list of
@@ -408,6 +416,7 @@ def resolve(
         max_rounds=max_rounds,
         verbose=verbose,
         pdt_output=pdt_output,
+        insecure=insecure,
     )
 
     initial_requirements = [d.to_dict() for d in direct_dependencies]

tests/data/insecure-setup/rdflib/__init__.py

@@ -0,0 +1,215 @@
+import logging
+import sys
+
+import six
+from rdflib import plugin
+from rdflib import query
+from rdflib import util
+from rdflib.graph import ConjunctiveGraph
+from rdflib.graph import Dataset
+from rdflib.graph import Graph
+from rdflib.namespace import CSVW
+from rdflib.namespace import DC
+from rdflib.namespace import DCAT
+from rdflib.namespace import DCTERMS
+from rdflib.namespace import DOAP
+from rdflib.namespace import FOAF
+from rdflib.namespace import ODRL2
+from rdflib.namespace import ORG
+from rdflib.namespace import OWL
+from rdflib.namespace import PROF
+from rdflib.namespace import PROV
+from rdflib.namespace import RDF
+from rdflib.namespace import RDFS
+from rdflib.namespace import SDO
+from rdflib.namespace import SH
+from rdflib.namespace import SKOS
+from rdflib.namespace import SOSA
+from rdflib.namespace import SSN
+from rdflib.namespace import TIME
+from rdflib.namespace import VOID
+from rdflib.namespace import XMLNS
+from rdflib.namespace import XSD
+from rdflib.namespace import Namespace
+from rdflib.term import BNode
+from rdflib.term import Literal
+from rdflib.term import URIRef
+from rdflib.term import Variable
+
+# tedious sop to flake8
+assert plugin
+assert query
+
+
+"""A pure Python package providing the core RDF constructs.
+
+The packages is intended to provide the core RDF types and interfaces
+for working with RDF. The package defines a plugin interface for
+parsers, stores, and serializers that other packages can use to
+implement parsers, stores, and serializers that will plug into the
+rdflib package.
+
+The primary interface `rdflib` exposes to work with RDF is
+`rdflib.graph.Graph`.
+
+A tiny example:
+
+    >>> from rdflib import Graph, URIRef, Literal
+
+    >>> g = Graph()
+    >>> result = g.parse("http://www.w3.org/2000/10/swap/test/meet/blue.rdf")
+
+    >>> print("graph has %s statements." % len(g))
+    graph has 4 statements.
+    >>>
+    >>> for s, p, o in g:
+    ...     if (s, p, o) not in g:
+    ...         raise Exception("It better be!")
+
+    >>> s = g.serialize(format='nt')
+    >>>
+    >>> sorted(g) == [
+    ...  (URIRef(u'http://meetings.example.com/cal#m1'),
+    ...   URIRef(u'http://www.example.org/meeting_organization#homePage'),
+    ...   URIRef(u'http://meetings.example.com/m1/hp')),
+    ...  (URIRef(u'http://www.example.org/people#fred'),
+    ...   URIRef(u'http://www.example.org/meeting_organization#attending'),
+    ...   URIRef(u'http://meetings.example.com/cal#m1')),
+    ...  (URIRef(u'http://www.example.org/people#fred'),
+    ...   URIRef(u'http://www.example.org/personal_details#GivenName'),
+    ...   Literal(u'Fred')),
+    ...  (URIRef(u'http://www.example.org/people#fred'),
+    ...   URIRef(u'http://www.example.org/personal_details#hasEmail'),
+    ...   URIRef(u'mailto:fred@example.com'))
+    ... ]
+    True
+
+"""
+__docformat__ = "restructuredtext en"
+
+# The format of the __version__ line is matched by a regex in setup.py
+__version__ = "5.0.0"
+__date__ = "2020-04-18"
+
+__all__ = [
+    "URIRef",
+    "BNode",
+    "Literal",
+    "Variable",
+    "Namespace",
+    "Dataset",
+    "Graph",
+    "ConjunctiveGraph",
+    "CSVW",
+    "DC",
+    "DCAT",
+    "DCTERMS",
+    "DOAP",
+    "FOAF",
+    "ODRL2",
+    "ORG",
+    "OWL",
+    "PROF",
+    "PROV",
+    "RDF",
+    "RDFS",
+    "SDO",
+    "SH",
+    "SKOS",
+    "SOSA",
+    "SSN",
+    "TIME",
+    "VOID",
+    "XMLNS",
+    "XSD",
+    "util",
+]
+
+
+assert sys.version_info >= (2, 7, 0), "rdflib requires Python 2.7 or higher"
+
+logger = logging.getLogger(__name__)
+_interactive_mode = False
+try:
+    import __main__
+
+    if not hasattr(__main__, "__file__") and sys.stdout is not None and sys.stderr.isatty():
+        # show log messages in interactive mode
+        _interactive_mode = True
+        logger.setLevel(logging.INFO)
+        logger.addHandler(logging.StreamHandler())
+    del __main__
+except ImportError:
+    # Main already imported from elsewhere
+    import warnings
+
+    warnings.warn("__main__ already imported", ImportWarning)
+    del warnings
+
+if _interactive_mode:
+    logger.info("RDFLib Version: %s" % __version__)
+else:
+    logger.debug("RDFLib Version: %s" % __version__)
+del _interactive_mode
+del sys
+
+try:
+    six.unichr(0x10FFFF)
+except ValueError:
+    import warnings
+
+    warnings.warn(
+        "You are using a narrow Python build!\n"
+        "This means that your Python does not properly support chars > 16bit.\n"
+        'On your system chars like c=u"\\U0010FFFF" will have a len(c)==2.\n'
+        "As this can cause hard to debug problems with string processing\n"
+        "(slicing, regexp, ...) later on, we strongly advise to use a wide\n"
+        "Python build in production systems.",
+        ImportWarning,
+    )
+    del warnings
+del six
+
+
+NORMALIZE_LITERALS = True
+"""
+If True - Literals lexical forms are normalized when created.
+I.e. the lexical forms is parsed according to data-type, then the
+stored lexical form is the re-serialized value that was parsed.
+
+Illegal values for a datatype are simply kept.  The normalized keyword
+for Literal.__new__ can override this.
+
+For example:
+
+>>> from rdflib import Literal,XSD
+>>> Literal("01", datatype=XSD.int)
+rdflib.term.Literal(u'1', datatype=rdflib.term.URIRef(u'http://www.w3.org/2001/XMLSchema#integer'))
+
+This flag may be changed at any time, but will only affect literals
+created after that time, previously created literals will remain
+(un)normalized.
+
+"""
+
+
+DAWG_LITERAL_COLLATION = False
+"""
+DAWG_LITERAL_COLLATION determines how literals are ordered or compared
+to each other.
+
+In SPARQL, applying the >,<,>=,<= operators to literals of
+incompatible data-types is an error, i.e:
+
+Literal(2)>Literal('cake') is neither true nor false, but an error.
+
+This is a problem in PY3, where lists of Literals of incompatible
+types can no longer be sorted.
+
+Setting this flag to True gives you strict DAWG/SPARQL compliance,
+setting it to False will order Literals with incompatible datatypes by
+datatype URI
+
+In particular, this determines how the rich comparison operators for
+Literal work, eq, __neq__, __lt__, etc.
+"""