Fix keyerror in keep_outdated when using VCS dependencies (#3768)

Fix keyerror in keep_outdated when using VCS dependencies

news/3768.bugfix.rst

@@ -0,0 +1 @@
+Fixed a ``KeyError`` which could occur when pinning outdated VCS dependencies via ``pipenv lock --keep-outdated``.

pipenv/resolver.py

@@ -237,7 +237,8 @@ def get_cleaned_dict(self, keep_outdated=False):
         if entry_hashes != locked_hashes and not self.is_updated:
             self.entry_dict["hashes"] = list(entry_hashes | locked_hashes)
         self.entry_dict["name"] = self.name
-        self.entry_dict["version"] = self.strip_version(self.entry_dict["version"])
+        if "version" in self.entry_dict:
+            self.entry_dict["version"] = self.strip_version(self.entry_dict["version"])
         _, self.entry_dict = self.get_markers_from_dict(self.entry_dict)
         return self.entry_dict
 
@@ -779,14 +780,6 @@ def main():
     warnings.simplefilter("ignore", category=ResourceWarning)
     replace_with_text_stream("stdout")
     replace_with_text_stream("stderr")
-    # from pipenv.vendor import colorama
-    # if os.name == "nt" and (
-    #     all(getattr(stream, method, None) for stream in [sys.stdout, sys.stderr] for method in ["write", "isatty"]) and
-    #     all(stream.isatty() for stream in [sys.stdout, sys.stderr])
-    # ):
-    #     colorama.init(wrap=False)
-    # elif os.name != "nt":
-    #     colorama.init()
     os.environ["PIP_DISABLE_PIP_VERSION_CHECK"] = str("1")
     os.environ["PYTHONIOENCODING"] = str("utf-8")
     os.environ["PYTHONUNBUFFERED"] = str("1")

pipenv/utils.py

@@ -985,8 +985,8 @@ def resolve(cmd, sp):
             _out = decode_output("{0}\n".format(_out))
             out += _out
             sp.text = to_native_string("{0}".format(_out[:100]))
-            # if environments.is_verbose():
-            #     sp.hide_and_write(_out.rstrip())
+            if environments.is_verbose():
+                sp.hide_and_write(_out.rstrip())
         _out = to_native_string("")
         if not result and not _out:
             break
@@ -2019,7 +2019,7 @@ def find_python(finder, line=None):
         )
     if line and os.path.isabs(line):
         if os.name == "nt":
-            line = posixpath.join(*line.split(os.path.sep))
+            line = make_posix(line)
         return line
     if not finder:
         from pipenv.vendor.pythonfinder import Finder

pipenv/vendor/requirementslib/models/requirements.py

@@ -814,22 +814,22 @@ def vcsrepo(self):
     @cached_property
     def metadata(self):
         # type: () -> Dict[Any, Any]
-        if self.is_local and is_installable_dir(self.path):
+        if self.is_local and self.path and is_installable_dir(self.path):
             return get_metadata(self.path)
         return {}
 
     @cached_property
     def parsed_setup_cfg(self):
         # type: () -> Dict[Any, Any]
-        if self.is_local and is_installable_dir(self.path):
+        if self.is_local and self.path and is_installable_dir(self.path):
             if self.setup_cfg:
                 return parse_setup_cfg(self.setup_cfg)
         return {}
 
     @cached_property
     def parsed_setup_py(self):
         # type: () -> Dict[Any, Any]
-        if self.is_local and is_installable_dir(self.path):
+        if self.is_local and self.path and is_installable_dir(self.path):
             if self.setup_py:
                 return ast_parse_setup_py(self.setup_py)
         return {}

pipenv/vendor/requirementslib/models/setup_info.py

@@ -160,6 +160,19 @@ def parse_special_directives(setup_entry, package_dir=None):
             sys.path.insert(0, package_dir)
             if "." in resource:
                 resource, _, attribute = resource.rpartition(".")
+            package, _, path = resource.partition(".")
+            base_path = os.path.join(package_dir, package)
+            if path:
+                path = os.path.join(base_path, os.path.join(*path.split(".")))
+            else:
+                path = base_path
+            if not os.path.exists(path) and os.path.exists("{0}.py".format(path)):
+                path = "{0}.py".format(path)
+            elif os.path.isdir(path):
+                path = os.path.join(path, "__init__.py")
+            rv = ast_parse_attribute_from_file(path, attribute)
+            if rv:
+                return str(rv)
             module = importlib.import_module(resource)
             rv = getattr(module, attribute)
             if not isinstance(rv, six.string_types):
@@ -203,10 +216,10 @@ def setuptools_parse_setup_cfg(path):
 
 def get_package_dir_from_setupcfg(parser, base_dir=None):
     # type: (configparser.ConfigParser, STRING_TYPE) -> Text
-    if not base_dir:
-        package_dir = os.getcwd()
-    else:
+    if base_dir is not None:
         package_dir = base_dir
+    else:
+        package_dir = os.getcwd()
     if parser.has_option("options", "packages.find"):
         pkg_dir = parser.get("options", "packages.find")
         if isinstance(package_dir, Mapping):
@@ -217,6 +230,15 @@ def get_package_dir_from_setupcfg(parser, base_dir=None):
             _, pkg_dir = pkg_dir.split("find:")
             pkg_dir = pkg_dir.strip()
         package_dir = os.path.join(package_dir, pkg_dir)
+    elif os.path.exists(os.path.join(package_dir, "setup.py")):
+        setup_py = ast_parse_setup_py(os.path.join(package_dir, "setup.py"))
+        if "package_dir" in setup_py:
+            package_lookup = setup_py["package_dir"]
+            if not isinstance(package_lookup, Mapping):
+                return package_lookup
+            return package_lookup.get(
+                next(iter(list(package_lookup.keys()))), package_dir
+            )
     return package_dir
 
 
@@ -638,7 +660,7 @@ def match_assignment_name(self, match):
 
 def ast_unparse(item, initial_mapping=False, analyzer=None, recurse=True):  # noqa:C901
     # type: (Any, bool, Optional[Analyzer], bool) -> Union[List[Any], Dict[Any, Any], Tuple[Any, ...], STRING_TYPE]
-    unparse = partial(ast_unparse, initial_mapping=initial_mapping, analyzer=analyzer)
+    unparse = partial(ast_unparse, initial_mapping=initial_mapping, analyzer=analyzer, recurse=recurse)
     if isinstance(item, ast.Dict):
         unparsed = dict(zip(unparse(item.keys), unparse(item.values)))
     elif isinstance(item, ast.List):
@@ -665,13 +687,35 @@ def ast_unparse(item, initial_mapping=False, analyzer=None, recurse=True):  # no
             unparsed = item
     elif six.PY3 and isinstance(item, ast.NameConstant):
         unparsed = item.value
+    elif isinstance(item, ast.Attribute):
+        attr_name = getattr(item, "value", None)
+        attr_attr = getattr(item, "attr", None)
+        name = None
+        if initial_mapping:
+            unparsed = item
+        elif attr_name and not recurse:
+            name = attr_name
+        else:
+            name = unparse(attr_name) if attr_name is not None else attr_attr
+        if name and attr_attr:
+            if not initial_mapping and isinstance(name, six.string_types):
+                unparsed = ".".join([item for item in (name, attr_attr) if item])
+            else:
+                unparsed = item
+        elif attr_attr and not name and not initial_mapping:
+            unparsed = attr_attr
+        else:
+            unparsed = name if not unparsed else unparsed
     elif isinstance(item, ast.Call):
         unparsed = {}
         if isinstance(item.func, ast.Name):
-            name = unparse(item.func)
-            unparsed[name] = {}
+            func_name = unparse(item.func)
+        elif isinstance(item.func, ast.Attribute):
+            func_name = unparse(item.func)
+        if func_name:
+            unparsed[func_name] = {}
             for keyword in item.keywords:
-                unparsed[name].update(unparse(keyword))
+                unparsed[func_name].update(unparse(keyword))
     elif isinstance(item, ast.keyword):
         unparsed = {unparse(item.arg): unparse(item.value)}
     elif isinstance(item, ast.Assign):
@@ -681,7 +725,7 @@ def ast_unparse(item, initial_mapping=False, analyzer=None, recurse=True):  # no
         # XXX: Original reference
         if not initial_mapping:
             target = unparse(next(iter(item.targets)), recurse=False)
-            val = unparse(item.value)
+            val = unparse(item.value, recurse=False)
             if isinstance(target, (tuple, set, list)):
                 unparsed = dict(zip(target, val))
             else:
@@ -704,15 +748,48 @@ def ast_unparse(item, initial_mapping=False, analyzer=None, recurse=True):  # no
     return unparsed
 
 
-def ast_parse_setup_py(path):
-    # type: (S) -> Dict[Any, Any]
+def ast_parse_attribute_from_file(path, attribute):
+    # type: (S) -> Any
+    analyzer = ast_parse_file(path)
+    target_value = None
+    for k, v in analyzer.assignments.items():
+        name = ""
+        if isinstance(k, ast.Name):
+            name = k.id
+        elif isinstance(k, ast.Attribute):
+            fn = ast_unparse(k)
+            if isinstance(fn, six.string_types):
+                _, _, name = fn.rpartition(".")
+        if name == attribute:
+            target_value = ast_unparse(v, analyzer=analyzer)
+            break
+    if isinstance(target_value, Mapping) and attribute in target_value:
+        return target_value[attribute]
+    return target_value
+
+
+def ast_parse_file(path):
+    # type: (S) -> Analyzer
     with open(path, "r") as fh:
         tree = ast.parse(fh.read())
     ast_analyzer = Analyzer()
     ast_analyzer.visit(tree)
+    return ast_analyzer
+
+
+def ast_parse_setup_py(path):
+    # type: (S) -> Dict[Any, Any]
+    ast_analyzer = ast_parse_file(path)
     setup = {}  # type: Dict[Any, Any]
     for k, v in ast_analyzer.function_map.items():
-        if isinstance(k, ast.Name) and k.id == "setup":
+        fn_name = ""
+        if isinstance(k, ast.Name):
+            fn_name = k.id
+        elif isinstance(k, ast.Attribute):
+            fn = ast_unparse(k)
+            if isinstance(fn, six.string_types):
+                _, _, fn_name = fn.rpartition(".")
+        if fn_name == "setup":
             setup = v
     cleaned_setup = ast_unparse(setup, analyzer=ast_analyzer)
     return cleaned_setup

setup.py

@@ -24,7 +24,7 @@
 required = [
     "pip>=18.0",
     "certifi",
-    "setuptools>=41.0.0",
+    "setuptools>=36.2.1",
     "virtualenv-clone>=0.2.5",
     "virtualenv",
     'enum34; python_version<"3"',