diff --git a/news/3768.bugfix.rst b/news/3768.bugfix.rst new file mode 100644 index 0000000000..8efe019787 --- /dev/null +++ b/news/3768.bugfix.rst @@ -0,0 +1 @@ +Fixed a ``KeyError`` which could occur when pinning outdated VCS dependencies via ``pipenv lock --keep-outdated``. diff --git a/pipenv/resolver.py b/pipenv/resolver.py index ca42b44cbc..c4f02607c5 100644 --- a/pipenv/resolver.py +++ b/pipenv/resolver.py @@ -237,7 +237,8 @@ def get_cleaned_dict(self, keep_outdated=False): if entry_hashes != locked_hashes and not self.is_updated: self.entry_dict["hashes"] = list(entry_hashes | locked_hashes) self.entry_dict["name"] = self.name - self.entry_dict["version"] = self.strip_version(self.entry_dict["version"]) + if "version" in self.entry_dict: + self.entry_dict["version"] = self.strip_version(self.entry_dict["version"]) _, self.entry_dict = self.get_markers_from_dict(self.entry_dict) return self.entry_dict @@ -779,14 +780,6 @@ def main(): warnings.simplefilter("ignore", category=ResourceWarning) replace_with_text_stream("stdout") replace_with_text_stream("stderr") - # from pipenv.vendor import colorama - # if os.name == "nt" and ( - # all(getattr(stream, method, None) for stream in [sys.stdout, sys.stderr] for method in ["write", "isatty"]) and - # all(stream.isatty() for stream in [sys.stdout, sys.stderr]) - # ): - # colorama.init(wrap=False) - # elif os.name != "nt": - # colorama.init() os.environ["PIP_DISABLE_PIP_VERSION_CHECK"] = str("1") os.environ["PYTHONIOENCODING"] = str("utf-8") os.environ["PYTHONUNBUFFERED"] = str("1") diff --git a/pipenv/utils.py b/pipenv/utils.py index 61d771ce74..b73b7fa737 100644 --- a/pipenv/utils.py +++ b/pipenv/utils.py @@ -985,8 +985,8 @@ def resolve(cmd, sp): _out = decode_output("{0}\n".format(_out)) out += _out sp.text = to_native_string("{0}".format(_out[:100])) - # if environments.is_verbose(): - # sp.hide_and_write(_out.rstrip()) + if environments.is_verbose(): + sp.hide_and_write(_out.rstrip()) _out = to_native_string("") if not result and not _out: break @@ -2019,7 +2019,7 @@ def find_python(finder, line=None): ) if line and os.path.isabs(line): if os.name == "nt": - line = posixpath.join(*line.split(os.path.sep)) + line = make_posix(line) return line if not finder: from pipenv.vendor.pythonfinder import Finder diff --git a/pipenv/vendor/requirementslib/models/requirements.py b/pipenv/vendor/requirementslib/models/requirements.py index b8534c697e..559ab424d1 100644 --- a/pipenv/vendor/requirementslib/models/requirements.py +++ b/pipenv/vendor/requirementslib/models/requirements.py @@ -814,14 +814,14 @@ def vcsrepo(self): @cached_property def metadata(self): # type: () -> Dict[Any, Any] - if self.is_local and is_installable_dir(self.path): + if self.is_local and self.path and is_installable_dir(self.path): return get_metadata(self.path) return {} @cached_property def parsed_setup_cfg(self): # type: () -> Dict[Any, Any] - if self.is_local and is_installable_dir(self.path): + if self.is_local and self.path and is_installable_dir(self.path): if self.setup_cfg: return parse_setup_cfg(self.setup_cfg) return {} @@ -829,7 +829,7 @@ def parsed_setup_cfg(self): @cached_property def parsed_setup_py(self): # type: () -> Dict[Any, Any] - if self.is_local and is_installable_dir(self.path): + if self.is_local and self.path and is_installable_dir(self.path): if self.setup_py: return ast_parse_setup_py(self.setup_py) return {} diff --git a/pipenv/vendor/requirementslib/models/setup_info.py b/pipenv/vendor/requirementslib/models/setup_info.py index b3251f71b2..0dd5b3c83e 100644 --- a/pipenv/vendor/requirementslib/models/setup_info.py +++ b/pipenv/vendor/requirementslib/models/setup_info.py @@ -160,6 +160,19 @@ def parse_special_directives(setup_entry, package_dir=None): sys.path.insert(0, package_dir) if "." in resource: resource, _, attribute = resource.rpartition(".") + package, _, path = resource.partition(".") + base_path = os.path.join(package_dir, package) + if path: + path = os.path.join(base_path, os.path.join(*path.split("."))) + else: + path = base_path + if not os.path.exists(path) and os.path.exists("{0}.py".format(path)): + path = "{0}.py".format(path) + elif os.path.isdir(path): + path = os.path.join(path, "__init__.py") + rv = ast_parse_attribute_from_file(path, attribute) + if rv: + return str(rv) module = importlib.import_module(resource) rv = getattr(module, attribute) if not isinstance(rv, six.string_types): @@ -203,10 +216,10 @@ def setuptools_parse_setup_cfg(path): def get_package_dir_from_setupcfg(parser, base_dir=None): # type: (configparser.ConfigParser, STRING_TYPE) -> Text - if not base_dir: - package_dir = os.getcwd() - else: + if base_dir is not None: package_dir = base_dir + else: + package_dir = os.getcwd() if parser.has_option("options", "packages.find"): pkg_dir = parser.get("options", "packages.find") if isinstance(package_dir, Mapping): @@ -217,6 +230,15 @@ def get_package_dir_from_setupcfg(parser, base_dir=None): _, pkg_dir = pkg_dir.split("find:") pkg_dir = pkg_dir.strip() package_dir = os.path.join(package_dir, pkg_dir) + elif os.path.exists(os.path.join(package_dir, "setup.py")): + setup_py = ast_parse_setup_py(os.path.join(package_dir, "setup.py")) + if "package_dir" in setup_py: + package_lookup = setup_py["package_dir"] + if not isinstance(package_lookup, Mapping): + return package_lookup + return package_lookup.get( + next(iter(list(package_lookup.keys()))), package_dir + ) return package_dir @@ -638,7 +660,7 @@ def match_assignment_name(self, match): def ast_unparse(item, initial_mapping=False, analyzer=None, recurse=True): # noqa:C901 # type: (Any, bool, Optional[Analyzer], bool) -> Union[List[Any], Dict[Any, Any], Tuple[Any, ...], STRING_TYPE] - unparse = partial(ast_unparse, initial_mapping=initial_mapping, analyzer=analyzer) + unparse = partial(ast_unparse, initial_mapping=initial_mapping, analyzer=analyzer, recurse=recurse) if isinstance(item, ast.Dict): unparsed = dict(zip(unparse(item.keys), unparse(item.values))) elif isinstance(item, ast.List): @@ -665,13 +687,35 @@ def ast_unparse(item, initial_mapping=False, analyzer=None, recurse=True): # no unparsed = item elif six.PY3 and isinstance(item, ast.NameConstant): unparsed = item.value + elif isinstance(item, ast.Attribute): + attr_name = getattr(item, "value", None) + attr_attr = getattr(item, "attr", None) + name = None + if initial_mapping: + unparsed = item + elif attr_name and not recurse: + name = attr_name + else: + name = unparse(attr_name) if attr_name is not None else attr_attr + if name and attr_attr: + if not initial_mapping and isinstance(name, six.string_types): + unparsed = ".".join([item for item in (name, attr_attr) if item]) + else: + unparsed = item + elif attr_attr and not name and not initial_mapping: + unparsed = attr_attr + else: + unparsed = name if not unparsed else unparsed elif isinstance(item, ast.Call): unparsed = {} if isinstance(item.func, ast.Name): - name = unparse(item.func) - unparsed[name] = {} + func_name = unparse(item.func) + elif isinstance(item.func, ast.Attribute): + func_name = unparse(item.func) + if func_name: + unparsed[func_name] = {} for keyword in item.keywords: - unparsed[name].update(unparse(keyword)) + unparsed[func_name].update(unparse(keyword)) elif isinstance(item, ast.keyword): unparsed = {unparse(item.arg): unparse(item.value)} elif isinstance(item, ast.Assign): @@ -681,7 +725,7 @@ def ast_unparse(item, initial_mapping=False, analyzer=None, recurse=True): # no # XXX: Original reference if not initial_mapping: target = unparse(next(iter(item.targets)), recurse=False) - val = unparse(item.value) + val = unparse(item.value, recurse=False) if isinstance(target, (tuple, set, list)): unparsed = dict(zip(target, val)) else: @@ -704,15 +748,48 @@ def ast_unparse(item, initial_mapping=False, analyzer=None, recurse=True): # no return unparsed -def ast_parse_setup_py(path): - # type: (S) -> Dict[Any, Any] +def ast_parse_attribute_from_file(path, attribute): + # type: (S) -> Any + analyzer = ast_parse_file(path) + target_value = None + for k, v in analyzer.assignments.items(): + name = "" + if isinstance(k, ast.Name): + name = k.id + elif isinstance(k, ast.Attribute): + fn = ast_unparse(k) + if isinstance(fn, six.string_types): + _, _, name = fn.rpartition(".") + if name == attribute: + target_value = ast_unparse(v, analyzer=analyzer) + break + if isinstance(target_value, Mapping) and attribute in target_value: + return target_value[attribute] + return target_value + + +def ast_parse_file(path): + # type: (S) -> Analyzer with open(path, "r") as fh: tree = ast.parse(fh.read()) ast_analyzer = Analyzer() ast_analyzer.visit(tree) + return ast_analyzer + + +def ast_parse_setup_py(path): + # type: (S) -> Dict[Any, Any] + ast_analyzer = ast_parse_file(path) setup = {} # type: Dict[Any, Any] for k, v in ast_analyzer.function_map.items(): - if isinstance(k, ast.Name) and k.id == "setup": + fn_name = "" + if isinstance(k, ast.Name): + fn_name = k.id + elif isinstance(k, ast.Attribute): + fn = ast_unparse(k) + if isinstance(fn, six.string_types): + _, _, fn_name = fn.rpartition(".") + if fn_name == "setup": setup = v cleaned_setup = ast_unparse(setup, analyzer=ast_analyzer) return cleaned_setup diff --git a/setup.py b/setup.py index d0beff70f9..c251e49adb 100644 --- a/setup.py +++ b/setup.py @@ -24,7 +24,7 @@ required = [ "pip>=18.0", "certifi", - "setuptools>=41.0.0", + "setuptools>=36.2.1", "virtualenv-clone>=0.2.5", "virtualenv", 'enum34; python_version<"3"',