init

.gitignore

@@ -0,0 +1,4 @@
+.*/
+**/__pycache__/
+interpreter
+debug/expr.nix

LICENSE

@@ -0,0 +1,9 @@
+MIT License
+
+Copyright (c) 2020 David Hauer
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

VERSION

@@ -0,0 +1 @@
+dev

debug/debug.py

@@ -0,0 +1,39 @@
+import os
+import subprocess as sp
+import tempfile
+from os.path import realpath, dirname
+
+from mach_nix.generate import main
+
+pwd = dirname(realpath(__file__))
+
+os.environ['py_ver_str'] = '3.7.5'
+os.environ['out_file'] = f'{pwd}/expr.nix'
+os.environ['disable_checks'] = 'true'
+os.environ['prefer_nixpkgs'] = 'true'
+
+nixpkgs_json = tempfile.mktemp()
+cmd = f'nix-build {pwd}/nixpkgs-json.nix -o {nixpkgs_json}'
+print(cmd)
+sp.check_call(cmd, shell=True)
+os.environ['nixpkgs_json'] = nixpkgs_json
+
+pypi_deps_db = tempfile.mktemp()
+
+cmd = f'nix-build {pwd}/pypi-deps-db.nix -o {pypi_deps_db}'
+sp.check_call(cmd, shell=True)
+os.environ['pypi_deps_db_data_dir'] = pypi_deps_db + "/data"
+
+for key in ('NIXPKGS_COMMIT', 'NIXPKGS_TARBALL_SHA256'):
+    with open(f"{pwd}/../mach_nix/nix/{key}") as f:
+        os.environ[key.lower()] = f.read()
+
+for key in ('PYPI_FETCHER_COMMIT', 'PYPI_FETCHER_TARBALL_SHA256'):
+    with open(f"{pypi_deps_db}/{key}") as f:
+        os.environ[key.lower()] = f.read()
+
+os.environ['requirements'] = 'requests'
+
+
+# generates and writes nix expression into ./debug/expr.nix
+main()

debug/nixpkgs-json.nix

@@ -0,0 +1,6 @@
+let
+  nixpkgs_src = (import ../mach_nix/nix/nixpkgs-src.nix).stable;
+  python = (import nixpkgs_src { config = {}; }).python37;
+in
+with import nixpkgs_src { config = {}; };
+import ../mach_nix/nix/nixpkgs-json.nix { inherit pkgs python; }

debug/pypi-deps-db.nix

@@ -0,0 +1,14 @@
+let
+  pkgs =  import (import ../mach_nix/nix/nixpkgs-src.nix).stable {};
+  commit = builtins.readFile ../mach_nix/nix/PYPI_DEPS_DB_COMMIT;
+  sha256 = builtins.readFile ../mach_nix/nix/PYPI_DEPS_DB_TARBALL_SHA256;
+  src = builtins.fetchTarball {
+    name = "pypi-deps-db-src";
+    url = "https://github.com/DavHau/pypi-deps-db/tarball/${commit}";
+    inherit sha256;
+  };
+in
+pkgs.buildEnv {
+  name = "pypi-deps-db-src";
+  paths = [ src ];
+}

default.nix

@@ -0,0 +1,13 @@
+let
+  pkgs = import (import ./mach_nix/nix/nixpkgs-src.nix).stable { config = {}; };
+  python = import ./mach_nix/nix/python.nix { inherit pkgs; };
+  python_deps = (pkgs.lib.attrValues (import ./mach_nix/nix/python-deps.nix { inherit python; fetchurl = pkgs.fetchurl; }));
+in
+python.pkgs.buildPythonPackage rec {
+  pname = "mach-nix";
+  version = builtins.readFile ./VERSION;
+  name = "${pname}-${version}";
+  src = ./.;
+  propagatedBuildInputs = python_deps;
+  doCheck = false;
+}

mach_nix/__init__.py

@@ -0,0 +1 @@
+from .run import main

mach_nix/data/bucket_dict.py

@@ -0,0 +1,85 @@
+import json
+import os
+from _sha256 import sha256
+from collections import UserDict, OrderedDict
+
+
+class LazyBucketDict(UserDict):
+
+    def __init__(self, directory, data=None):
+        super().__init__()
+        self.directory = directory
+        self.data = {}
+        if data:
+            for key, val in data.items():
+                self.__setitem__(key, val)
+
+    def __getitem__(self, key):
+        bucket = self.bucket(key)
+        self.ensure_bucket_loaded(bucket)
+        return self.data[bucket][key]
+
+    def __setitem__(self, key, val):
+        bucket = self.bucket(key)
+        self.ensure_bucket_loaded(bucket)
+        self.data[bucket][key] = val
+
+    def __contains__(self, key):
+        bucket = self.bucket(key)
+        self.ensure_bucket_loaded(bucket)
+        return key in self.data[bucket]
+
+    def __delitem__(self, key):
+        bucket = self.bucket(key)
+        self.ensure_bucket_loaded(bucket)
+        del self.data[bucket][key]
+
+    @staticmethod
+    def bucket_keys():
+        hexdigits = "0123456789abcdef"
+        for a in hexdigits:
+            for b in hexdigits:
+                yield a + b
+
+    def by_bucket(self, bucket):
+        self.ensure_bucket_loaded(bucket)
+        return self.data[bucket]
+
+    def keys(self, bucket=None):
+        if bucket is None:
+            for bucket in self.bucket_keys():
+                self.ensure_bucket_loaded(bucket)
+                for k in self.data[bucket].keys():
+                    yield k
+        else:
+            self.ensure_bucket_loaded(bucket)
+            for k in self.data[bucket].keys():
+                yield k
+
+    @staticmethod
+    def bucket(key):
+        return sha256(key.encode()).hexdigest()[:2]
+
+    def save_bucket(self, bucket, directory_path):
+        self.ensure_bucket_loaded(bucket)
+        save = OrderedDict(sorted(self.data[bucket].items(), key=lambda item: item[0]))
+        with open(f"{directory_path}/{bucket}.json", 'w') as f:
+            json.dump(save, f, indent=2)
+
+    def save(self):
+        if not os.path.isdir(self.directory):
+            os.mkdir(self.directory)
+        for bucket in self.data.keys():
+            self.save_bucket(bucket, self.directory)
+
+    def load_bucket(self, bucket):
+        file = f"{self.directory}/{bucket}.json"
+        if not os.path.isfile(file):
+            self.data[bucket] = {}
+        else:
+            with open(file) as f:
+                self.data[bucket] = json.load(f)
+
+    def ensure_bucket_loaded(self, bucket):
+        if bucket not in self.data:
+            self.load_bucket(bucket)

mach_nix/data/data_interface.py

@@ -0,0 +1,184 @@
+import json
+import sys
+from collections import UserDict
+from dataclasses import dataclass
+from typing import List, Tuple
+
+import distlib.markers
+from packaging.version import Version, parse
+
+from mach_nix.requirements import strip_reqs_by_marker, Requirement, parse_reqs, context
+from mach_nix.versions import PyVer
+from .bucket_dict import LazyBucketDict
+
+
+class DependencyDB(UserDict):
+    def __init__(self, py_ver: PyVer, data_dir, *args, **kwargs):
+        super(DependencyDB, self).__init__(*args, **kwargs)
+        self.data = LazyBucketDict(data_dir)
+        self.context = context(py_ver)
+        self.py_ver_digits = py_ver.digits()
+        for name, releases in self.data.items():
+            key = self._unify_key(name)
+            if key != name:
+                self.data[key] = self.data[name]
+                del self.data[name]
+
+    def __getitem__(self, item) -> dict:
+        result = {}
+        for ver, pyvers in self.data[self._unify_key(item)].items():
+            if self.py_ver_digits in pyvers:
+                if isinstance(pyvers[self.py_ver_digits], str):
+                    result[ver] = pyvers[pyvers[self.py_ver_digits]]
+                else:
+                    result[ver] = pyvers[self.py_ver_digits]
+        return result
+
+    def exists(self, name, ver=None):
+        try:
+            key = self._unify_key(name)
+            pkg = self[key]
+        except KeyError:
+            return False
+        if ver:
+            return pkg['ver'] == ver
+        return True
+
+    def _unify_key(self, key: str) -> str:
+        return key.replace('_', '-').lower()
+
+    def get_pkg_reqs(self, pkg_name, pkg_version, extras=None) -> Tuple[List[Requirement], List[Requirement]]:
+        """
+        Get requirements for package
+        """
+        ver_str = str(pkg_version)
+        if not self.exists(pkg_name) or ver_str not in self[pkg_name]:
+            raise Exception(f'Cannot find {pkg_name}:{pkg_version} in db')
+        pkg = self[pkg_name][ver_str]
+        requirements = dict(
+            setup_requires=[],
+            install_requires=[]
+        )
+        for t in ("setup_requires", "install_requires"):
+            if t not in pkg:
+                requirements[t] = []
+            else:
+                reqs_raw = pkg[t]
+                reqs = list(parse_reqs(reqs_raw))
+                requirements[t] = list(strip_reqs_by_marker(reqs, self.context))
+        extras = set(extras) if extras else []
+        if 'extras_require' in pkg:
+            for name, reqs_str in pkg['extras_require'].items():
+                # handle extras with marker in key
+                if ':' in name:
+                    name, marker = name.split(':')
+                    if not distlib.markers.interpret(marker, self.context):
+                        continue
+                # handle if extra's key only contains marker. like ':python_version < "3.7"'
+                if name == '' or name in extras:
+                    requirements['install_requires'] += list(strip_reqs_by_marker(list(parse_reqs(reqs_str)), self.context))
+
+        return requirements['install_requires'], requirements['setup_requires']
+
+    def available_versions(self, pkg_name: str) -> List[Version]:
+        name = pkg_name.replace("_", "-").lower()
+        if self.exists(name):
+            return [parse(ver) for ver in self[name].keys()]
+        error_text = \
+            f"\nThe Package '{pkg_name}' cannot be found in the dependency DB used by mach-nix. \n" \
+            f"Please check the following:\n" \
+            f"  1. Does the package actually exist on pypi? Please check https://pypi.org/project/{pkg_name}/\n" \
+            f"  2. Does the package's initial release date predate the mach-nix release date? \n" \
+            f"     If so, either upgrade mach-nix itself or manually specify 'pypi_deps_db_commit' and\n" \
+            f"     'pypi_deps_db_sha256 for a newer commit of https://github.com/DavHau/pypi-deps-db/commits/master\n" \
+            f"If none of that works, there was probably a problem while extracting dependency information by the crawler " \
+            f"maintaining the database.\n" \
+            f"Please open an issue here: https://github.com/DavHau/pypi-crawlers/issues/new\n"
+        print(error_text, file=sys.stderr)
+        exit(1)
+
+
+@dataclass
+class NixpkgsPyPkg:
+    nix_key: str
+    ver: Version
+
+
+class NixpkgsDirectory(UserDict):
+    def __init__(self, nixpkgs_json_file, **kwargs):
+        with open(nixpkgs_json_file) as f:
+            data = json.load(f)
+        self.by_nix_key = {}
+        for nix_key, version in data.items():
+            if not version:
+                continue
+            self.by_nix_key[nix_key] = NixpkgsPyPkg(
+                nix_key=nix_key,
+                ver=parse(version)
+            )
+        self.data = {}
+        for nix_key, pkg in self.by_nix_key.items():
+            key = self._unify_key(nix_key)
+            if key not in self.data:
+                self.data[key] = []
+            # Skip if version already exists. Prevents infinite recursions in nix (see 'pytest' + 'pytest_5')
+            elif any(existing_pkg.ver == pkg.ver for existing_pkg in self.data[key]):
+                continue
+            self.data[key].append(pkg)
+        super(NixpkgsDirectory, self).__init__(self.data, **kwargs)
+
+    def __getitem__(self, item) -> NixpkgsPyPkg:
+        return self.data[self._unify_key(item)][-1]
+
+    def has_multiple_candidates(self, name):
+        return len(self.data[self._unify_key(name)]) > 1
+
+    def get_all_candidates(self, name) -> List[NixpkgsPyPkg]:
+        return self.data[self._unify_key(name)]
+
+    def get_highest_ver(self, pkgs: List[NixpkgsPyPkg]):
+        return max(pkgs, key=lambda p: p.ver)
+
+    @staticmethod
+    def is_same_ver(ver1, ver2, ver_idx):
+        if any(not ver.release or len(ver.release) <= ver_idx for ver in (ver1, ver2)):
+            return False
+        return ver1.release[ver_idx] == ver2.release[ver_idx]
+
+    def choose_nix_pkg_for_py_pkg(self, name, ver):
+        """
+        In case a python package has more than one definition in nixpkgs
+        like `django` and `django_2_2`, this algo will select the right one.
+        """
+        pkgs: List[NixpkgsPyPkg] = sorted(self.get_all_candidates(name), key=lambda pkg: pkg.ver)
+        if len(pkgs) == 1:
+            return self[name].nix_key
+        # try to find nixpkgs pkg with best matching version
+        remaining_pkgs = pkgs
+        for i in range(7):  # usually there are not more than 4 parts in a version
+            same_ver = list(filter(lambda p: self.is_same_ver(ver, p.ver, i), remaining_pkgs))
+            if len(same_ver) == 1:
+                return same_ver[0].nix_key
+            elif len(same_ver) == 0:
+                highest = self.get_highest_ver(remaining_pkgs).nix_key
+                print(f'WARNING: Unable to decide which of nixpkgs\'s definitions {[p.nix_key for p in remaining_pkgs]}'
+                      f' is best as base for {name}:{ver}. Picking {highest}')
+                return highest
+            remaining_pkgs = same_ver
+        # In every case we should have returned by now
+        raise Exception("Dude... Check yor code!")
+
+    def get_by_nix_key(self, nix_key):
+        return self.data[nix_key]
+
+    def _unify_key(self, key) -> str:
+        return key.replace('-', '').replace('_', '').lower().rstrip('0123456789')
+
+    def exists(self, name, ver=None):
+        try:
+            pkg = self[self._unify_key(name)]
+        except KeyError:
+            return False
+        if ver:
+            return pkg.ver == ver
+        return True