version 2.2.0 (#60)

# 2.2.0 (09 Aug 2020)
Improved success rate, MacOS support, bugfixes, optimizations

### Features
 - Improved selection of wheel releases. MacOS is now supported and architectures besides x86_64 should be handled correctly.
 - Whenever mach-nix resolves dependencies, a visualization of the resulting dependency tree is printed on the terminal. 
 - The dependency DB is now accessed through a caching layer which reduces the resolver's CPU time significantly for larger environments.
 - The python platform context is now generated from the nix build environment variable `system`. This should decrease the chance of impurities during dependency resolution.
 
### Fixes
 - The requires_python attribute of wheels was not respected. This lead to failing builds especially for older python versions. Now `requires_python` is part of the dependency graph and affects resolution.
 - Detecting the correct package name for python packages in nixpkgs often failed since the attribute names don't follow a fixed schema. This lead to a handful of different errors in different situations. Now the package names are extracted from the pypi `url` inside the `src` attribute which is much more reliable. For packages which are not fetched from pypi, the `pname` attribute is used as fallback.
 - Fixed bug which lead to the error `attribute 'sdist' missing` if a package from the nixpkgs provider was used which doesn't publish it's source on pypi. (For example `tensorflow`)
 
### Other Changes
 - Mach-nix now uses a revision of the nixpkgs-unstable branch instead of nixos-20.03 as base fo the tool and the nixpkgs provider.
 - Updated revision of the dependency DB

Author: DavHau

Changelog.md

@@ -1,3 +1,21 @@
+# 2.2.0 (09 Aug 2020)
+Improved success rate, MacOS support, bugfixes, optimizations
+
+### Features
+ - Improved selection of wheel releases. MacOS is now supported and architectures besides x86_64 should be handled correctly.
+ - Whenever mach-nix resolves dependencies, a visualization of the resulting dependency tree is printed on the terminal. 
+ - The dependency DB is now accessed through a caching layer which reduces the resolver's CPU time significantly for larger environments.
+ - The python platform context is now generated from the nix build environment variable `system`. This should decrease the chance of impurities during dependency resolution.
+ 
+### Fixes
+ - The requires_python attribute of wheels was not respected. This lead to failing builds especially for older python versions. Now `requires_python` is part of the dependency graph and affects resolution.
+ - Detecting the correct package name for python packages in nixpkgs often failed since the attribute names don't follow a fixed schema. This lead to a handful of different errors in different situations. Now the package names are extracted from the pypi `url` inside the `src` attribute which is much more reliable. For packages which are not fetched from pypi, the `pname` attribute is used as fallback.
+ - Fixed bug which lead to the error `attribute 'sdist' missing` if a package from the nixpkgs provider was used which doesn't publish it's source on pypi. (For example `tensorflow`)
+ 
+### Other Changes
+ - Mach-nix now uses a revision of the nixpkgs-unstable branch instead of nixos-20.03 as base fo the tool and the nixpkgs provider.
+ - Updated revision of the dependency DB
+
 # 2.1.1 (30 Jul 2020)
 Fix broken wheel packages
 ### Fixes:

Readme.md

@@ -55,11 +55,11 @@ Table of Contents
 You can either install mach-nix via pip or by using nix in case you already have the nix package manager installed.
 #### Installing via pip
 ```shell
-pip install git+git://github.com/DavHau/mach-nix@2.1.1
+pip install git+git://github.com/DavHau/mach-nix@2.2.0
 ```
 #### Installing via nix
 ```shell
-nix-env -if https://github.com/DavHau/mach-nix/tarball/2.1.1 -A mach-nix
+nix-env -if https://github.com/DavHau/mach-nix/tarball/2.2.0 -A mach-nix
 ```
 
 ---
@@ -91,7 +91,7 @@ You can call mach-nix directly from a nix expression
 let
   mach-nix = import (builtins.fetchGit {
     url = "https://github.com/DavHau/mach-nix/";
-    ref = "2.1.1";
+    ref = "2.2.0";
   });
 in
 mach-nix.mkPython {

debug/nixpkgs-json.nix

@@ -10,4 +10,8 @@ let
   python = (import target_nixpkgs_src { config = {}; }).python37;
 in
 with import target_nixpkgs_src { config = {}; overlays = []; };
-import ../mach_nix/nix/nixpkgs-json.nix { inherit pkgs python; }
+import ../mach_nix/nix/nixpkgs-json.nix rec {
+  inherit pkgs python;
+  overrides = [(import ./overrides_pre.nix)];
+  mergeOverrides = lib.foldr lib.composeExtensions (self: super: { });
+}

debug/overrides_pre.nix.example

@@ -0,0 +1,19 @@
+pySelf: pySuper: {
+  pulsar = pySuper.buildPythonPackage {
+    src = fetchGit {
+      url = "https://github.com/DavHau/pulsar";
+      ref = "dev";
+      rev = "4b01fa19cc28ac912a6ffb18ef64d46bae77f800";
+    };
+    pname = "pulsar";
+    version = "2.0.2";  # not really
+    doCheck = false;
+    doInstallCheck = false;
+  };
+  asyncssh = pySuper.asyncssh.overridePythonAttrs (oldAttrs: {
+    patches = [];
+  });
+  pytest-dependency = pySuper.pytest-dependency.overridePythonAttrs (oldAttrs: {
+    patches = [];
+  });
+}

default.nix

@@ -2,18 +2,7 @@ let
   pkgs = import (import ./mach_nix/nix/nixpkgs-src.nix) { config = {}; overlays = []; };
   python = import ./mach_nix/nix/python.nix { inherit pkgs; };
   python_deps = (builtins.attrValues (import ./mach_nix/nix/python-deps.nix { inherit python; fetchurl = pkgs.fetchurl; }));
-  mergeOverrides = with pkgs.lib; overrides:
-    if length overrides == 0
-    then a: b: {}  # return dummy overrides
-    else
-      if length overrides == 1
-      then elemAt overrides 0
-      else
-        let
-          last = head ( reverseList overrides );
-          rest = reverseList (tail ( reverseList overrides ));
-        in
-          composeExtensions (mergeOverrides rest) last;
+  mergeOverrides = with pkgs.lib; foldr composeExtensions (self: super: { });
   autoPatchelfHook = import ./mach_nix/nix/auto_patchelf_hook.nix {inherit (pkgs) fetchurl makeSetupHook writeText;};
 in
 rec {
@@ -37,9 +26,15 @@ rec {
 
   # Returns `overrides` and `select_pkgs` which satisfy your requirements
   machNix = args:
-    let result = import "${machNixFile args}/share/mach_nix_file.nix";
+    let
+      result = import "${machNixFile args}/share/mach_nix_file.nix";
+      manylinux =
+        if pkgs.stdenv.hostPlatform.system == "x86_64-darwin" then
+          []
+        else
+          pkgs.pythonManylinuxPackages.manylinux1;
     in {
-      overrides = result.overrides pkgs.pythonManylinuxPackages.manylinux1 autoPatchelfHook;
+      overrides = result.overrides manylinux autoPatchelfHook;
       select_pkgs = result.select_pkgs;
     };
 
@@ -57,18 +52,19 @@ rec {
       disable_checks ? true,  # Disable tests wherever possible to decrease build time.
       overrides_pre ? [],  # list of pythonOverrides to apply before the machnix overrides
       overrides_post ? [],  # list of pythonOverrides to apply after the machnix overrides
-      pkgs ? nixpkgs,  # pass custom nixpkgs. Only used for manylinux wheel dependencies
+      pkgs ? nixpkgs,  # pass custom nixpkgs.
       providers ? {},  # define provider preferences
       pypi_deps_db_commit ? builtins.readFile ./mach_nix/nix/PYPI_DEPS_DB_COMMIT,  # python dependency DB version
       pypi_deps_db_sha256 ? builtins.readFile ./mach_nix/nix/PYPI_DEPS_DB_SHA256,
-      python ? pkgs.python3,  # select custom python to base overrides on. Should be from nixpkgs >= 20.03
+      python ? pkgs.python3,  # select custom python to base overrides onto. Should be from nixpkgs >= 20.03
       _provider_defaults ? with builtins; fromTOML (readFile ./mach_nix/provider_defaults.toml),
       ...
     }:
     let
       py = python.override { packageOverrides = mergeOverrides overrides_pre; };
       result = machNix {
         inherit requirements disable_checks providers pypi_deps_db_commit pypi_deps_db_sha256 _provider_defaults;
+        overrides = overrides_pre;
         python = py;
       };
       py_final = python.override { packageOverrides = mergeOverrides (
@@ -88,17 +84,18 @@ rec {
       disable_checks ? true,  # Disable tests wherever possible to decrease build time.
       overrides_pre ? [],  # list of pythonOverrides to apply before the machnix overrides
       overrides_post ? [],  # list of pythonOverrides to apply after the machnix overrides
-      pkgs ? nixpkgs,  # pass custom nixpkgs. Only used for manylinux wheel dependencies
+      pkgs ? nixpkgs,  # pass custom nixpkgs.
       providers ? {},  # define provider preferences
       pypi_deps_db_commit ? builtins.readFile ./mach_nix/nix/PYPI_DEPS_DB_COMMIT,  # python dependency DB version
       pypi_deps_db_sha256 ? builtins.readFile ./mach_nix/nix/PYPI_DEPS_DB_SHA256,
-      python ? pkgs.python3,  # select custom python to base overrides on. Should be from nixpkgs >= 20.03
+      python ? pkgs.python3,  # select custom python to base overrides onto. Should be from nixpkgs >= 20.03
       _provider_defaults ? with builtins; fromTOML (readFile ./mach_nix/provider_defaults.toml)
     }:
     let
       py = python.override { packageOverrides = mergeOverrides overrides_pre; };
       result = machNix {
         inherit requirements disable_checks providers pypi_deps_db_commit pypi_deps_db_sha256 _provider_defaults;
+        overrides = overrides_pre;
         python = py;
       };
       py_final = python.override { packageOverrides = mergeOverrides (

examples.md

@@ -24,7 +24,7 @@ build a python environment from a list of requirements
 let
   mach-nix = import (builtins.fetchGit {
     url = "https://github.com/DavHau/mach-nix/";
-    ref = "2.1.1";
+    ref = "2.2.0";
   });
 in mach-nix.mkPython {
   requirements = builtins.readFile ./requirements.txt;
@@ -37,7 +37,7 @@ Build a python package from its source code and a list of requirements
 let
   mach-nix = import (builtins.fetchGit {
     url = "https://github.com/DavHau/mach-nix/";
-    ref = "2.1.1";
+    ref = "2.2.0";
   });
 in mach-nix.buildPythonPackage {
   pname = "my-package";
@@ -53,7 +53,7 @@ Build a python package from its source code and a list of requirements
 let
   mach-nix = import (builtins.fetchGit {
     url = "https://github.com/DavHau/mach-nix/";
-    ref = "2.1.1";
+    ref = "2.2.0";
   });
 in mach-nix.buildPythonPackage rec {
   pname = "projectname";
@@ -77,7 +77,7 @@ I have a complex set of requirements including tensorflow. I'd like to have tens
 let
   mach-nix = import (builtins.fetchGit {
     url = "https://github.com/DavHau/mach-nix/";
-    ref = "2.1.1";
+    ref = "2.2.0";
   });
 in mach-nix.mkPython {
 
@@ -100,7 +100,7 @@ I'd like to install a more recent version of tensorflow which is not available f
 let
   mach-nix = import (builtins.fetchGit {
     url = "https://github.com/DavHau/mach-nix/";
-    ref = "2.1.1";
+    ref = "2.2.0";
   });
 in mach-nix.mkPython {
 
@@ -129,7 +129,7 @@ I'd like to use a recent version of Pytorch from wheel, but I'd like to build th
 let
   mach-nix = import (builtins.fetchGit {
     url = "https://github.com/DavHau/mach-nix/";
-    ref = "2.1.1";
+    ref = "2.2.0";
   });
 overlays = []; # some very useful overlays
 in mach-nix.mkPython rec {
@@ -164,7 +164,7 @@ I have a complex requirements.txt which includes `imagecodecs`. It is available
 let
   mach-nix = import (builtins.fetchGit {
     url = "https://github.com/DavHau/mach-nix/";
-    ref = "2.1.1";
+    ref = "2.2.0";
   });
 in mach-nix.mkPython rec {
 

mach_nix/VERSION

@@ -1 +1 @@
-2.1.0
+2.2.0

mach_nix/data/nixpkgs.py

@@ -5,6 +5,8 @@
 
 from packaging.version import Version, parse
 
+from mach_nix.cache import cached
+
 
 @dataclass
 class NixpkgsPyPkg:
@@ -48,6 +50,7 @@ def is_same_ver(ver1, ver2, ver_idx):
             return False
         return ver1.release[ver_idx] == ver2.release[ver_idx]
 
+    @cached(lambda args: (args[1], args[2]))
     def find_best_nixpkgs_candidate(self, name, ver):
         """
         In case a python package has more than one candidate in nixpkgs

mach_nix/data/providers.py

@@ -8,6 +8,7 @@
 
 import distlib.markers
 from packaging.version import Version, parse
+from pkg_resources import RequirementParseError
 
 from .nixpkgs import NixpkgsIndex
 from mach_nix.requirements import filter_reqs_by_eval_marker, Requirement, parse_reqs, context
@@ -65,7 +66,7 @@ def __init__(self, pkg_name, pkg_ver, provider_name, *args, **kwargs):
 
 class DependencyProviderBase(ABC):
     def __init__(self, py_ver: PyVer, platform, system, *args, **kwargs):
-        self.context = context(py_ver)
+        self.context = context(py_ver, platform, system)
         self.context_wheel = self.context.copy()
         self.context_wheel['extra'] = None
         self.py_ver_digits = py_ver.digits()
@@ -202,11 +203,6 @@ def _available_versions(self, pkg_name: str) -> Iterable[Version]:
 
 class NixpkgsDependencyProvider(DependencyProviderBase):
     name = 'nixpkgs'
-    # mapping from pypi name to nix key
-    # _aliases = dict(
-    #     torch='pytorch',
-    #     tensorboard='tensorflow-tensorboard'
-    # )
 
     # TODO: implement extras by looking them up via the equivalent wheel
     def __init__(
@@ -261,22 +257,24 @@ class WheelDependencyProvider(DependencyProviderBase):
     def __init__(self, data_dir: str, *args, **kwargs):
         super(WheelDependencyProvider, self).__init__(*args, **kwargs)
         self.data = LazyBucketDict(data_dir)
-        m = self.py_ver_digits[-1]
+        maj = self.py_ver_digits[0]  # major version
+        min = self.py_ver_digits[1]  # minor version
         if self.system == "linux":
+            cp_abi = f"cp{maj}{min}mu" if int(maj) == 2 else f"cp{maj}{min}m"
             self.preferred_wheels = (
-                re.compile(rf"(py3|cp3){m}?-(cp3{m}m|abi3|none)-manylinux2014_{self.platform}"),
-                re.compile(rf"(py3|cp3){m}?-(cp3{m}m|abi3|none)-manylinux2010_{self.platform}"),
-                re.compile(rf"(py3|cp3){m}?-(cp3{m}m|abi3|none)-manylinux1_{self.platform}"),
-                re.compile(rf"(py3|cp3){m}?-(cp3{m}m|abi3|none)-linux_{self.platform}"),
-                re.compile(rf"(py3|cp3){m}?-(cp3{m}m|abi3|none)-any"),
+                re.compile(rf"(py{maj}|cp{maj}){min}?-({cp_abi}|abi3|none)-manylinux2014_{self.platform}"),
+                re.compile(rf"(py{maj}|cp{maj}){min}?-({cp_abi}|abi3|none)-manylinux2010_{self.platform}"),
+                re.compile(rf"(py{maj}|cp{maj}){min}?-({cp_abi}|abi3|none)-manylinux1_{self.platform}"),
+                re.compile(rf"(py{maj}|cp{maj}){min}?-({cp_abi}|abi3|none)-linux_{self.platform}"),
+                re.compile(rf"(py{maj}|cp{maj}){min}?-({cp_abi}|abi3|none)-any"),
             )
         elif self.system == "darwin":
             self.preferred_wheels = (
-                re.compile(rf"(py3|cp3){m}?-(cp3{m}|abi3|none)-macosx_\d*_\d*_universal"),
-                re.compile(rf"(py3|cp3){m}?-(cp3{m}|abi3|none)-macosx_\d*_\d*_x86_64"),
-                re.compile(rf"(py3|cp3){m}?-(cp3{m}|abi3|none)-macosx_\d*_\d*_intel"),
-                re.compile(rf"(py3|cp3){m}?-(cp3{m}|abi3|none)-macosx_\d*_\d*_(fat64|fat32)"),
-                re.compile(rf"(py3|cp3){m}?-(cp3{m}|abi3|none)-any"),)
+                re.compile(rf"(py{maj}|cp{maj}){min}?-(cp{maj}{min}|abi3|none)-macosx_\d*_\d*_universal"),
+                re.compile(rf"(py{maj}|cp{maj}){min}?-(cp{maj}{min}|abi3|none)-macosx_\d*_\d*_x86_64"),
+                re.compile(rf"(py{maj}|cp{maj}){min}?-(cp{maj}{min}|abi3|none)-macosx_\d*_\d*_intel"),
+                re.compile(rf"(py{maj}|cp{maj}){min}?-(cp{maj}{min}|abi3|none)-macosx_\d*_\d*_(fat64|fat32)"),
+                re.compile(rf"(py{maj}|cp{maj}){min}?-(cp{maj}{min}|abi3|none)-any"),)
         else:
             raise Exception(f"Unsupported Platform {platform.system()}")
 
@@ -317,7 +315,7 @@ def _all_releases(self, pkg_name):
                         fn,
                         deps['requires_dist'] if 'requires_dist' in deps else None,
                         deps['requires_extras'] if 'requires_extras' in deps else None,
-                        deps['requires_python'] if 'requires_python' in deps else None,
+                        deps['requires_python'].strip(',') if 'requires_python' in deps else None,
                     )
 
     def _apply_filters(self, filters: List[callable], objects: Iterable):
@@ -362,7 +360,12 @@ def _python_requires_ok(self, wheel: WheelRelease):
         if not wheel.requires_python:
             return True
         ver = parse('.'.join(self.py_ver_digits))
-        return bool(filter_versions([ver], list(parse_reqs(f"python{wheel.requires_python}"))[0].specs))
+        try:
+            parsed_py_requires = list(parse_reqs(f"python{wheel.requires_python}"))
+            return bool(filter_versions([ver], parsed_py_requires[0].specs))
+        except RequirementParseError:
+            print(f"WARNING: `requires_python` attribute of wheel {wheel.name}:{wheel.ver} could not be parsed")
+            return False
 
 
 class SdistDependencyProvider(DependencyProviderBase):

mach_nix/generate.py

@@ -48,7 +48,7 @@ def main():
         disable_checks,
         ResolvelibResolver(nixpkgs, deps_provider),
     )
-    reqs = filter_reqs_by_eval_marker(parse_reqs(requirements), context(py_ver))
+    reqs = filter_reqs_by_eval_marker(parse_reqs(requirements), context(py_ver, platform, system))
     expr = generator.generate(reqs)
     with open(out_file, 'w') as f:
         f.write(expr)

mach_nix/generators/overides_generator.py

@@ -68,11 +68,13 @@ def _gen_prop_build_inputs(self, prop_build_inputs_local, prop_build_inputs_nixp
             f"{b}" for b in sorted(prop_build_inputs_local | prop_build_inputs_nixpkgs))
         return prop_build_inputs_str
 
-    def _gen_overrideAttrs(self, name, ver, nix_name, build_inputs_str, prop_build_inputs_str):
+    def _gen_overrideAttrs(self, name, ver, nix_name, build_inputs_str, prop_build_inputs_str, keep_src=False):
         out = f"""
             {nix_name} = python-super.{nix_name}.overridePythonAttrs ( oldAttrs: {{
               pname = "{name}";
-              version = "{ver}";
+              version = "{ver}";"""
+        if not keep_src:
+            out += f"""
               src = fetchPypi "{name}" "{ver}";"""
         if build_inputs_str:
             out += f"""
@@ -183,7 +185,9 @@ def _gen_overrides(self, pkgs: Dict[str, ResolvedPkg], overlay_keys, pkgs_names:
                     out += self._unify_nixpkgs_keys(pkg.name)
             elif pkg.provider_info.provider == NixpkgsDependencyProvider.name:
                 nix_name = self.nixpkgs.find_best_nixpkgs_candidate(pkg.name, pkg.ver)
-                out += self._gen_overrideAttrs(pkg.name, pkg.ver, nix_name, build_inputs_str, prop_build_inputs_str)
+                out += self._gen_overrideAttrs(
+                    pkg.name, pkg.ver, nix_name, build_inputs_str, prop_build_inputs_str,
+                    keep_src=True)
                 out += self._unify_nixpkgs_keys(pkg.name, main_key=nix_name)
         end_overlay_section = f"""
                 }};

mach_nix/nix/NIXPKGS_COMMIT

@@ -1 +1 @@
-f8248ab6d9e69ea9c07950d73d48807ec595e923
+1365b9ac700aabe8dc959e9897bc1376e416c553

mach_nix/nix/NIXPKGS_SHA256

@@ -1 +1 @@
-009i9j6mbq6i481088jllblgdnci105b2q4mscprdawg3knlyahk
+180bq107z7qc2h4dxs3fxmrag3l8p9lvgjf3530qknp2h3ji8vqh

mach_nix/nix/PYPI_DEPS_DB_COMMIT

@@ -1 +1 @@
-2a2501aab1fe4eb50763652c4f74c5327f976f85
+521ca251792514f1a8170dd89b11edd158d7bfbe

mach_nix/nix/PYPI_DEPS_DB_SHA256

@@ -1 +1 @@
-03c64w1x4q2li57cn6zsb2shjk2fc4f5ksmsz3695dmvckp0lj14
+01hi8r7pzvpimw282zsnk1giysznarx8a067dnh8m9h7wz506lgc