From 79c713bc1498ccceed9f8d2fc0b8d29b1b45a209 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 6 Sep 2018 14:28:44 -0400 Subject: [PATCH] release-lib: Cache cross nixpkgs evals too This will help with release-cross.nix eval time. It also allowed me to share code between the cross and native helpers. --- pkgs/top-level/release-cross.nix | 6 +- pkgs/top-level/release-lib.nix | 94 ++++++++++++++++++++------------ 2 files changed, 62 insertions(+), 38 deletions(-) diff --git a/pkgs/top-level/release-cross.nix b/pkgs/top-level/release-cross.nix index 0d15d817a66c..ee69dd5a43c5 100644 --- a/pkgs/top-level/release-cross.nix +++ b/pkgs/top-level/release-cross.nix @@ -81,11 +81,11 @@ in # good idea lest there be some irrelevant pass-through debug attrs that # cause false negatives. testEqualOne = path: system: let - f = path: attrs: builtins.toString (lib.getAttrFromPath path (allPackages attrs)); + f = path: crossSystem: system: builtins.toString (lib.getAttrFromPath path (pkgsForCross crossSystem system)); in assertTrue ( - f path { inherit system; } + f path null system == - f (["buildPackages"] ++ path) { inherit system crossSystem; } + f (["buildPackages"] ++ path) crossSystem system ); testEqual = path: systems: forMatchingSystems systems (testEqualOne path); diff --git a/pkgs/top-level/release-lib.nix b/pkgs/top-level/release-lib.nix index 623c9d9cb000..2b72de7c7ef5 100644 --- a/pkgs/top-level/release-lib.nix +++ b/pkgs/top-level/release-lib.nix @@ -11,8 +11,6 @@ in with lib; rec { - allPackages = args: packageSet (args // nixpkgsArgs); - pkgs = packageSet (lib.recursiveUpdate { system = "x86_64-linux"; config.allowUnsupportedSystem = true; } nixpkgsArgs); inherit lib; @@ -23,29 +21,53 @@ rec { /* !!! Hack: poor man's memoisation function. Necessary to prevent Nixpkgs from being evaluated again and again for every job/platform pair. */ - pkgsFor = system: - if system == "x86_64-linux" then pkgs_x86_64_linux - else if system == "i686-linux" then pkgs_i686_linux - else if system == "aarch64-linux" then pkgs_aarch64_linux - else if system == "armv6l-linux" then pkgs_armv6l_linux - else if system == "armv7l-linux" then pkgs_armv7l_linux - else if system == "x86_64-darwin" then pkgs_x86_64_darwin - else if system == "x86_64-freebsd" then pkgs_x86_64_freebsd - else if system == "i686-freebsd" then pkgs_i686_freebsd - else if system == "i686-cygwin" then pkgs_i686_cygwin - else if system == "x86_64-cygwin" then pkgs_x86_64_cygwin - else abort "unsupported system type: ${system}"; + mkPkgsFor = crossSystem: let + packageSet' = args: packageSet (args // { inherit crossSystem; } // nixpkgsArgs); - pkgs_x86_64_linux = allPackages { system = "x86_64-linux"; }; - pkgs_i686_linux = allPackages { system = "i686-linux"; }; - pkgs_aarch64_linux = allPackages { system = "aarch64-linux"; }; - pkgs_armv6l_linux = allPackages { system = "armv6l-linux"; }; - pkgs_armv7l_linux = allPackages { system = "armv7l-linux"; }; - pkgs_x86_64_darwin = allPackages { system = "x86_64-darwin"; }; - pkgs_x86_64_freebsd = allPackages { system = "x86_64-freebsd"; }; - pkgs_i686_freebsd = allPackages { system = "i686-freebsd"; }; - pkgs_i686_cygwin = allPackages { system = "i686-cygwin"; }; - pkgs_x86_64_cygwin = allPackages { system = "x86_64-cygwin"; }; + pkgs_x86_64_linux = packageSet' { system = "x86_64-linux"; }; + pkgs_i686_linux = packageSet' { system = "i686-linux"; }; + pkgs_aarch64_linux = packageSet' { system = "aarch64-linux"; }; + pkgs_armv6l_linux = packageSet' { system = "armv6l-linux"; }; + pkgs_armv7l_linux = packageSet' { system = "armv7l-linux"; }; + pkgs_x86_64_darwin = packageSet' { system = "x86_64-darwin"; }; + pkgs_x86_64_freebsd = packageSet' { system = "x86_64-freebsd"; }; + pkgs_i686_freebsd = packageSet' { system = "i686-freebsd"; }; + pkgs_i686_cygwin = packageSet' { system = "i686-cygwin"; }; + pkgs_x86_64_cygwin = packageSet' { system = "x86_64-cygwin"; }; + + in system: + if system == "x86_64-linux" then pkgs_x86_64_linux + else if system == "i686-linux" then pkgs_i686_linux + else if system == "aarch64-linux" then pkgs_aarch64_linux + else if system == "armv6l-linux" then pkgs_armv6l_linux + else if system == "armv7l-linux" then pkgs_armv7l_linux + else if system == "x86_64-darwin" then pkgs_x86_64_darwin + else if system == "x86_64-freebsd" then pkgs_x86_64_freebsd + else if system == "i686-freebsd" then pkgs_i686_freebsd + else if system == "i686-cygwin" then pkgs_i686_cygwin + else if system == "x86_64-cygwin" then pkgs_x86_64_cygwin + else abort "unsupported system type: ${system}"; + + inherit (pkgsForCross null) pkgsFor; + + + # More poor man's memoisation + pkgsForCross = let + examplesByConfig = lib.flip lib.mapAttrs' + (builtins.removeAttrs lib.systems.examples [ "riscv" ]) + (_: crossSystem: nameValuePair crossSystem.config { + inherit crossSystem; + pkgsFor = mkPkgsFor crossSystem; + }); + native = mkPkgsFor null; + in crossSystem: let + errorMsg = "unsupported crossSystem: ${toString crossSystem.config or ""}"; + candidate = examplesByConfig.${crossSystem.config} or (throw errorMsg); + in if crossSystem == null + then native + else if lib.matchAttrs crossSystem candidate.crossSystem + then candidate.pkgsFor + else throw errorMsg; # Given a list of 'meta.platforms'-style patterns, return the sublist of @@ -90,30 +112,32 @@ rec { platform as an argument . We return an attribute set containing a derivation for each supported platform, i.e. ‘{ x86_64-linux = f pkgs_x86_64_linux; i686-linux = f pkgs_i686_linux; ... }’. */ - testOn = metaPatterns: f: forMatchingSystems metaPatterns - (system: hydraJob' (f (pkgsFor system))); + testOn = testOnCross null; /* Similar to the testOn function, but with an additional - 'crossSystem' parameter for allPackages, defining the target + 'crossSystem' parameter for packageSet', defining the target platform for cross builds. */ testOnCross = crossSystem: metaPatterns: f: forMatchingSystems metaPatterns - (system: hydraJob' (f (allPackages { inherit system crossSystem; }))); + (system: hydraJob' (f (pkgsForCross crossSystem system))); /* Given a nested set where the leaf nodes are lists of platforms, map each leaf node to `testOn [platforms...] (pkgs: pkgs.)'. */ - mapTestOn = mapAttrsRecursive - (path: metaPatterns: testOn metaPatterns (pkgs: getAttrFromPath path pkgs)); + mapTestOn = _mapTestOnHelper id null; + + + _mapTestOnHelper = f: crossSystem: mapAttrsRecursive + (path: metaPatterns: testOnCross crossSystem metaPatterns + (pkgs: f (getAttrFromPath path pkgs))); /* Similar to the testOn function, but with an additional 'crossSystem' - * parameter for allPackages, defining the target platform for cross builds, - * and triggering the build of the host derivation (cross built - crossDrv). */ - mapTestOnCross = crossSystem: mapAttrsRecursive - (path: metaPatterns: testOnCross crossSystem metaPatterns - (pkgs: addMetaAttrs { maintainers = crossMaintainers; } (getAttrFromPath path pkgs))); + * parameter for packageSet', defining the target platform for cross builds, + * and triggering the build of the host derivation. */ + mapTestOnCross = _mapTestOnHelper + (addMetaAttrs { maintainers = crossMaintainers; }); /* Recursively map a (nested) set of derivations to an isomorphic