on iko/ghcjs-patch-master help sign in
Better 'required' job that includes everything (#976)

This requires explicitly identifying the things tht don't work and removing them. I used our own `disabled` attribute, because `broken` is weird and breaks evaluation, which isn't really what we want. Cuts the number of jobs from over 2000 to 216. This seems to greatly speeds up the time it takes small changes (where most stuff remains the same) to be processed by hydra. Uses https://github.com/NixOS/hydra/issues/715 to reference the jobs in required by name avoiding OOM issues. Co-authored-by: Hamish Mackenzie <Hamish.Mackenzie@iohk.io>

commitf6bc2f7315af6c630f78f4350499af40b404bc87 authorMichael Peyton Jones <michael.peyton-jones@iohk.io> dateWed 6 Jan 2021, 03:44 (6 years ago) parentfea239b
10 files changed+81−59
ci.nix+5−2 release.nix+30−7 test/builder-haddock/default.nix+5−8 test/cabal-simple-prof/default.nix+5−1 test/fully-static/default.nix+4−1 test/lookup-sha256/default.nix+3−2 test/setup-deps/default.nix+11−15 test/shell-for-setup-deps/default.nix+10−19 test/shell-for/default.nix+4−2 test/with-packages/default.nix+4−2
ci.nix+5−2
@@ -52,6 +52,9 @@
5252 # aarch64 cross only works on linux
5353 inherit (lib.systems.examples) musl64 aarch64-multiplatform;
5454 };
55+ isDisabled = d:
56+ let meta = d.meta or {};
57+ in meta.disabled or false;
5558in
5659dimension "Nixpkgs version" nixpkgsVersions (nixpkgsName: nixpkgs-pin:
5760 let pinnedNixpkgsSrc = sources.${nixpkgs-pin};
@@ -62,7 +65,7 @@ dimension "Nixpkgs version" nixpkgsVersions (nixpkgsName: nixpkgs-pin:
6265 let pkgs = import pinnedNixpkgsSrc (nixpkgsArgs // { inherit system; });
6366 build = import ./build.nix { inherit pkgs ifdLevel compiler-nix-name; };
6467 platformFilter = platformFilterGeneric pkgs system;
65− in filterAttrsOnlyRecursive (_: v: platformFilter v) {
68+ in filterAttrsOnlyRecursive (_: v: platformFilter v && !(isDisabled v)) ({
6669 # Native builds
6770 # TODO: can we merge this into the general case by picking an appropriate "cross system" to mean native?
6871 native = pkgs.recurseIntoAttrs ({
@@ -98,7 +101,7 @@ dimension "Nixpkgs version" nixpkgsVersions (nixpkgsName: nixpkgs-pin:
98101 } // pkgs.lib.optionalAttrs (ifdLevel >= 3) {
99102 hello = (pkgs.haskell-nix.hackage-package { name = "hello"; version = "1.0.0.2"; inherit compiler-nix-name; }).components.exes.hello;
100103 })
101− )
104+ ))
102105 )
103106 )
104107)
release.nix+30−7
@@ -10,16 +10,39 @@ let
1010 lib = genericPkgs.lib;
1111 ci = import ./ci.nix { inherit supportedSystems ifdLevel checkMaterialization; restrictEval = true; };
1212 allJobs = stripAttrsForHydra (filterDerivations ci);
13−in allJobs // {
14− # On IOHK Hydra, "required" is a special job that updates the
15− # GitHub CI status.
13+ latestJobs = {
14+ # All the jobs are included in the `requiredJobs`, but the ones
15+ # added here will also included without aggregation, making it easier
16+ # to find a failing test. Keep in mind though that adding too many
17+ # of these will slow down eval times.
18+ linux = allJobs.R2009.ghc8102.linux.native or {};
19+ darwin = allJobs.R2009.ghc8102.darwin.native or {};
20+ };
21+ names = x: lib.filter (n: n != "recurseForDerivations" && n != "meta")
22+ (builtins.attrNames x);
23+ requiredJobs =
24+ builtins.listToAttrs (
25+ lib.concatMap (nixpkgsVer:
26+ let nixpkgsJobs = allJobs.${nixpkgsVer};
27+ in lib.concatMap (compiler-nix-name:
28+ let ghcJobs = nixpkgsJobs.${compiler-nix-name};
29+ in builtins.map (platform: {
30+ name = "required-${nixpkgsVer}-${compiler-nix-name}-${platform}";
31+ value = genericPkgs.releaseTools.aggregate {
32+ name = "haskell.nix-${nixpkgsVer}-${compiler-nix-name}-${platform}";
33+ meta.description = "All ${nixpkgsVer} ${compiler-nix-name} ${platform} jobs";
34+ constituents = lib.collect (d: lib.isDerivation d) ghcJobs.${platform};
35+ };
36+ }) (names ghcJobs)
37+ ) (names nixpkgsJobs)
38+ ) (names allJobs));
39+in latestJobs // requiredJobs // {
1640 required = genericPkgs.releaseTools.aggregate {
1741 name = "haskell.nix-required";
1842 meta.description = "All jobs required to pass CI";
19− # Hercules will require all of these, we just require the 20.09 jobs
20− # to avoid stressing Hydra too much
21− constituents = lib.collect lib.isDerivation allJobs.R2009.ghc865.linux.native;
43+ # Using the names here requires https://github.com/NixOS/hydra/issues/715
44+ constituents = builtins.attrNames requiredJobs;
2245 };
23− }
46+ }
2447
2548
test/builder-haddock/default.nix+5−8
@@ -35,12 +35,7 @@ in
3535 buildCommand = let
3636 inherit (packages.test-haddock.components) library;
3737 noDocLibrary = packages.stm.components.library;
38− in if (stdenv.hostPlatform != stdenv.buildPlatform)
39− then ''
40− echo "Skipping haddock tests when cross compiling" >& 2
41− touch $out
42− ''
43− else ''
38+ in ''
4439 ########################################################################
4540 # test haddock
4641
@@ -82,6 +77,8 @@ in
8277 touch $out
8378 '';
8479
85− meta.platforms = platforms.all;
86− meta.disabled = stdenv.hostPlatform.isMusl;
80+ meta = {
81+ platforms = platforms.all;
82+ disabled = stdenv.hostPlatform != stdenv.buildPlatform || stdenv.hostPlatform.isMusl;
83+ };
8784} // { inherit packages pkgSet; }
test/cabal-simple-prof/default.nix+5−1
@@ -46,7 +46,11 @@ in recurseIntoAttrs {
4646 touch $out
4747 '';
4848
49− meta.platforms = platforms.all;
49+ meta = {
50+ platforms = platforms.all;
51+ # This test seeems to be broken on 8.6 and 8.8
52+ disabled = compiler-nix-name == "ghc865" || compiler-nix-name == "ghc884";
53+ };
5054
5155 passthru = {
5256 # Used for debugging with nix repl
test/fully-static/default.nix+4−1
@@ -52,7 +52,10 @@ in recurseIntoAttrs {
5252
5353 '') + "touch $out";
5454
55− meta.platforms = platforms.all;
55+ meta = {
56+ # A dependency is broken on Windows, just run on unix
57+ platforms = platforms.unix;
58+ };
5659
5760 passthru = {
5861 # Attributes used for debugging with nix repl
test/lookup-sha256/default.nix+3−2
@@ -1,5 +1,6 @@
11{ pkgs, lib, stdenv, haskell-nix, testSrc, zlib, compiler-nix-name } :
2− (haskell-nix.hackage-package {
2+ # A dependency is broken on windows, just run this on unix.
3+ lib.addMetaAttrs { platforms = lib.platforms.unix; } ((haskell-nix.hackage-package {
34 inherit compiler-nix-name;
45 name = "pandoc";
56 version = "2.9.2.1";
@@ -10,4 +11,4 @@
1011 { "https://github.com/jgm/pandoc-citeproc"."0.17"
1112 = "0dxx8cp2xndpw3jwiawch2dkrkp15mil7pyx7dvd810pwc22pm2q"; }
1213 ."${location}"."${tag}";
13− }).components.exes.pandoc
14+ }).components.exes.pandoc)
test/setup-deps/default.nix+11−15
@@ -16,19 +16,17 @@ let
1616 };
1717
1818 packages = project.hsPkgs;
19−in recurseIntoAttrs (if stdenv.buildPlatform != stdenv.hostPlatform
20− then
21− let skip = pkgs.runCommand "skip-test-setup-deps" {} ''
22− echo "Skipping setup-deps test when cross compiling as it needs the ghc lib" >& 2
23− touch $out
24− '';
25− in {
26− ifdInputs = { plan-nix = skip; };
27− run = skip;
28− }
29− else {
19+ meta = {
20+ platforms = platforms.unix;
21+ # Building reinstallable lib GHC is broken on 8.10, and we require lib ghc so this won't work with cross-compiling.
22+ # Moreover, even building the plan doesn't seem to work in these circumstances.
23+ disabled = stdenv.buildPlatform != stdenv.hostPlatform || stdenv.hostPlatform.isMusl || compiler-nix-name == "ghc8102";
24+ };
25+in
26+
27+recurseIntoAttrs ({
3028 ifdInputs = {
31− inherit (project) plan-nix;
29+ plan-nix = addMetaAttrs meta project.plan-nix;
3230 };
3331 run = pkgs.stdenv.mkDerivation {
3432 name = "setup-deps-test";
@@ -42,9 +40,7 @@ in recurseIntoAttrs (if stdenv.buildPlatform != stdenv.hostPlatform
4240 touch $out
4341 '';
4442
45− meta.platforms = platforms.unix;
46− meta.disabled = stdenv.hostPlatform.isMusl || stdenv.hostPlatform.isWindows;
47−
43+ inherit meta;
4844 passthru = {
4945 # Attributes used for debugging with nix repl
5046 inherit project packages;
test/shell-for-setup-deps/default.nix+10−19
@@ -15,23 +15,7 @@ let
1515
1616 env = project.shellFor {};
1717
18−# Making this work for cross compilers will be difficult as setup-deps are
19−# built for the build platform and the shell will be for the host platform.
20−# We probably need a shell that provides both build and host ghc
21−# and corresponding package DBs and a way to use them.
22−# This problem affects musl as well as the build libraries are linked to glibc.
23−in recurseIntoAttrs (if stdenv.buildPlatform != stdenv.hostPlatform
24− then
25− let skip = runCommand "skipping-test-shell-for-setup-deps" {} ''
26− echo "Skipping shell-for-setup-deps test on cross compilers (does not work yet)" >& 2
27− touch $out
28− '';
29− in {
30− ifdInputs = { plan-nix = skip; };
31− env = skip;
32− run = skip;
33− }
34− else {
18+in recurseIntoAttrs ({
3519 ifdInputs = {
3620 inherit (project) plan-nix;
3721 };
@@ -52,8 +36,15 @@ in recurseIntoAttrs (if stdenv.buildPlatform != stdenv.hostPlatform
5236 touch $out
5337 '';
5438
55− meta.platforms = platforms.all;
56− meta.disabled = stdenv.buildPlatform != stdenv.hostPlatform;
39+ meta = {
40+ platforms = platforms.all;
41+ # Making this work for cross compilers will be difficult as setup-deps are
42+ # built for the build platform and the shell will be for the host platform.
43+ # We probably need a shell that provides both build and host ghc
44+ # and corresponding package DBs and a way to use them.
45+ # This problem affects musl as well as the build libraries are linked to glibc.
46+ disabled = stdenv.buildPlatform != stdenv.hostPlatform;
47+ };
5748
5849 passthru = {
5950 # Used for debugging with nix repl
test/shell-for/default.nix+4−2
@@ -64,8 +64,10 @@ in recurseIntoAttrs {
6464 touch $out
6565 '';
6666
67− meta.platforms = platforms.all;
68− meta.disabled = stdenv.hostPlatform.isMusl || stdenv.hostPlatform.isWindows;
67+ meta = {
68+ platforms = platforms.unix;
69+ disabled = stdenv.hostPlatform.isMusl;
70+ };
6971
7072 passthru = {
7173 # Used for debugging with nix repl
test/with-packages/default.nix+4−2
@@ -82,8 +82,10 @@ in recurseIntoAttrs {
8282 touch $out
8383 '';
8484
85− meta.platforms = platforms.all;
86− meta.disabled = stdenv.hostPlatform.isMusl;
85+ meta = {
86+ platforms = platforms.all;
87+ disabled = stdenv.hostPlatform.isMusl;
88+ };
8789
8890 passthru = {
8991 # Used for debugging with nix repl