Better 'required' job that includes everything (#976)

This requires explicitly identifying the things tht don't work and
removing them. I used our own `disabled` attribute, because `broken` is
weird and breaks evaluation, which isn't really what we want.

Cuts the number of jobs from over 2000 to 216. This seems to greatly
speeds up the time it takes small changes (where most stuff remains the
same) to be processed by hydra.

Uses https://github.com/NixOS/hydra/issues/715 to reference the
jobs in required by name avoiding OOM issues.

Co-authored-by: Hamish Mackenzie <Hamish.Mackenzie@iohk.io>

ci.nix

@@ -52,6 +52,9 @@
     # aarch64 cross only works on linux
     inherit (lib.systems.examples) musl64 aarch64-multiplatform;
   };
+  isDisabled = d:
+    let meta = d.meta or {};
+    in meta.disabled or false;
 in
 dimension "Nixpkgs version" nixpkgsVersions (nixpkgsName: nixpkgs-pin:
   let pinnedNixpkgsSrc = sources.${nixpkgs-pin};
@@ -62,7 +65,7 @@ dimension "Nixpkgs version" nixpkgsVersions (nixpkgsName: nixpkgs-pin:
       let pkgs = import pinnedNixpkgsSrc (nixpkgsArgs // { inherit system; });
           build = import ./build.nix { inherit pkgs ifdLevel compiler-nix-name; };
           platformFilter = platformFilterGeneric pkgs system;
-      in filterAttrsOnlyRecursive (_: v: platformFilter v) {
+      in filterAttrsOnlyRecursive (_: v: platformFilter v && !(isDisabled v)) ({
         # Native builds
         # TODO: can we merge this into the general case by picking an appropriate "cross system" to mean native?
         native = pkgs.recurseIntoAttrs ({
@@ -98,7 +101,7 @@ dimension "Nixpkgs version" nixpkgsVersions (nixpkgsName: nixpkgs-pin:
         } // pkgs.lib.optionalAttrs (ifdLevel >= 3) {
           hello = (pkgs.haskell-nix.hackage-package { name = "hello"; version = "1.0.0.2"; inherit compiler-nix-name; }).components.exes.hello;
         })
-      )
+      ))
     )
   )
 )

release.nix

@@ -10,16 +10,39 @@ let
   lib = genericPkgs.lib;
   ci = import ./ci.nix { inherit supportedSystems ifdLevel checkMaterialization; restrictEval = true; };
   allJobs = stripAttrsForHydra (filterDerivations ci);
-in allJobs // {
-    # On IOHK Hydra, "required" is a special job that updates the
-    # GitHub CI status.
+  latestJobs = {
+    # All the jobs are included in the `requiredJobs`, but the ones
+    # added here will also included without aggregation, making it easier
+    # to find a failing test.  Keep in mind though that adding too many
+    # of these will slow down eval times.
+    linux = allJobs.R2009.ghc8102.linux.native or {};
+    darwin = allJobs.R2009.ghc8102.darwin.native or {};
+  };
+  names = x: lib.filter (n: n != "recurseForDerivations" && n != "meta")
+    (builtins.attrNames x);
+  requiredJobs =
+    builtins.listToAttrs (
+      lib.concatMap (nixpkgsVer:
+        let nixpkgsJobs = allJobs.${nixpkgsVer};
+        in lib.concatMap (compiler-nix-name:
+          let ghcJobs = nixpkgsJobs.${compiler-nix-name};
+          in builtins.map (platform: {
+            name = "required-${nixpkgsVer}-${compiler-nix-name}-${platform}";
+            value = genericPkgs.releaseTools.aggregate {
+              name = "haskell.nix-${nixpkgsVer}-${compiler-nix-name}-${platform}";
+              meta.description = "All ${nixpkgsVer} ${compiler-nix-name} ${platform} jobs";
+              constituents = lib.collect (d: lib.isDerivation d) ghcJobs.${platform};
+            };
+          }) (names ghcJobs)
+        ) (names nixpkgsJobs)
+      ) (names allJobs));
+in latestJobs // requiredJobs // {
     required = genericPkgs.releaseTools.aggregate {
       name = "haskell.nix-required";
       meta.description = "All jobs required to pass CI";
-      # Hercules will require all of these, we just require the 20.09 jobs
-      # to avoid stressing Hydra too much
-      constituents = lib.collect lib.isDerivation allJobs.R2009.ghc865.linux.native;
+      # Using the names here requires https://github.com/NixOS/hydra/issues/715
+      constituents = builtins.attrNames requiredJobs;
     };
-  }
+ }
 
 

test/builder-haddock/default.nix

@@ -35,12 +35,7 @@ in
     buildCommand = let
       inherit (packages.test-haddock.components) library;
       noDocLibrary = packages.stm.components.library;
-    in if (stdenv.hostPlatform != stdenv.buildPlatform)
-      then ''
-        echo "Skipping haddock tests when cross compiling" >& 2
-        touch $out
-      ''
-      else ''
+    in ''
       ########################################################################
       # test haddock
 
@@ -82,6 +77,8 @@ in
       touch $out
     '';
 
-    meta.platforms = platforms.all;
-    meta.disabled = stdenv.hostPlatform.isMusl;
+    meta = {
+      platforms = platforms.all;
+      disabled = stdenv.hostPlatform != stdenv.buildPlatform || stdenv.hostPlatform.isMusl;
+    };
 } // { inherit packages pkgSet; }

test/cabal-simple-prof/default.nix

@@ -46,7 +46,11 @@ in recurseIntoAttrs {
       touch $out
     '';
 
-    meta.platforms = platforms.all;
+    meta = {
+      platforms = platforms.all;
+      # This test seeems to be broken on 8.6 and 8.8
+      disabled = compiler-nix-name == "ghc865" || compiler-nix-name == "ghc884";
+    };
 
     passthru = {
       # Used for debugging with nix repl

test/fully-static/default.nix

@@ -52,7 +52,10 @@ in recurseIntoAttrs {
 
       '') + "touch $out";
 
-    meta.platforms = platforms.all;
+    meta = {
+      # A dependency is broken on Windows, just run on unix
+      platforms = platforms.unix;
+    };
 
     passthru = {
       # Attributes used for debugging with nix repl

test/lookup-sha256/default.nix

@@ -1,5 +1,6 @@
 { pkgs, lib, stdenv, haskell-nix, testSrc, zlib, compiler-nix-name } :
-  (haskell-nix.hackage-package {
+  # A dependency is broken on windows, just run this on unix.
+  lib.addMetaAttrs { platforms = lib.platforms.unix; } ((haskell-nix.hackage-package {
     inherit compiler-nix-name;
     name         = "pandoc";
     version      = "2.9.2.1";
@@ -10,4 +11,4 @@
       { "https://github.com/jgm/pandoc-citeproc"."0.17"
           = "0dxx8cp2xndpw3jwiawch2dkrkp15mil7pyx7dvd810pwc22pm2q"; }
         ."${location}"."${tag}";
-  }).components.exes.pandoc
+  }).components.exes.pandoc)

test/setup-deps/default.nix

@@ -16,19 +16,17 @@ let
   };
 
   packages = project.hsPkgs;
-in recurseIntoAttrs (if stdenv.buildPlatform != stdenv.hostPlatform
- then
-    let skip = pkgs.runCommand "skip-test-setup-deps" {} ''
-      echo "Skipping setup-deps test when cross compiling as it needs the ghc lib" >& 2
-      touch $out
-    '';
-    in {
-      ifdInputs = { plan-nix = skip; };
-      run = skip;
-    }
- else {
+  meta = {
+    platforms = platforms.unix;
+    # Building reinstallable lib GHC is broken on 8.10, and we require lib ghc so this won't work with cross-compiling.
+    # Moreover, even building the plan doesn't seem to work in these circumstances.
+    disabled = stdenv.buildPlatform != stdenv.hostPlatform || stdenv.hostPlatform.isMusl || compiler-nix-name == "ghc8102";
+  };
+in 
+
+recurseIntoAttrs ({
   ifdInputs = {
-    inherit (project) plan-nix;
+    plan-nix = addMetaAttrs meta project.plan-nix;
   };
   run = pkgs.stdenv.mkDerivation {
     name = "setup-deps-test";
@@ -42,9 +40,7 @@ in recurseIntoAttrs (if stdenv.buildPlatform != stdenv.hostPlatform
       touch $out
     '';
 
-    meta.platforms = platforms.unix;
-    meta.disabled = stdenv.hostPlatform.isMusl || stdenv.hostPlatform.isWindows;
-
+    inherit meta;
     passthru = {
       # Attributes used for debugging with nix repl
       inherit project packages;

test/shell-for-setup-deps/default.nix

@@ -15,23 +15,7 @@ let
 
   env = project.shellFor {};
 
-# Making this work for cross compilers will be difficult as setup-deps are
-# built for the build platform and the shell will be for the host platform.
-# We probably need a shell that provides both build and host ghc
-# and corresponding package DBs and a way to use them.
-# This problem affects musl as well as the build libraries are linked to glibc.
-in recurseIntoAttrs (if stdenv.buildPlatform != stdenv.hostPlatform
- then
-    let skip = runCommand "skipping-test-shell-for-setup-deps" {} ''
-      echo "Skipping shell-for-setup-deps test on cross compilers (does not work yet)" >& 2
-      touch $out
-    '';
-    in {
-      ifdInputs = { plan-nix = skip; };
-      env = skip;
-      run = skip;
-    }
- else {
+in recurseIntoAttrs ({
   ifdInputs = {
     inherit (project) plan-nix;
   };
@@ -52,8 +36,15 @@ in recurseIntoAttrs (if stdenv.buildPlatform != stdenv.hostPlatform
       touch $out
     '';
 
-    meta.platforms = platforms.all;
-    meta.disabled = stdenv.buildPlatform != stdenv.hostPlatform;
+    meta = {
+      platforms = platforms.all;
+      # Making this work for cross compilers will be difficult as setup-deps are
+      # built for the build platform and the shell will be for the host platform.
+      # We probably need a shell that provides both build and host ghc
+      # and corresponding package DBs and a way to use them.
+      # This problem affects musl as well as the build libraries are linked to glibc.
+      disabled = stdenv.buildPlatform != stdenv.hostPlatform;
+    };
 
     passthru = {
       # Used for debugging with nix repl

test/shell-for/default.nix

@@ -64,8 +64,10 @@ in recurseIntoAttrs {
       touch $out
     '';
 
-    meta.platforms = platforms.all;
-    meta.disabled = stdenv.hostPlatform.isMusl || stdenv.hostPlatform.isWindows;
+    meta = {
+      platforms = platforms.unix;
+      disabled = stdenv.hostPlatform.isMusl;
+    };
 
     passthru = {
       # Used for debugging with nix repl

test/with-packages/default.nix

@@ -82,8 +82,10 @@ in recurseIntoAttrs {
       touch $out
     '';
 
-    meta.platforms = platforms.all;
-    meta.disabled = stdenv.hostPlatform.isMusl;
+    meta = {
+      platforms = platforms.all;
+      disabled = stdenv.hostPlatform.isMusl;
+    };
 
     passthru = {
       # Used for debugging with nix repl