mirror of
https://github.com/ilyakooo0/haskell.nix.git
synced 2024-09-19 02:47:34 +03:00
Better 'required' job that includes everything (#976)
This requires explicitly identifying the things tht don't work and removing them. I used our own `disabled` attribute, because `broken` is weird and breaks evaluation, which isn't really what we want. Cuts the number of jobs from over 2000 to 216. This seems to greatly speeds up the time it takes small changes (where most stuff remains the same) to be processed by hydra. Uses https://github.com/NixOS/hydra/issues/715 to reference the jobs in required by name avoiding OOM issues. Co-authored-by: Hamish Mackenzie <Hamish.Mackenzie@iohk.io>
This commit is contained in:
parent
fea239b6ba
commit
f6bc2f7315
7
ci.nix
7
ci.nix
@ -52,6 +52,9 @@
|
||||
# aarch64 cross only works on linux
|
||||
inherit (lib.systems.examples) musl64 aarch64-multiplatform;
|
||||
};
|
||||
isDisabled = d:
|
||||
let meta = d.meta or {};
|
||||
in meta.disabled or false;
|
||||
in
|
||||
dimension "Nixpkgs version" nixpkgsVersions (nixpkgsName: nixpkgs-pin:
|
||||
let pinnedNixpkgsSrc = sources.${nixpkgs-pin};
|
||||
@ -62,7 +65,7 @@ dimension "Nixpkgs version" nixpkgsVersions (nixpkgsName: nixpkgs-pin:
|
||||
let pkgs = import pinnedNixpkgsSrc (nixpkgsArgs // { inherit system; });
|
||||
build = import ./build.nix { inherit pkgs ifdLevel compiler-nix-name; };
|
||||
platformFilter = platformFilterGeneric pkgs system;
|
||||
in filterAttrsOnlyRecursive (_: v: platformFilter v) {
|
||||
in filterAttrsOnlyRecursive (_: v: platformFilter v && !(isDisabled v)) ({
|
||||
# Native builds
|
||||
# TODO: can we merge this into the general case by picking an appropriate "cross system" to mean native?
|
||||
native = pkgs.recurseIntoAttrs ({
|
||||
@ -98,7 +101,7 @@ dimension "Nixpkgs version" nixpkgsVersions (nixpkgsName: nixpkgs-pin:
|
||||
} // pkgs.lib.optionalAttrs (ifdLevel >= 3) {
|
||||
hello = (pkgs.haskell-nix.hackage-package { name = "hello"; version = "1.0.0.2"; inherit compiler-nix-name; }).components.exes.hello;
|
||||
})
|
||||
)
|
||||
))
|
||||
)
|
||||
)
|
||||
)
|
||||
|
37
release.nix
37
release.nix
@ -10,16 +10,39 @@ let
|
||||
lib = genericPkgs.lib;
|
||||
ci = import ./ci.nix { inherit supportedSystems ifdLevel checkMaterialization; restrictEval = true; };
|
||||
allJobs = stripAttrsForHydra (filterDerivations ci);
|
||||
in allJobs // {
|
||||
# On IOHK Hydra, "required" is a special job that updates the
|
||||
# GitHub CI status.
|
||||
latestJobs = {
|
||||
# All the jobs are included in the `requiredJobs`, but the ones
|
||||
# added here will also included without aggregation, making it easier
|
||||
# to find a failing test. Keep in mind though that adding too many
|
||||
# of these will slow down eval times.
|
||||
linux = allJobs.R2009.ghc8102.linux.native or {};
|
||||
darwin = allJobs.R2009.ghc8102.darwin.native or {};
|
||||
};
|
||||
names = x: lib.filter (n: n != "recurseForDerivations" && n != "meta")
|
||||
(builtins.attrNames x);
|
||||
requiredJobs =
|
||||
builtins.listToAttrs (
|
||||
lib.concatMap (nixpkgsVer:
|
||||
let nixpkgsJobs = allJobs.${nixpkgsVer};
|
||||
in lib.concatMap (compiler-nix-name:
|
||||
let ghcJobs = nixpkgsJobs.${compiler-nix-name};
|
||||
in builtins.map (platform: {
|
||||
name = "required-${nixpkgsVer}-${compiler-nix-name}-${platform}";
|
||||
value = genericPkgs.releaseTools.aggregate {
|
||||
name = "haskell.nix-${nixpkgsVer}-${compiler-nix-name}-${platform}";
|
||||
meta.description = "All ${nixpkgsVer} ${compiler-nix-name} ${platform} jobs";
|
||||
constituents = lib.collect (d: lib.isDerivation d) ghcJobs.${platform};
|
||||
};
|
||||
}) (names ghcJobs)
|
||||
) (names nixpkgsJobs)
|
||||
) (names allJobs));
|
||||
in latestJobs // requiredJobs // {
|
||||
required = genericPkgs.releaseTools.aggregate {
|
||||
name = "haskell.nix-required";
|
||||
meta.description = "All jobs required to pass CI";
|
||||
# Hercules will require all of these, we just require the 20.09 jobs
|
||||
# to avoid stressing Hydra too much
|
||||
constituents = lib.collect lib.isDerivation allJobs.R2009.ghc865.linux.native;
|
||||
# Using the names here requires https://github.com/NixOS/hydra/issues/715
|
||||
constituents = builtins.attrNames requiredJobs;
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
@ -35,12 +35,7 @@ in
|
||||
buildCommand = let
|
||||
inherit (packages.test-haddock.components) library;
|
||||
noDocLibrary = packages.stm.components.library;
|
||||
in if (stdenv.hostPlatform != stdenv.buildPlatform)
|
||||
then ''
|
||||
echo "Skipping haddock tests when cross compiling" >& 2
|
||||
touch $out
|
||||
''
|
||||
else ''
|
||||
in ''
|
||||
########################################################################
|
||||
# test haddock
|
||||
|
||||
@ -82,6 +77,8 @@ in
|
||||
touch $out
|
||||
'';
|
||||
|
||||
meta.platforms = platforms.all;
|
||||
meta.disabled = stdenv.hostPlatform.isMusl;
|
||||
meta = {
|
||||
platforms = platforms.all;
|
||||
disabled = stdenv.hostPlatform != stdenv.buildPlatform || stdenv.hostPlatform.isMusl;
|
||||
};
|
||||
} // { inherit packages pkgSet; }
|
||||
|
@ -46,7 +46,11 @@ in recurseIntoAttrs {
|
||||
touch $out
|
||||
'';
|
||||
|
||||
meta.platforms = platforms.all;
|
||||
meta = {
|
||||
platforms = platforms.all;
|
||||
# This test seeems to be broken on 8.6 and 8.8
|
||||
disabled = compiler-nix-name == "ghc865" || compiler-nix-name == "ghc884";
|
||||
};
|
||||
|
||||
passthru = {
|
||||
# Used for debugging with nix repl
|
||||
|
@ -52,7 +52,10 @@ in recurseIntoAttrs {
|
||||
|
||||
'') + "touch $out";
|
||||
|
||||
meta.platforms = platforms.all;
|
||||
meta = {
|
||||
# A dependency is broken on Windows, just run on unix
|
||||
platforms = platforms.unix;
|
||||
};
|
||||
|
||||
passthru = {
|
||||
# Attributes used for debugging with nix repl
|
||||
|
@ -1,5 +1,6 @@
|
||||
{ pkgs, lib, stdenv, haskell-nix, testSrc, zlib, compiler-nix-name } :
|
||||
(haskell-nix.hackage-package {
|
||||
# A dependency is broken on windows, just run this on unix.
|
||||
lib.addMetaAttrs { platforms = lib.platforms.unix; } ((haskell-nix.hackage-package {
|
||||
inherit compiler-nix-name;
|
||||
name = "pandoc";
|
||||
version = "2.9.2.1";
|
||||
@ -10,4 +11,4 @@
|
||||
{ "https://github.com/jgm/pandoc-citeproc"."0.17"
|
||||
= "0dxx8cp2xndpw3jwiawch2dkrkp15mil7pyx7dvd810pwc22pm2q"; }
|
||||
."${location}"."${tag}";
|
||||
}).components.exes.pandoc
|
||||
}).components.exes.pandoc)
|
||||
|
@ -16,19 +16,17 @@ let
|
||||
};
|
||||
|
||||
packages = project.hsPkgs;
|
||||
in recurseIntoAttrs (if stdenv.buildPlatform != stdenv.hostPlatform
|
||||
then
|
||||
let skip = pkgs.runCommand "skip-test-setup-deps" {} ''
|
||||
echo "Skipping setup-deps test when cross compiling as it needs the ghc lib" >& 2
|
||||
touch $out
|
||||
'';
|
||||
in {
|
||||
ifdInputs = { plan-nix = skip; };
|
||||
run = skip;
|
||||
}
|
||||
else {
|
||||
meta = {
|
||||
platforms = platforms.unix;
|
||||
# Building reinstallable lib GHC is broken on 8.10, and we require lib ghc so this won't work with cross-compiling.
|
||||
# Moreover, even building the plan doesn't seem to work in these circumstances.
|
||||
disabled = stdenv.buildPlatform != stdenv.hostPlatform || stdenv.hostPlatform.isMusl || compiler-nix-name == "ghc8102";
|
||||
};
|
||||
in
|
||||
|
||||
recurseIntoAttrs ({
|
||||
ifdInputs = {
|
||||
inherit (project) plan-nix;
|
||||
plan-nix = addMetaAttrs meta project.plan-nix;
|
||||
};
|
||||
run = pkgs.stdenv.mkDerivation {
|
||||
name = "setup-deps-test";
|
||||
@ -42,9 +40,7 @@ in recurseIntoAttrs (if stdenv.buildPlatform != stdenv.hostPlatform
|
||||
touch $out
|
||||
'';
|
||||
|
||||
meta.platforms = platforms.unix;
|
||||
meta.disabled = stdenv.hostPlatform.isMusl || stdenv.hostPlatform.isWindows;
|
||||
|
||||
inherit meta;
|
||||
passthru = {
|
||||
# Attributes used for debugging with nix repl
|
||||
inherit project packages;
|
||||
|
@ -15,23 +15,7 @@ let
|
||||
|
||||
env = project.shellFor {};
|
||||
|
||||
# Making this work for cross compilers will be difficult as setup-deps are
|
||||
# built for the build platform and the shell will be for the host platform.
|
||||
# We probably need a shell that provides both build and host ghc
|
||||
# and corresponding package DBs and a way to use them.
|
||||
# This problem affects musl as well as the build libraries are linked to glibc.
|
||||
in recurseIntoAttrs (if stdenv.buildPlatform != stdenv.hostPlatform
|
||||
then
|
||||
let skip = runCommand "skipping-test-shell-for-setup-deps" {} ''
|
||||
echo "Skipping shell-for-setup-deps test on cross compilers (does not work yet)" >& 2
|
||||
touch $out
|
||||
'';
|
||||
in {
|
||||
ifdInputs = { plan-nix = skip; };
|
||||
env = skip;
|
||||
run = skip;
|
||||
}
|
||||
else {
|
||||
in recurseIntoAttrs ({
|
||||
ifdInputs = {
|
||||
inherit (project) plan-nix;
|
||||
};
|
||||
@ -52,8 +36,15 @@ in recurseIntoAttrs (if stdenv.buildPlatform != stdenv.hostPlatform
|
||||
touch $out
|
||||
'';
|
||||
|
||||
meta.platforms = platforms.all;
|
||||
meta.disabled = stdenv.buildPlatform != stdenv.hostPlatform;
|
||||
meta = {
|
||||
platforms = platforms.all;
|
||||
# Making this work for cross compilers will be difficult as setup-deps are
|
||||
# built for the build platform and the shell will be for the host platform.
|
||||
# We probably need a shell that provides both build and host ghc
|
||||
# and corresponding package DBs and a way to use them.
|
||||
# This problem affects musl as well as the build libraries are linked to glibc.
|
||||
disabled = stdenv.buildPlatform != stdenv.hostPlatform;
|
||||
};
|
||||
|
||||
passthru = {
|
||||
# Used for debugging with nix repl
|
||||
|
@ -64,8 +64,10 @@ in recurseIntoAttrs {
|
||||
touch $out
|
||||
'';
|
||||
|
||||
meta.platforms = platforms.all;
|
||||
meta.disabled = stdenv.hostPlatform.isMusl || stdenv.hostPlatform.isWindows;
|
||||
meta = {
|
||||
platforms = platforms.unix;
|
||||
disabled = stdenv.hostPlatform.isMusl;
|
||||
};
|
||||
|
||||
passthru = {
|
||||
# Used for debugging with nix repl
|
||||
|
@ -82,8 +82,10 @@ in recurseIntoAttrs {
|
||||
touch $out
|
||||
'';
|
||||
|
||||
meta.platforms = platforms.all;
|
||||
meta.disabled = stdenv.hostPlatform.isMusl;
|
||||
meta = {
|
||||
platforms = platforms.all;
|
||||
disabled = stdenv.hostPlatform.isMusl;
|
||||
};
|
||||
|
||||
passthru = {
|
||||
# Used for debugging with nix repl
|
||||
|
Loading…
Reference in New Issue
Block a user