64700ae · haskell.nix · git.iko.soy

git.iko.soy/haskell.nix

on iko/ghcjs-patch-master help sign in

CI: share between ci.nix and release.nix (#504)

64700aeMichael Peyton Jones7 years ago

build.nix+4−13

compare with iko/ghcjs-patch-master

unifiedsplit

@@ -3,25 +3,16 @@

3# It is separate from default.nix because that file is the public API

4# of Haskell.nix, which shouldn't have tests, etc.

6−{ nixpkgs ? ./nixpkgs

6+{ nixpkgs ? (import ./nixpkgs/default.nix)

7−# Provide args to the nixpkgs instantiation.

7+, nixpkgsArgs ? (import ./default.nix)

8−, system ? builtins.currentSystem

8+, pkgs ? (nixpkgs nixpkgsArgs)

9−, crossSystem ? null

10−, config ? {}

11−, nixpkgsArgs ? { inherit system crossSystem; }

12, ifdLevel ? 1000

9, ifdLevel ? 1000

13}:

10}:

15let

12let

16− haskellNixArgs = import ./default.nix;

17− pkgs = import nixpkgs ({

18− config = haskellNixArgs.config // config;

19− inherit (haskellNixArgs) overlays;

20− } // nixpkgsArgs);

21 haskell = pkgs.haskell-nix;

13 haskell = pkgs.haskell-nix;

22−

23in rec {

14in rec {

24− tests = import ./test/default.nix { inherit nixpkgs nixpkgsArgs ifdLevel; };

15+ tests = import ./test/default.nix { inherit pkgs ifdLevel; };

26 # Scripts for keeping Hackage and Stackage up to date, and CI tasks.

17 # Scripts for keeping Hackage and Stackage up to date, and CI tasks.

27 # The dontRecurseIntoAttrs prevents these from building on hydra

18 # The dontRecurseIntoAttrs prevents these from building on hydra

ci-lib.nixnew file+45

@@ -0,0 +1,45 @@

1+let

2+ # Generic nixpkgs, use *only* for lib functions that are stable across versions

3+ pkgs = import ./nixpkgs/default.nix {};

4+ lib = pkgs.lib;

5+in rec {

6+ inherit (import ./dimension.nix) dimension;

8+ # A filter for removing packages that aren't supported on the current platform

9+ # according to 'meta.platforms'.

10+ platformFilterGeneric = pkgs: system:

11+ # This needs to use the correct nixpkgs version so all the systems line up

12+ let lib = pkgs.lib;

13+ platform = lib.systems.elaborate { inherit system; };

14+ # Can't just default to [] for platforms, since no meta.platforms

15+ # means "all platforms" not "no platforms"

16+ in drv : if drv ? meta && drv.meta ? platforms then

17+ lib.any (lib.meta.platformMatch platform) drv.meta.platforms

18+ else true;

19+

20+ # Hydra doesn't like these attributes hanging around in "jobsets": it thinks they're jobs!

21+ stripAttrsForHydra = filterAttrsOnlyRecursive (n: _: n != "recurseForDerivations" && n != "dimension");

22+

23+ # Keep derivations and attrsets with 'recurseForDerivations'. This ensures that we match the

24+ # derivations that Hercules will see, and prevents Hydra from trying to pick up all sorts of bad stuff

25+ # (like attrsets that contain themselves!).

26+ filterDerivations = filterAttrsOnlyRecursive (n: attrs: lib.isDerivation attrs || attrs.recurseForDerivations or false);

27+

28+ # A version of 'filterAttrsRecursive' that doesn't recurse into derivations. This prevents us from going into an infinite

29+ # loop with the 'out' attribute on derivations.

30+ # TODO: Surely this shouldn't be necessary. I think normal 'filterAttrsRecursive' will effectively cause infinite loops

31+ # if you keep derivations and your predicate forces the value of the attribute, as this then triggers a loop on the

32+ # 'out' attribute. Weird.

33+ filterAttrsOnlyRecursive = pred: set:

34+ lib.listToAttrs (

35+ lib.concatMap (name:

36+ let v = set.${name}; in

37+ if pred name v then [

38+ (lib.nameValuePair name (

39+ if builtins.isAttrs v && !lib.isDerivation v then filterAttrsOnlyRecursive pred v

40+ else v

41+ ))

42+ ] else []

43+ ) (builtins.attrNames set)

44+ );

45+}

ci.nix+51−27

@@ -1,44 +1,68 @@

1−# these are patched nixpkgs that include the following PRs:

1+# 'supportedSystems' restricts the set of systems that we will evaluate for. Useful when you're evaluting

2−# - https://github.com/NixOS/nixpkgs/pull/71216

2+# on a machine with e.g. no way to build the Darwin IFDs you need!

3−# - https://github.com/NixOS/nixpkgs/pull/68398

3+{ supportedSystems ? [ "x86_64-linux" "x86_64-darwin" ]

4−

4+, ifdLevel ? 3

5+# Whether or not we are evaluating in restricted mode. This is true in Hydra, but not in Hercules.

6+, restrictEval ? false }:

5let

7let

6− inherit (import ./dimension.nix) dimension;

8+ inherit (import ./ci-lib.nix) dimension platformFilterGeneric filterAttrsOnlyRecursive;

7 nixpkgsVersions = {

9 nixpkgsVersions = {

8−# "release-18.09" = builtins.fetchTarball "https://github.com/input-output-hk/nixpkgs/archive/7e4dcacbf066a8e2d12693a9de1fb30c77081c5d.tar.gz";

10+ "R1903" = "release-19.03";

9− "release-19.03" = builtins.fetchTarball "https://github.com/input-output-hk/nixpkgs/archive/a8f81dc037a5977414a356dd068f2621b3c89b60.tar.gz";

11+ "R1909" = "release-19.09";

10− # "release-19.09" = builtins.fetchTarball "https://github.com/input-output-hk/nixpkgs/archive/3d623a406cec9052ae0a16a79ce3ce9de11236bb.tar.gz";

11 };

12 };

12− systems = {

13+ systems = nixpkgs: nixpkgs.lib.filterAttrs (_: v: builtins.elem v supportedSystems) {

14+ # I wanted to take these from 'lib.systems.examples', but apparently there isn't one for linux!

13− "x86_64-linux" = {};

15+ linux = "x86_64-linux";

16+ darwin = "x86_64-darwin";

14 };

17 };

15− crossSystems = {

18+ crossSystems = nixpkgsName: nixpkgs: system:

16− "x86_64-pc-mingw32" = {};

19+ # We need to use the actual nixpkgs version we're working with here, since the values

20+ # of 'lib.systems.examples' are not understood between all versions

21+ let lib = nixpkgs.lib;

22+ in lib.optionalAttrs (system == "x86_64-linux" || nixpkgsName == "R1903") {

23+ # Windows cross compilation is currently broken on macOS for nixpkgs 19.09 (works on 19.03)

24+ inherit (lib.systems.examples) mingwW64;

25+ } // lib.optionalAttrs (system == "x86_64-linux") {

26+ # Musl cross only works on linux

27+ # aarch64 cross only works on linux

28+ inherit (lib.systems.examples) musl64 aarch64-multiplatform;

17 };

29 };

18 haskellNixArgs = import ./.;

30 haskellNixArgs = import ./.;

19in

31in

20−dimension "Nixpkgs version" nixpkgsVersions (nixpkgsName: nixpkgsSrc:

32+dimension "Nixpkgs version" nixpkgsVersions (nixpkgsName: nixpkgs-pin:

33+ # We need this for generic nixpkgs stuff at the right version

34+ let genericPkgs = import ./nixpkgs { inherit nixpkgs-pin; };

21− dimension "System" systems (system: _:

35+ in dimension "System" (systems genericPkgs) (systemName: system:

36+ let pkgs = import ./nixpkgs (haskellNixArgs // { inherit nixpkgs-pin system; });

37+ build = import ./build.nix { inherit pkgs ifdLevel; };

38+ platformFilter = platformFilterGeneric pkgs system;

39+ blacklisted = n:

40+ # update-hackage accesses the hackage index at eval time (!), which doesn't work in restricted mode

41+ # https://github.com/input-output-hk/haskell.nix/issues/507

42+ (restrictEval && (n == "update-hackage")) ;

43+ in filterAttrsOnlyRecursive (n: v: !(blacklisted n) && platformFilter v) {

22− # Native builds

44+ # Native builds

23− # TODO: can we merge this into the general case by picking an appropriate "cross system" to mean native?

45+ # TODO: can we merge this into the general case by picking an appropriate "cross system" to mean native?

24− let pkgs = import nixpkgsSrc (haskellNixArgs // { inherit system; });

46+ native = pkgs.recurseIntoAttrs {

25− in pkgs.recurseIntoAttrs {

47+ inherit (build) tests maintainer-scripts maintainer-script-cache;

26− hello = (pkgs.haskell-nix.hackage-package { name = "hello"; version = "1.0.0.2";}).components.exes.hello;

48+ hello = (pkgs.haskell-nix.hackage-package { name = "hello"; version = "1.0.0.2"; }).components.exes.hello;

27− iserv-proxy = pkgs.ghc-extra-packages.ghc865.iserv-proxy.components.exes.iserv-proxy;

49+ iserv-proxy = pkgs.ghc-extra-packages.ghc865.iserv-proxy.components.exes.iserv-proxy;

28− ghc = pkgs.recurseIntoAttrs pkgs.haskell-nix.compiler;

50+ ghc = pkgs.recurseIntoAttrs pkgs.haskell-nix.compiler;

29− tests = pkgs.lib.optionalAttrs (system == "x86_64-linux") (import ./test { inherit pkgs; });

51+ };

30 }

52 }

31 //

53 //

32− dimension "Cross system" crossSystems (crossSystem: _:

54+ dimension "Cross system" (crossSystems nixpkgsName genericPkgs system) (crossSystemName: crossSystem:

33 # Cross builds

55 # Cross builds

34− let pkgs = import nixpkgsSrc (haskellNixArgs // { inherit system; crossSystem.config = crossSystem; });

56+ let pkgs = import ./nixpkgs (haskellNixArgs // { inherit nixpkgs-pin system crossSystem; });

57+ build = import ./build.nix { inherit pkgs ifdLevel; };

35 in pkgs.recurseIntoAttrs {

58 in pkgs.recurseIntoAttrs {

36− hello = (pkgs.haskell-nix.hackage-package { name = "hello"; version = "1.0.0.2";}).components.exes.hello;

59+ hello = (pkgs.haskell-nix.hackage-package { name = "hello"; version = "1.0.0.2"; }).components.exes.hello;

37−

38 iserv-proxy = pkgs.ghc-extra-packages.ghc865.iserv-proxy.components.exes.iserv-proxy;

60 iserv-proxy = pkgs.ghc-extra-packages.ghc865.iserv-proxy.components.exes.iserv-proxy;

39−

40 remote-iserv = pkgs.ghc-extra-packages.ghc865.remote-iserv.components.exes.remote-iserv;

61 remote-iserv = pkgs.ghc-extra-packages.ghc865.remote-iserv.components.exes.remote-iserv;

41 }

62 }

63+ //

64+ # Tests are broken on aarch64 cross https://github.com/input-output-hk/haskell.nix/issues/513

65+ pkgs.lib.optionalAttrs (crossSystemName != "aarch64-multiplatform") build.tests

42 )

66 )

43 )

67 )

44)

68)

release.nix+12−70

@@ -1,81 +1,23 @@

1+# 'supportedSystems' restricts the set of systems that we will evaluate for. Useful when you're evaluting

2+# on a machine with e.g. no way to build the Darwin IFDs you need!

1{ supportedSystems ? [ "x86_64-linux" "x86_64-darwin" ]

3{ supportedSystems ? [ "x86_64-linux" "x86_64-darwin" ]

2−, scrubJobs ? true

3−, haskell-nix ? { outPath = ./.; rev = "abcdef"; }

4−, nixpkgsArgs ? {}

5−, ifdLevel ? 3

4+, ifdLevel ? 3 }:

6−}:

8−let defaultNixpkgs = import ./nixpkgs {}; in

9−with defaultNixpkgs.lib;

10let

6let

11− # Remove tests which have meta.disabled = true

7+ inherit (import ./ci-lib.nix) stripAttrsForHydra filterDerivations;

12− filterTests = let

8+ genericPkgs = import ./nixpkgs {};

13− nonEmpty = attrs: length (attrValues attrs) != 0;

9+ lib = genericPkgs.lib;

14− removeDisabled = filterAttrs (system: test: !(test.meta.disabled or false));

10+ ci = import ./ci.nix { inherit supportedSystems ifdLevel; restrictEval = true; };

15− in jobs: jobs // {

11+ allJobs = stripAttrsForHydra (filterDerivations ci);

16− tests = filterAttrs (_: nonEmpty) (mapAttrs (name: removeDisabled) jobs.tests);

17− };

18−

19− jobs = nixpkgs-pin:

20− # Now we know what nixpkgs-pin we are testing use that one.

21− # This fixes and issue where the vendor has changed from `x86_64-pc-mingw32`

22− # to `x86_64-w64-mingw32` in 19.09.

23− let pinnedNixpkgs = import ./nixpkgs { inherit nixpkgs-pin; }; in

24− with pinnedNixpkgs.lib;

25− let

26− inherit (systems.examples) musl64 mingwW64 aarch64-multiplatform raspberryPi;

27− packages = supportedSystems:

28− with (import (pinnedNixpkgs.path + "/pkgs/top-level/release-lib.nix") {

29− inherit supportedSystems scrubJobs nixpkgsArgs;

30− packageSet = {

31− system ? builtins.currentSystem

32− , crossSystem ? null

33− , nixpkgsArgs ? { inherit system crossSystem; }

34− , ...}@args:

35− import (haskell-nix + /build.nix) (args // {

36− nixpkgsArgs = nixpkgsArgs // {

37− inherit nixpkgs-pin;

38− };

39− inherit ifdLevel;

40− });

41− });

42− {

43− mapTestOn = mapTestOn (packagePlatforms pkgs);

44− mapTestOnCross = p: mapTestOnCross p (packagePlatforms pkgs);

45− };

46− in

47− {

48− native = filterTests ((packages supportedSystems).mapTestOn);

49− # Musl cross compile does not work on macOS

50− "${musl64.config}" = filterTests ((packages (filter (x: x == "x86_64-linux") supportedSystems)).mapTestOnCross musl64);

51− # Windows cross compilation is currently broken on macOS for nixpkgs 19.09 (works on 19.03)

52− "${mingwW64.config}" = filterTests ((packages (filter

53− (x: x == "x86_64-linux" || nixpkgs-pin == "release-19.03") supportedSystems)).mapTestOnCross mingwW64);

54− "${aarch64-multiplatform.config}" = filterTests ((packages (filter (x: x == "x86_64-linux") supportedSystems))

55− .mapTestOnCross aarch64-multiplatform);

56− # "${raspberryPi.config}" = filterTests ((packages (filter (x: x == "x86_64-linux") supportedSystems))

57− # .mapTestOnCross raspberryPi);

58− };

59−

60− allJobs =

61− builtins.mapAttrs (_: nixpkgs-pin: jobs nixpkgs-pin) {

62− "R1903" = "release-19.03";

63− "R1909" = "release-19.09";

64− };

65−

66in allJobs // {

12in allJobs // {

67 # On IOHK Hydra, "required" is a special job that updates the

13 # On IOHK Hydra, "required" is a special job that updates the

68 # GitHub CI status.

14 # GitHub CI status.

69− required = defaultNixpkgs.releaseTools.aggregate {

15+ required = genericPkgs.releaseTools.aggregate {

70 name = "haskell.nix-required";

16 name = "haskell.nix-required";

71 meta.description = "All jobs required to pass CI";

17 meta.description = "All jobs required to pass CI";

72− constituents =

18+ # Hercules will require all of these, we just require the 1909 jobs

73− collect isDerivation allJobs.R1903.native

19+ # to avoid stressing Hydra too much

74− ++ collect isDerivation allJobs.R1909.native

75− ++ collect isDerivation allJobs.R1903.x86_64-pc-mingw32

76− ++ collect isDerivation allJobs.R1909.x86_64-w64-mingw32

20+ constituents = lib.collect lib.isDerivation allJobs.R1909.linux.native;

77− ++ collect isDerivation allJobs.R1903.x86_64-unknown-linux-musl

78− ++ collect isDerivation allJobs.R1909.x86_64-unknown-linux-musl;

79 };

21 };

80 }

22 }

scripts/update-docs.nix+3−2

@@ -7,7 +7,8 @@ let

7 repo = "git@github.com:input-output-hk/haskell.nix.git";

8 sshKey = "/run/keys/buildkite-haskell-dot-nix-ssh-private";

9in

10+ # update-docs depends on glibc which doesn't build on darwin

10− writeScript "update-docs.sh" ''

11+ meta.addMetaAttrs { platforms = platforms.linux; } (writeScript "update-docs.sh" ''

11 #!${stdenv.shell}

12 #!${stdenv.shell}

13 set -euo pipefail

14 set -euo pipefail

@@ -46,4 +47,4 @@ in

46 if [ "''${BUILDKITE_BRANCH:-}" = master ]; then

47 if [ "''${BUILDKITE_BRANCH:-}" = master ]; then

47 git push ${repo} HEAD:gh-pages

48 git push ${repo} HEAD:gh-pages

48 fi

49 fi

49− ''

50+ '')

test/cabal-source-repo/default.nix+1−1

@@ -28,7 +28,7 @@ in recurseIntoAttrs {

29 passthru = {

30 # Attributes used for debugging with nix repl

31− inherit pkgSet packages;

31+ inherit packages;

32 };

33 };

34}

test/cabal-sublib/default.nix+1−1

@@ -59,7 +59,7 @@ in recurseIntoAttrs {

60 passthru = {

61 # Used for debugging with nix repl

62− inherit pkgSet packages;

62+ inherit packages;

64 # Used for testing externally with nix-shell (../tests.sh).

65 # This just adds cabal-install to the existing shells.

test/call-stack-to-nix/default.nix+1−1

@@ -36,4 +36,4 @@ in recurseIntoAttrs {

36 inherit pkgSet packages;

37 };

38 };

39−}

39+}

test/exe-only/default.nix+1−1

@@ -52,7 +52,7 @@ in recurseIntoAttrs {

53 passthru = {

54 # Used for debugging with nix repl

55− inherit pkgSet packages;

55+ inherit packages;

57 # Used for testing externally with nix-shell (../tests.sh).

58 # This just adds cabal-install to the existing shells.