Test setup deps issue depending on ghc and Cabal (#278)

Projects like asterius and leksah that include dependencies on ghc. Sometimes the are needed by a setup.hs file (as is the case for the binaryen package used by asterius). The recent overlays merge broke this and #279 was needed to allow reinstallableLibGhc to be used to fix it. This PR adds a test case to make sure it is not inadvertently broken again. Also gives us a place to try out better ways to make this type of package work.* Test setup deps issue depending on ghc and Cabal

* Uses nixpkgs 19.09 + macOS wine fix on hydra/buildkite

* Enables macOS hydra tests

* disables musl tests for now

* Adds haskellNixRoots to tests

* Uses reinstallableLibGhc to fix the new setup-depends test

* Adds meta.platforms to haskell.nix components

* Set meta.platforms on cabal-install

* Builds some maintainer-scripts on hydra

* Pins hackage index-state for ghc-extra-packages

build.nix

@@ -20,17 +20,20 @@ let
         darcs = (self.haskell-nix.hackage-package {
           name = "darcs";
           version = "2.14.2";
+          index-state = "2019-10-28T00:00:00Z";
           # Apply the latest darcs.net Setup.hs patches
           modules = [{packages.darcs.patches = [ ./patches/darcs-setup.patch ];}];
         }).components.exes.darcs;
       })]; } // nixpkgsArgs);
   haskell = pkgs.haskell-nix;
 
-in {
-  inherit (haskell) haskellNixRoots;
+in rec {
   tests = import ./test/default.nix { inherit nixpkgs nixpkgsArgs; };
 
   # Scripts for keeping Hackage and Stackage up to date, and CI tasks.
+  # The dontRecurseIntoAttrs prevents these from building on hydra
+  # as not all of them can work in restricted eval mode (as they
+  # are not pure).
   maintainer-scripts = pkgs.dontRecurseIntoAttrs {
     update-hackage = haskell.callPackage ./scripts/update-hackage.nix {};
     update-stackage = haskell.callPackage ./scripts/update-stackage.nix {};
@@ -51,4 +54,14 @@ in {
     check-hydra = haskell.callPackage ./scripts/check-hydra.nix {};
     check-closure-size = haskell.callPackage ./scripts/check-closure-size.nix {};
   };
+
+  # These are pure parts of maintainer-script so they can be built by hydra
+  # and added to the cache to speed up buildkite.
+  maintainer-script-cache = pkgs.recurseIntoAttrs {
+    inherit (maintainer-scripts) update-docs check-hydra check-closure-size;
+    # Some of the dependencies of the impure scripts so that they will
+    # will be in the cache too for buildkite.
+    inherit (pkgs) coreutils glibc git openssh cabal-install nix-prefetch-git;
+    inherit (haskell) nix-tools;
+  };
 }

builder/comp-builder.nix

@@ -146,6 +146,7 @@ stdenv.mkDerivation ({
         license-map = import ../lib/cabal-licenses.nix lib;
       in license-map.${package.license} or
         (builtins.trace "WARNING: license \"${package.license}\" not found" license-map.LicenseRef-OtherLicense);
+    platforms = if component.platforms == null then stdenv.lib.platforms.all else component.platforms;
   };
 
   LANG = "en_US.UTF-8";         # GHC needs the locale configured during the Haddock phase.

builder/setup-builder.nix

@@ -41,6 +41,17 @@ in
         inherit configFiles cleanSrc;
       };
 
+      meta = {
+        homepage = package.homepage;
+        description = package.synopsis;
+        license =
+          let
+            license-map = import ../lib/cabal-licenses.nix lib;
+          in license-map.${package.license} or
+            (builtins.trace "WARNING: license \"${package.license}\" not found" license-map.LicenseRef-OtherLicense);
+        platforms = if component.platforms == null then stdenv.lib.platforms.all else component.platforms;
+      };
+
       phases = ["unpackPhase" "patchPhase" "buildPhase" "installPhase"];
       buildPhase = ''
         if [[ ! -f ./Setup.hs  && ! -f ./Setup.lhs ]]; then

compiler/bootstrap/cabal-install.nix

@@ -41,6 +41,10 @@ in stdenv.mkDerivation ({
       inherit version;
   };
 
+  meta = {
+    platforms = stdenv.lib.platforms.all;
+  };
+
   nativeBuildInputs = [ ghc zlib ];
 
   inherit src;

modules/package.nix

@@ -210,6 +210,10 @@ in {
             type = listOfFilteringNulls unspecified;
             default = [];
           };
+          platforms = mkOption {
+            type = nullOr (listOfFilteringNulls unspecified);
+            default = null;
+          };
         };
       };
     in {
@@ -232,6 +236,7 @@ in {
           cxxSources = [];
           jsSources = [];
           extraSrcFiles = [ "Setup.hs" "Setup.lhs" ];
+          platforms = null;
         };
       };
       library = mkOption {

nixpkgs/github.json

@@ -1,7 +1,7 @@
 {
-  "url": "https://github.com/NixOS/nixpkgs",
-  "rev": "5000b1478a11acbff7d95519c7300f3e6691885d",
-  "date": "2019-10-13T18:41:59-04:00",
-  "sha256": "0yyhkf48b15dcnq0gbiyi4ypjn09a0mpy6v7yilidwqydmkc0wxr",
+  "url": "https://github.com/input-output-hk/nixpkgs",
+  "rev": "3d623a406cec9052ae0a16a79ce3ce9de11236bb",
+  "date": "2019-10-18T17:51:24+13:00",
+  "sha256": "1a4yd980ylh1ygvmws15wldifz7jxjy8z07vc914kam1p25yjpj1",
   "fetchSubmodules": false
 }

overlays/ghc-packages.nix

@@ -84,6 +84,7 @@ in rec {
   ghc-extra-projects = builtins.mapAttrs (name: proj: self.haskell-nix.cabalProject' {
       name = "ghc-extra-packages";
       src = proj;
+      index-state = "2019-10-31T00:00:00Z";
       ghc = self.buildPackages.haskell.compiler.${name};
     })
     ghc-extra-pkgs-cabal-projects;

release.nix

@@ -1,4 +1,4 @@
-{ supportedSystems ? [ "x86_64-linux" ] # spare our "x86_64-darwin" builders for now
+{ supportedSystems ? [ "x86_64-linux" "x86_64-darwin" ]
 , scrubJobs ? true
 , haskell-nix ? { outPath = ./.; rev = "abcdef"; }
 , nixpkgsArgs ? {}
@@ -24,7 +24,8 @@ let
 
   jobs = {
     native = filterTests (mapTestOn (packagePlatforms pkgs));
-    "${musl64.config}" = filterTests (mapTestOnCross musl64 (packagePlatforms pkgs));
+    # Disabled for now. Something is wrong and this would require `allowBroken`
+    # "${musl64.config}" = filterTests (mapTestOnCross musl64 (packagePlatforms pkgs));
   } // {
     # On IOHK Hydra, "required" is a special job that updates the
     # GitHub CI status.

scripts/check-hydra.nix

@@ -13,6 +13,7 @@ writeScript "check-hydra.sh" ''
   command time --format '%e' -o eval-time.txt \
       hydra-eval-jobs \
       --option allowed-uris "https://github.com/NixOS https://github.com/input-output-hk" \
+      --arg supportedSystems '[ builtins.currentSystem ]' \
       -I . release.nix > eval.json
   EVAL_EXIT_CODE="$?"
   if [ "$EVAL_EXIT_CODE" != 0 ]

test/default.nix

@@ -1,5 +1,5 @@
 { pkgs ? import nixpkgs ((import ../.) // nixpkgsArgs)
-, nixpkgs ? <nixpkgs>
+, nixpkgs ? ../nixpkgs
 , nixpkgsArgs ? { }
 }:
 
@@ -8,6 +8,7 @@ with pkgs;
 let
   util = callPackage ./util.nix {};
 in pkgs.recurseIntoAttrs {
+  inherit (haskell-nix) haskellNixRoots;
   cabal-simple = haskell-nix.callPackage ./cabal-simple { inherit util; };
   cabal-simple-prof = haskell-nix.callPackage ./cabal-simple-prof { inherit util; };
   cabal-sublib = haskell-nix.callPackage ./cabal-sublib { inherit util; };
@@ -18,6 +19,7 @@ in pkgs.recurseIntoAttrs {
   snapshots = haskell-nix.callPackage ./snapshots {};
   shell-for = haskell-nix.callPackage ./shell-for {};
   shell-for-setup-deps = haskell-nix.callPackage ./shell-for-setup-deps {};
+  setup-deps = import ./setup-deps { inherit pkgs; };
   callStackToNix = haskell-nix.callPackage ./call-stack-to-nix {};
   callCabalProjectToNix = haskell-nix.callPackage ./call-cabal-project-to-nix {};
   cabal-source-repo = haskell-nix.callPackage ./cabal-source-repo {};

test/setup-deps/cabal.project

@@ -0,0 +1,3 @@
+index-state: 2019-10-01T00:00:00Z
+
+packages: pkg

test/setup-deps/default.nix

@@ -0,0 +1,39 @@
+{ pkgs }:
+
+with pkgs;
+with stdenv.lib;
+
+let
+  project = haskell-nix.cabalProject' {
+    src = ./.;
+    modules = [{
+      # Package has no exposed modules which causes
+      #   haddock: No input file(s)
+      packages.bytestring-builder.doHaddock = false;
+    }
+    { reinstallableLibGhc = true; } ];
+  };
+
+  packages = project.hsPkgs;
+in recurseIntoAttrs {
+  plan = haskell-nix.withInputs project.plan-nix;
+  run = pkgs.stdenv.mkDerivation {
+    name = "call-cabal-project-to-nix-test";
+
+    buildCommand = ''
+      exe="${packages.pkg.components.exes.pkg}/bin/pkg"
+
+      printf "checking whether executable runs... " >& 2
+      $exe
+
+      touch $out
+    '';
+
+    meta.platforms = platforms.all;
+
+    passthru = {
+      # Attributes used for debugging with nix repl
+      inherit project packages;
+    };
+  };
+}

test/setup-deps/pkg/Setup.hs

@@ -0,0 +1,27 @@
+import Distribution.Simple
+
+-- https://github.com/snoyberg/conduit#readme
+
+import Conduit
+import System.Directory (removeFile)
+
+-- Make sure Cabal and ghc were included
+import Distribution.Package ()
+import Ar ()
+
+example = do
+    -- Pure operations: summing numbers.
+    print $ runConduitPure $ yieldMany [1..10] .| sumC
+
+    -- Exception safe file access: copy a file.
+    writeFile "input.txt" "This is a test." -- create the source file
+    runConduitRes $ sourceFileBS "input.txt" .| sinkFile "output.txt" -- actual copying
+    readFile "output.txt" >>= putStrLn -- prove that it worked
+
+    -- Perform transformations.
+    print $ runConduitPure $ yieldMany [1..10] .| mapC (+ 1) .| sinkList
+
+    removeFile "input.txt"
+    removeFile "output.txt"
+
+main = defaultMain

test/setup-deps/pkg/app/Main.hs

@@ -0,0 +1,6 @@
+module Main where
+
+import qualified Data.Text.IO as T
+
+main :: IO ()
+main = return ()

test/setup-deps/pkg/pkg.cabal

@@ -0,0 +1,36 @@
+cabal-version:       2.2
+-- Initial package description 'pkgb.cabal' generated by 'cabal init'.  For
+--  further documentation, see http://haskell.org/cabal/users-guide/
+
+name:                pkg
+version:             0.1.0.0
+-- synopsis:
+-- description:
+-- bug-reports:
+license:             LicenseRef-PublicDomain
+author:              Rodney Lorrimar
+maintainer:          rodney.lorrimar@iohk.io
+category:            Testing
+
+custom-setup
+  setup-depends:       conduit
+                     , conduit-extra
+                     , Cabal
+                     , ghc
+
+library
+  exposed-modules:     Pkg
+  build-depends:       base
+  hs-source-dirs:      src
+  default-language:    Haskell2010
+  build-tools: alex, happy
+
+executable pkg
+  main-is:             Main.hs
+  -- other-modules:
+  -- other-extensions:
+  build-depends:       base
+                     , optparse-applicative
+                     , text
+  hs-source-dirs:      app
+  default-language:    Haskell2010

test/setup-deps/pkg/src/conduit-test.hs

@@ -0,0 +1,5 @@
+module Main where
+
+import Conduit
+
+main = return ()