External hackages (#535)

* Add support for external Hackage repositories

Currently haskell.nix is not able to build Cabal projects that depend on
packages from private Hackage repositories, as it make only main Hackage
available to cabal. This is unfortunate.

This commit adds this functionality, by allowing the user to pass
`extra-hackages` and `extra-hackage-tarballs` to `mkPkgSet` and
`callCabalToNix` respectively, to add as much extra repositories as
needed.

This repositories are first made available to Cabal when calling
`v2-configure`, resulting in correct plans. Later they are combined with
global Hackage when building dependencies of the local packages.

* Use cabal.project.freeze if available

Currently callCabalProjectToNix does not copy `cabal.project.freeze`
from source directory, leading to different build plans when building
components with nix and when building project with `cabal new-build`
inside `nix-shell`.

This behavior is undesired, so this commits fixes it.

* Add tests for extra-hackages functionality

Co-authored-by: Moritz Angermann <moritz.angermann@gmail.com>

lib/call-cabal-project-to-nix.nix

@@ -23,6 +23,7 @@
                      #     { "https://github.com/jgm/pandoc-citeproc"."0.17"
                      #         = "0dxx8cp2xndpw3jwiawch2dkrkp15mil7pyx7dvd810pwc22pm2q"; }
                      #       ."${repo.location}"."${repo.tag}";
+, extra-hackage-tarballs ? []
 , ...
 }@args:
 # cabal-install versions before 2.4 will generate insufficient plan information.
@@ -41,7 +42,7 @@ let
       inherit src;
       filter = path: type:
         type == "directory" ||
-        pkgs.lib.any (i: (pkgs.lib.hasSuffix i path)) [ ".project" ".cabal" "package.yaml" ]; }
+        pkgs.lib.any (i: (pkgs.lib.hasSuffix i path)) [ ".project" ".cabal" ".freeze" "package.yaml" ]; }
     else src;
 
   # Using origSrcSubDir bypasses any cleanSourceWith so that it will work when
@@ -225,7 +226,7 @@ let
     export SSL_CERT_FILE=${cacert}/etc/ssl/certs/ca-bundle.crt
     export GIT_SSL_CAINFO=${cacert}/etc/ssl/certs/ca-bundle.crt
     HOME=${dotCabal {
-      inherit cabal-install nix-tools;
+      inherit cabal-install nix-tools extra-hackage-tarballs;
       index-state =
         builtins.trace ("Using index-state: ${index-state-found}" + (if name == null then "" else " for " + name))
           index-state-found;

mk-local-hackage-repo/default.nix

@@ -14,10 +14,10 @@
 # nix would be pointless as we'd have to hardcode them to produce the same output
 # reproducably.
 #
-{ pkgs, hackageTarball }:
-{ index-state, sha256, ... }@args:
-let index = hackageTarball args; in
-pkgs.runCommand "hackage-repo-${builtins.replaceStrings [":"] [""] index-state}" { nativeBuildInputs = [ pkgs.buildPackages.nix ]; } ''
+pkgs:
+{ name, index }:
+
+pkgs.runCommand "hackage-repo-${name}" { nativeBuildInputs = [ pkgs.buildPackages.nix ]; } ''
 mkdir -p $out
 export expires="4000-01-01T00:00:00Z"
 

overlays/haskell.nix

@@ -70,13 +70,19 @@ self: super: {
             { pkg-def  # Base package set. Either from stackage (via stack-to-nix) or from a cabal projects plan file (via plan-to-nix)
             , pkg-def-extras ? [] # Additional packages to augment the Base package set `pkg-def` with.
             , modules ? []
+            , extra-hackages ? [] # Extra Hackage repositories to use besides main one.
             }@args:
 
-            import ../package-set.nix (args // {
+            let
+              hackageAll = builtins.foldl' (base: extra: base // extra) hackage extra-hackages;
+            in
+
+            import ../package-set.nix {
+                inherit (args) pkg-def pkg-def-extras;
                 modules = defaultModules ++ modules;
                 pkgs = self;
-                inherit hackage pkg-def;
-            });
+                hackage = hackageAll;
+            };
 
         # Some boot packages (libiserv) are in lts, but not in hackage,
         # so we should not try to get it from hackage based on the stackage
@@ -128,6 +134,7 @@ self: super: {
             { plan-pkgs  # Path to the output of plan-to-nix
             , pkg-def-extras ? []
             , modules ? []
+            , extra-hackages ? []
             }@args:
 
             let
@@ -146,6 +153,7 @@ self: super: {
                 modules = [ { doExactConfig = true; } patchesModule ]
                        ++ modules
                        ++ plan-pkgs.modules or [];
+                inherit extra-hackages;
             };
 
         # Package sets for all stackage snapshots.
@@ -169,33 +177,60 @@ self: super: {
         nix-tools = self.buildPackages.haskell-nix.nix-tools-cross-compiled;
         # TODO perhaps there is a cleaner way to get a suitable nix-tools.
 
-            # Produce a fixed output derivation from a moving target (hackage index tarball)
+        # Produce a fixed output derivation from a moving target (hackage index tarball)
+        # Takes desired index-state and sha256 and produces a set { name, index }, where
+        # index points to "01-index.tar.gz" file downloaded from hackage.haskell.org.
         hackageTarball = { index-state, sha256, nix-tools ? self.haskell-nix.nix-tools, ... }:
             assert sha256 != null;
-            self.fetchurl {
-                name = "01-index.tar.gz-at-${builtins.replaceStrings [":"] [""] index-state}";
+            let at = builtins.replaceStrings [":"] [""] index-state; in
+            { name = "hackage.haskell.org-at-${at}";
+              index = self.fetchurl {
+                name = "01-index.tar.gz-at-${at}";
                 url = "https://hackage.haskell.org/01-index.tar.gz";
                 downloadToTemp = true;
                 postFetch = "${nix-tools}/bin/truncate-index -o $out -i $downloadedFile -s ${index-state}";
 
                 outputHashAlgo = "sha256";
                 outputHash = sha256;
+              };
             };
 
-        mkLocalHackageRepo = import ../mk-local-hackage-repo { inherit hackageTarball; pkgs = self; };
+        # Creates Cabal local repository from { name, index } set.
+        mkLocalHackageRepo = import ../mk-local-hackage-repo self;
 
-        dotCabal = { index-state, sha256, cabal-install, ... }@args:
-            self.runCommand "dot-cabal-at-${builtins.replaceStrings [":"] [""] index-state}" { nativeBuildInputs = [ cabal-install ]; } ''
+        dotCabal = { index-state, sha256, cabal-install, extra-hackage-tarballs ? [], ... }@args:
+            let
+              allTarballs = [ (hackageTarball args) ] ++ extra-hackage-tarballs;
+              allNames = self.lib.concatMapStringsSep "-" (tarball: tarball.name) allTarballs;
+              # Main Hackage index-state is embedded in its name and thus will propagate to
+              # dotCabalName anyway.
+              dotCabalName = "dot-cabal-" + allNames;
+            in
+            self.runCommand dotCabalName { nativeBuildInputs = [ cabal-install ]; } ''
                 mkdir -p $out/.cabal
                 cat <<EOF > $out/.cabal/config
-                repository cached
-                    url: file:${mkLocalHackageRepo args}
-                    secure: True
-                    root-keys:
-                    key-threshold: 0
+                ${self.lib.concatStrings (
+                  map (tarball:
+                ''
+                repository ${tarball.name}
+                  url: file:${mkLocalHackageRepo tarball}
+                  secure: True
+                  root-keys:
+                  key-threshold: 0
+
+                '') allTarballs
+                )}
                 EOF
-                mkdir -p $out/.cabal/packages/cached
-                HOME=$out cabal new-update cached
+
+                # All repositories must be mkdir'ed before calling new-update on any repo,
+                # otherwise it fails.
+                ${self.lib.concatStrings (map ({ name, ... }: ''
+                  mkdir -p $out/.cabal/packages/${name}
+                '') allTarballs)}
+
+                ${self.lib.concatStrings (map ({ name, ... }: ''
+                  HOME=$out cabal new-update ${name}
+                '') allTarballs)}
             '';
 
         # Some of features of haskell.nix rely on using a hackage index
@@ -429,6 +464,7 @@ self: super: {
                   pkg-def-extras = args.pkg-def-extras or [];
                   modules = (args.modules or [])
                           ++ self.lib.optional (args ? ghc) { ghc.package = args.ghc; };
+                  extra-hackages = args.extra-hackages or [];
                 };
             in { inherit (pkg-set.config) hsPkgs; inherit pkg-set; plan-nix = plan.nix; };
 

test/default.nix

@@ -175,6 +175,7 @@ let
     exe-only = callTest ./exe-only { inherit util; };
     stack-source-repo = callTest ./stack-source-repo {};
     lookup-sha256 = callTest ./lookup-sha256 {};
+    extra-hackage = callTest ./extra-hackage {};
 
     unit = unitTests;
   };

test/extra-hackage/README.md

@@ -0,0 +1,11 @@
+# Tests for extra Hackage functionality
+
+This directory contains two packages, `external-package-demo` and `external-package-user`, the
+second one depends on the first one. Both packages were created with `cabal init`.
+
+`external-package-demo` was uploaded to local Hackage at `localhost` and `01-index.tar.gz` from that
+Hackage was downloaded to this directory. Then the index file was processed with `hackage-to-nix`,
+the result is in `hackage/` directory.
+
+The tests check that `cabalProject'` is able to construct plan with dependencies from extra Hackage
+and then build the package itself.

test/extra-hackage/default.nix

@@ -0,0 +1,69 @@
+{ stdenv, cabalProject', haskellLib, recurseIntoAttrs, testSrc }:
+
+with stdenv.lib;
+
+let  
+  
+  hackage = import ./hackage;
+
+  tarball = {
+    name = "extra-hackage-demo";
+    index = ./01-index.tar.gz;
+  };
+
+  demo-src = ./external-package-demo-0.1.0.0.tar.gz;
+
+  project = cabalProject' {
+    src = testSrc "extra-hackage/external-package-user";
+
+    extra-hackages = [ hackage ];
+    extra-hackage-tarballs = [ tarball ];
+
+    modules = [
+      # To prevent nix-build from trying to download it from the
+      # actual Hackage.
+      { packages.external-package-demo.src = demo-src; }
+    ];
+  };
+  packages = project.hsPkgs;
+
+in recurseIntoAttrs {
+  ifdInputs = {
+    inherit (project) plan-nix;
+  };
+  run = stdenv.mkDerivation {
+    name = "external-hackage-test";
+
+    buildCommand = ''
+      exe="${packages.external-package-user.components.exes.external-package-user}/bin/external-package-user${stdenv.hostPlatform.extensions.executable}"
+      size=$(command stat --format '%s' "$exe")
+      printf "size of executable $exe is $size. \n" >& 2
+      # fixme: run on target platform when cross-compiled
+      printf "checking whether executable runs... " >& 2
+      cat ${haskellLib.check packages.external-package-user.components.exes.external-package-user}
+    '' + (if stdenv.hostPlatform.isMusl
+      then ''
+        printf "checking that executable is statically linked... " >& 2
+        (ldd $exe 2>&1 || true) | grep -i "not a"
+      ''
+      else
+        # Skip this on aarch as we do not have an `ldd` tool
+        optionalString (!stdenv.hostPlatform.isAarch32 && !stdenv.hostPlatform.isAarch64) (''
+          printf "checking that executable is dynamically linked to system libraries... " >& 2
+        '' + optionalString stdenv.isLinux ''
+          ldd $exe | grep libpthread
+        '' + optionalString stdenv.isDarwin ''
+          otool -L $exe |grep .dylib
+      '')) + ''
+      printf "Checking that \"all\" component has the programs... " >& 2
+      all_exe="${packages.external-package-user.components.all}/bin/external-package-user${stdenv.hostPlatform.extensions.executable}"
+      test -f "$all_exe"
+      echo "$all_exe" >& 2
+      touch $out
+    '';    
+    meta.platforms = platforms.all;
+    passthru = {
+      inherit project;
+    };  
+  };
+}

test/extra-hackage/external-package-demo/CHANGELOG.md

@@ -0,0 +1,5 @@
+# Revision history for external-package-demo
+
+## 0.1.0.0 -- YYYY-mm-dd
+
+* First version. Released on an unsuspecting world.

test/extra-hackage/external-package-demo/Main.hs

@@ -0,0 +1,8 @@
+module Main where
+
+import qualified MyLib (someFunc)
+
+main :: IO ()
+main = do
+  putStrLn "Hello, Haskell!"
+  MyLib.someFunc

test/extra-hackage/external-package-demo/MyLib.hs

@@ -0,0 +1,4 @@
+module MyLib (someFunc) where
+
+someFunc :: IO ()
+someFunc = putStrLn "someFunc"

test/extra-hackage/external-package-demo/Setup.hs

@@ -0,0 +1,2 @@
+import Distribution.Simple
+main = defaultMain

test/extra-hackage/external-package-demo/external-package-demo.cabal

@@ -0,0 +1,32 @@
+cabal-version:       2.4
+-- Initial package description 'external-package-demo.cabal' generated by
+-- 'cabal init'.  For further documentation, see
+-- http://haskell.org/cabal/users-guide/
+
+name:                external-package-demo
+version:             0.1.0.0
+synopsis:            Demo package.
+description:         Just a demo package...
+-- bug-reports:
+license:             BSD-3-Clause
+author:              Maxim Koltsov
+maintainer:          kolmax94@gmail.com
+-- copyright:
+-- category:
+extra-source-files:  CHANGELOG.md
+
+library
+  exposed-modules:     MyLib
+  -- other-modules:
+  -- other-extensions:
+  build-depends:       base < 5
+  -- hs-source-dirs:
+  default-language:    Haskell2010
+
+executable external-package-demo
+  main-is:             Main.hs
+  other-modules:       MyLib
+  -- other-extensions:
+  build-depends:       base, external-package-demo
+  -- hs-source-dirs:
+  default-language:    Haskell2010

test/extra-hackage/external-package-user/CHANGELOG.md

@@ -0,0 +1,5 @@
+# Revision history for external-package-user
+
+## 0.1.0.0 -- YYYY-mm-dd
+
+* First version. Released on an unsuspecting world.

test/extra-hackage/external-package-user/Main.hs

@@ -0,0 +1,8 @@
+module Main where
+
+import qualified MyLib (someFunc)
+
+main :: IO ()
+main = do
+  putStrLn "Hello, Haskell!"
+  MyLib.someFunc

test/extra-hackage/external-package-user/MyLib.hs

@@ -0,0 +1,4 @@
+module MyLib (someFunc) where
+
+someFunc :: IO ()
+someFunc = putStrLn "someFunc"

test/extra-hackage/external-package-user/Setup.hs

@@ -0,0 +1,2 @@
+import Distribution.Simple
+main = defaultMain

test/extra-hackage/external-package-user/cabal.project

@@ -0,0 +1,4 @@
+packages: *.cabal
+
+repository local
+    url: http://127.0.0.1:7777

test/extra-hackage/external-package-user/external-package-user.cabal

@@ -0,0 +1,33 @@
+cabal-version:       2.4
+-- Initial package description 'external-package-user.cabal' generated by
+-- 'cabal init'.  For further documentation, see
+-- http://haskell.org/cabal/users-guide/
+
+name:                external-package-user
+version:             0.1.0.0
+synopsis:            User package
+description:         A user of external-package-demo
+-- bug-reports:
+license:             BSD-3-Clause
+author:              Maxim Koltsov
+maintainer:          kolmax94@gmail.com
+-- copyright:
+-- category:
+extra-source-files:  CHANGELOG.md
+
+library
+  exposed-modules:     MyLib
+  -- other-modules:
+  -- other-extensions:
+  build-depends:       base < 5,
+                       external-package-demo
+  -- hs-source-dirs:
+  default-language:    Haskell2010
+
+executable external-package-user
+  main-is:             Main.hs
+  other-modules:       MyLib
+  -- other-extensions:
+  build-depends:       base, external-package-user
+  -- hs-source-dirs:
+  default-language:    Haskell2010

test/extra-hackage/hackage/default.nix

@@ -0,0 +1,9 @@
+with builtins; mapAttrs (_: mapAttrs (_: data: rec {
+ inherit (data) sha256;
+ revisions = (mapAttrs (rev: rdata: {
+  inherit (rdata) revNum sha256;
+  outPath = ./. + "/hackage/${rdata.outPath}";
+ }) data.revisions) // {
+  default = revisions."${data.revisions.default}";
+ };
+})) (fromJSON (readFile ./hackage.json))

test/extra-hackage/hackage/hackage.json

@@ -0,0 +1,15 @@
+{
+ "external-package-demo": {
+  "0.1.0.0": {
+   "revisions": {
+    "default": "r0",
+    "r0": {
+     "outPath": "external-package-demo-0.1.0.0-r0-3230db0813f2b468afb3ff7d8bbfcf570019a7faa4a0f59b2ea96743932105e7.nix",
+     "revNum": 0,
+     "sha256": "3230db0813f2b468afb3ff7d8bbfcf570019a7faa4a0f59b2ea96743932105e7"
+    }
+   },
+   "sha256": "1fce0685dcb89200ed286b9ae0983322ebc8a2d5de0541da55a5b82797dba740"
+  }
+ }
+}

test/extra-hackage/hackage/hackage/external-package-demo-0.1.0.0-r0-3230db0813f2b468afb3ff7d8bbfcf570019a7faa4a0f59b2ea96743932105e7.nix

@@ -0,0 +1,76 @@
+let
+  buildDepError = pkg:
+    builtins.throw ''
+      The Haskell package set does not contain the package: ${pkg} (build dependency).
+      
+      If you are using Stackage, make sure that you are using a snapshot that contains the package. Otherwise you may need to update the Hackage snapshot you are using, usually by updating haskell.nix.
+      '';
+  sysDepError = pkg:
+    builtins.throw ''
+      The Nixpkgs package set does not contain the package: ${pkg} (system dependency).
+      
+      You may need to augment the system package mapping in haskell.nix so that it can be found.
+      '';
+  pkgConfDepError = pkg:
+    builtins.throw ''
+      The pkg-conf packages does not contain the package: ${pkg} (pkg-conf dependency).
+      
+      You may need to augment the pkg-conf package mapping in haskell.nix so that it can be found.
+      '';
+  exeDepError = pkg:
+    builtins.throw ''
+      The local executable components do not include the component: ${pkg} (executable dependency).
+      '';
+  legacyExeDepError = pkg:
+    builtins.throw ''
+      The Haskell package set does not contain the package: ${pkg} (executable dependency).
+      
+      If you are using Stackage, make sure that you are using a snapshot that contains the package. Otherwise you may need to update the Hackage snapshot you are using, usually by updating haskell.nix.
+      '';
+  buildToolDepError = pkg:
+    builtins.throw ''
+      Neither the Haskell package set or the Nixpkgs package set contain the package: ${pkg} (build tool dependency).
+      
+      If this is a system dependency:
+      You may need to augment the system package mapping in haskell.nix so that it can be found.
+      
+      If this is a Haskell dependency:
+      If you are using Stackage, make sure that you are using a snapshot that contains the package. Otherwise you may need to update the Hackage snapshot you are using, usually by updating haskell.nix.
+      '';
+in { system, compiler, flags, pkgs, hsPkgs, pkgconfPkgs, config, ... }:
+  {
+    flags = {};
+    package = {
+      specVersion = "2.4";
+      identifier = { name = "external-package-demo"; version = "0.1.0.0"; };
+      license = "BSD-3-Clause";
+      copyright = "";
+      maintainer = "kolmax94@gmail.com";
+      author = "Maxim Koltsov";
+      homepage = "";
+      url = "";
+      synopsis = "Demo package.";
+      description = "Just a demo package...";
+      buildType = "Simple";
+      };
+    components = {
+      "library" = {
+        depends = [ (hsPkgs."base" or (buildDepError "base")) ];
+        buildable = true;
+        };
+      exes = {
+        "external-package-demo" = {
+          depends = [
+            (hsPkgs."base" or (buildDepError "base"))
+            (hsPkgs."external-package-demo" or (buildDepError "external-package-demo"))
+            ];
+          buildable = true;
+          };
+        };
+      };
+    } // {
+    src = (pkgs.lib).mkDefault (pkgs.fetchurl {
+      url = "http://not-there//package/external-package-demo-0.1.0.0/external-package-demo-0.1.0.0.tar.gz";
+      sha256 = config.sha256;
+      });
+    }

test/tests.sh

@@ -104,4 +104,16 @@ printf "*** Checking the maintainer scripts...\n" >& 2
 nix build $NIX_BUILD_ARGS --no-link --keep-going -f ../build.nix maintainer-scripts
 echo >& 2
 
+printf "*** Checking that plan construction works with extra Hackages...\n" >& 2
+nix build $NIX_BUILD_ARGS --no-link \
+    -f /default.nix \
+    extra-hackage.plan-nix
+echo >& 2
+
+printf "*** Checking that package with extra Hackages can be build...\n" >& 2
+nix build $NIX_BUILD_ARGS --no-link \
+    -f /default.nix \
+    extra-hackage.hsPkgs.external-package-user.components.all
+echo >& 2
+
 printf "\n*** Finished successfully\n" >& 2