on iko/ghcjs-patch-master help sign in
External hackages (#535)

* Add support for external Hackage repositories Currently haskell.nix is not able to build Cabal projects that depend on packages from private Hackage repositories, as it make only main Hackage available to cabal. This is unfortunate. This commit adds this functionality, by allowing the user to pass `extra-hackages` and `extra-hackage-tarballs` to `mkPkgSet` and `callCabalToNix` respectively, to add as much extra repositories as needed. This repositories are first made available to Cabal when calling `v2-configure`, resulting in correct plans. Later they are combined with global Hackage when building dependencies of the local packages. * Use cabal.project.freeze if available Currently callCabalProjectToNix does not copy `cabal.project.freeze` from source directory, leading to different build plans when building components with nix and when building project with `cabal new-build` inside `nix-shell`. This behavior is undesired, so this commits fixes it. * Add tests for extra-hackages functionality Co-authored-by: Moritz Angermann <moritz.angermann@gmail.com>

commit07031ee22402c225206a9ad3ae120dee8af60214 authorMaxim Koltsov <kolmax94@gmail.com> dateSat 2 May 2020, 13:16 (6 years ago) parent78f4e25
23 files changed+359−22
lib/call-cabal-project-to-nix.nix+3−2 mk-local-hackage-repo/default.nix+4−4 overlays/haskell.nix+52−16 test/default.nix+1 test/extra-hackage/01-index.tar.gz test/extra-hackage/README.md+11 test/extra-hackage/default.nix+69 test/extra-hackage/external-package-demo-0.1.0.0.tar.gz test/extra-hackage/external-package-demo/CHANGELOG.md+5 test/extra-hackage/external-package-demo/Main.hs+8 test/extra-hackage/external-package-demo/MyLib.hs+4 test/extra-hackage/external-package-demo/Setup.hs+2 test/extra-hackage/external-package-demo/external-package-demo.cabal+32 test/extra-hackage/external-package-user/CHANGELOG.md+5 test/extra-hackage/external-package-user/Main.hs+8 test/extra-hackage/external-package-user/MyLib.hs+4 test/extra-hackage/external-package-user/Setup.hs+2 test/extra-hackage/external-package-user/cabal.project+4 test/extra-hackage/external-package-user/external-package-user.cabal+33 test/extra-hackage/hackage/default.nix+9 test/extra-hackage/hackage/hackage.json+15 test/extra-hackage/hackage/hackage/external-package-demo-0.1.0.0-r0-3230db0813f2b468afb3ff7d8bbfcf570019a7faa4a0f59b2ea96743932105e7.nix+76 test/tests.sh+12
lib/call-cabal-project-to-nix.nix+3−2
@@ -23,6 +23,7 @@
2323 # { "https://github.com/jgm/pandoc-citeproc"."0.17"
2424 # = "0dxx8cp2xndpw3jwiawch2dkrkp15mil7pyx7dvd810pwc22pm2q"; }
2525 # ."${repo.location}"."${repo.tag}";
26+, extra-hackage-tarballs ? []
2627, ...
2728}@args:
2829# cabal-install versions before 2.4 will generate insufficient plan information.
@@ -41,7 +42,7 @@ let
4142 inherit src;
4243 filter = path: type:
4344 type == "directory" ||
44− pkgs.lib.any (i: (pkgs.lib.hasSuffix i path)) [ ".project" ".cabal" "package.yaml" ]; }
45+ pkgs.lib.any (i: (pkgs.lib.hasSuffix i path)) [ ".project" ".cabal" ".freeze" "package.yaml" ]; }
4546 else src;
4647
4748 # Using origSrcSubDir bypasses any cleanSourceWith so that it will work when
@@ -225,7 +226,7 @@ let
225226 export SSL_CERT_FILE=${cacert}/etc/ssl/certs/ca-bundle.crt
226227 export GIT_SSL_CAINFO=${cacert}/etc/ssl/certs/ca-bundle.crt
227228 HOME=${dotCabal {
228− inherit cabal-install nix-tools;
229+ inherit cabal-install nix-tools extra-hackage-tarballs;
229230 index-state =
230231 builtins.trace ("Using index-state: ${index-state-found}" + (if name == null then "" else " for " + name))
231232 index-state-found;
mk-local-hackage-repo/default.nix+4−4
@@ -14,10 +14,10 @@
1414# nix would be pointless as we'd have to hardcode them to produce the same output
1515# reproducably.
1616#
17−{ pkgs, hackageTarball }:
18−{ index-state, sha256, ... }@args:
19−let index = hackageTarball args; in
20−pkgs.runCommand "hackage-repo-${builtins.replaceStrings [":"] [""] index-state}" { nativeBuildInputs = [ pkgs.buildPackages.nix ]; } ''
17+pkgs:
18+{ name, index }:
19+
20+pkgs.runCommand "hackage-repo-${name}" { nativeBuildInputs = [ pkgs.buildPackages.nix ]; } ''
2121mkdir -p $out
2222export expires="4000-01-01T00:00:00Z"
2323
overlays/haskell.nix+52−16
@@ -70,13 +70,19 @@ self: super: {
7070 { pkg-def # Base package set. Either from stackage (via stack-to-nix) or from a cabal projects plan file (via plan-to-nix)
7171 , pkg-def-extras ? [] # Additional packages to augment the Base package set `pkg-def` with.
7272 , modules ? []
73+ , extra-hackages ? [] # Extra Hackage repositories to use besides main one.
7374 }@args:
7475
75− import ../package-set.nix (args // {
76+ let
77+ hackageAll = builtins.foldl' (base: extra: base // extra) hackage extra-hackages;
78+ in
79+
80+ import ../package-set.nix {
81+ inherit (args) pkg-def pkg-def-extras;
7682 modules = defaultModules ++ modules;
7783 pkgs = self;
78− inherit hackage pkg-def;
79− });
84+ hackage = hackageAll;
85+ };
8086
8187 # Some boot packages (libiserv) are in lts, but not in hackage,
8288 # so we should not try to get it from hackage based on the stackage
@@ -128,6 +134,7 @@ self: super: {
128134 { plan-pkgs # Path to the output of plan-to-nix
129135 , pkg-def-extras ? []
130136 , modules ? []
137+ , extra-hackages ? []
131138 }@args:
132139
133140 let
@@ -146,6 +153,7 @@ self: super: {
146153 modules = [ { doExactConfig = true; } patchesModule ]
147154 ++ modules
148155 ++ plan-pkgs.modules or [];
156+ inherit extra-hackages;
149157 };
150158
151159 # Package sets for all stackage snapshots.
@@ -169,33 +177,60 @@ self: super: {
169177 nix-tools = self.buildPackages.haskell-nix.nix-tools-cross-compiled;
170178 # TODO perhaps there is a cleaner way to get a suitable nix-tools.
171179
172− # Produce a fixed output derivation from a moving target (hackage index tarball)
180+ # Produce a fixed output derivation from a moving target (hackage index tarball)
181+ # Takes desired index-state and sha256 and produces a set { name, index }, where
182+ # index points to "01-index.tar.gz" file downloaded from hackage.haskell.org.
173183 hackageTarball = { index-state, sha256, nix-tools ? self.haskell-nix.nix-tools, ... }:
174184 assert sha256 != null;
175− self.fetchurl {
176− name = "01-index.tar.gz-at-${builtins.replaceStrings [":"] [""] index-state}";
185+ let at = builtins.replaceStrings [":"] [""] index-state; in
186+ { name = "hackage.haskell.org-at-${at}";
187+ index = self.fetchurl {
188+ name = "01-index.tar.gz-at-${at}";
177189 url = "https://hackage.haskell.org/01-index.tar.gz";
178190 downloadToTemp = true;
179191 postFetch = "${nix-tools}/bin/truncate-index -o $out -i $downloadedFile -s ${index-state}";
180192
181193 outputHashAlgo = "sha256";
182194 outputHash = sha256;
195+ };
183196 };
184197
185− mkLocalHackageRepo = import ../mk-local-hackage-repo { inherit hackageTarball; pkgs = self; };
198+ # Creates Cabal local repository from { name, index } set.
199+ mkLocalHackageRepo = import ../mk-local-hackage-repo self;
186200
187− dotCabal = { index-state, sha256, cabal-install, ... }@args:
188− self.runCommand "dot-cabal-at-${builtins.replaceStrings [":"] [""] index-state}" { nativeBuildInputs = [ cabal-install ]; } ''
201+ dotCabal = { index-state, sha256, cabal-install, extra-hackage-tarballs ? [], ... }@args:
202+ let
203+ allTarballs = [ (hackageTarball args) ] ++ extra-hackage-tarballs;
204+ allNames = self.lib.concatMapStringsSep "-" (tarball: tarball.name) allTarballs;
205+ # Main Hackage index-state is embedded in its name and thus will propagate to
206+ # dotCabalName anyway.
207+ dotCabalName = "dot-cabal-" + allNames;
208+ in
209+ self.runCommand dotCabalName { nativeBuildInputs = [ cabal-install ]; } ''
189210 mkdir -p $out/.cabal
190211 cat <<EOF > $out/.cabal/config
191− repository cached
192− url: file:${mkLocalHackageRepo args}
193− secure: True
194− root-keys:
195− key-threshold: 0
212+ ${self.lib.concatStrings (
213+ map (tarball:
214+ ''
215+ repository ${tarball.name}
216+ url: file:${mkLocalHackageRepo tarball}
217+ secure: True
218+ root-keys:
219+ key-threshold: 0
220+
221+ '') allTarballs
222+ )}
196223 EOF
197− mkdir -p $out/.cabal/packages/cached
198− HOME=$out cabal new-update cached
224+
225+ # All repositories must be mkdir'ed before calling new-update on any repo,
226+ # otherwise it fails.
227+ ${self.lib.concatStrings (map ({ name, ... }: ''
228+ mkdir -p $out/.cabal/packages/${name}
229+ '') allTarballs)}
230+
231+ ${self.lib.concatStrings (map ({ name, ... }: ''
232+ HOME=$out cabal new-update ${name}
233+ '') allTarballs)}
199234 '';
200235
201236 # Some of features of haskell.nix rely on using a hackage index
@@ -429,6 +464,7 @@ self: super: {
429464 pkg-def-extras = args.pkg-def-extras or [];
430465 modules = (args.modules or [])
431466 ++ self.lib.optional (args ? ghc) { ghc.package = args.ghc; };
467+ extra-hackages = args.extra-hackages or [];
432468 };
433469 in { inherit (pkg-set.config) hsPkgs; inherit pkg-set; plan-nix = plan.nix; };
434470
test/default.nix+1
@@ -175,6 +175,7 @@ let
175175 exe-only = callTest ./exe-only { inherit util; };
176176 stack-source-repo = callTest ./stack-source-repo {};
177177 lookup-sha256 = callTest ./lookup-sha256 {};
178+ extra-hackage = callTest ./extra-hackage {};
178179
179180 unit = unitTests;
180181 };
test/extra-hackage/01-index.tar.gznew file

Binary file not shown.

test/extra-hackage/README.mdnew file+11
@@ -0,0 +1,11 @@
1+# Tests for extra Hackage functionality
2+
3+This directory contains two packages, `external-package-demo` and `external-package-user`, the
4+second one depends on the first one. Both packages were created with `cabal init`.
5+
6+`external-package-demo` was uploaded to local Hackage at `localhost` and `01-index.tar.gz` from that
7+Hackage was downloaded to this directory. Then the index file was processed with `hackage-to-nix`,
8+the result is in `hackage/` directory.
9+
10+The tests check that `cabalProject'` is able to construct plan with dependencies from extra Hackage
11+and then build the package itself.
test/extra-hackage/default.nixnew file+69
@@ -0,0 +1,69 @@
1+{ stdenv, cabalProject', haskellLib, recurseIntoAttrs, testSrc }:
2+
3+with stdenv.lib;
4+
5+let
6+
7+ hackage = import ./hackage;
8+
9+ tarball = {
10+ name = "extra-hackage-demo";
11+ index = ./01-index.tar.gz;
12+ };
13+
14+ demo-src = ./external-package-demo-0.1.0.0.tar.gz;
15+
16+ project = cabalProject' {
17+ src = testSrc "extra-hackage/external-package-user";
18+
19+ extra-hackages = [ hackage ];
20+ extra-hackage-tarballs = [ tarball ];
21+
22+ modules = [
23+ # To prevent nix-build from trying to download it from the
24+ # actual Hackage.
25+ { packages.external-package-demo.src = demo-src; }
26+ ];
27+ };
28+ packages = project.hsPkgs;
29+
30+in recurseIntoAttrs {
31+ ifdInputs = {
32+ inherit (project) plan-nix;
33+ };
34+ run = stdenv.mkDerivation {
35+ name = "external-hackage-test";
36+
37+ buildCommand = ''
38+ exe="${packages.external-package-user.components.exes.external-package-user}/bin/external-package-user${stdenv.hostPlatform.extensions.executable}"
39+ size=$(command stat --format '%s' "$exe")
40+ printf "size of executable $exe is $size. \n" >& 2
41+ # fixme: run on target platform when cross-compiled
42+ printf "checking whether executable runs... " >& 2
43+ cat ${haskellLib.check packages.external-package-user.components.exes.external-package-user}
44+ '' + (if stdenv.hostPlatform.isMusl
45+ then ''
46+ printf "checking that executable is statically linked... " >& 2
47+ (ldd $exe 2>&1 || true) | grep -i "not a"
48+ ''
49+ else
50+ # Skip this on aarch as we do not have an `ldd` tool
51+ optionalString (!stdenv.hostPlatform.isAarch32 && !stdenv.hostPlatform.isAarch64) (''
52+ printf "checking that executable is dynamically linked to system libraries... " >& 2
53+ '' + optionalString stdenv.isLinux ''
54+ ldd $exe | grep libpthread
55+ '' + optionalString stdenv.isDarwin ''
56+ otool -L $exe |grep .dylib
57+ '')) + ''
58+ printf "Checking that \"all\" component has the programs... " >& 2
59+ all_exe="${packages.external-package-user.components.all}/bin/external-package-user${stdenv.hostPlatform.extensions.executable}"
60+ test -f "$all_exe"
61+ echo "$all_exe" >& 2
62+ touch $out
63+ '';
64+ meta.platforms = platforms.all;
65+ passthru = {
66+ inherit project;
67+ };
68+ };
69+}
test/extra-hackage/external-package-demo-0.1.0.0.tar.gznew file

Binary file not shown.

test/extra-hackage/external-package-demo/CHANGELOG.mdnew file+5
@@ -0,0 +1,5 @@
1+# Revision history for external-package-demo
2+
3+## 0.1.0.0 -- YYYY-mm-dd
4+
5+* First version. Released on an unsuspecting world.
test/extra-hackage/external-package-demo/Main.hsnew file+8
@@ -0,0 +1,8 @@
1+module Main where
2+
3+import qualified MyLib (someFunc)
4+
5+main :: IO ()
6+main = do
7+ putStrLn "Hello, Haskell!"
8+ MyLib.someFunc
test/extra-hackage/external-package-demo/MyLib.hsnew file+4
@@ -0,0 +1,4 @@
1+module MyLib (someFunc) where
2+
3+someFunc :: IO ()
4+someFunc = putStrLn "someFunc"
test/extra-hackage/external-package-demo/Setup.hsnew file+2
@@ -0,0 +1,2 @@
1+import Distribution.Simple
2+main = defaultMain
test/extra-hackage/external-package-demo/external-package-demo.cabalnew file+32
@@ -0,0 +1,32 @@
1+cabal-version: 2.4
2+-- Initial package description 'external-package-demo.cabal' generated by
3+-- 'cabal init'. For further documentation, see
4+-- http://haskell.org/cabal/users-guide/
5+
6+name: external-package-demo
7+version: 0.1.0.0
8+synopsis: Demo package.
9+description: Just a demo package...
10+-- bug-reports:
11+license: BSD-3-Clause
12+author: Maxim Koltsov
13+maintainer: kolmax94@gmail.com
14+-- copyright:
15+-- category:
16+extra-source-files: CHANGELOG.md
17+
18+library
19+ exposed-modules: MyLib
20+ -- other-modules:
21+ -- other-extensions:
22+ build-depends: base < 5
23+ -- hs-source-dirs:
24+ default-language: Haskell2010
25+
26+executable external-package-demo
27+ main-is: Main.hs
28+ other-modules: MyLib
29+ -- other-extensions:
30+ build-depends: base, external-package-demo
31+ -- hs-source-dirs:
32+ default-language: Haskell2010
test/extra-hackage/external-package-user/CHANGELOG.mdnew file+5
@@ -0,0 +1,5 @@
1+# Revision history for external-package-user
2+
3+## 0.1.0.0 -- YYYY-mm-dd
4+
5+* First version. Released on an unsuspecting world.
test/extra-hackage/external-package-user/Main.hsnew file+8
@@ -0,0 +1,8 @@
1+module Main where
2+
3+import qualified MyLib (someFunc)
4+
5+main :: IO ()
6+main = do
7+ putStrLn "Hello, Haskell!"
8+ MyLib.someFunc
test/extra-hackage/external-package-user/MyLib.hsnew file+4
@@ -0,0 +1,4 @@
1+module MyLib (someFunc) where
2+
3+someFunc :: IO ()
4+someFunc = putStrLn "someFunc"
test/extra-hackage/external-package-user/Setup.hsnew file+2
@@ -0,0 +1,2 @@
1+import Distribution.Simple
2+main = defaultMain
test/extra-hackage/external-package-user/cabal.projectnew file+4
@@ -0,0 +1,4 @@
1+packages: *.cabal
2+
3+repository local
4+ url: http://127.0.0.1:7777
test/extra-hackage/external-package-user/external-package-user.cabalnew file+33
@@ -0,0 +1,33 @@
1+cabal-version: 2.4
2+-- Initial package description 'external-package-user.cabal' generated by
3+-- 'cabal init'. For further documentation, see
4+-- http://haskell.org/cabal/users-guide/
5+
6+name: external-package-user
7+version: 0.1.0.0
8+synopsis: User package
9+description: A user of external-package-demo
10+-- bug-reports:
11+license: BSD-3-Clause
12+author: Maxim Koltsov
13+maintainer: kolmax94@gmail.com
14+-- copyright:
15+-- category:
16+extra-source-files: CHANGELOG.md
17+
18+library
19+ exposed-modules: MyLib
20+ -- other-modules:
21+ -- other-extensions:
22+ build-depends: base < 5,
23+ external-package-demo
24+ -- hs-source-dirs:
25+ default-language: Haskell2010
26+
27+executable external-package-user
28+ main-is: Main.hs
29+ other-modules: MyLib
30+ -- other-extensions:
31+ build-depends: base, external-package-user
32+ -- hs-source-dirs:
33+ default-language: Haskell2010
test/extra-hackage/hackage/default.nixnew file+9
@@ -0,0 +1,9 @@
1+with builtins; mapAttrs (_: mapAttrs (_: data: rec {
2+ inherit (data) sha256;
3+ revisions = (mapAttrs (rev: rdata: {
4+ inherit (rdata) revNum sha256;
5+ outPath = ./. + "/hackage/${rdata.outPath}";
6+ }) data.revisions) // {
7+ default = revisions."${data.revisions.default}";
8+ };
9+})) (fromJSON (readFile ./hackage.json))
test/extra-hackage/hackage/hackage.jsonnew file+15
@@ -0,0 +1,15 @@
1+{
2+ "external-package-demo": {
3+ "0.1.0.0": {
4+ "revisions": {
5+ "default": "r0",
6+ "r0": {
7+ "outPath": "external-package-demo-0.1.0.0-r0-3230db0813f2b468afb3ff7d8bbfcf570019a7faa4a0f59b2ea96743932105e7.nix",
8+ "revNum": 0,
9+ "sha256": "3230db0813f2b468afb3ff7d8bbfcf570019a7faa4a0f59b2ea96743932105e7"
10+ }
11+ },
12+ "sha256": "1fce0685dcb89200ed286b9ae0983322ebc8a2d5de0541da55a5b82797dba740"
13+ }
14+ }
15+}
test/extra-hackage/hackage/hackage/external-package-demo-0.1.0.0-r0-3230db0813f2b468afb3ff7d8bbfcf570019a7faa4a0f59b2ea96743932105e7.nixnew file+76
@@ -0,0 +1,76 @@
1+let
2+ buildDepError = pkg:
3+ builtins.throw ''
4+ The Haskell package set does not contain the package: ${pkg} (build dependency).
5+
6+ If you are using Stackage, make sure that you are using a snapshot that contains the package. Otherwise you may need to update the Hackage snapshot you are using, usually by updating haskell.nix.
7+ '';
8+ sysDepError = pkg:
9+ builtins.throw ''
10+ The Nixpkgs package set does not contain the package: ${pkg} (system dependency).
11+
12+ You may need to augment the system package mapping in haskell.nix so that it can be found.
13+ '';
14+ pkgConfDepError = pkg:
15+ builtins.throw ''
16+ The pkg-conf packages does not contain the package: ${pkg} (pkg-conf dependency).
17+
18+ You may need to augment the pkg-conf package mapping in haskell.nix so that it can be found.
19+ '';
20+ exeDepError = pkg:
21+ builtins.throw ''
22+ The local executable components do not include the component: ${pkg} (executable dependency).
23+ '';
24+ legacyExeDepError = pkg:
25+ builtins.throw ''
26+ The Haskell package set does not contain the package: ${pkg} (executable dependency).
27+
28+ If you are using Stackage, make sure that you are using a snapshot that contains the package. Otherwise you may need to update the Hackage snapshot you are using, usually by updating haskell.nix.
29+ '';
30+ buildToolDepError = pkg:
31+ builtins.throw ''
32+ Neither the Haskell package set or the Nixpkgs package set contain the package: ${pkg} (build tool dependency).
33+
34+ If this is a system dependency:
35+ You may need to augment the system package mapping in haskell.nix so that it can be found.
36+
37+ If this is a Haskell dependency:
38+ If you are using Stackage, make sure that you are using a snapshot that contains the package. Otherwise you may need to update the Hackage snapshot you are using, usually by updating haskell.nix.
39+ '';
40+in { system, compiler, flags, pkgs, hsPkgs, pkgconfPkgs, config, ... }:
41+ {
42+ flags = {};
43+ package = {
44+ specVersion = "2.4";
45+ identifier = { name = "external-package-demo"; version = "0.1.0.0"; };
46+ license = "BSD-3-Clause";
47+ copyright = "";
48+ maintainer = "kolmax94@gmail.com";
49+ author = "Maxim Koltsov";
50+ homepage = "";
51+ url = "";
52+ synopsis = "Demo package.";
53+ description = "Just a demo package...";
54+ buildType = "Simple";
55+ };
56+ components = {
57+ "library" = {
58+ depends = [ (hsPkgs."base" or (buildDepError "base")) ];
59+ buildable = true;
60+ };
61+ exes = {
62+ "external-package-demo" = {
63+ depends = [
64+ (hsPkgs."base" or (buildDepError "base"))
65+ (hsPkgs."external-package-demo" or (buildDepError "external-package-demo"))
66+ ];
67+ buildable = true;
68+ };
69+ };
70+ };
71+ } // {
72+ src = (pkgs.lib).mkDefault (pkgs.fetchurl {
73+ url = "http://not-there//package/external-package-demo-0.1.0.0/external-package-demo-0.1.0.0.tar.gz";
74+ sha256 = config.sha256;
75+ });
76+ }
test/tests.sh+12
@@ -104,4 +104,16 @@ printf "*** Checking the maintainer scripts...\n" >& 2
104104nix build $NIX_BUILD_ARGS --no-link --keep-going -f ../build.nix maintainer-scripts
105105echo >& 2
106106
107+printf "*** Checking that plan construction works with extra Hackages...\n" >& 2
108+nix build $NIX_BUILD_ARGS --no-link \
109+ -f /default.nix \
110+ extra-hackage.plan-nix
111+echo >& 2
112+
113+printf "*** Checking that package with extra Hackages can be build...\n" >& 2
114+nix build $NIX_BUILD_ARGS --no-link \
115+ -f /default.nix \
116+ extra-hackage.hsPkgs.external-package-user.components.all
117+echo >& 2
118+
107119printf "\n*** Finished successfully\n" >& 2