git.iko.soy/haskell.nix
on iko/ghcjs-patch-master help sign in
Call cabal project to nix (now with hackage-truncate) (#135)

* This adds more logic for the cabalProjectToNix idf That is, something like this: ``` nix-build --expr 'with import ./. {}; callCabalProjectToNix { index-state = "2019-04-30T00:00:00Z"; src = /some/path; }' ``` should produce something that can be build with mkCabalProjectPkgSet. * Make sure the hackageTarball's store path doesn't change previously the fetchurl would produce a different store path each and every time as hackage's index is a moving target. With this impurity setup, we can ignore this. * Fix test * Proper name * Re-enable test * Allow to parameterize over the `system` for the test default.nix * Copy instead of link for ifds. This makes me really sad.

commitdfefba5be7aa55e522fb682caeb2c8cefb5a8d35 authorMoritz Angermann <moritz.angermann@gmail.com> dateTue 21 May 2019, 20:05 (7 years ago) parent11d5ca7
10 files changed+249−17
compare with iko/ghcjs-patch-master
unifiedsplit
default.nix+35−6 lib/cabalProjectToNix.nix+22−6 lib/index-state-hashes.nix+51 mk-local-hackage-repo/default.nix+45 mk-local-hackage-repo/mirrors.json+9 mk-local-hackage-repo/root.json+34 mk-local-hackage-repo/snapshot.json+31 mk-local-hackage-repo/timestamp.json+17 test/call-cabal-project-to-nix/default.nix+2−2 test/default.nix+3−3
default.nix+35−6
@@ -38,11 +38,6 @@ let
3838 # overridden with NIX_PATH.
3939 fetchExternal = import ./lib/fetch-external.nix;
4040
41− mkHackageIndex = indexState: import ./lib/hackageIndex.nix {
42− inherit (pkgs) runCommand cabal-install;
43− inherit indexState;
44− };
45−
4641 # All packages from Hackage as Nix expressions
4742 hackage = import (fetchExternal {
4843 name = "hackage-exprs-source";
@@ -132,10 +127,44 @@ let
132127 # Make this handy overridable fetch function available.
133128 inherit fetchExternal;
134129
130+ # Produce a fixed output derivation from a moving target (hackage index tarball)
131+ hackageTarball = { index-state, sha256 }:
132+ pkgs.runCommand "01-index.tar.gz-at-${builtins.replaceStrings [":"] [""] index-state}" {
133+ nativeBuildInputs = [ pkgs.curl ];
134+ outputHashAlgo = "sha256";
135+ outputHash = sha256;
136+ # We'll use fetchurl's result in an env var ...
137+ HACKAGE_INDEX = builtins.fetchurl "https://hackage.haskell.org/01-index.tar.gz";
138+ # ... and mark that impure. That way we can
139+ # ensure the sore path stays the same and doesn't
140+ # depend on the fetchurl result path.
141+ impureEnvVars = [ "HACKAGE_INDEX" ];
142+ }
143+ ''
144+ ${self.nix-tools}/bin/truncate-index -o $out -i $HACKAGE_INDEX -s ${index-state}
145+ '';
146+
147+ mkLocalHackageRepo = import ./mk-local-hackage-repo { inherit (self) hackageTarball; inherit pkgs; };
148+
149+ dotCabal = { index-state, sha256 }@args:
150+ pkgs.runCommand "dot-cabal-at-${builtins.replaceStrings [":"] [""] index-state}" { nativeBuildInputs = [ pkgs.cabal-install ]; } ''
151+ mkdir -p $out/.cabal
152+ cat <<EOF > $out/.cabal/config
153+ repository cached
154+ url: file:${self.mkLocalHackageRepo args}
155+ secure: True
156+ root-keys:
157+ key-threshold: 0
158+ EOF
159+ mkdir -p $out/.cabal/packages/cached
160+ HOME=$out cabal new-update cached
161+ '';
162+
163+
135164 # Takes a haskell src directory runs cabal new-configure and plan-to-nix.
136165 # Resulting nix files are added to nix-plan subdirectory.
137166 callCabalProjectToNix = import ./lib/cabalProjectToNix.nix {
138− inherit mkHackageIndex;
167+ inherit (self) dotCabal;
139168 inherit pkgs;
140169 inherit (pkgs) runCommand cabal-install ghc;
141170 inherit (pkgs.haskellPackages) hpack;
lib/cabalProjectToNix.nix+22−6
@@ -1,7 +1,7 @@
1−{ mkHackageIndex, pkgs, runCommand, nix-tools, cabal-install, ghc, hpack, symlinkJoin }:
1+{ dotCabal, pkgs, runCommand, nix-tools, cabal-install, ghc, hpack, symlinkJoin }:
22let defaultGhc = ghc;
33 defaultCabalInstall = cabal-install;
4−in { hackageIndexState, src, ghc ? defaultGhc, cabal-install ? defaultCabalInstall }:
4+in { index-state, index-sha256 ? import ./index-state-hashes.nix index-state, src, ghc ? defaultGhc, cabal-install ? defaultCabalInstall }:
55let
66 cabalFiles =
77 pkgs.lib.cleanSourceWith {
@@ -14,7 +14,7 @@ let
1414 # cabal-install versions before 2.4 will generate insufficient plan information.
1515 then throw "cabal-install (current version: ${cabal-install.version}) needs to be at least 2.4 for plan-to-nix to work without cabal-to-nix"
1616 else runCommand "plan" {
17− nativeBuildInputs = [ nix-tools ghc hpack cabal-install pkgs.rsync ];
17+ nativeBuildInputs = [ nix-tools ghc hpack cabal-install pkgs.rsync pkgs.git ];
1818 } ''
1919 tmp=$(mktemp -d)
2020 cd $tmp
@@ -25,7 +25,7 @@ let
2525 # without the source available (we cleaneSourceWith'd it),
2626 # this may not produce the right result.
2727 find . -name package.yaml -exec hpack "{}" \;
28− HOME=${mkHackageIndex hackageIndexState} cabal new-configure
28+ HOME=${dotCabal { inherit index-state; sha256 = index-sha256; }} cabal new-configure
2929
3030 export LANG=C.utf8 # Needed or stack-to-nix will die on unicode inputs
3131 mkdir -p $out
@@ -54,9 +54,25 @@ let
5454 mv $out/pkgs.nix $out/default.nix
5555 '';
5656in
57− runCommand "plan-and-src" { nativeBuildInputs = [ pkgs.xorg.lndir pkgs.rsync ]; } ''
57+ # TODO: We really want this (symlinks) instead of copying the source over each and
58+ # every time. However this will not work with sandboxed builds. They won't
59+ # have access to `plan` or `src` paths. So while they will see all the
60+ # links, they won't be able to read any of them.
61+ #
62+ # We should be able to fix this if we propagaed the build inputs properly.
63+ # As we are `import`ing the produced nix-path here, we seem to be losing the
64+ # dependencies though.
65+ #
66+ # I guess the end-result is that ifd's don't work well with symlinks.
67+ #
68+ # symlinkJoin {
69+ # name = "plan-and-src";
70+ # # todo: should we clean `src` to drop any .git, .nix, ... other irelevant files?
71+ # buildInputs = [ plan src ];
72+ # }
73+ runCommand "plan-and-src" { nativeBuildInputs = [ pkgs.rsync ]; } ''
5874 mkdir $out
5975 # todo: should we clean `src` to drop any .git, .nix, ... other irelevant files?
60− lndir -silent "${src}" "$out"
76+ rsync -a "${src}/" "$out/"
6177 rsync -a ${plan}/ $out/
6278 ''
lib/index-state-hashes.nixnew file+51
@@ -0,0 +1,51 @@
1+dt: {
2+ "2019-04-01-00T00:00:00Z" = "a46ed96a1aa0510d1b54d508ae14dc6e460ff89ae41f0bd075aef87ba8c8ae73 ";
3+ "2019-04-02-00T00:00:00Z" = "56dca813015122e0d633cb53bf8d11fb0c43a2e518b93cbbf93b0bdbf00e0c99 ";
4+ "2019-04-03-00T00:00:00Z" = "b7a162996feec90e9acdff8bfb85a46ccbcafbbf0772739da45aef4e56480198 ";
5+ "2019-04-04-00T00:00:00Z" = "6c890e98501539389807c7f7353b40af9bdd9e84a816adaccb79244ab1e506a0 ";
6+ "2019-04-05-00T00:00:00Z" = "905b0ae09f4929b19b0885dcef785a45142bd6a7673e26979d6bc7044dedfd29 ";
7+ "2019-04-06-00T00:00:00Z" = "2face70f256bc0a556e645f1f843c196411a97c78ded610443f3814b0783e5dc ";
8+ "2019-04-07-00T00:00:00Z" = "f837f74181bd81120904c0e72b7c479faf178d1f2172a1442e167e7213b4a433 ";
9+ "2019-04-08-00T00:00:00Z" = "86ff75e355e8b72757becb7761a1cd90b45b60f0f4777f8a137f30a27c766d65 ";
10+ "2019-04-09-00T00:00:00Z" = "c00dc1146184db0743dce73d7bb53dca0fe75dff6634f61f11a6326984dba4ad ";
11+ "2019-04-10-00T00:00:00Z" = "09cbd82c5f4a42e69299564a1494f5c2ff9fefde0663441a1878d351833a278c ";
12+ "2019-04-11-00T00:00:00Z" = "072da9339e8c60b5e911659c110b0317a4e7890e30ce77e113edce1c373434d5 ";
13+ "2019-04-12-00T00:00:00Z" = "9b14849542745d86529afa0f2cd2db989ea32a22a9a46d56fd3bc659339bdc28 ";
14+ "2019-04-13-00T00:00:00Z" = "4aeba1860afc057b3faca85a289313a148a3392b80e5feab2263abfe8f9af155 ";
15+ "2019-04-14-00T00:00:00Z" = "63a804787e63db9283b948ccc8ffafb487809efe60151295499e9d95230146fa ";
16+ "2019-04-15-00T00:00:00Z" = "ef8607a4bf9e62052a428111b6decb5ef99e6da9dc885bf1bb51e593c4a97f9f ";
17+ "2019-04-16-00T00:00:00Z" = "451781d772888fbabc9b3efa5b612972ada8b10d9f1e3e4ef9e13e0055b26d52 ";
18+ "2019-04-17-00T00:00:00Z" = "af1169f894221a57d0c19a24ae656e65b37f571256f24586ee288f1848b781a4 ";
19+ "2019-04-18-00T00:00:00Z" = "686c07e94d86c3b3bfc7faa98f8e92123cc338a3cccb961aab69cce26cb9b8df ";
20+ "2019-04-19-00T00:00:00Z" = "da95cb1cb086671a046db5c1d49e81a641975ac875d87a1760b730fd44d592c3 ";
21+ "2019-04-20-00T00:00:00Z" = "1e215bdab78c14262f34e4c49e1f21af4bcb307cd87cd6dcb13a148c38b9e2a1 ";
22+ "2019-04-21-00T00:00:00Z" = "0fa42b89bdd55becc51eb6e0697d491d4c2f0205acc929f1f6ec46031ea4e37f ";
23+ "2019-04-22-00T00:00:00Z" = "2d8925ef46b1ebb2c5d5a4b2084e24ecc700aece3d459b07b33997e2dae557e0 ";
24+ "2019-04-23-00T00:00:00Z" = "b8c1ed659457be7d9b36f6f912d60b4e0960f19675315ede84503829346155d6 ";
25+ "2019-04-24-00T00:00:00Z" = "6342a0cdc02a3d4d49cc35a71f3a7f8e50814deb7610440a66837df69f13d6c2 ";
26+ "2019-04-25-00T00:00:00Z" = "b0624fb282a7540dc696325d95c81efdfcc00f2ad12738dc6e368609ebeb2258 ";
27+ "2019-04-26-00T00:00:00Z" = "8990dba9bf5657b5b263a28ef5f0a94593977fb44e7306a332c011e672c95aa1 ";
28+ "2019-04-27-00T00:00:00Z" = "42928f2ebc26e97f265be57a49b57fe0a0f64e55cd7e1a2ac4cafe2923a07011 ";
29+ "2019-04-28-00T00:00:00Z" = "1f1062646f08f565f58df6db14293a2f6faceecae5be4938d8ac9c223a9c37cc ";
30+ "2019-04-29-00T00:00:00Z" = "895bbce8a69b4fa4f0238dae9fc171eb57e9f56e71d9b31a5bb59f379db5298e ";
31+ "2019-04-30-00T00:00:00Z" = "e793c2ee42dc48c738b7e014bdf23d3b5106f0d2a943cc1a3ab6b39e24b092b2 ";
32+ "2019-05-01-00T00:00:00Z" = "1bfd413657fc405af03a6be09813a51b4a06a58aca3d3b1309de48b68ab7989e ";
33+ "2019-05-02-00T00:00:00Z" = "0c15661a4d602db8bc6ceeb637daeaa3513a97881a0690770bf16e6b5117096f ";
34+ "2019-05-03-00T00:00:00Z" = "6ec0bde7ba7efa8878f0a0ecf292a2ee5f439ae1687d00e201f2ad1bdb1f00d9 ";
35+ "2019-05-04-00T00:00:00Z" = "e279e2f613a6609874a29192e7e2c8dd4cbb98bc9d4da1ede8a4347f1484fcf5 ";
36+ "2019-05-05-00T00:00:00Z" = "6e47ddad859c1a848a45a9cf480be7c00966fd8b1497cf778acad344443165c8 ";
37+ "2019-05-06-00T00:00:00Z" = "e67d97b349980006c2041225900d2d7fc672c7ec64fb4e2f1276403a468d72a9 ";
38+ "2019-05-07-00T00:00:00Z" = "436d95d1de9c79c28081ad739f026ac1c9629549f4dcc153ec2fbfa57998c072 ";
39+ "2019-05-08-00T00:00:00Z" = "c75741834dc028e5e9968ce68d3d3381e22182246750cb6810f7b72e4803b39b ";
40+ "2019-05-09-00T00:00:00Z" = "8e65a6b0460b7cdffacab757e86c8c479bacc329523ebb1044dd7eb1e3c51c37 ";
41+ "2019-05-10-00T00:00:00Z" = "2cbcd3df939e0def72fc425753bd24bb803031d10bb5a9259f456c6e33d9d68d ";
42+ "2019-05-11-00T00:00:00Z" = "f8fd06d5571990bf7787f8aaff9b233f53ae816c6a8b2e3c193556dd376355f4 ";
43+ "2019-05-12-00T00:00:00Z" = "4651555d1bee27ee57643e0046eac1ec0b7a37c2da20be25b40b1775fb524402 ";
44+ "2019-05-13-00T00:00:00Z" = "3cda810efb958168fe8e01dc67f550b43b052c42c2d481e4941476b9165d2fd4 ";
45+ "2019-05-14-00T00:00:00Z" = "6a26a74e2cf1f8cbf110c35967befbfa4c3b941c56d69f33981444a432bba9b6 ";
46+ "2019-05-15-00T00:00:00Z" = "617f36845a950ed52192326d016a7c072e53cf62f30a0dcdad552fbf02966110 ";
47+ "2019-05-16-00T00:00:00Z" = "b5017246f47d403a5f3ed887bab30f6c39da411d8397477f6962375741a07935 ";
48+ "2019-05-17-00T00:00:00Z" = "2b269610ca0cdaeef36d70bd7bbe13a2a3425dda3873c89c9752704796281a78 ";
49+ "2019-05-18-00T00:00:00Z" = "d8ec971679393cb24a20035c752f2295e0ee8aabd1177e3902a40a94dd167529 ";
50+ "2019-05-19-00T00:00:00Z" = "8a61ec8f6748e92763fd35e36d3fabcde205ff1f8cdf7c08c942a69c6344f040 ";
51+}.dt or null
mk-local-hackage-repo/default.nixnew file+45
@@ -0,0 +1,45 @@
1+# Create a local hackare repo, we can use as a repository in a cabal config
2+#
3+# This will include:
4+# - 01-index.tar.gz (the index file)
5+# - root.json and
6+# - mirrors.json as metadata items.
7+# - snapshot.json that records the index, root and mirrors.
8+# - timestamp.json that will record the snapshot.json
9+#
10+# This is all part of The Update Framework (TUF) and the specific implementation
11+# cabal-install (via hackage-security) does of it.
12+#
13+# We will create a completely unsigned bare repository. Using signing keys within
14+# nix would be pointless as we'd have to hardcode them to produce the same output
15+# reproducably.
16+#
17+{ pkgs, hackageTarball }:
18+{ index-state, sha256 }@args:
19+let index = hackageTarball args; in
20+pkgs.runCommand "hackage-repo-${builtins.replaceStrings [":"] [""] index-state}" { nativeBuildInputs = [ pkgs.nix ]; } ''
21+mkdir -p $out
22+export expires="4000-01-01T00:00:00Z"
23+
24+ln -sf ${index} $out/01-index.tar.gz
25+export index_md5=$(nix-hash --flat --type md5 ${index})
26+export index_sha256=$(nix-hash --flat --type sha256 ${index})
27+export index_length=$(stat --printf="%s" ${index})
28+
29+substituteAll ${./root.json} $out/root.json
30+export root_md5=$(nix-hash --flat --type md5 $out/root.json)
31+export root_sha256=$(nix-hash --flat --type sha256 $out/root.json)
32+export root_length=$(stat --printf="%s" $out/root.json)
33+
34+substituteAll ${./mirrors.json} $out/mirrors.json
35+export mirrors_md5=$(nix-hash --flat --type md5 $out/mirrors.json)
36+export mirrors_sha256=$(nix-hash --flat --type sha256 $out/mirrors.json)
37+export mirrors_length=$(stat --printf="%s" $out/mirrors.json)
38+
39+substituteAll ${./snapshot.json} $out/snapshot.json
40+export snapshot_md5=$(nix-hash --flat --type md5 $out/snapshot.json)
41+export snapshot_sha256=$(nix-hash --flat --type sha256 $out/snapshot.json)
42+export snapshot_length=$(stat --printf="%s" $out/snapshot.json)
43+
44+substituteAll ${./timestamp.json} $out/timestamp.json
45+''
mk-local-hackage-repo/mirrors.jsonnew file+9
@@ -0,0 +1,9 @@
1+{
2+ "signatures": [ ],
3+ "signed": {
4+ "_type": "Mirrorlist",
5+ "expires": "@expires@",
6+ "mirrors": [],
7+ "version": 1
8+ }
9+}
mk-local-hackage-repo/root.jsonnew file+34
@@ -0,0 +1,34 @@
1+{
2+ "signatures": [],
3+ "signed": {
4+ "_type": "Root",
5+ "expires": "@expires@",
6+ "keys": {
7+ },
8+ "roles": {
9+ "mirrors": {
10+ "keyids": [ ],
11+ "threshold": 0
12+ },
13+ "root": {
14+ "keyids": [ ],
15+ "threshold": 0
16+ },
17+ "snapshot": {
18+ "keyids": [ ],
19+ "threshold": 0
20+ },
21+ "targets": {
22+ "keyids": [
23+ ],
24+ "threshold": 0
25+ },
26+ "timestamp": {
27+ "keyids": [
28+ ],
29+ "threshold": 0
30+ }
31+ },
32+ "version": 1
33+ }
34+}
mk-local-hackage-repo/snapshot.jsonnew file+31
@@ -0,0 +1,31 @@
1+{
2+ "signatures": [],
3+ "signed": {
4+ "_type": "Snapshot",
5+ "expires": "@expires@",
6+ "meta": {
7+ "<repo>/01-index.tar.gz": {
8+ "hashes": {
9+ "md5": "@index_md5@",
10+ "sha256": "@index_sha256@"
11+ },
12+ "length": @index_length@
13+ },
14+ "<repo>/root.json": {
15+ "hashes": {
16+ "md5": "@root_md5@",
17+ "sha256": "@root_sha256@"
18+ },
19+ "length": @root_length@
20+ },
21+ "<repo>/mirrors.json": {
22+ "hashes": {
23+ "md5": "@mirrors_md5@",
24+ "sha256": "@mirrors_sha256@"
25+ },
26+ "length": @mirrors_length@
27+ }
28+ },
29+ "version": 1
30+ }
31+}
mk-local-hackage-repo/timestamp.jsonnew file+17
@@ -0,0 +1,17 @@
1+{
2+ "signatures": [],
3+ "signed": {
4+ "_type": "Timestamp",
5+ "expires": "@expires@",
6+ "meta": {
7+ "<repo>/snapshot.json": {
8+ "hashes": {
9+ "md5": "@snapshot_md5@",
10+ "sha256": "@snapshot_sha256@"
11+ },
12+ "length": @snapshot_length@
13+ }
14+ },
15+ "version": 1
16+ }
17+}
test/call-cabal-project-to-nix/default.nix+2−2
@@ -5,7 +5,7 @@ with stdenv.lib;
55let
66 pkgSet = mkCabalProjectPkgSet {
77 plan-pkgs = import (callCabalProjectToNix {
8− hackageIndexState = "2019-04-24T21:34:04Z";
8+ index-state = "2019-04-30T00:00:00Z";
99 # reuse the cabal-simple test project
1010 src = ../cabal-simple;
1111 });
@@ -13,7 +13,7 @@ let
1313 packages = pkgSet.config.hsPkgs;
1414in
1515 stdenv.mkDerivation {
16− name = "callStackToNix-test";
16+ name = "call-cabal-project-to-nix-test";
1717
1818 buildCommand = ''
1919 exe="${packages.cabal-simple.components.exes.cabal-simple}/bin/cabal-simple"
test/default.nix+3−3
@@ -1,4 +1,5 @@
1−{ pkgs ? import nixpkgs {}
1+{ system ? builtins.currentSystem
2+, pkgs ? import nixpkgs { inherit system; }
23, nixpkgs ? ../nixpkgs
34, haskell ? pkgs.callPackage ../. { }
45}:
@@ -17,8 +18,7 @@ in {
1718 stack-simple = haskell.callPackage ./stack-simple {};
1819 callStackToNix = haskell.callPackage ./callStackToNix {};
1920
20− # Disabled -- doesn't work in a sandboxed build
21− # callCabalProjectToNix = haskell.callPackage ./call-cabal-project-to-nix {};
21+ callCabalProjectToNix = haskell.callPackage ./call-cabal-project-to-nix {};
2222
2323 # Run unit tests with: nix-instantiate --eval --strict -A unit
2424 # An empty list means success.