diff --git a/default.nix b/default.nix index b612e796..06b6bec9 100644 --- a/default.nix +++ b/default.nix @@ -38,11 +38,6 @@ let # overridden with NIX_PATH. fetchExternal = import ./lib/fetch-external.nix; - mkHackageIndex = indexState: import ./lib/hackageIndex.nix { - inherit (pkgs) runCommand cabal-install; - inherit indexState; - }; - # All packages from Hackage as Nix expressions hackage = import (fetchExternal { name = "hackage-exprs-source"; @@ -132,10 +127,44 @@ let # Make this handy overridable fetch function available. inherit fetchExternal; + # Produce a fixed output derivation from a moving target (hackage index tarball) + hackageTarball = { index-state, sha256 }: + pkgs.runCommand "01-index.tar.gz-at-${builtins.replaceStrings [":"] [""] index-state}" { + nativeBuildInputs = [ pkgs.curl ]; + outputHashAlgo = "sha256"; + outputHash = sha256; + # We'll use fetchurl's result in an env var ... + HACKAGE_INDEX = builtins.fetchurl "https://hackage.haskell.org/01-index.tar.gz"; + # ... and mark that impure. That way we can + # ensure the sore path stays the same and doesn't + # depend on the fetchurl result path. + impureEnvVars = [ "HACKAGE_INDEX" ]; + } + '' + ${self.nix-tools}/bin/truncate-index -o $out -i $HACKAGE_INDEX -s ${index-state} + ''; + + mkLocalHackageRepo = import ./mk-local-hackage-repo { inherit (self) hackageTarball; inherit pkgs; }; + + dotCabal = { index-state, sha256 }@args: + pkgs.runCommand "dot-cabal-at-${builtins.replaceStrings [":"] [""] index-state}" { nativeBuildInputs = [ pkgs.cabal-install ]; } '' + mkdir -p $out/.cabal + cat < $out/.cabal/config + repository cached + url: file:${self.mkLocalHackageRepo args} + secure: True + root-keys: + key-threshold: 0 + EOF + mkdir -p $out/.cabal/packages/cached + HOME=$out cabal new-update cached + ''; + + # Takes a haskell src directory runs cabal new-configure and plan-to-nix. # Resulting nix files are added to nix-plan subdirectory. callCabalProjectToNix = import ./lib/cabalProjectToNix.nix { - inherit mkHackageIndex; + inherit (self) dotCabal; inherit pkgs; inherit (pkgs) runCommand cabal-install ghc; inherit (pkgs.haskellPackages) hpack; diff --git a/lib/cabalProjectToNix.nix b/lib/cabalProjectToNix.nix index 26fd5da0..bfe74877 100644 --- a/lib/cabalProjectToNix.nix +++ b/lib/cabalProjectToNix.nix @@ -1,7 +1,7 @@ -{ mkHackageIndex, pkgs, runCommand, nix-tools, cabal-install, ghc, hpack, symlinkJoin }: +{ dotCabal, pkgs, runCommand, nix-tools, cabal-install, ghc, hpack, symlinkJoin }: let defaultGhc = ghc; defaultCabalInstall = cabal-install; -in { hackageIndexState, src, ghc ? defaultGhc, cabal-install ? defaultCabalInstall }: +in { index-state, index-sha256 ? import ./index-state-hashes.nix index-state, src, ghc ? defaultGhc, cabal-install ? defaultCabalInstall }: let cabalFiles = pkgs.lib.cleanSourceWith { @@ -14,7 +14,7 @@ let # cabal-install versions before 2.4 will generate insufficient plan information. then throw "cabal-install (current version: ${cabal-install.version}) needs to be at least 2.4 for plan-to-nix to work without cabal-to-nix" else runCommand "plan" { - nativeBuildInputs = [ nix-tools ghc hpack cabal-install pkgs.rsync ]; + nativeBuildInputs = [ nix-tools ghc hpack cabal-install pkgs.rsync pkgs.git ]; } '' tmp=$(mktemp -d) cd $tmp @@ -25,7 +25,7 @@ let # without the source available (we cleaneSourceWith'd it), # this may not produce the right result. find . -name package.yaml -exec hpack "{}" \; - HOME=${mkHackageIndex hackageIndexState} cabal new-configure + HOME=${dotCabal { inherit index-state; sha256 = index-sha256; }} cabal new-configure export LANG=C.utf8 # Needed or stack-to-nix will die on unicode inputs mkdir -p $out @@ -54,9 +54,25 @@ let mv $out/pkgs.nix $out/default.nix ''; in - runCommand "plan-and-src" { nativeBuildInputs = [ pkgs.xorg.lndir pkgs.rsync ]; } '' + # TODO: We really want this (symlinks) instead of copying the source over each and + # every time. However this will not work with sandboxed builds. They won't + # have access to `plan` or `src` paths. So while they will see all the + # links, they won't be able to read any of them. + # + # We should be able to fix this if we propagaed the build inputs properly. + # As we are `import`ing the produced nix-path here, we seem to be losing the + # dependencies though. + # + # I guess the end-result is that ifd's don't work well with symlinks. + # + # symlinkJoin { + # name = "plan-and-src"; + # # todo: should we clean `src` to drop any .git, .nix, ... other irelevant files? + # buildInputs = [ plan src ]; + # } + runCommand "plan-and-src" { nativeBuildInputs = [ pkgs.rsync ]; } '' mkdir $out # todo: should we clean `src` to drop any .git, .nix, ... other irelevant files? - lndir -silent "${src}" "$out" + rsync -a "${src}/" "$out/" rsync -a ${plan}/ $out/ '' diff --git a/lib/index-state-hashes.nix b/lib/index-state-hashes.nix new file mode 100644 index 00000000..db0a99e9 --- /dev/null +++ b/lib/index-state-hashes.nix @@ -0,0 +1,51 @@ +dt: { + "2019-04-01-00T00:00:00Z" = "a46ed96a1aa0510d1b54d508ae14dc6e460ff89ae41f0bd075aef87ba8c8ae73 "; + "2019-04-02-00T00:00:00Z" = "56dca813015122e0d633cb53bf8d11fb0c43a2e518b93cbbf93b0bdbf00e0c99 "; + "2019-04-03-00T00:00:00Z" = "b7a162996feec90e9acdff8bfb85a46ccbcafbbf0772739da45aef4e56480198 "; + "2019-04-04-00T00:00:00Z" = "6c890e98501539389807c7f7353b40af9bdd9e84a816adaccb79244ab1e506a0 "; + "2019-04-05-00T00:00:00Z" = "905b0ae09f4929b19b0885dcef785a45142bd6a7673e26979d6bc7044dedfd29 "; + "2019-04-06-00T00:00:00Z" = "2face70f256bc0a556e645f1f843c196411a97c78ded610443f3814b0783e5dc "; + "2019-04-07-00T00:00:00Z" = "f837f74181bd81120904c0e72b7c479faf178d1f2172a1442e167e7213b4a433 "; + "2019-04-08-00T00:00:00Z" = "86ff75e355e8b72757becb7761a1cd90b45b60f0f4777f8a137f30a27c766d65 "; + "2019-04-09-00T00:00:00Z" = "c00dc1146184db0743dce73d7bb53dca0fe75dff6634f61f11a6326984dba4ad "; + "2019-04-10-00T00:00:00Z" = "09cbd82c5f4a42e69299564a1494f5c2ff9fefde0663441a1878d351833a278c "; + "2019-04-11-00T00:00:00Z" = "072da9339e8c60b5e911659c110b0317a4e7890e30ce77e113edce1c373434d5 "; + "2019-04-12-00T00:00:00Z" = "9b14849542745d86529afa0f2cd2db989ea32a22a9a46d56fd3bc659339bdc28 "; + "2019-04-13-00T00:00:00Z" = "4aeba1860afc057b3faca85a289313a148a3392b80e5feab2263abfe8f9af155 "; + "2019-04-14-00T00:00:00Z" = "63a804787e63db9283b948ccc8ffafb487809efe60151295499e9d95230146fa "; + "2019-04-15-00T00:00:00Z" = "ef8607a4bf9e62052a428111b6decb5ef99e6da9dc885bf1bb51e593c4a97f9f "; + "2019-04-16-00T00:00:00Z" = "451781d772888fbabc9b3efa5b612972ada8b10d9f1e3e4ef9e13e0055b26d52 "; + "2019-04-17-00T00:00:00Z" = "af1169f894221a57d0c19a24ae656e65b37f571256f24586ee288f1848b781a4 "; + "2019-04-18-00T00:00:00Z" = "686c07e94d86c3b3bfc7faa98f8e92123cc338a3cccb961aab69cce26cb9b8df "; + "2019-04-19-00T00:00:00Z" = "da95cb1cb086671a046db5c1d49e81a641975ac875d87a1760b730fd44d592c3 "; + "2019-04-20-00T00:00:00Z" = "1e215bdab78c14262f34e4c49e1f21af4bcb307cd87cd6dcb13a148c38b9e2a1 "; + "2019-04-21-00T00:00:00Z" = "0fa42b89bdd55becc51eb6e0697d491d4c2f0205acc929f1f6ec46031ea4e37f "; + "2019-04-22-00T00:00:00Z" = "2d8925ef46b1ebb2c5d5a4b2084e24ecc700aece3d459b07b33997e2dae557e0 "; + "2019-04-23-00T00:00:00Z" = "b8c1ed659457be7d9b36f6f912d60b4e0960f19675315ede84503829346155d6 "; + "2019-04-24-00T00:00:00Z" = "6342a0cdc02a3d4d49cc35a71f3a7f8e50814deb7610440a66837df69f13d6c2 "; + "2019-04-25-00T00:00:00Z" = "b0624fb282a7540dc696325d95c81efdfcc00f2ad12738dc6e368609ebeb2258 "; + "2019-04-26-00T00:00:00Z" = "8990dba9bf5657b5b263a28ef5f0a94593977fb44e7306a332c011e672c95aa1 "; + "2019-04-27-00T00:00:00Z" = "42928f2ebc26e97f265be57a49b57fe0a0f64e55cd7e1a2ac4cafe2923a07011 "; + "2019-04-28-00T00:00:00Z" = "1f1062646f08f565f58df6db14293a2f6faceecae5be4938d8ac9c223a9c37cc "; + "2019-04-29-00T00:00:00Z" = "895bbce8a69b4fa4f0238dae9fc171eb57e9f56e71d9b31a5bb59f379db5298e "; + "2019-04-30-00T00:00:00Z" = "e793c2ee42dc48c738b7e014bdf23d3b5106f0d2a943cc1a3ab6b39e24b092b2 "; + "2019-05-01-00T00:00:00Z" = "1bfd413657fc405af03a6be09813a51b4a06a58aca3d3b1309de48b68ab7989e "; + "2019-05-02-00T00:00:00Z" = "0c15661a4d602db8bc6ceeb637daeaa3513a97881a0690770bf16e6b5117096f "; + "2019-05-03-00T00:00:00Z" = "6ec0bde7ba7efa8878f0a0ecf292a2ee5f439ae1687d00e201f2ad1bdb1f00d9 "; + "2019-05-04-00T00:00:00Z" = "e279e2f613a6609874a29192e7e2c8dd4cbb98bc9d4da1ede8a4347f1484fcf5 "; + "2019-05-05-00T00:00:00Z" = "6e47ddad859c1a848a45a9cf480be7c00966fd8b1497cf778acad344443165c8 "; + "2019-05-06-00T00:00:00Z" = "e67d97b349980006c2041225900d2d7fc672c7ec64fb4e2f1276403a468d72a9 "; + "2019-05-07-00T00:00:00Z" = "436d95d1de9c79c28081ad739f026ac1c9629549f4dcc153ec2fbfa57998c072 "; + "2019-05-08-00T00:00:00Z" = "c75741834dc028e5e9968ce68d3d3381e22182246750cb6810f7b72e4803b39b "; + "2019-05-09-00T00:00:00Z" = "8e65a6b0460b7cdffacab757e86c8c479bacc329523ebb1044dd7eb1e3c51c37 "; + "2019-05-10-00T00:00:00Z" = "2cbcd3df939e0def72fc425753bd24bb803031d10bb5a9259f456c6e33d9d68d "; + "2019-05-11-00T00:00:00Z" = "f8fd06d5571990bf7787f8aaff9b233f53ae816c6a8b2e3c193556dd376355f4 "; + "2019-05-12-00T00:00:00Z" = "4651555d1bee27ee57643e0046eac1ec0b7a37c2da20be25b40b1775fb524402 "; + "2019-05-13-00T00:00:00Z" = "3cda810efb958168fe8e01dc67f550b43b052c42c2d481e4941476b9165d2fd4 "; + "2019-05-14-00T00:00:00Z" = "6a26a74e2cf1f8cbf110c35967befbfa4c3b941c56d69f33981444a432bba9b6 "; + "2019-05-15-00T00:00:00Z" = "617f36845a950ed52192326d016a7c072e53cf62f30a0dcdad552fbf02966110 "; + "2019-05-16-00T00:00:00Z" = "b5017246f47d403a5f3ed887bab30f6c39da411d8397477f6962375741a07935 "; + "2019-05-17-00T00:00:00Z" = "2b269610ca0cdaeef36d70bd7bbe13a2a3425dda3873c89c9752704796281a78 "; + "2019-05-18-00T00:00:00Z" = "d8ec971679393cb24a20035c752f2295e0ee8aabd1177e3902a40a94dd167529 "; + "2019-05-19-00T00:00:00Z" = "8a61ec8f6748e92763fd35e36d3fabcde205ff1f8cdf7c08c942a69c6344f040 "; +}.dt or null diff --git a/mk-local-hackage-repo/default.nix b/mk-local-hackage-repo/default.nix new file mode 100644 index 00000000..b8f39314 --- /dev/null +++ b/mk-local-hackage-repo/default.nix @@ -0,0 +1,45 @@ +# Create a local hackare repo, we can use as a repository in a cabal config +# +# This will include: +# - 01-index.tar.gz (the index file) +# - root.json and +# - mirrors.json as metadata items. +# - snapshot.json that records the index, root and mirrors. +# - timestamp.json that will record the snapshot.json +# +# This is all part of The Update Framework (TUF) and the specific implementation +# cabal-install (via hackage-security) does of it. +# +# We will create a completely unsigned bare repository. Using signing keys within +# nix would be pointless as we'd have to hardcode them to produce the same output +# reproducably. +# +{ pkgs, hackageTarball }: +{ index-state, sha256 }@args: +let index = hackageTarball args; in +pkgs.runCommand "hackage-repo-${builtins.replaceStrings [":"] [""] index-state}" { nativeBuildInputs = [ pkgs.nix ]; } '' +mkdir -p $out +export expires="4000-01-01T00:00:00Z" + +ln -sf ${index} $out/01-index.tar.gz +export index_md5=$(nix-hash --flat --type md5 ${index}) +export index_sha256=$(nix-hash --flat --type sha256 ${index}) +export index_length=$(stat --printf="%s" ${index}) + +substituteAll ${./root.json} $out/root.json +export root_md5=$(nix-hash --flat --type md5 $out/root.json) +export root_sha256=$(nix-hash --flat --type sha256 $out/root.json) +export root_length=$(stat --printf="%s" $out/root.json) + +substituteAll ${./mirrors.json} $out/mirrors.json +export mirrors_md5=$(nix-hash --flat --type md5 $out/mirrors.json) +export mirrors_sha256=$(nix-hash --flat --type sha256 $out/mirrors.json) +export mirrors_length=$(stat --printf="%s" $out/mirrors.json) + +substituteAll ${./snapshot.json} $out/snapshot.json +export snapshot_md5=$(nix-hash --flat --type md5 $out/snapshot.json) +export snapshot_sha256=$(nix-hash --flat --type sha256 $out/snapshot.json) +export snapshot_length=$(stat --printf="%s" $out/snapshot.json) + +substituteAll ${./timestamp.json} $out/timestamp.json +'' diff --git a/mk-local-hackage-repo/mirrors.json b/mk-local-hackage-repo/mirrors.json new file mode 100644 index 00000000..4f749df2 --- /dev/null +++ b/mk-local-hackage-repo/mirrors.json @@ -0,0 +1,9 @@ +{ + "signatures": [ ], + "signed": { + "_type": "Mirrorlist", + "expires": "@expires@", + "mirrors": [], + "version": 1 + } +} diff --git a/mk-local-hackage-repo/root.json b/mk-local-hackage-repo/root.json new file mode 100644 index 00000000..d8856ed3 --- /dev/null +++ b/mk-local-hackage-repo/root.json @@ -0,0 +1,34 @@ +{ + "signatures": [], + "signed": { + "_type": "Root", + "expires": "@expires@", + "keys": { + }, + "roles": { + "mirrors": { + "keyids": [ ], + "threshold": 0 + }, + "root": { + "keyids": [ ], + "threshold": 0 + }, + "snapshot": { + "keyids": [ ], + "threshold": 0 + }, + "targets": { + "keyids": [ + ], + "threshold": 0 + }, + "timestamp": { + "keyids": [ + ], + "threshold": 0 + } + }, + "version": 1 + } +} diff --git a/mk-local-hackage-repo/snapshot.json b/mk-local-hackage-repo/snapshot.json new file mode 100644 index 00000000..e1373499 --- /dev/null +++ b/mk-local-hackage-repo/snapshot.json @@ -0,0 +1,31 @@ +{ + "signatures": [], + "signed": { + "_type": "Snapshot", + "expires": "@expires@", + "meta": { + "/01-index.tar.gz": { + "hashes": { + "md5": "@index_md5@", + "sha256": "@index_sha256@" + }, + "length": @index_length@ + }, + "/root.json": { + "hashes": { + "md5": "@root_md5@", + "sha256": "@root_sha256@" + }, + "length": @root_length@ + }, + "/mirrors.json": { + "hashes": { + "md5": "@mirrors_md5@", + "sha256": "@mirrors_sha256@" + }, + "length": @mirrors_length@ + } + }, + "version": 1 + } +} diff --git a/mk-local-hackage-repo/timestamp.json b/mk-local-hackage-repo/timestamp.json new file mode 100644 index 00000000..d9916707 --- /dev/null +++ b/mk-local-hackage-repo/timestamp.json @@ -0,0 +1,17 @@ +{ + "signatures": [], + "signed": { + "_type": "Timestamp", + "expires": "@expires@", + "meta": { + "/snapshot.json": { + "hashes": { + "md5": "@snapshot_md5@", + "sha256": "@snapshot_sha256@" + }, + "length": @snapshot_length@ + } + }, + "version": 1 + } +} diff --git a/test/call-cabal-project-to-nix/default.nix b/test/call-cabal-project-to-nix/default.nix index c18b6a33..e8e69bc6 100644 --- a/test/call-cabal-project-to-nix/default.nix +++ b/test/call-cabal-project-to-nix/default.nix @@ -5,7 +5,7 @@ with stdenv.lib; let pkgSet = mkCabalProjectPkgSet { plan-pkgs = import (callCabalProjectToNix { - hackageIndexState = "2019-04-24T21:34:04Z"; + index-state = "2019-04-30T00:00:00Z"; # reuse the cabal-simple test project src = ../cabal-simple; }); @@ -13,7 +13,7 @@ let packages = pkgSet.config.hsPkgs; in stdenv.mkDerivation { - name = "callStackToNix-test"; + name = "call-cabal-project-to-nix-test"; buildCommand = '' exe="${packages.cabal-simple.components.exes.cabal-simple}/bin/cabal-simple" diff --git a/test/default.nix b/test/default.nix index dc152ae3..cfc58cea 100644 --- a/test/default.nix +++ b/test/default.nix @@ -1,4 +1,5 @@ -{ pkgs ? import nixpkgs {} +{ system ? builtins.currentSystem +, pkgs ? import nixpkgs { inherit system; } , nixpkgs ? ../nixpkgs , haskell ? pkgs.callPackage ../. { } }: @@ -17,8 +18,7 @@ in { stack-simple = haskell.callPackage ./stack-simple {}; callStackToNix = haskell.callPackage ./callStackToNix {}; - # Disabled -- doesn't work in a sandboxed build - # callCabalProjectToNix = haskell.callPackage ./call-cabal-project-to-nix {}; + callCabalProjectToNix = haskell.callPackage ./call-cabal-project-to-nix {}; # Run unit tests with: nix-instantiate --eval --strict -A unit # An empty list means success.