From 2d54f2c30335b5133a34a2f578afb2bcf05e5331 Mon Sep 17 00:00:00 2001 From: Moritz Angermann Date: Thu, 23 May 2019 13:13:04 +0800 Subject: [PATCH] Use pkgs.fetchurl to build a fixed output derivation (#145) * Use pkgs.fetchurl to build a fixed output derivation This should hopefully get us around the restricted limitations. * Adds asserts * Fix index-state-hashes file * Fix index-state-hashes Also adds a regeneration tool: ``` nix build -f ./. generateHackageIndexStateHashes cp result lib/index-state-hashes.nix ``` * Fix off-by-one * better errors --- default.nix | 39 ++++++++----- lib/cabalProjectToNix.nix | 14 +++-- lib/index-state-hashes.nix | 106 ++++++++++++++++++----------------- nix-tools/nix-tools-src.json | 6 +- 4 files changed, 92 insertions(+), 73 deletions(-) diff --git a/default.nix b/default.nix index f459f09d..2e29b784 100644 --- a/default.nix +++ b/default.nix @@ -1,7 +1,8 @@ -{ pkgs ? import nixpkgs {} +{ pkgs ? import nixpkgs nixpkgsArgs # Use a pinned nixpkgs rather than the one on NIX_PATH , nixpkgs ? ./nixpkgs - +# Provide args to the nixpkgs instantiation. +, nixpkgsArgs ? {} # You can provide different pins for hackage.nix and stackage.nix if required. # It's also possible to override these sources with NIX_PATH. , hackageSourceJSON ? ./hackage-src.json @@ -135,20 +136,16 @@ let # Produce a fixed output derivation from a moving target (hackage index tarball) hackageTarball = { index-state, sha256 }: - pkgs.runCommand "01-index.tar.gz-at-${builtins.replaceStrings [":"] [""] index-state}" { - nativeBuildInputs = [ pkgs.curl ]; + assert sha256 != null; + pkgs.fetchurl { + name = "01-index.tar.gz-at-${builtins.replaceStrings [":"] [""] index-state}"; + url = "https://hackage.haskell.org/01-index.tar.gz"; + downloadToTemp = true; + postFetch = "${self.nix-tools}/bin/truncate-index -o $out -i $downloadedFile -s ${index-state}"; + outputHashAlgo = "sha256"; outputHash = sha256; - # We'll use fetchurl's result in an env var ... - HACKAGE_INDEX = builtins.fetchurl "https://hackage.haskell.org/01-index.tar.gz"; - # ... and mark that impure. That way we can - # ensure the sore path stays the same and doesn't - # depend on the fetchurl result path. - impureEnvVars = [ "HACKAGE_INDEX" ]; - } - '' - ${self.nix-tools}/bin/truncate-index -o $out -i $HACKAGE_INDEX -s ${index-state} - ''; + }; mkLocalHackageRepo = import ./mk-local-hackage-repo { inherit (self) hackageTarball; inherit pkgs; }; @@ -166,6 +163,20 @@ let HOME=$out cabal new-update cached ''; + generateHackageIndexStateHashes = pkgs.runCommand "index-state-hashes" {} '' + truncate=${self.nix-tools}/bin/truncate-index + start=${let ls = builtins.attrNames (import ./lib/index-state-hashes.nix); in builtins.elemAt ls (builtins.length ls - 1)} + cat ${./lib/index-state-hashes.nix} | head -n -1 >> $out + for d in $(seq -f '%.f' $(date -u +%s -d $start) 86400 $(date -u +%s)) ; do + dt=$(date -u +%Y-%m-%d -d @$d) + if [[ "''${dt}T00:00:00Z" != "$start" ]]; then + ''${truncate} -o ''${dt}-01-index.tar.gz -i ${builtins.fetchurl "https://hackage.haskell.org/01-index.tar.gz"} -s "''${dt}T00:00:00Z" + sha256=$(${pkgs.nix}/bin/nix-hash --flat --type sha256 ''${dt}-01-index.tar.gz) + echo " \"''${dt}T00:00:00Z\" = \"''${sha256}\";" >> $out + fi + done + echo '}' >> $out + ''; # Takes a haskell src directory runs cabal new-configure and plan-to-nix. # Resulting nix files are added to nix-plan subdirectory. diff --git a/lib/cabalProjectToNix.nix b/lib/cabalProjectToNix.nix index bfe74877..6f353f94 100644 --- a/lib/cabalProjectToNix.nix +++ b/lib/cabalProjectToNix.nix @@ -1,7 +1,14 @@ { dotCabal, pkgs, runCommand, nix-tools, cabal-install, ghc, hpack, symlinkJoin }: let defaultGhc = ghc; defaultCabalInstall = cabal-install; -in { index-state, index-sha256 ? import ./index-state-hashes.nix index-state, src, ghc ? defaultGhc, cabal-install ? defaultCabalInstall }: +in { index-state, index-sha256 ? (import ./index-state-hashes.nix).${index-state} or null, src, ghc ? defaultGhc, cabal-install ? defaultCabalInstall }: + +# better error message than just assert failed. +assert (if index-sha256 == null then throw "provided sha256 for index-state ${index-state} is null!" else true); +# cabal-install versions before 2.4 will generate insufficient plan information. +assert (if (builtins.compareVersions cabal-install.version "2.4.0.0") < 0 + then throw "cabal-install (current version: ${cabal-install.version}) needs to be at least 2.4 for plan-to-nix to work without cabal-to-nix" + else true); let cabalFiles = pkgs.lib.cleanSourceWith { @@ -10,10 +17,7 @@ let type == "directory" || pkgs.lib.any (i: (pkgs.lib.hasSuffix i path)) [ ".project" ".cabal" "package.yaml" ]; }; - plan = if (builtins.compareVersions cabal-install.version "2.4.0.0") < 0 - # cabal-install versions before 2.4 will generate insufficient plan information. - then throw "cabal-install (current version: ${cabal-install.version}) needs to be at least 2.4 for plan-to-nix to work without cabal-to-nix" - else runCommand "plan" { + plan = runCommand "plan" { nativeBuildInputs = [ nix-tools ghc hpack cabal-install pkgs.rsync pkgs.git ]; } '' tmp=$(mktemp -d) diff --git a/lib/index-state-hashes.nix b/lib/index-state-hashes.nix index db0a99e9..6774cfd9 100644 --- a/lib/index-state-hashes.nix +++ b/lib/index-state-hashes.nix @@ -1,51 +1,55 @@ -dt: { - "2019-04-01-00T00:00:00Z" = "a46ed96a1aa0510d1b54d508ae14dc6e460ff89ae41f0bd075aef87ba8c8ae73 "; - "2019-04-02-00T00:00:00Z" = "56dca813015122e0d633cb53bf8d11fb0c43a2e518b93cbbf93b0bdbf00e0c99 "; - "2019-04-03-00T00:00:00Z" = "b7a162996feec90e9acdff8bfb85a46ccbcafbbf0772739da45aef4e56480198 "; - "2019-04-04-00T00:00:00Z" = "6c890e98501539389807c7f7353b40af9bdd9e84a816adaccb79244ab1e506a0 "; - "2019-04-05-00T00:00:00Z" = "905b0ae09f4929b19b0885dcef785a45142bd6a7673e26979d6bc7044dedfd29 "; - "2019-04-06-00T00:00:00Z" = "2face70f256bc0a556e645f1f843c196411a97c78ded610443f3814b0783e5dc "; - "2019-04-07-00T00:00:00Z" = "f837f74181bd81120904c0e72b7c479faf178d1f2172a1442e167e7213b4a433 "; - "2019-04-08-00T00:00:00Z" = "86ff75e355e8b72757becb7761a1cd90b45b60f0f4777f8a137f30a27c766d65 "; - "2019-04-09-00T00:00:00Z" = "c00dc1146184db0743dce73d7bb53dca0fe75dff6634f61f11a6326984dba4ad "; - "2019-04-10-00T00:00:00Z" = "09cbd82c5f4a42e69299564a1494f5c2ff9fefde0663441a1878d351833a278c "; - "2019-04-11-00T00:00:00Z" = "072da9339e8c60b5e911659c110b0317a4e7890e30ce77e113edce1c373434d5 "; - "2019-04-12-00T00:00:00Z" = "9b14849542745d86529afa0f2cd2db989ea32a22a9a46d56fd3bc659339bdc28 "; - "2019-04-13-00T00:00:00Z" = "4aeba1860afc057b3faca85a289313a148a3392b80e5feab2263abfe8f9af155 "; - "2019-04-14-00T00:00:00Z" = "63a804787e63db9283b948ccc8ffafb487809efe60151295499e9d95230146fa "; - "2019-04-15-00T00:00:00Z" = "ef8607a4bf9e62052a428111b6decb5ef99e6da9dc885bf1bb51e593c4a97f9f "; - "2019-04-16-00T00:00:00Z" = "451781d772888fbabc9b3efa5b612972ada8b10d9f1e3e4ef9e13e0055b26d52 "; - "2019-04-17-00T00:00:00Z" = "af1169f894221a57d0c19a24ae656e65b37f571256f24586ee288f1848b781a4 "; - "2019-04-18-00T00:00:00Z" = "686c07e94d86c3b3bfc7faa98f8e92123cc338a3cccb961aab69cce26cb9b8df "; - "2019-04-19-00T00:00:00Z" = "da95cb1cb086671a046db5c1d49e81a641975ac875d87a1760b730fd44d592c3 "; - "2019-04-20-00T00:00:00Z" = "1e215bdab78c14262f34e4c49e1f21af4bcb307cd87cd6dcb13a148c38b9e2a1 "; - "2019-04-21-00T00:00:00Z" = "0fa42b89bdd55becc51eb6e0697d491d4c2f0205acc929f1f6ec46031ea4e37f "; - "2019-04-22-00T00:00:00Z" = "2d8925ef46b1ebb2c5d5a4b2084e24ecc700aece3d459b07b33997e2dae557e0 "; - "2019-04-23-00T00:00:00Z" = "b8c1ed659457be7d9b36f6f912d60b4e0960f19675315ede84503829346155d6 "; - "2019-04-24-00T00:00:00Z" = "6342a0cdc02a3d4d49cc35a71f3a7f8e50814deb7610440a66837df69f13d6c2 "; - "2019-04-25-00T00:00:00Z" = "b0624fb282a7540dc696325d95c81efdfcc00f2ad12738dc6e368609ebeb2258 "; - "2019-04-26-00T00:00:00Z" = "8990dba9bf5657b5b263a28ef5f0a94593977fb44e7306a332c011e672c95aa1 "; - "2019-04-27-00T00:00:00Z" = "42928f2ebc26e97f265be57a49b57fe0a0f64e55cd7e1a2ac4cafe2923a07011 "; - "2019-04-28-00T00:00:00Z" = "1f1062646f08f565f58df6db14293a2f6faceecae5be4938d8ac9c223a9c37cc "; - "2019-04-29-00T00:00:00Z" = "895bbce8a69b4fa4f0238dae9fc171eb57e9f56e71d9b31a5bb59f379db5298e "; - "2019-04-30-00T00:00:00Z" = "e793c2ee42dc48c738b7e014bdf23d3b5106f0d2a943cc1a3ab6b39e24b092b2 "; - "2019-05-01-00T00:00:00Z" = "1bfd413657fc405af03a6be09813a51b4a06a58aca3d3b1309de48b68ab7989e "; - "2019-05-02-00T00:00:00Z" = "0c15661a4d602db8bc6ceeb637daeaa3513a97881a0690770bf16e6b5117096f "; - "2019-05-03-00T00:00:00Z" = "6ec0bde7ba7efa8878f0a0ecf292a2ee5f439ae1687d00e201f2ad1bdb1f00d9 "; - "2019-05-04-00T00:00:00Z" = "e279e2f613a6609874a29192e7e2c8dd4cbb98bc9d4da1ede8a4347f1484fcf5 "; - "2019-05-05-00T00:00:00Z" = "6e47ddad859c1a848a45a9cf480be7c00966fd8b1497cf778acad344443165c8 "; - "2019-05-06-00T00:00:00Z" = "e67d97b349980006c2041225900d2d7fc672c7ec64fb4e2f1276403a468d72a9 "; - "2019-05-07-00T00:00:00Z" = "436d95d1de9c79c28081ad739f026ac1c9629549f4dcc153ec2fbfa57998c072 "; - "2019-05-08-00T00:00:00Z" = "c75741834dc028e5e9968ce68d3d3381e22182246750cb6810f7b72e4803b39b "; - "2019-05-09-00T00:00:00Z" = "8e65a6b0460b7cdffacab757e86c8c479bacc329523ebb1044dd7eb1e3c51c37 "; - "2019-05-10-00T00:00:00Z" = "2cbcd3df939e0def72fc425753bd24bb803031d10bb5a9259f456c6e33d9d68d "; - "2019-05-11-00T00:00:00Z" = "f8fd06d5571990bf7787f8aaff9b233f53ae816c6a8b2e3c193556dd376355f4 "; - "2019-05-12-00T00:00:00Z" = "4651555d1bee27ee57643e0046eac1ec0b7a37c2da20be25b40b1775fb524402 "; - "2019-05-13-00T00:00:00Z" = "3cda810efb958168fe8e01dc67f550b43b052c42c2d481e4941476b9165d2fd4 "; - "2019-05-14-00T00:00:00Z" = "6a26a74e2cf1f8cbf110c35967befbfa4c3b941c56d69f33981444a432bba9b6 "; - "2019-05-15-00T00:00:00Z" = "617f36845a950ed52192326d016a7c072e53cf62f30a0dcdad552fbf02966110 "; - "2019-05-16-00T00:00:00Z" = "b5017246f47d403a5f3ed887bab30f6c39da411d8397477f6962375741a07935 "; - "2019-05-17-00T00:00:00Z" = "2b269610ca0cdaeef36d70bd7bbe13a2a3425dda3873c89c9752704796281a78 "; - "2019-05-18-00T00:00:00Z" = "d8ec971679393cb24a20035c752f2295e0ee8aabd1177e3902a40a94dd167529 "; - "2019-05-19-00T00:00:00Z" = "8a61ec8f6748e92763fd35e36d3fabcde205ff1f8cdf7c08c942a69c6344f040 "; -}.dt or null +{ + "2019-04-01T00:00:00Z" = "c474efaba33ac2e6b737a2ac694a74e326884eacd01708eb1f42200854c7a8c9"; + "2019-04-02T00:00:00Z" = "f720c323879e5b5f4d6fe23ff5aa6a93b9f0f828096096523b48d8c718d3893c"; + "2019-04-03T00:00:00Z" = "eedd2c025b5a59037698d24dc2a035cc93f1f464a1d738bf0b9a70c20837b485"; + "2019-04-04T00:00:00Z" = "78dd777f80a34c9ad01783bbae7fe648e564d838bed971126596d52aba8faee7"; + "2019-04-05T00:00:00Z" = "3f892e7be48e2723910aa5b3f7ed8c5e7ca247275a3e03ccc3ce84eb5e239a90"; + "2019-04-06T00:00:00Z" = "5698ef0c65688760b8a5fb97acd4f7e85f904f6d150fdf72b401c1b6c40846bd"; + "2019-04-07T00:00:00Z" = "0800715fcdbbd83fba59f38e2b2f27d6a5ababd477bcfa89d13b27961522a6fe"; + "2019-04-08T00:00:00Z" = "a7b73aa3891079ebcc42d8ce8d2e05624ff3ea3cec81f199731f4a88b5eaf17b"; + "2019-04-09T00:00:00Z" = "75bffd47f7c6f3cb80b0808f0bda19dbc7f2e1dfdb0bc867259d4e0d0ffcc6fe"; + "2019-04-10T00:00:00Z" = "35aae6a9fcb71b98a1e6a84511e40769cf6ed141a22778235770e5d1c7e46982"; + "2019-04-11T00:00:00Z" = "094321bbafa5150b3e397b2fee0052e441e16c971357d182b12628e24475b03c"; + "2019-04-12T00:00:00Z" = "29092b8546eea6f22e1169216bcefd3544d15f7f3f1549cb0d0f7a6717a2d2cb"; + "2019-04-13T00:00:00Z" = "e7cd063aaeb24d0a4604db4a7f827259852a9fe4f1bc67f3c171a8fbcb5b8707"; + "2019-04-14T00:00:00Z" = "e1db5ce15198d5c98473d32dd74d9532fb064a042d71cb078d0c4f97daa05351"; + "2019-04-15T00:00:00Z" = "7df0a5d157b6e730c68f614644ae7863de32d19260730683ad685cc24691ed93"; + "2019-04-16T00:00:00Z" = "76763ec23fd99e41a42d6a6c06afebc34c8319ba7bb03433cc7db868cab31d8b"; + "2019-04-17T00:00:00Z" = "96232013324793960158bf7512fc0cc4855afbefdcc98e510bcbaae99ec11feb"; + "2019-04-18T00:00:00Z" = "605c20dc88f98e0821ad8913f198e690a6a9ff4128d13cc91cfb695ee5d25b3f"; + "2019-04-19T00:00:00Z" = "a5bad701ce83d51a0d4de910d6502ae835bdffd947bbbbbac8adf9cfd3e1e9ce"; + "2019-04-20T00:00:00Z" = "5727553a452213ed41bde3eb651b78998dfd1fa5a19fedaecf3c28462e8bb175"; + "2019-04-21T00:00:00Z" = "81dbfcfa7f33beee9f6e2d57f07b7757e1390d2d389c4763e8e7b3f767ec7b60"; + "2019-04-22T00:00:00Z" = "9c0051b3d1023e7118087f0c8914fbf8dcae1e62545cae47c82c12a37012a00a"; + "2019-04-23T00:00:00Z" = "52232c52172f61113a839ac3511051cfb13ba91089fe458360fde193d00e86ca"; + "2019-04-24T00:00:00Z" = "f8c9eb039f16c95c825109469937c1a8ef270f56f0d376ca1cd19f6b61ba8ab4"; + "2019-04-25T00:00:00Z" = "b67e1a5e9a88beb2c741e47349cbc74fec01f2cb3279f982161e3f63e51a4576"; + "2019-04-26T00:00:00Z" = "4de880d417d118fe296121b2f3ac3bb2d4b08d56ed5490f30b81ae0ac2388016"; + "2019-04-27T00:00:00Z" = "20dd0ba939636081482b9bbc6fac0bcb2b1f5710ad70713945e7af2e9ee1b510"; + "2019-04-28T00:00:00Z" = "33df24a7827a6504944a6546fb32c22bee1af05e8d441f30d0a7abe6487eb65f"; + "2019-04-29T00:00:00Z" = "50fe3ffa024e0a131ac9440b3fec099ca83d298163f71731b25bbdb84b7681ac"; + "2019-04-30T00:00:00Z" = "f0dedbd5e75d581fc163d94ddba92a62f785a90183ccfaa426131de6218ad7cb"; + "2019-05-01T00:00:00Z" = "6ee044506ff9befb4709526f15e2858fce30a0c40b27ca1e518f9466f0f81c5e"; + "2019-05-02T00:00:00Z" = "0077eb4cdd1a9649dad8196dfd2d8f9db9183b66f0049e0f7c4fd843f43a4664"; + "2019-05-03T00:00:00Z" = "c1a0e3c1c3854911ffa5e49d9e1aa6b43235cbc5ec272ff5be675c8f8d231b4a"; + "2019-05-04T00:00:00Z" = "7c2e4bec642de2e6b5afd3aa57ee53323f6b1bec36d9af78a432199034abe254"; + "2019-05-05T00:00:00Z" = "b4ff7a89b7f93c88597119656cf938fdf2b72454991fd8b683b5f7a5bdba0fd4"; + "2019-05-06T00:00:00Z" = "822220587de7ee36d633e9bc72df760064237b13c286d0c52efdc5d5048297de"; + "2019-05-07T00:00:00Z" = "2927619e7b636780296f06d3f816c159d121b507bb2c81c7abda40926d76df12"; + "2019-05-08T00:00:00Z" = "05e3bed075a0a9095a7ddfee1a096e56bea41948eeec1a7d3a673980a257d4a3"; + "2019-05-09T00:00:00Z" = "501bddbc5cdac16f85724066df5be4e2f40e327b5f5fe0999e70c2901ce2d89a"; + "2019-05-10T00:00:00Z" = "4e6e357aa2f820ca3883dee8f2af7e359f640e1639036408d6e2b8af9751a028"; + "2019-05-11T00:00:00Z" = "8203d571863706fb4fa5ec121b689d8601135a63ca12456c86554029270b9ad0"; + "2019-05-12T00:00:00Z" = "99fb942cabcda72770adec289d4f138a985e76fb2e36dea39f54734ef74c5bb5"; + "2019-05-13T00:00:00Z" = "fef0a5e9170ac226ed47a483057f7e3f84ae374102431e5f46165140c516aa1f"; + "2019-05-14T00:00:00Z" = "2c08a3a145485279f396521374ebe1efd2022497809ae957259fea2bdb5cf829"; + "2019-05-15T00:00:00Z" = "864143953a41b64d5263ce67ec476396adafc92e77f77748916cefa76b9dd849"; + "2019-05-16T00:00:00Z" = "d5bb19b8019b617794dcb2a4cff72ebfb8edb96b232f040c486dd570e1785528"; + "2019-05-17T00:00:00Z" = "4c06e58c1698f650eb28ed545f9d243a564b30a4c0e3097df62e18dfffe9c72b"; + "2019-05-18T00:00:00Z" = "a3fd7461d73c2c393e9ed37c029b4b92482b9aa9df7790c826cb0a0408fb4522"; + "2019-05-19T00:00:00Z" = "84b578c748049bcba2b5ab35fea77d1278662b4fb003ee1c903aa88c167c19ab"; + "2019-05-20T00:00:00Z" = "87e8c61faf75005929650fde538d3a63d0db9e28ea3a40db79dc4f3ab9a5347f"; + "2019-05-21T00:00:00Z" = "27f4ad652a0880a26d8f8242611f6112296f078b45868f704cf628f9b6610d00"; + "2019-05-22T00:00:00Z" = "f6c2c8a3b3c44a893138042eb7ff8470511156ea2fabafc633eb7cb4b53af236"; + "2019-05-23T00:00:00Z" = "9587cd6b2a317e608e8b34bafc13e0ef9d22b5332f21cc5544cea0d765223cba"; +} diff --git a/nix-tools/nix-tools-src.json b/nix-tools/nix-tools-src.json index 9433975d..19c5b8f4 100644 --- a/nix-tools/nix-tools-src.json +++ b/nix-tools/nix-tools-src.json @@ -1,7 +1,7 @@ { "url": "https://github.com/input-output-hk/nix-tools", - "rev": "09b4ec4ea17673204786f554f63544e6af5eb83c", - "date": "2019-05-15T08:44:38+08:00", - "sha256": "0gcb8pi9rfc4bddkdh857vfa370r11631mkmap8x52q8rirammbl", + "rev": "69ca9bd0c1994f7bddb12bc1a99ac980e7365c14", + "date": "2019-05-23T11:35:17+08:00", + "sha256": "0f9ds6i4qni6xfgh8bq5kr469wr8j7a2zlb9l9g6gqy0y3hpihqm", "fetchSubmodules": false }