From 03fd50f0b75e3e49e1d427197d08f1266d5c335d Mon Sep 17 00:00:00 2001 From: Moritz Angermann Date: Sat, 8 Jun 2019 15:21:02 +0800 Subject: [PATCH] Run the nightly scripts to also update the index-state-hashes (#146) * Run the nightly scripts to also update the index-state-hashes This moves the index-state-hashes to `hackage.nix` as well. Thus we need to have a least one hackage.nix update in. * fix import * Update update-index-state-hashes.nix Add comments. --- default.nix | 22 ++++------- lib/cabalProjectToNix.nix | 4 +- lib/index-state-hashes.nix | 55 --------------------------- scripts/update-hackage.nix | 6 ++- scripts/update-index-state-hashes.nix | 42 ++++++++++++++++++++ 5 files changed, 57 insertions(+), 72 deletions(-) delete mode 100644 lib/index-state-hashes.nix create mode 100644 scripts/update-index-state-hashes.nix diff --git a/default.nix b/default.nix index 9c4c15a5..450f356d 100644 --- a/default.nix +++ b/default.nix @@ -50,6 +50,10 @@ let }; hackage = import hackageSrc; + # Contains the hashes of the cabal 01-index.tar.gz for given + # index states. Starting from April 1st 2019. + indexStateHashesPath = hackageSrc + "/index-state-hashes.nix"; + # The set of all Stackage snapshots stackageSrc = fetchExternal { name = "stackage-snapshot-source"; @@ -130,7 +134,9 @@ let # Snapshots of Hackage and Stackage, converted to Nix expressions, # regularly updated. + inherit hackageSrc stackageSrc; inherit hackage stackage; + inherit indexStateHashesPath; # Make this handy overridable fetch function available. inherit fetchExternal; @@ -167,24 +173,12 @@ let HOME=$out cabal new-update cached ''; - generateHackageIndexStateHashes = pkgs.runCommand "index-state-hashes" {} '' - truncate=${self.nix-tools}/bin/truncate-index - start=${let ls = builtins.attrNames (import ./lib/index-state-hashes.nix); in builtins.elemAt ls (builtins.length ls - 1)} - cat ${./lib/index-state-hashes.nix} | head -n -1 >> $out - for d in $(seq -f '%.f' $(date -u +%s -d $start) 86400 $(date -u +%s)) ; do - dt=$(date -u +%Y-%m-%d -d @$d) - if [[ "''${dt}T00:00:00Z" != "$start" ]]; then - ''${truncate} -o ''${dt}-01-index.tar.gz -i ${builtins.fetchurl "https://hackage.haskell.org/01-index.tar.gz"} -s "''${dt}T00:00:00Z" - sha256=$(${pkgs.nix}/bin/nix-hash --flat --type sha256 ''${dt}-01-index.tar.gz) - echo " \"''${dt}T00:00:00Z\" = \"''${sha256}\";" >> $out - fi - done - echo '}' >> $out - ''; + update-index-state-hashes = self.callPackage ./scripts/update-index-state-hashes.nix {}; # Takes a haskell src directory runs cabal new-configure and plan-to-nix. # Resulting nix files are added to nix-plan subdirectory. callCabalProjectToNix = import ./lib/cabalProjectToNix.nix { + index-state-hashes = import indexStateHashesPath; inherit (self) dotCabal; inherit pkgs; inherit (pkgs) runCommand cabal-install ghc symlinkJoin cacert; diff --git a/lib/cabalProjectToNix.nix b/lib/cabalProjectToNix.nix index a8df0d5e..4d7a95c1 100644 --- a/lib/cabalProjectToNix.nix +++ b/lib/cabalProjectToNix.nix @@ -1,7 +1,7 @@ -{ dotCabal, pkgs, runCommand, nix-tools, cabal-install, ghc, hpack, symlinkJoin, cacert }: +{ dotCabal, pkgs, runCommand, nix-tools, cabal-install, ghc, hpack, symlinkJoin, cacert, index-state-hashes }: let defaultGhc = ghc; defaultCabalInstall = cabal-install; -in { index-state, index-sha256 ? (import ./index-state-hashes.nix).${index-state} or null, src, ghc ? defaultGhc, cabal-install ? defaultCabalInstall }: +in { index-state, index-sha256 ? index-state-hashes.${index-state} or null, src, ghc ? defaultGhc, cabal-install ? defaultCabalInstall }: # better error message than just assert failed. assert (if index-sha256 == null then throw "provided sha256 for index-state ${index-state} is null!" else true); diff --git a/lib/index-state-hashes.nix b/lib/index-state-hashes.nix deleted file mode 100644 index 6774cfd9..00000000 --- a/lib/index-state-hashes.nix +++ /dev/null @@ -1,55 +0,0 @@ -{ - "2019-04-01T00:00:00Z" = "c474efaba33ac2e6b737a2ac694a74e326884eacd01708eb1f42200854c7a8c9"; - "2019-04-02T00:00:00Z" = "f720c323879e5b5f4d6fe23ff5aa6a93b9f0f828096096523b48d8c718d3893c"; - "2019-04-03T00:00:00Z" = "eedd2c025b5a59037698d24dc2a035cc93f1f464a1d738bf0b9a70c20837b485"; - "2019-04-04T00:00:00Z" = "78dd777f80a34c9ad01783bbae7fe648e564d838bed971126596d52aba8faee7"; - "2019-04-05T00:00:00Z" = "3f892e7be48e2723910aa5b3f7ed8c5e7ca247275a3e03ccc3ce84eb5e239a90"; - "2019-04-06T00:00:00Z" = "5698ef0c65688760b8a5fb97acd4f7e85f904f6d150fdf72b401c1b6c40846bd"; - "2019-04-07T00:00:00Z" = "0800715fcdbbd83fba59f38e2b2f27d6a5ababd477bcfa89d13b27961522a6fe"; - "2019-04-08T00:00:00Z" = "a7b73aa3891079ebcc42d8ce8d2e05624ff3ea3cec81f199731f4a88b5eaf17b"; - "2019-04-09T00:00:00Z" = "75bffd47f7c6f3cb80b0808f0bda19dbc7f2e1dfdb0bc867259d4e0d0ffcc6fe"; - "2019-04-10T00:00:00Z" = "35aae6a9fcb71b98a1e6a84511e40769cf6ed141a22778235770e5d1c7e46982"; - "2019-04-11T00:00:00Z" = "094321bbafa5150b3e397b2fee0052e441e16c971357d182b12628e24475b03c"; - "2019-04-12T00:00:00Z" = "29092b8546eea6f22e1169216bcefd3544d15f7f3f1549cb0d0f7a6717a2d2cb"; - "2019-04-13T00:00:00Z" = "e7cd063aaeb24d0a4604db4a7f827259852a9fe4f1bc67f3c171a8fbcb5b8707"; - "2019-04-14T00:00:00Z" = "e1db5ce15198d5c98473d32dd74d9532fb064a042d71cb078d0c4f97daa05351"; - "2019-04-15T00:00:00Z" = "7df0a5d157b6e730c68f614644ae7863de32d19260730683ad685cc24691ed93"; - "2019-04-16T00:00:00Z" = "76763ec23fd99e41a42d6a6c06afebc34c8319ba7bb03433cc7db868cab31d8b"; - "2019-04-17T00:00:00Z" = "96232013324793960158bf7512fc0cc4855afbefdcc98e510bcbaae99ec11feb"; - "2019-04-18T00:00:00Z" = "605c20dc88f98e0821ad8913f198e690a6a9ff4128d13cc91cfb695ee5d25b3f"; - "2019-04-19T00:00:00Z" = "a5bad701ce83d51a0d4de910d6502ae835bdffd947bbbbbac8adf9cfd3e1e9ce"; - "2019-04-20T00:00:00Z" = "5727553a452213ed41bde3eb651b78998dfd1fa5a19fedaecf3c28462e8bb175"; - "2019-04-21T00:00:00Z" = "81dbfcfa7f33beee9f6e2d57f07b7757e1390d2d389c4763e8e7b3f767ec7b60"; - "2019-04-22T00:00:00Z" = "9c0051b3d1023e7118087f0c8914fbf8dcae1e62545cae47c82c12a37012a00a"; - "2019-04-23T00:00:00Z" = "52232c52172f61113a839ac3511051cfb13ba91089fe458360fde193d00e86ca"; - "2019-04-24T00:00:00Z" = "f8c9eb039f16c95c825109469937c1a8ef270f56f0d376ca1cd19f6b61ba8ab4"; - "2019-04-25T00:00:00Z" = "b67e1a5e9a88beb2c741e47349cbc74fec01f2cb3279f982161e3f63e51a4576"; - "2019-04-26T00:00:00Z" = "4de880d417d118fe296121b2f3ac3bb2d4b08d56ed5490f30b81ae0ac2388016"; - "2019-04-27T00:00:00Z" = "20dd0ba939636081482b9bbc6fac0bcb2b1f5710ad70713945e7af2e9ee1b510"; - "2019-04-28T00:00:00Z" = "33df24a7827a6504944a6546fb32c22bee1af05e8d441f30d0a7abe6487eb65f"; - "2019-04-29T00:00:00Z" = "50fe3ffa024e0a131ac9440b3fec099ca83d298163f71731b25bbdb84b7681ac"; - "2019-04-30T00:00:00Z" = "f0dedbd5e75d581fc163d94ddba92a62f785a90183ccfaa426131de6218ad7cb"; - "2019-05-01T00:00:00Z" = "6ee044506ff9befb4709526f15e2858fce30a0c40b27ca1e518f9466f0f81c5e"; - "2019-05-02T00:00:00Z" = "0077eb4cdd1a9649dad8196dfd2d8f9db9183b66f0049e0f7c4fd843f43a4664"; - "2019-05-03T00:00:00Z" = "c1a0e3c1c3854911ffa5e49d9e1aa6b43235cbc5ec272ff5be675c8f8d231b4a"; - "2019-05-04T00:00:00Z" = "7c2e4bec642de2e6b5afd3aa57ee53323f6b1bec36d9af78a432199034abe254"; - "2019-05-05T00:00:00Z" = "b4ff7a89b7f93c88597119656cf938fdf2b72454991fd8b683b5f7a5bdba0fd4"; - "2019-05-06T00:00:00Z" = "822220587de7ee36d633e9bc72df760064237b13c286d0c52efdc5d5048297de"; - "2019-05-07T00:00:00Z" = "2927619e7b636780296f06d3f816c159d121b507bb2c81c7abda40926d76df12"; - "2019-05-08T00:00:00Z" = "05e3bed075a0a9095a7ddfee1a096e56bea41948eeec1a7d3a673980a257d4a3"; - "2019-05-09T00:00:00Z" = "501bddbc5cdac16f85724066df5be4e2f40e327b5f5fe0999e70c2901ce2d89a"; - "2019-05-10T00:00:00Z" = "4e6e357aa2f820ca3883dee8f2af7e359f640e1639036408d6e2b8af9751a028"; - "2019-05-11T00:00:00Z" = "8203d571863706fb4fa5ec121b689d8601135a63ca12456c86554029270b9ad0"; - "2019-05-12T00:00:00Z" = "99fb942cabcda72770adec289d4f138a985e76fb2e36dea39f54734ef74c5bb5"; - "2019-05-13T00:00:00Z" = "fef0a5e9170ac226ed47a483057f7e3f84ae374102431e5f46165140c516aa1f"; - "2019-05-14T00:00:00Z" = "2c08a3a145485279f396521374ebe1efd2022497809ae957259fea2bdb5cf829"; - "2019-05-15T00:00:00Z" = "864143953a41b64d5263ce67ec476396adafc92e77f77748916cefa76b9dd849"; - "2019-05-16T00:00:00Z" = "d5bb19b8019b617794dcb2a4cff72ebfb8edb96b232f040c486dd570e1785528"; - "2019-05-17T00:00:00Z" = "4c06e58c1698f650eb28ed545f9d243a564b30a4c0e3097df62e18dfffe9c72b"; - "2019-05-18T00:00:00Z" = "a3fd7461d73c2c393e9ed37c029b4b92482b9aa9df7790c826cb0a0408fb4522"; - "2019-05-19T00:00:00Z" = "84b578c748049bcba2b5ab35fea77d1278662b4fb003ee1c903aa88c167c19ab"; - "2019-05-20T00:00:00Z" = "87e8c61faf75005929650fde538d3a63d0db9e28ea3a40db79dc4f3ab9a5347f"; - "2019-05-21T00:00:00Z" = "27f4ad652a0880a26d8f8242611f6112296f078b45868f704cf628f9b6610d00"; - "2019-05-22T00:00:00Z" = "f6c2c8a3b3c44a893138042eb7ff8470511156ea2fabafc633eb7cb4b53af236"; - "2019-05-23T00:00:00Z" = "9587cd6b2a317e608e8b34bafc13e0ef9d22b5332f21cc5544cea0d765223cba"; -} diff --git a/scripts/update-hackage.nix b/scripts/update-hackage.nix index 6bad4c10..7e6ae5e4 100644 --- a/scripts/update-hackage.nix +++ b/scripts/update-hackage.nix @@ -1,4 +1,4 @@ -{ stdenv, writeScript, coreutils, glibc, git, openssh, nix-tools, cabal-install, nix-prefetch-git }@args: +{ stdenv, writeScript, coreutils, glibc, git, openssh, nix-tools, cabal-install, nix-prefetch-git, update-index-state-hashes }@args: import ./update-external.nix args { name = "hackage"; @@ -20,6 +20,10 @@ import ./update-external.nix args { hackage-to-nix hackage.nix + echo "Running update-index-state-hashes..." + + ${update-index-state-hashes}/bin/update-index-state-hashes > ./index-state-hashes.nix + cd hackage.nix ''; } diff --git a/scripts/update-index-state-hashes.nix b/scripts/update-index-state-hashes.nix new file mode 100644 index 00000000..8935a6fb --- /dev/null +++ b/scripts/update-index-state-hashes.nix @@ -0,0 +1,42 @@ +{ indexStateHashesPath, nix-tools, coreutils, nix, writeShellScriptBin, stdenv }: +with builtins; +with stdenv.lib; +writeShellScriptBin "update-index-state-hashes" '' + # get all the tools we need. I wonder if there is a better way? But I couldn't find + # any buildInputs for writeShellScriptBin :-/ + export PATH="${getBin coreutils}/bin:${getBin nix-tools}/bin:${getBin nix}/bin:$PATH" + + # We'll take the last element from the indexStatesHashes file via nix and get the name. + # This is the last timestamp recorded in the file (implicit assumption: the file is + # ordered, and nix preserved that order when parsing it into a attributeset). + start=${let ls = attrNames (import indexStateHashesPath); in elemAt ls (length ls - 1)} + + # The indexStatesHashesPath looks like + # { + # ... + # } + # Idea: take everything but drop the last line, and can then just append each new + # entry and finally close the file with "}". We'll do this by echoing to STDOUT! + + # Old file without the closing curly brace. + cat ${indexStateHashesPath} | head -n -1 + + # Parse the $start date, and now into seconds with the date command. Then walk + # them by 86400 (24*60*60) days. We need to format the output with '%.f' as we + # don't want fractional values. + for d in $(seq -f '%.f' $(date -u +%s -d $start) 86400 $(date -u +%s)) ; do + # turn the step date $d into a YYYY-MM-DD string, and generate the truncated + # index, compute it's hash and echo "$dT00:00:00Z" = "$sha256"; to STDOUT. + dt=$(date -u +%Y-%m-%d -d @$d) + + # ensure we don't generate the $start date twice (skip the first invocation). + if [[ "''${dt}T00:00:00Z" != "$start" ]]; then + truncate-index -o ''${dt}-01-index.tar.gz -i ${fetchurl "https://hackage.haskell.org/01-index.tar.gz"} -s "''${dt}T00:00:00Z" + sha256=$(nix-hash --flat --type sha256 ''${dt}-01-index.tar.gz) + echo " \"''${dt}T00:00:00Z\" = \"''${sha256}\";" + fi + done + + # emit the finall closing brace. + echo '}' + ''