git.iko.soy/haskell.nix
on iko/ghcjs-patch-master help sign in
Do not use evalPackages to create sourceRepo for builds (#918)

* Do not use evalPackages to create sourceRepo for builds but still use it when creating source repo for plan calculation. This might be a way to fix #917 * Update unit tests * Refactor this patch, and add some comments this is much less convoluted. I moved the fetchgit invocation from `cabal-project-parser.nix` to `call-cabal-project-to-nix` (which makes sense in a way, if a parser should return parsed data, and not do stuff), and then some simplifications could happen. Also added some comments. * Update unit tests * Dead code * Less repetition in tests * Comment tweaks Co-authored-by: Hamish Mackenzie <Hamish.Mackenzie@iohk.io>

commitbdd69693b9597c4d893b6d134cf2f3f8b1809e09 authorJoachim Breitner <mail@joachim-breitner.de> dateWed 11 Nov 2020, 12:49 (6 years ago) parent654063d
3 files changed+53−39
compare with iko/ghcjs-patch-master
unifiedsplit
lib/cabal-project-parser.nix+13−25 lib/call-cabal-project-to-nix.nix+34−8 test/unit.nix+6−6
lib/cabal-project-parser.nix+13−25
@@ -69,36 +69,24 @@ let
6969 else __trace "Expected attribute but found `${s}`" { inherit name attrs; }
7070 ) { name = null; attrs = {}; } (stripComments (unindent blockLines))).attrs;
7171
72− hashPath = path:
73− builtins.readFile (pkgs.runCommand "hash-path" { preferLocalBuild = true; }
74− "echo -n $(${pkgs.nix}/bin/nix-hash --type sha256 --base32 ${path}) > $out");
75−
76− # Use pkgs.fetchgit if we have a sha256. Add comment like this
72+ # Gets data for all the repositories to fetch
73+ # A comment like
7774 # --shar256: 003lm3pm0000hbfmii7xcdd9v20000flxf7gdl2pyxia7p014i8z
78− # otherwise use __fetchGit.
79− fetchRepo = cabalProjectFileName: lookupSha256: repo:
75+ # will be trated like a field and returned here
76+ # (used in call-cabal-project-to-nix.nix to create a fixed-output derivation)
77+ extractRepoData = cabalProjectFileName: lookupSha256: repo:
8078 builtins.map (subdir:
81− let sha256 = repo."--sha256" or (lookupSha256 repo);
82− in (if sha256 != null
83− then pkgs.evalPackages.fetchgit {
84− url = repo.location;
85− rev = repo.tag;
86− inherit sha256;
87− }
88− else
89− let drv = builtins.fetchGit {
90− url = repo.location;
91− ref = repo.tag;
92− };
93− in __trace "WARNING: No sha256 found for source-repository-package ${repo.location} ${repo.tag} download may fail in restricted mode (hydra)"
94− (__trace "Consider adding `--sha256: ${hashPath drv}` to the ${cabalProjectFileName} file or passing in a lookupSha256 argument"
95− drv)
96− ) + (if subdir == "." then "" else "/" + subdir))
79+ {
80+ url = repo.location;
81+ ref = repo.tag;
82+ sha256 = repo."--sha256" or (lookupSha256 repo);
83+ inherit subdir;
84+ })
9785 (if repo ? subdir
9886 then pkgs.lib.filter (x: x != "") (pkgs.lib.splitString " " repo.subdir)
9987 else ["."]);
10088
101− # Parse a source-repository-package and fetch it if has `type: git`
89+ # Parse a source-repository-package and return data of `type: git` repositories
10290 parseBlock = cabalProjectFileName: lookupSha256: block:
10391 let
10492 x = span (pkgs.lib.strings.hasPrefix " ") (pkgs.lib.splitString "\n" block);
@@ -110,7 +98,7 @@ let
11098 otherText = "\nsource-repository-package\n" + block;
11199 }
112100 else {
113− sourceRepo = fetchRepo cabalProjectFileName lookupSha256 attrs;
101+ sourceRepo = extractRepoData cabalProjectFileName lookupSha256 attrs;
114102 otherText = pkgs.lib.strings.concatStringsSep "\n" x.snd;
115103 };
116104
lib/call-cabal-project-to-nix.nix+34−8
@@ -157,17 +157,46 @@ let
157157 # restricted-eval mode (as long as a sha256 is included).
158158 # Replace source-repository-package blocks that have a sha256 with
159159 # packages: block containing nix store paths of the fetched repos.
160+
161+ hashPath = path:
162+ builtins.readFile (pkgs.runCommand "hash-path" { preferLocalBuild = true; }
163+ "echo -n $(${pkgs.nix}/bin/nix-hash --type sha256 --base32 ${path}) > $out");
164+
160165 replaceSoureRepos = projectFile:
161166 let
167+ fetchRepo = fetchgit: repoData:
168+ let
169+ fetched =
170+ if repoData.sha256 != null
171+ then fetchgit { inherit (repoData) url sha256; rev = repoData.ref; }
172+ else
173+ let drv = builtins.fetchGit { inherit (repoData) url ref; };
174+ in __trace "WARNING: No sha256 found for source-repository-package ${repoData.url} ${repoData.ref} download may fail in restricted mode (hydra)"
175+ (__trace "Consider adding `--sha256: ${hashPath drv}` to the ${cabalProjectFileName} file or passing in a lookupSha256 argument"
176+ drv);
177+ in fetched + (if repoData.subdir == "." then "" else "/" + repoData.subdir);
178+
162179 blocks = pkgs.lib.splitString "\nsource-repository-package\n" ("\n" + projectFile);
163180 initialText = pkgs.lib.lists.take 1 blocks;
164181 repoBlocks = builtins.map (pkgs.haskell-nix.haskellLib.parseBlock cabalProjectFileName lookupSha256) (pkgs.lib.lists.drop 1 blocks);
165− sourceRepos = pkgs.lib.lists.concatMap (x: x.sourceRepo) repoBlocks;
182+ sourceRepoData = pkgs.lib.lists.concatMap (x: x.sourceRepo) repoBlocks;
166183 otherText = pkgs.evalPackages.writeText "cabal.project" (pkgs.lib.strings.concatStringsSep "\n" (
167184 initialText
168185 ++ (builtins.map (x: x.otherText) repoBlocks)));
186+
187+ # we need the repository content twice:
188+ # * at eval time (below to build the fixed project file)
189+ # Here we want to use pkgs.evalPackages.fetchgit, so one can calculate
190+ # the build plan for any target without a remote builder
191+ # * at built time (passed out)
192+ # Here we want to use plain pkgs.fetchgit, which is what a builder
193+ # on the target system would use, so that the derivation is unaffected
194+ # and, say, a linux release build job can identify the derivation
195+ # as built by a darwin builder, and fetch it from a cache
196+ sourceReposBuild = builtins.map (fetchRepo pkgs.evalPackages.fetchgit) sourceRepoData ;
197+ sourceReposEval = builtins.map (fetchRepo pkgs.fetchgit) sourceRepoData;
169198 in {
170− inherit sourceRepos;
199+ sourceRepos = sourceReposBuild;
171200 makeFixedProjectFile = ''
172201 cp -f ${otherText} ./cabal.project
173202 chmod +w -R ./cabal.project
@@ -184,18 +213,15 @@ let
184213 "${f}/" "./.source-repository-packages/${builtins.toString n}/"
185214 echo " ./.source-repository-packages/${builtins.toString n}" >> ./cabal.project
186215 '')
187− (pkgs.lib.lists.range 0 ((builtins.length fixedProject.sourceRepos) - 1))
188− sourceRepos
216+ (pkgs.lib.lists.range 0 ((builtins.length sourceReposEval) - 1))
217+ sourceReposEval
189218 )
190219 );
191220 };
192221
193222 fixedProject =
194223 if rawCabalProject == null
195− then {
196− sourceRepos = [];
197− makeFixedProjectFile = "";
198− }
224+ then { sourceRepos = []; makeFixedProjectFile = ""; }
199225 else replaceSoureRepos rawCabalProject;
200226
201227 # The use of the actual GHC can cause significant problems:
test/unit.nix+6−6
@@ -25,9 +25,9 @@ let
2525 package.identifier.name = "nnn";
2626 };
2727
28− testRepo = pkgs.fetchgit {
28+ testRepoData = {
2929 url = "https://github.com/input-output-hk/haskell.nix.git";
30− rev = "487eea1c249537d34c27f6143dff2b9d5586c657";
30+ ref = "487eea1c249537d34c27f6143dff2b9d5586c657";
3131 sha256 = "077j5j3j86qy1wnabjlrg4dmqy1fv037dyq3xb8ch4ickpxxs123";
3232 };
3333in
@@ -65,7 +65,7 @@ lib.runTests {
6565 '');
6666 expected = __toJSON {
6767 otherText = "-- end of block\n";
68− sourceRepo = [testRepo];
68+ sourceRepo = [(testRepoData // { subdir = "."; })];
6969 };
7070 };
7171
@@ -80,7 +80,7 @@ lib.runTests {
8080 '');
8181 expected = __toJSON {
8282 otherText = "-- end of block\n";
83− sourceRepo = [(testRepo + "/dir")];
83+ sourceRepo = [(testRepoData // { subdir = "dir"; })];
8484 };
8585 };
8686
@@ -95,7 +95,7 @@ lib.runTests {
9595 '');
9696 expected = __toJSON {
9797 otherText = "-- end of block\n";
98− sourceRepo = [(testRepo + "/dir1") (testRepo + "/dir2")];
98+ sourceRepo = [(testRepoData // { subdir = "dir1"; }) (testRepoData // { subdir = "dir2"; })];
9999 };
100100 };
101101
@@ -112,7 +112,7 @@ lib.runTests {
112112 '');
113113 expected = __toJSON {
114114 otherText = "-- end of block\n";
115− sourceRepo = [(testRepo + "/dir1") (testRepo + "/dir2")];
115+ sourceRepo = [(testRepoData // { subdir = "dir1"; }) (testRepoData // { subdir = "dir2"; })];
116116 };
117117 };
118118}