git.iko.soy/haskell.nix
on iko/ghcjs-patch-master help sign in
Add `cleanGits` and support it in`cabalProject` (#631)

Some times it is handy to temporarily use a relative path between git repos. If the repos are individually cleaned this is not possible (since the cleaned version of one repo will never include the files of the other). `cleanGits` allows us to specify a root directory and any number of sub directories containing git repos.

commitecc72c194988c27ccceb1db501d320472a5ddcb2 authorHamish Mackenzie <Hamish.Mackenzie@iohk.io> dateThu 28 May 2020, 21:59 (6 years ago) parent5eacdc8
7 files changed+135−31
compare with iko/ghcjs-patch-master
unifiedsplit
docs/user-guide/clean-git.md+50 lib/call-cabal-project-to-nix.nix+18−14 lib/call-stack-to-nix.nix+5−4 lib/clean-git.nix+3−1 lib/clean-source-with.nix+15−7 lib/default.nix+25 lib/import-and-filter-project.nix+19−5
docs/user-guide/clean-git.md+50
@@ -38,3 +38,53 @@ components.tests.test.src = haskell-nix.haskellLib.cleanSourceWith {
3838};
3939```
4040
41+## Multiple Git Repositories with cleanGits
42+
43+Some times it is handy to temporarily use a relative path between git
44+repos. If the repos are individually cleaned this is not possible
45+(since the cleaned version of one repo will never include the files
46+of the other).
47+
48+There are 3 options:
49+
50+* We could `symlinkJoin` the cleaned directories together, but the
51+ result could not be cleaned and any change would to either
52+ repo would result in a rebuild of everything.
53+
54+* We could add one repo to the other as a submodule,
55+ but adding and then removing a submodule is a pain and it does not
56+ work well if you have more than one repo that needs to share the
57+ submodule.
58+
59+* We could add a `source-repository-package` but then we would have
60+ to commit each change before testing.
61+
62+`cleanGits` allows us to specify a root directory and any number of
63+sub directories containing git repos.
64+
65+For example if `repoA` and `repoB` are two git repos with
66+cabal packages and want to use the `repoB` package when building
67+`repoA. First we can add `../repoB` to `repoA/cabal.project`:
68+
69+```
70+packages:
71+ ./.
72+ ../repoB
73+```
74+
75+Then in `repoA/default.nix` we can use:
76+
77+```
78+haskell-nix.cabalProject {
79+ src = haskell-nix.haskellLib.cleanSourceWith {
80+ src = haskell-nix.haskellLib.cleanGits {
81+ name = "root";
82+ src = ../.; # Parent dir that contains repoA and repoB
83+ gitDirs = [ "repoA" "repoB" ];
84+ };
85+ subDir = "repoA"; # Where to look for the `cabal.project`
86+ includeSiblings = true; # Tells it not to exclude `repoB` dir
87+ };
88+}
89+```
90+
lib/call-cabal-project-to-nix.nix+18−14
@@ -54,20 +54,23 @@ in
5454
5555let
5656 ghc = ghc';
57+ subDir' = src.origSubDir or "";
58+ subDir = pkgs.lib.strings.removePrefix "/" subDir';
5759 maybeCleanedSource =
5860 if haskellLib.canCleanSource src
59− then haskellLib.cleanSourceWith {
60− inherit src;
61− filter = path: type:
62− type == "directory" ||
63− pkgs.lib.any (i: (pkgs.lib.hasSuffix i path)) [ ".project" ".cabal" ".freeze" "package.yaml" ]; }
64− else src;
61+ then (haskellLib.cleanSourceWith {
62+ name = src.name + "-root-cabal-files";
63+ src = src.origSrc;
64+ filter = path: type: src.filter path type && (
65+ type == "directory" ||
66+ pkgs.lib.any (i: (pkgs.lib.hasSuffix i path)) [ ".project" ".cabal" ".freeze" "package.yaml" ]); })
67+ else src.origSrc or src;
6568
6669 # Using origSrcSubDir bypasses any cleanSourceWith so that it will work when
6770 # access to the store is restricted. If origSrc was already in the store
6871 # you can pass the project in as a string.
6972 rawCabalProject =
70− let origSrcDir = maybeCleanedSource.origSrcSubDir or maybeCleanedSource;
73+ let origSrcDir = (maybeCleanedSource.origSrcSubDir or maybeCleanedSource) + subDir';
7174 in if cabalProject != null
7275 then cabalProject
7376 else
@@ -321,12 +324,13 @@ let
321324 fi
322325 cp -r ${maybeCleanedSource}/* .
323326 chmod +w -R .
324− ${fixedProject.makeFixedProjectFile}
325327 # warning: this may not generate the proper cabal file.
326328 # hpack allows globbing, and turns that into module lists
327329 # without the source available (we cleaneSourceWith'd it),
328330 # this may not produce the right result.
329331 find . -name package.yaml -exec hpack "{}" \;
332+ ${pkgs.lib.optionalString (subDir != "") "cd ${subDir}"}
333+ ${fixedProject.makeFixedProjectFile}
330334 export SSL_CERT_FILE=${cacert}/etc/ssl/certs/ca-bundle.crt
331335 export GIT_SSL_CAINFO=${cacert}/etc/ssl/certs/ca-bundle.crt
332336 HOME=${dotCabal {
@@ -357,24 +361,24 @@ let
357361
358362 # make sure the path's in the plan.json are relative to $out instead of $tmp
359363 # this is necessary so that plan-to-nix relative path logic can work.
360− substituteInPlace $tmp/dist-newstyle/cache/plan.json --replace "$tmp" "$out"
364+ substituteInPlace $tmp${subDir'}/dist-newstyle/cache/plan.json --replace "$tmp" "$out"
361365
362366 # run `plan-to-nix` in $out. This should produce files right there with the
363367 # proper relative paths.
364− (cd $out && plan-to-nix --full --plan-json $tmp/dist-newstyle/cache/plan.json -o .)
368+ (cd $out${subDir'} && plan-to-nix --full --plan-json $tmp${subDir'}/dist-newstyle/cache/plan.json -o .)
365369
366370 # Remove the non nix files ".project" ".cabal" "package.yaml" files
367371 # as they should not be in the output hash (they may change slightly
368372 # without affecting the nix).
369− if [ -d $out/.source-repository-packages ]; then
370− chmod +w -R $out/.source-repository-packages
371− rm -rf $out/.source-repository-packages
373+ if [ -d $out${subDir'}/.source-repository-packages ]; then
374+ chmod +w -R $out${subDir'}/.source-repository-packages
375+ rm -rf $out${subDir'}/.source-repository-packages
372376 fi
373377 find $out \( -type f -or -type l \) ! -name '*.nix' -delete
374378 # Remove empty dirs
375379 find $out -type d -empty -delete
376380
377381 # move pkgs.nix to default.nix ensure we can just nix `import` the result.
378− mv $out/pkgs.nix $out/default.nix
382+ mv $out${subDir'}/pkgs.nix $out${subDir'}/default.nix
379383 '');
380384in { projectNix = plan-nix; inherit src; inherit (fixedProject) sourceRepos; }
lib/call-stack-to-nix.nix+5−4
@@ -16,6 +16,7 @@
1616, checkMaterialization ? null # If true the nix files will be generated used to check plan-sha256 and material
1717, ... }:
1818let
19+ subDir' = src.origSubDir or "";
1920 stackToNixArgs = builtins.concatStringsSep " " [
2021 "--full"
2122 "--stack-yaml=${src}/${if stackYaml == null then "stack.yaml" else stackYaml}"
@@ -37,11 +38,11 @@ let
3738 LC_ALL = "en_US.UTF-8";
3839 preferLocalBuild = false;
3940 } (''
40− mkdir -p $out
41+ mkdir -p $out${subDir'}
4142 '' + pkgs.lib.optionalString (cache != null) ''
42− cp ${mkCacheFile cache}/.stack-to-nix.cache* $out
43+ cp ${mkCacheFile cache}/.stack-to-nix.cache* $out${subDir'}
4344 '' + ''
44− (cd $out && stack-to-nix ${stackToNixArgs})
45+ (cd $out${subDir'} && stack-to-nix ${stackToNixArgs})
4546
4647 # We need to strip out any references to $src, as those won't
4748 # be accessable in restricted mode.
@@ -50,6 +51,6 @@ let
5051 done
5152
5253 # move pkgs.nix to default.nix ensure we can just nix `import` the result.
53− mv $out/pkgs.nix $out/default.nix
54+ mv $out${subDir'}/pkgs.nix $out${subDir'}/default.nix
5455 ''));
5556in { projectNix = stack; inherit src; sourceRepos = []; }
lib/clean-git.nix+3−1
@@ -33,7 +33,9 @@ let
3333
3434 whitelist_set = listToAttrs (
3535 concatMap (p:
36− let full_path = toString (root + "/${p}"); in
36+ # Using `origSrcSubDir` (if present) makes it possible to cleanGit src that
37+ # has already been cleaned with cleanSrcWith.
38+ let full_path = root.origSrcSubDir or (toString root) + "/${p}"; in
3739 map (p': { name = p'; value = true; }) (all_paren_dirs full_path)
3840 ) files
3941 );
lib/clean-source-with.nix+15−7
@@ -35,6 +35,10 @@
3535 # It will be as if `src = src + "/${subDir}` and filters
3636 # already applied to `src` will be respected.
3737 #
38+ # includeSiblings: By default the siblings trees of `subDir` are excluded.
39+ # In some cases it is useful to include these so that
40+ # relative references to those siblings will work.
41+ #
3842 # name: Optional name to use as part of the store path.
3943 # If you do not provide a `name` it wil be derived
4044 # from the `subDir`. You should provide `name` or
@@ -47,7 +51,7 @@
4751 # the message to the use more meaningful.
4852 #
4953 cleanSourceWith = { filter ? _path: _type: true, src, subDir ? "", name ? null
50− , caller ? "cleanSourceWith" }:
54+ , caller ? "cleanSourceWith", includeSiblings ? false }:
5155 let
5256 subDir' = if subDir == "" then "" else "/" + subDir;
5357 subDirName = __replaceStrings ["/"] ["-"] subDir;
@@ -61,12 +65,16 @@
6165 then path: type: src.filter path type
6266 else path: type: true;
6367 filter' = path: type:
64− # Include parent paths based on the parent filter
65− (lib.strings.hasPrefix (path + "/") (origSrcSubDir + "/")
66− && parentFilter path type)
67− # Children only if both filters return true
68− || (lib.strings.hasPrefix (origSrcSubDir + "/") path
69− && (filter path type && parentFilter path type));
68+ # Respect the parent filter
69+ parentFilter path type && (
70+ # Must include parent paths of the subdir.
71+ (lib.strings.hasPrefix (path + "/") (origSrcSubDir + "/"))
72+ ||
73+ # Everything else is either the child tree or sibling tree.
74+ ((includeSiblings || lib.strings.hasPrefix (origSrcSubDir + "/") path)
75+ && filter path type # Use the filter function to decide if we need it
76+ )
77+ );
7078 name' = if name != null
7179 then name
7280 else
lib/default.nix+25
@@ -208,6 +208,31 @@ in {
208208 inherit (pkgs.evalPackages) runCommand;
209209 };
210210
211+ # Some times it is handy to temporarily use a relative path between git
212+ # repos. If the repos are individually cleaned this is not possible
213+ # (since the cleaned version of one repo will never include the files
214+ # of the other).
215+ #
216+ # `cleanGits` allows us to specify a root directory and any number of
217+ # sub directories containing git repos.
218+ #
219+ # See docs/user-guide/clean-git.md for details of how to use this
220+ # with `cabalProject`.
221+ cleanGits = { src, gitDirs, name ? null, caller ? "cleanGits" }@args:
222+ let
223+ # List of filters, one for each git directory.
224+ filters = builtins.map (subDir:
225+ (pkgs.haskell-nix.haskellLib.cleanGit {
226+ src = pkgs.haskell-nix.haskellLib.cleanSourceWith {
227+ inherit src subDir;
228+ };
229+ }).filter) gitDirs;
230+ in pkgs.haskell-nix.haskellLib.cleanSourceWith {
231+ inherit src name caller;
232+ # Keep files that match any of the filters
233+ filter = path: type: pkgs.lib.any (f: f path type) filters;
234+ };
235+
211236 # Check a test component
212237 check = import ./check.nix {
213238 inherit stdenv lib haskellLib srcOnly;
lib/import-and-filter-project.nix+19−5
@@ -4,7 +4,21 @@
44{ pkgs, haskellLib }:
55{ projectNix, sourceRepos, src }:
66let
7− project = import "${projectNix}";
7+ # Full source including possible relartive paths form the
8+ # project directory.
9+ srcRoot =
10+ if haskellLib.canCleanSource src
11+ then haskellLib.cleanSourceWith {
12+ name = src.name + "-root";
13+ src = src.origSrc or src;
14+ inherit (src) filter;
15+ }
16+ else src.origSrc or src;
17+ # The sub directory containing the cabal.project or stack.yaml file
18+ projectSubDir' = src.origSubDir or ""; # With leading /
19+ projectSubDir = pkgs.lib.strings.removePrefix "/" projectSubDir'; # Without /
20+ projectSubDir'' = if projectSubDir == "" then "" else projectSubDir + "/"; # With trailing /
21+ project = import "${projectNix}${projectSubDir'}";
822in {
923 nix = projectNix;
1024 pkgs = project // {
@@ -20,14 +34,14 @@ in {
2034 subDir = pkgs.lib.strings.removePrefix "/" (
2135 pkgs.lib.strings.removePrefix (toString projectNix)
2236 (toString oldPkg.src.content));
23− srcRepoPrefix = ".source-repository-packages/";
37+ srcRepoPrefix = projectSubDir'' + ".source-repository-packages/";
2438 in if pkgs.lib.strings.hasPrefix srcRepoPrefix subDir
2539 then
2640 pkgs.lib.lists.elemAt sourceRepos (
2741 pkgs.lib.strings.toInt (pkgs.lib.strings.removePrefix srcRepoPrefix subDir))
28− else if haskellLib.canCleanSource src
29− then haskellLib.cleanSourceWith { inherit src subDir; }
30− else src + (if subDir == "" then "" else "/" + subDir);
42+ else if haskellLib.canCleanSource srcRoot
43+ then haskellLib.cleanSourceWith { src = srcRoot; inherit subDir; }
44+ else srcRoot + (if subDir == "" then "" else "/" + subDir);
3145 in oldPkg // {
3246 src = (pkgs.lib).mkDefault packageSrc;
3347 }) old;