ecc72c1 · haskell.nix · git.iko.soy

git.iko.soy/haskell.nix

on iko/ghcjs-patch-master help sign in

Add `cleanGits` and support it in`cabalProject` (#631)

ecc72c1Hamish Mackenzie6 years ago

docs/user-guide/clean-git.md+50

compare with iko/ghcjs-patch-master

unifiedsplit

@@ -38,3 +38,53 @@ components.tests.test.src = haskell-nix.haskellLib.cleanSourceWith {

38};

39```

41+## Multiple Git Repositories with cleanGits

42+

43+Some times it is handy to temporarily use a relative path between git

44+repos. If the repos are individually cleaned this is not possible

45+(since the cleaned version of one repo will never include the files

46+of the other).

47+

48+There are 3 options:

49+

50+* We could `symlinkJoin` the cleaned directories together, but the

51+ result could not be cleaned and any change would to either

52+ repo would result in a rebuild of everything.

53+

54+* We could add one repo to the other as a submodule,

55+ but adding and then removing a submodule is a pain and it does not

56+ work well if you have more than one repo that needs to share the

57+ submodule.

58+

59+* We could add a `source-repository-package` but then we would have

60+ to commit each change before testing.

61+

62+`cleanGits` allows us to specify a root directory and any number of

63+sub directories containing git repos.

64+

65+For example if `repoA` and `repoB` are two git repos with

66+cabal packages and want to use the `repoB` package when building

67+`repoA. First we can add `../repoB` to `repoA/cabal.project`:

68+

69+```

70+packages:

71+ ./.

72+ ../repoB

73+```

74+

75+Then in `repoA/default.nix` we can use:

76+

77+```

78+haskell-nix.cabalProject {

79+ src = haskell-nix.haskellLib.cleanSourceWith {

80+ src = haskell-nix.haskellLib.cleanGits {

81+ name = "root";

82+ src = ../.; # Parent dir that contains repoA and repoB

83+ gitDirs = [ "repoA" "repoB" ];

84+ };

85+ subDir = "repoA"; # Where to look for the `cabal.project`

86+ includeSiblings = true; # Tells it not to exclude `repoB` dir

87+ };

88+}

89+```

90+

lib/call-cabal-project-to-nix.nix+18−14

@@ -54,20 +54,23 @@ in

55let

56 ghc = ghc';

57+ subDir' = src.origSubDir or "";

58+ subDir = pkgs.lib.strings.removePrefix "/" subDir';

57 maybeCleanedSource =

59 maybeCleanedSource =

58 if haskellLib.canCleanSource src

60 if haskellLib.canCleanSource src

59− then haskellLib.cleanSourceWith {

61+ then (haskellLib.cleanSourceWith {

60− inherit src;

62+ name = src.name + "-root-cabal-files";

63+ src = src.origSrc;

61− filter = path: type:

64+ filter = path: type: src.filter path type && (

62− type == "directory" ||

65+ type == "directory" ||

63− pkgs.lib.any (i: (pkgs.lib.hasSuffix i path)) [ ".project" ".cabal" ".freeze" "package.yaml" ]; }

66+ pkgs.lib.any (i: (pkgs.lib.hasSuffix i path)) [ ".project" ".cabal" ".freeze" "package.yaml" ]); })

64− else src;

67+ else src.origSrc or src;

66 # Using origSrcSubDir bypasses any cleanSourceWith so that it will work when

69 # Using origSrcSubDir bypasses any cleanSourceWith so that it will work when

67 # access to the store is restricted. If origSrc was already in the store

70 # access to the store is restricted. If origSrc was already in the store

68 # you can pass the project in as a string.

71 # you can pass the project in as a string.

69 rawCabalProject =

72 rawCabalProject =

70− let origSrcDir = maybeCleanedSource.origSrcSubDir or maybeCleanedSource;

73+ let origSrcDir = (maybeCleanedSource.origSrcSubDir or maybeCleanedSource) + subDir';

71 in if cabalProject != null

74 in if cabalProject != null

72 then cabalProject

75 then cabalProject

73 else

76 else

@@ -321,12 +324,13 @@ let

321 fi

324 fi

322 cp -r ${maybeCleanedSource}/* .

325 cp -r ${maybeCleanedSource}/* .

323 chmod +w -R .

326 chmod +w -R .

324− ${fixedProject.makeFixedProjectFile}

325 # warning: this may not generate the proper cabal file.

327 # warning: this may not generate the proper cabal file.

326 # hpack allows globbing, and turns that into module lists

328 # hpack allows globbing, and turns that into module lists

327 # without the source available (we cleaneSourceWith'd it),

329 # without the source available (we cleaneSourceWith'd it),

328 # this may not produce the right result.

330 # this may not produce the right result.

329 find . -name package.yaml -exec hpack "{}" \;

331 find . -name package.yaml -exec hpack "{}" \;

332+ ${pkgs.lib.optionalString (subDir != "") "cd ${subDir}"}

333+ ${fixedProject.makeFixedProjectFile}

330 export SSL_CERT_FILE=${cacert}/etc/ssl/certs/ca-bundle.crt

334 export SSL_CERT_FILE=${cacert}/etc/ssl/certs/ca-bundle.crt

331 export GIT_SSL_CAINFO=${cacert}/etc/ssl/certs/ca-bundle.crt

335 export GIT_SSL_CAINFO=${cacert}/etc/ssl/certs/ca-bundle.crt

332 HOME=${dotCabal {

336 HOME=${dotCabal {

@@ -357,24 +361,24 @@ let

357

361

358 # make sure the path's in the plan.json are relative to $out instead of $tmp

362 # make sure the path's in the plan.json are relative to $out instead of $tmp

359 # this is necessary so that plan-to-nix relative path logic can work.

363 # this is necessary so that plan-to-nix relative path logic can work.

360− substituteInPlace $tmp/dist-newstyle/cache/plan.json --replace "$tmp" "$out"

364+ substituteInPlace $tmp${subDir'}/dist-newstyle/cache/plan.json --replace "$tmp" "$out"

361

365

362 # run `plan-to-nix` in $out. This should produce files right there with the

366 # run `plan-to-nix` in $out. This should produce files right there with the

363 # proper relative paths.

367 # proper relative paths.

364− (cd $out && plan-to-nix --full --plan-json $tmp/dist-newstyle/cache/plan.json -o .)

368+ (cd $out${subDir'} && plan-to-nix --full --plan-json $tmp${subDir'}/dist-newstyle/cache/plan.json -o .)

365

369

366 # Remove the non nix files ".project" ".cabal" "package.yaml" files

370 # Remove the non nix files ".project" ".cabal" "package.yaml" files

367 # as they should not be in the output hash (they may change slightly

371 # as they should not be in the output hash (they may change slightly

368 # without affecting the nix).

372 # without affecting the nix).

369− if [ -d $out/.source-repository-packages ]; then

373+ if [ -d $out${subDir'}/.source-repository-packages ]; then

370− chmod +w -R $out/.source-repository-packages

374+ chmod +w -R $out${subDir'}/.source-repository-packages

371− rm -rf $out/.source-repository-packages

375+ rm -rf $out${subDir'}/.source-repository-packages

372 fi

376 fi

373 find $out $ -type f -or -type l $ ! -name '*.nix' -delete

377 find $out $ -type f -or -type l $ ! -name '*.nix' -delete

374 # Remove empty dirs

378 # Remove empty dirs

375 find $out -type d -empty -delete

379 find $out -type d -empty -delete

376

380

377 # move pkgs.nix to default.nix ensure we can just nix `import` the result.

381 # move pkgs.nix to default.nix ensure we can just nix `import` the result.

378− mv $out/pkgs.nix $out/default.nix

382+ mv $out${subDir'}/pkgs.nix $out${subDir'}/default.nix

379 '');

383 '');

380in { projectNix = plan-nix; inherit src; inherit (fixedProject) sourceRepos; }

384in { projectNix = plan-nix; inherit src; inherit (fixedProject) sourceRepos; }

lib/call-stack-to-nix.nix+5−4

@@ -16,6 +16,7 @@

16, checkMaterialization ? null # If true the nix files will be generated used to check plan-sha256 and material

17, ... }:

18let

19+ subDir' = src.origSubDir or "";

19 stackToNixArgs = builtins.concatStringsSep " " [

20 stackToNixArgs = builtins.concatStringsSep " " [

20 "--full"

21 "--full"

21 "--stack-yaml=${src}/${if stackYaml == null then "stack.yaml" else stackYaml}"

22 "--stack-yaml=${src}/${if stackYaml == null then "stack.yaml" else stackYaml}"

@@ -37,11 +38,11 @@ let

37 LC_ALL = "en_US.UTF-8";

38 LC_ALL = "en_US.UTF-8";

38 preferLocalBuild = false;

39 preferLocalBuild = false;

39 } (''

40 } (''

40− mkdir -p $out

41+ mkdir -p $out${subDir'}

41 '' + pkgs.lib.optionalString (cache != null) ''

42 '' + pkgs.lib.optionalString (cache != null) ''

42− cp ${mkCacheFile cache}/.stack-to-nix.cache* $out

43+ cp ${mkCacheFile cache}/.stack-to-nix.cache* $out${subDir'}

43 '' + ''

44 '' + ''

44− (cd $out && stack-to-nix ${stackToNixArgs})

45+ (cd $out${subDir'} && stack-to-nix ${stackToNixArgs})

46 # We need to strip out any references to $src, as those won't

47 # We need to strip out any references to $src, as those won't

47 # be accessable in restricted mode.

48 # be accessable in restricted mode.

@@ -50,6 +51,6 @@ let

50 done

51 done

52 # move pkgs.nix to default.nix ensure we can just nix `import` the result.

53 # move pkgs.nix to default.nix ensure we can just nix `import` the result.

53− mv $out/pkgs.nix $out/default.nix

54+ mv $out${subDir'}/pkgs.nix $out${subDir'}/default.nix

54 ''));

55 ''));

55in { projectNix = stack; inherit src; sourceRepos = []; }

56in { projectNix = stack; inherit src; sourceRepos = []; }

lib/clean-git.nix+3−1

@@ -33,7 +33,9 @@ let

34 whitelist_set = listToAttrs (

35 concatMap (p:

36+ # Using `origSrcSubDir` (if present) makes it possible to cleanGit src that

37+ # has already been cleaned with cleanSrcWith.

36− let full_path = toString (root + "/${p}"); in

38+ let full_path = root.origSrcSubDir or (toString root) + "/${p}"; in

37 map (p': { name = p'; value = true; }) (all_paren_dirs full_path)

39 map (p': { name = p'; value = true; }) (all_paren_dirs full_path)

38 ) files

40 ) files

39 );

41 );

lib/clean-source-with.nix+15−7

@@ -35,6 +35,10 @@

35 # It will be as if `src = src + "/${subDir}` and filters

36 # already applied to `src` will be respected.

37 #

38+ # includeSiblings: By default the siblings trees of `subDir` are excluded.

39+ # In some cases it is useful to include these so that

40+ # relative references to those siblings will work.

41+ #

38 # name: Optional name to use as part of the store path.

42 # name: Optional name to use as part of the store path.

39 # If you do not provide a `name` it wil be derived

43 # If you do not provide a `name` it wil be derived

40 # from the `subDir`. You should provide `name` or

44 # from the `subDir`. You should provide `name` or

@@ -47,7 +51,7 @@

47 # the message to the use more meaningful.

51 # the message to the use more meaningful.

48 #

52 #

49 cleanSourceWith = { filter ? _path: _type: true, src, subDir ? "", name ? null

53 cleanSourceWith = { filter ? _path: _type: true, src, subDir ? "", name ? null

50− , caller ? "cleanSourceWith" }:

54+ , caller ? "cleanSourceWith", includeSiblings ? false }:

51 let

55 let

52 subDir' = if subDir == "" then "" else "/" + subDir;

56 subDir' = if subDir == "" then "" else "/" + subDir;

53 subDirName = __replaceStrings ["/"] ["-"] subDir;

57 subDirName = __replaceStrings ["/"] ["-"] subDir;

@@ -61,12 +65,16 @@

61 then path: type: src.filter path type

65 then path: type: src.filter path type

62 else path: type: true;

66 else path: type: true;

63 filter' = path: type:

67 filter' = path: type:

64− # Include parent paths based on the parent filter

68+ # Respect the parent filter

69+ parentFilter path type && (

70+ # Must include parent paths of the subdir.

65− (lib.strings.hasPrefix (path + "/") (origSrcSubDir + "/")

71+ (lib.strings.hasPrefix (path + "/") (origSrcSubDir + "/"))

66− && parentFilter path type)

72+ ||

67− # Children only if both filters return true

73+ # Everything else is either the child tree or sibling tree.

68− || (lib.strings.hasPrefix (origSrcSubDir + "/") path

74+ ((includeSiblings || lib.strings.hasPrefix (origSrcSubDir + "/") path)

69− && (filter path type && parentFilter path type));

75+ && filter path type # Use the filter function to decide if we need it

76+ )

77+ );

70 name' = if name != null

78 name' = if name != null

71 then name

79 then name

72 else

80 else

lib/default.nix+25

@@ -208,6 +208,31 @@ in {

208 inherit (pkgs.evalPackages) runCommand;

209 };

210

211+ # Some times it is handy to temporarily use a relative path between git

212+ # repos. If the repos are individually cleaned this is not possible

213+ # (since the cleaned version of one repo will never include the files

214+ # of the other).

215+ #

216+ # `cleanGits` allows us to specify a root directory and any number of

217+ # sub directories containing git repos.

218+ #

219+ # See docs/user-guide/clean-git.md for details of how to use this

220+ # with `cabalProject`.

221+ cleanGits = { src, gitDirs, name ? null, caller ? "cleanGits" }@args:

222+ let

223+ # List of filters, one for each git directory.

224+ filters = builtins.map (subDir:

225+ (pkgs.haskell-nix.haskellLib.cleanGit {

226+ src = pkgs.haskell-nix.haskellLib.cleanSourceWith {

227+ inherit src subDir;

228+ };

229+ }).filter) gitDirs;

230+ in pkgs.haskell-nix.haskellLib.cleanSourceWith {

231+ inherit src name caller;

232+ # Keep files that match any of the filters

233+ filter = path: type: pkgs.lib.any (f: f path type) filters;

234+ };

235+

211 # Check a test component

236 # Check a test component

212 check = import ./check.nix {

237 check = import ./check.nix {

213 inherit stdenv lib haskellLib srcOnly;

238 inherit stdenv lib haskellLib srcOnly;

lib/import-and-filter-project.nix+19−5

@@ -4,7 +4,21 @@

4{ pkgs, haskellLib }:

5{ projectNix, sourceRepos, src }:

6let

7+ # Full source including possible relartive paths form the

8+ # project directory.

9+ srcRoot =

10+ if haskellLib.canCleanSource src

11+ then haskellLib.cleanSourceWith {

12+ name = src.name + "-root";

13+ src = src.origSrc or src;

14+ inherit (src) filter;

15+ }

16+ else src.origSrc or src;

17+ # The sub directory containing the cabal.project or stack.yaml file

18+ projectSubDir' = src.origSubDir or ""; # With leading /

19+ projectSubDir = pkgs.lib.strings.removePrefix "/" projectSubDir'; # Without /

20+ projectSubDir'' = if projectSubDir == "" then "" else projectSubDir + "/"; # With trailing /

7− project = import "${projectNix}";

21+ project = import "${projectNix}${projectSubDir'}";

8in {

22in {

9 nix = projectNix;

23 nix = projectNix;

10 pkgs = project // {

24 pkgs = project // {

@@ -20,14 +34,14 @@ in {

20 subDir = pkgs.lib.strings.removePrefix "/" (

34 subDir = pkgs.lib.strings.removePrefix "/" (

21 pkgs.lib.strings.removePrefix (toString projectNix)

35 pkgs.lib.strings.removePrefix (toString projectNix)

22 (toString oldPkg.src.content));

36 (toString oldPkg.src.content));

23− srcRepoPrefix = ".source-repository-packages/";

37+ srcRepoPrefix = projectSubDir'' + ".source-repository-packages/";

24 in if pkgs.lib.strings.hasPrefix srcRepoPrefix subDir

38 in if pkgs.lib.strings.hasPrefix srcRepoPrefix subDir

25 then

39 then

26 pkgs.lib.lists.elemAt sourceRepos (

40 pkgs.lib.lists.elemAt sourceRepos (

27 pkgs.lib.strings.toInt (pkgs.lib.strings.removePrefix srcRepoPrefix subDir))

41 pkgs.lib.strings.toInt (pkgs.lib.strings.removePrefix srcRepoPrefix subDir))

28− else if haskellLib.canCleanSource src

42+ else if haskellLib.canCleanSource srcRoot

29− then haskellLib.cleanSourceWith { inherit src subDir; }

43+ then haskellLib.cleanSourceWith { src = srcRoot; inherit subDir; }

30− else src + (if subDir == "" then "" else "/" + subDir);

44+ else srcRoot + (if subDir == "" then "" else "/" + subDir);

31 in oldPkg // {

45 in oldPkg // {

32 src = (pkgs.lib).mkDefault packageSrc;

46 src = (pkgs.lib).mkDefault packageSrc;

33 }) old;

47 }) old;