0366537 · haskell.nix · git.iko.soy

git.iko.soy/haskell.nix

on iko/ghcjs-patch-master help sign in

Use new cleanSourceWith to filter component source (#209)

When trying to build haskell packages from local source using nix ( source code you have used `git clone` to retrieve and have modified) it is often annoying that small changes can trigger a lot of components to be built unnecessarily. For instance if you change the code for a test in a package it will often trigger builds of the library and all its dependencies. To avoid this problem we could manually set up `cleanSourceWith` calls in nix to filter out files that are not need, but to do this every component would be difficult and error prone. This change automates this process using the information in the packages `.cabal` file to filter the source of each component automatically when using `callCabalPlanToNix` (we should be able to add it to `callStackToNix` too). ## How does it work? * A new version on `cleanSourceWith` is used that adds a `subDir` argument. It allows descending into a subdirectory to compose with cleaning the source. * A new `--full` argument is passed to `plan-to-nix` by `callCabalPlanToNix` telling it to output extra information about the cabal packages in the plan (directories and filenames). * `callCabalProjectToNix` uses the new `subDir` argument of `cleanSourceWith` when setting the packageSrc. * `comp-builder.nix` and `setup-builder.nix` use a new `cleanCabalComponent` function to further clean the packageSrc using the new information in the `package` and `component`. ## hpack wildcards `package.yaml` files can include wild cards and these are not handled yet (they do not map to anything in the `.cabal` file and so are lost). They will need to be added manually to the appropriate field (eg. `packages.xyz.package.dataFiles`) If your hpack file does not contain wildcards or if you are willing to add the wildcards entries manually in nix it is still useful to enable cleaning on hpack. For example: ``` packages.ghc-toolkit.package.cleanHpack = true; packages.ghc-toolkit.components.library.extraSrcFiles = [ "genapply/**/**" "boot-libs/**/**" "ghc-libdir/**/**" ]; ```

commit0366537651975a278377a52e6746e86e91ad7dee authorHamish Mackenzie <Hamish.Mackenzie@iohk.io> dateMon 26 Aug 2019, 12:27 (7 years ago) parentb4afacb

16 files changed+381−94

compare with iko/ghcjs-patch-master

unifiedsplit

builder/comp-builder.nix+9−2 builder/setup-builder.nix+29−9 default.nix+12−6 docs/reference/library.md+41 lib/call-cabal-project-to-nix.nix+11−46 lib/call-stack-to-nix.nix+2−11 lib/clean-cabal-component.nix+85 lib/clean-source-with.nix+41 lib/default.nix+7 lib/import-and-filter-project.nix+36 modules/package.nix+89−1 nix-tools/nix-tools-src.json+3−3 nix-tools/pkgs.nix+7−7 test/cabal-source-repo/default.nix+3−3 test/call-cabal-project-to-nix/default.nix+3−3 test/call-stack-to-nix/default.nix+3−3

builder/comp-builder.nix+9−2

@@ -43,6 +43,11 @@

4343}:

4444

4545let

46+ # TODO fix cabal wildcard support so hpack wildcards can be mapped to cabal wildcards

47+ cleanSrc = if cabal-generator == "hpack" && !(package.cleanHpack or false)

48+ then builtins.trace ("Cleaning component source not supported for hpack package: " + name) src

49+ else haskellLib.cleanCabalComponent package component src;

50+

4651 fullName = if haskellLib.isAll componentId

4752 then "${name}-all"

4853 else "${name}-${componentId.ctype}-${componentId.cname}";

@@ -116,12 +121,14 @@ in stdenv.lib.fix (drv:

116121stdenv.mkDerivation ({

117122 name = fullName;

118123

119− inherit src doCheck doCrossCheck dontPatchELF dontStrip;

124+ src = cleanSrc;

125+

126+ inherit doCheck doCrossCheck dontPatchELF dontStrip;

120127

121128 passthru = {

122129 inherit (package) identifier;

123130 config = component;

124− inherit configFiles executableToolDepends;

131+ inherit configFiles executableToolDepends cleanSrc;

125132 env = shellWrappers;

126133

127134 # The directory containing the haddock documentation.

builder/setup-builder.nix+29−9

@@ -3,28 +3,48 @@

33{ setup-depends, package, name, src, flags }:

55let

6+ component = {

7+ depends = setup-depends;

8+ libs = [];

9+ frameworks = [];

10+ doExactConfig = false;

11+ # We have to set hsSourceDirs or cleanCabalComponent will

12+ # include everything (and as a result all the components of

13+ # the package will depend on eveything in the package).

14+ # TODO find a better way

15+ hsSourceDirs = ["setup-src"];

16+ includeDirs = [];

17+ asmSources = [];

18+ cSources = [];

19+ cmmSources = [];

20+ cxxSources = [];

21+ jsSources = [];

22+ extraSrcFiles = [ "Setup.hs" "Setup.lhs" ];

23+ };

24+ cleanSrc = haskellLib.cleanCabalComponent package component src;

25+

626 fullName = "${name}-setup";

727

828 includeGhcPackage = lib.any (p: p.identifier.name == "ghc") setup-depends;

929

1030 configFiles = makeSetupConfigFiles {

1131 inherit (package) identifier;

12− inherit fullName flags;

13− component = {

14− depends = setup-depends;

15− libs = [];

16− frameworks = [];

17− doExactConfig = false;

18− };

19− };

32+ inherit fullName flags component;

33+ };

2034

2135in

2236 stdenv.lib.fix (drv:

2337 stdenv.mkDerivation {

2438 name = "${fullName}";

25− inherit src;

39+ src = cleanSrc;

2640 nativeBuildInputs = [ghc];

2741

42+ passthru = {

43+ inherit (package) identifier;

44+ config = component;

45+ inherit configFiles cleanSrc;

46+ };

47+

2848 CABAL_CONFIG = configFiles + /cabal.config;

2949 phases = ["unpackPhase" "buildPhase" "installPhase"];

3050 buildPhase = ''

default.nix+12−6

@@ -178,12 +178,13 @@ let

178178

179179 mkLocalHackageRepo = import ./mk-local-hackage-repo { inherit (self) hackageTarball; inherit pkgs; };

180180

181− dotCabal = { index-state, sha256 }@args:

182− pkgs.runCommand "dot-cabal-at-${builtins.replaceStrings [":"] [""] index-state}" { nativeBuildInputs = [ pkgs.cabal-install ]; } ''

181+ dotCabal = { index-state, sha256, cabal-install ? pkgs.cabal-install }@args:

182+ pkgs.runCommand "dot-cabal-at-${builtins.replaceStrings [":"] [""] index-state}" { nativeBuildInputs = [ cabal-install ]; } ''

183183 mkdir -p $out/.cabal

184184 cat <<EOF > $out/.cabal/config

185185 repository cached

186− url: file:${self.mkLocalHackageRepo args}

186+ url: file:${self.mkLocalHackageRepo

187+ (builtins.removeAttrs args ["cabal-install"])}

187188 secure: True

188189 root-keys:

189190 key-threshold: 0

@@ -205,13 +206,18 @@ let

205206 # Resulting nix files are added to nix-plan subdirectory.

206207 callCabalProjectToNix = import ./lib/call-cabal-project-to-nix.nix {

207208 index-state-hashes = import indexStateHashesPath;

208− inherit (buildPackages) dotCabal;

209− pkgs = buildPackages.pkgs; # buildPackages;

209+ inherit (buildPackages) dotCabal haskellLib;

210+ pkgs = buildPackages.pkgs;

210211 inherit (buildPackages.pkgs.haskellPackages) hpack;

211212 inherit (buildPackages.pkgs) runCommand cabal-install ghc symlinkJoin cacert;

212213 inherit (buildPackages) nix-tools;

213214 };

214215

216+ # Loads a plan and filters the package directories using cleanSourceWith

217+ importAndFilterProject = import ./lib/import-and-filter-project.nix {

218+ inherit pkgs haskellLib;

219+ };

220+

215221 # References to the unpacked sources, for caching in a Hydra jobset.

216222 source-pins = self.callPackage ./lib/make-source-pins.nix {

217223 sources = [ hackageSrc stackageSrc pkgs.path ];

@@ -238,7 +244,7 @@ let

238244 tar xzf ${tarball}

239245 mv "${name}-${version}" $out

240246 '';

241− in let plan-pkgs = import (callCabalProjectToNix { inherit src; index-state = builtins.trace "Using index-state: ${index-state}" index-state; });

247+ in let plan-pkgs = (callCabalProjectToNix { inherit src; index-state = builtins.trace "Using index-state: ${index-state}" index-state; }).pkgs;

242248 in let pkg-set = mkCabalProjectPkgSet { inherit plan-pkgs; };

243249 in pkg-set.config.hsPkgs.${name};

244250 });

docs/reference/library.md+41

@@ -177,8 +177,49 @@ A derivation containing the `nix-tools` [command-line tools](commands.md).

177177

178178## callStackToNix

179179

180+Runs `stack-to-nix` and produces the output needed for

181+`importAndFilterProject`.

182+

183+**Example**:

184+

185+```nix

186+ pkgSet = mkStackPkgSet {

187+ stack-pkgs = (importAndFilterProject (callStackToNix {

188+ src = ./.;

189+ })).pkgs;

190+ pkg-def-extras = [];

191+ modules = [];

192+ };

193+```

194+

195+

180196## callCabalProjectToNix

181197

198+Runs `cabal new-configure` and `plan-to-nix` and produces the output

199+needed for `importAndFilterProject`.

200+

201+**Example**:

202+

203+```nix

204+ pkgSet = mkCabalProjectPkgSet {

205+ plan-pkgs = (importAndFilterProject (callCabalProjectToNix {

206+ index-state = "2019-04-30T00:00:00Z";

207+ src = ./.;

208+ })).pkgs;

209+```

210+

211+## importAndFilterProject

212+

213+Imports from a derivation created by `callStackToNix`

214+or `callCabalProjectToNix`.

215+

216+The result is an attrset with the following values:

217+

218+| Attribute | Type | Description |

219+|----------------|------|-----------------------------------------------------|

220+| `pkgs` | attrset | that can be passed to `mkStackPkgSet` (as `stack-pkgs`) or `mkCabalProjectPkgSet` (as `plan-pkgs`). |

221+| `nix` | | this can be built and cached so that the amount built in the evaluation phase is not too great (helps to avoid timeouts on Hydra). |

222+

182223## hackage

183224## stackage

184225

lib/call-cabal-project-to-nix.nix+11−46

@@ -1,4 +1,4 @@

1−{ dotCabal, pkgs, runCommand, nix-tools, cabal-install, ghc, hpack, symlinkJoin, cacert, index-state-hashes }:

1+{ dotCabal, pkgs, runCommand, nix-tools, cabal-install, ghc, hpack, symlinkJoin, cacert, index-state-hashes, haskellLib }:

22let defaultGhc = ghc;

33 defaultCabalInstall = cabal-install;

44in { index-state ? null, index-sha256 ? null, src, ghc ? defaultGhc,

@@ -10,19 +10,19 @@ assert (if (builtins.compareVersions cabal-install.version "2.4.0.0") < 0

1010 else true);

1111let

1212 maybeCleanedSource =

13− if pkgs.lib.canCleanSource src

14− then pkgs.lib.cleanSourceWith {

13+ if haskellLib.canCleanSource src

14+ then haskellLib.cleanSourceWith {

1515 inherit src;

1616 filter = path: type:

1717 type == "directory" ||

1818 pkgs.lib.any (i: (pkgs.lib.hasSuffix i path)) [ ".project" ".cabal" "package.yaml" ]; }

1919 else src;

2020

21− # Using origSrc bypasses any cleanSourceWith so that it will work when

21+ # Using origSrcSubDir bypasses any cleanSourceWith so that it will work when

2222 # access to the store is restricted. If origSrc was already in the store

2323 # you can pass the project in as a string.

2424 rawCabalProject =

25− let origSrcDir = maybeCleanedSource.origSrc or maybeCleanedSource;

25+ let origSrcDir = maybeCleanedSource.origSrcSubDir or maybeCleanedSource;

2626 in if cabalProject != null

2727 then cabalProject

2828 else

@@ -81,7 +81,7 @@ let

8181 parseBlock = block:

8282 let

8383 x = span (pkgs.lib.strings.hasPrefix " ") (pkgs.lib.splitString "\n" block);

84− attrs = parseBlockLines x.fst;

84+ attrs = parseBlockLines x.fst;

8585 in

8686 if attrs."--sha256" or "" == ""

8787 then {

@@ -137,7 +137,7 @@ let

137137 }

138138 else replaceSoureRepos rawCabalProject;

139139

140− plan = runCommand "plan-to-nix-pkgs" {

140+ plan-nix = runCommand "plan-to-nix-pkgs" {

141141 nativeBuildInputs = [ nix-tools ghc hpack cabal-install pkgs.rsync pkgs.git ];

142142 } (''

143143 tmp=$(mktemp -d)

@@ -154,7 +154,8 @@ let

154154 export GIT_SSL_CAINFO=${cacert}/etc/ssl/certs/ca-bundle.crt

155155 HOME=${dotCabal {

156156 index-state = index-state-found;

157− sha256 = index-sha256-found; }} cabal new-configure \

157+ sha256 = index-sha256-found;

158+ inherit cabal-install; }} cabal new-configure \

158159 --with-ghc=${ghc.targetPrefix}ghc \

159160 --with-ghc-pkg=${ghc.targetPrefix}ghc-pkg

160161

@@ -179,45 +180,9 @@ let

179180

180181 # run `plan-to-nix` in $out. This should produce files right there with the

181182 # proper relative paths.

182− (cd $out && plan-to-nix --plan-json $tmp/dist-newstyle/cache/plan.json -o .)

183+ (cd $out && plan-to-nix --full --plan-json $tmp/dist-newstyle/cache/plan.json -o .)

183184

184185 # move pkgs.nix to default.nix ensure we can just nix `import` the result.

185186 mv $out/pkgs.nix $out/default.nix

186187 '');

187−in

188− # TODO: We really want this (symlinks) instead of copying the source over each and

189− # every time. However this will not work with sandboxed builds. They won't

190− # have access to `plan` or `src` paths. So while they will see all the

191− # links, they won't be able to read any of them.

192− #

193− # We should be able to fix this if we propagaed the build inputs properly.

194− # As we are `import`ing the produced nix-path here, we seem to be losing the

195− # dependencies though.

196− #

197− # I guess the end-result is that ifd's don't work well with symlinks.

198− #

199− # symlinkJoin {

200− # name = "plan-and-src";

201− # # todo: should we clean `src` to drop any .git, .nix, ... other irelevant files?

202− # buildInputs = [ plan src ];

203− # }

204− runCommand "plan-to-nix-pkgs-with-src" { nativeBuildInputs = [ pkgs.rsync ]; } (''

205− mkdir $out

206− # todo: should we clean `src` to drop any .git, .nix, ... other irelevant files?

207− rsync -a "${src}/" "$out/"

208− rsync -a ${plan}/ $out/

209− '' +

210− ( pkgs.lib.strings.concatStrings (

211− pkgs.lib.lists.zipListsWith (n: f: ''

212− mkdir -p $out/.source-repository-packages/${builtins.toString n}

213− rsync -a "${f}/" "$out/.source-repository-packages/${builtins.toString n}/"

214− '')

215− (pkgs.lib.lists.range 0 ((builtins.length fixedProject.sourceRepos) - 1))

216− fixedProject.sourceRepos

217− )

218− ) + ''

219− # Rsync will have made $out read only and that can cause problems when

220− # nix sandboxing is enabled (since it can prevent nix from moving the directory

221− # out of the chroot sandbox).

222− chmod +w $out

223− '')

188+in { projectNix = plan-nix; inherit src; inherit (fixedProject) sourceRepos; }

lib/call-stack-to-nix.nix+2−11

@@ -9,6 +9,7 @@

99{ src, stackYaml ? null, ignorePackageYaml ? false }:

1010let

1111 stackToNixArgs = builtins.concatStringsSep " " [

12+ "--full"

1213 "--stack-yaml=${src}/${if stackYaml == null then "stack.yaml" else stackYaml}"

1314 (if ignorePackageYaml then "--ignore-package-yaml" else "")

1415 "-o ."

@@ -30,14 +31,4 @@ let

3031 # move pkgs.nix to default.nix ensure we can just nix `import` the result.

3132 mv $out/pkgs.nix $out/default.nix

3233 '';

33−in

34−runCommand "stack-to-nix-pkgs-with-src" { nativeBuildInputs = [ pkgs.rsync ]; } ''

35− mkdir $out

36− # todo: should we clean `src` to drop any .git, .nix, ... other irelevant files?

37− rsync -a "${src}/" "$out/"

38− rsync -a ${stack}/ $out/

39− # Rsync will have made $out read only and that can cause problems when

40− # nix sandboxing is enabled (since it can prevent nix from moving the directory

41− # out of the chroot sandbox).

42− chmod +w $out

43−''

34+in { projectNix = stack; inherit src; sourceRepos = []; }

lib/clean-cabal-component.nixnew file+85

@@ -0,0 +1,85 @@

1+# Use cleanSourceWith to filter just the files needed for a particular

2+# component of the package

3+{ lib, cleanSourceWith }: package: component: src:

4+let

5+ srcStr' = src.origSrcSubDir or src.origSrc or null;

6+ # Transform

7+ # "." -> ""

8+ # "./." -> ""

9+ # "./xyz" -> "xyz"

10+ normalizeRelativePath = rel:

11+ if rel == "." || rel == "./."

12+ then ""

13+ else lib.strings.removePrefix "./" rel;

14+ # Like normalizeRelativePath but with a trailing / when needed

15+ normalizeRelativeDir = dir:

16+ let p = normalizeRelativePath dir;

17+ in if p == "" then "" else p + "/";

18+in

19+ if srcStr' == null || package.detailLevel != "FullDetails"

20+ then src

21+ else

22+ let

23+ srcStr = toString srcStr';

24+ dataDir = normalizeRelativeDir package.dataDir;

25+ hsSourceDirs = builtins.map normalizeRelativeDir component.hsSourceDirs

26+ ++ (if component.hsSourceDirs == [] then [""] else []);

27+ includeDirs = builtins.map normalizeRelativeDir component.includeDirs;

28+ dirsNeeded = [dataDir]

29+ ++ hsSourceDirs

30+ ++ includeDirs;

31+ fileMatch = dir: list:

32+ let

33+ prefixes = builtins.map (f: dir + f) (

34+ lib.lists.remove null (lib.lists.flatten (

35+ builtins.map (f: builtins.match "([^*]*)[*].*" f) list)));

36+ exactMatches = builtins.map (f: dataDir + f) (

37+ lib.lists.remove null (lib.lists.flatten (

38+ builtins.map (f: builtins.match "([^*]*)" f) list)));

39+ in rPath: lib.any (d: lib.strings.hasPrefix d rPath) prefixes

40+ || lib.any (d: d == rPath) exactMatches;

41+ dataFileMatch = fileMatch dataDir package.dataFiles;

42+ licenseMatch = fileMatch "" package.licenseFiles;

43+ extraSrcMatch = fileMatch "" (

44+ package.extraSrcFiles

45+ ++ component.extraSrcFiles);

46+ extraDocMatch = fileMatch "" package.extraDocFiles;

47+ otherSourceFiles =

48+ component.asmSources

49+ ++ component.cmmSources

50+ ++ component.cSources

51+ ++ component.cxxSources

52+ ++ component.jsSources;

53+ in cleanSourceWith {

54+ inherit src;

55+ filter = path: type:

56+ assert (if !lib.strings.hasPrefix (srcStr + "/") (path + "/")

57+ then throw ("Unexpected path " + path + " (expected something in " + srcStr + "/)")

58+ else true);

59+ let

60+ srcStrLen = lib.strings.stringLength srcStr;

61+ rPath = lib.strings.substring (srcStrLen + 1) (lib.strings.stringLength path - srcStrLen - 1) path;

62+ # This is a handy way to find out why different files are included

63+ # traceReason = reason: v: if v then builtins.trace (rPath + " : " + reason) true else false;

64+ traceReason = reason: v: v;

65+ in

66+ traceReason "directory is needed" (

67+ lib.any (d: lib.strings.hasPrefix (rPath + "/") d) (

68+ dirsNeeded

69+ ++ package.extraSrcFiles

70+ ++ component.extraSrcFiles

71+ ++ package.extraDocFiles

72+ ++ builtins.map (f:

73+ dataDir + (if dataDir == "" then "" else "/") + f) package.dataFiles

74+ ++ otherSourceFiles))

75+ || traceReason "cabal package definition" (lib.strings.hasSuffix ".cabal" rPath)

76+ || traceReason "hpack package defintion" (rPath == "package.yaml")

77+ || traceReason "data file" (lib.strings.hasPrefix dataDir rPath

78+ && dataFileMatch rPath)

79+ || traceReason "haskell source dir" (lib.any (d: lib.strings.hasPrefix d rPath) hsSourceDirs)

80+ || traceReason "include dir" (lib.any (d: lib.strings.hasPrefix d rPath) includeDirs)

81+ || traceReason "license file" (licenseMatch rPath)

82+ || traceReason "extra source file" (extraSrcMatch rPath)

83+ || traceReason "extra doc file" (extraDocMatch rPath)

84+ || traceReason "other source file" (lib.any (f: f == rPath) otherSourceFiles);

85+ }

lib/clean-source-with.nixnew file+41

@@ -0,0 +1,41 @@

1+# This is a replacement for the cleanSourceWith function in nixpkgs

2+# https://github.com/NixOS/nixpkgs/blob/1d9d31a0eb8e8358830528538a391df52f6a075a/lib/sources.nix#L41

3+# It adds a subDir argument in a way that allows descending into a subdirectory

4+# to compose with cleaning the source with a filter.

5+{ lib }: rec {

6+ cleanSourceWith = { filter ? path: type: true, src, subDir ? "" }:

7+ let

8+ subDir' = if subDir == "" then "" else "/" + subDir;

9+ # In case this is mixed with older versions of cleanSourceWith

10+ isFiltered = src ? _isLibCleanSourceWith;

11+ isFilteredEx = src ? _isLibCleanSourceWithEx;

12+ origSrc = if isFiltered || isFilteredEx then src.origSrc else src;

13+ origSubDir = if isFilteredEx then src.origSubDir + subDir' else subDir';

14+ origSrcSubDir = toString origSrc + origSubDir;

15+ parentFilter = if isFiltered || isFilteredEx

16+ then path: type: src.filter path type

17+ else path: type: true;

18+ filter' = path: type:

19+ # Include parent paths based on the parent filter

20+ (lib.strings.hasPrefix (path + "/") (origSrcSubDir + "/")

21+ && parentFilter path type)

22+ # Children only if both filters return true

23+ || (lib.strings.hasPrefix (origSrcSubDir + "/") path

24+ && (filter path type && parentFilter path type));

25+ in {

26+ inherit origSrc origSubDir origSrcSubDir;

27+ filter = filter';

28+ outPath = (builtins.filterSource filter' origSrc) + origSubDir;

29+ _isLibCleanSourceWithEx = true;

30+ # It is only safe for older cleanSourceWith to filter this one

31+ # if the we are still looking at the root of origSrc

32+ _isLibCleanSourceWith = origSubDir == "";

33+ };

34+

35+ pathHasContext = builtins.hasContext or (lib.hasPrefix builtins.storeDir);

36+

37+ canCleanSource = src:

38+ src ? _isLibCleanSourceWithEx

39+ || src ? _isLibCleanSourceWith

40+ || !(pathHasContext (toString src));

41+}

lib/default.nix+7

@@ -112,4 +112,11 @@ with haskellLib;

112112 (mapAttrs (_: package: package.components.${group} // { recurseForDerivations = true; })

113113 (filterAttrs (name: package: (package.isHaskell or false) && packageSel package) haskellPackages))

114114 // { recurseForDerivations = true; };

115+

116+ # Replacement for lib.cleanSourceWith that has a subDir argument.

117+ inherit (import ./clean-source-with.nix { inherit lib; }) cleanSourceWith canCleanSource;

118+

119+ # Use cleanSourceWith to filter just the files needed for a particular

120+ # component of a package

121+ cleanCabalComponent = import ./clean-cabal-component.nix { inherit lib cleanSourceWith; };

115122}

lib/import-and-filter-project.nixnew file+36

@@ -0,0 +1,36 @@

1+# This function takes the output of `callCabalProjectToNix` or

2+# `callStackToNix`. It imports the projectNix that was produced

3+# and combines it with the original source.

4+{ pkgs, haskellLib }:

5+{ projectNix, sourceRepos, src }:

6+let

7+ project = import "${projectNix}";

8+in {

9+ nix = projectNix;

10+ pkgs = project // {

11+ extras = hackage: let old = (project.extras hackage).packages; in {

12+ packages = pkgs.lib.attrsets.mapAttrs (name: value:

13+ if builtins.isFunction value

14+ then value

15+ else {...}@args:

16+ let oldPkg = import value args;

17+ packageSrc = if !pkgs.lib.strings.hasPrefix (toString projectNix) (toString oldPkg.src.content)

18+ then toString oldPkg.src.content

19+ else let

20+ subDir = pkgs.lib.strings.removePrefix "/" (

21+ pkgs.lib.strings.removePrefix (toString projectNix)

22+ (toString oldPkg.src.content));

23+ srcRepoPrefix = ".source-repository-packages/";

24+ in if pkgs.lib.strings.hasPrefix srcRepoPrefix subDir

25+ then

26+ pkgs.lib.lists.elemAt sourceRepos (

27+ pkgs.lib.strings.toInt (pkgs.lib.strings.removePrefix srcRepoPrefix subDir))

28+ else if haskellLib.canCleanSource src

29+ then haskellLib.cleanSourceWith { inherit src subDir; }

30+ else src + (if subDir == "" then "" else "/" + subDir);

31+ in oldPkg // {

32+ src = (pkgs.lib).mkDefault packageSrc;

33+ }) old;

34+ };

35+ };

36+}

modules/package.nix+89−1

@@ -19,7 +19,11 @@

1919with lib;

2020with types;

2121

22−{

22+# Work around issue that can cause _lots_ of files to be copied into the store.

23+# See https://github.com/NixOS/nixpkgs/pull/64691

24+let path = types.path // { check = x: types.path.check (x.origSrc or x); };

25+

26+in {

2327 # This is how the Nix expressions generated by *-to-nix receive

2428 # their flags argument.

2529 config._module.args = mod_args // { flags = config.flags; };

@@ -82,6 +86,46 @@ with types;

8286 type = listOfFilteringNulls unspecified;

8387 default = [];

8488 };

89+

90+ detailLevel = mkOption {

91+ type = str;

92+ default = "MinimalDetails";

93+ };

94+

95+ licenseFiles = mkOption {

96+ type = listOfFilteringNulls unspecified;

97+ default = [];

98+ };

99+

100+ dataDir = mkOption {

101+ type = str;

102+ default = "";

103+ };

104+

105+ dataFiles = mkOption {

106+ type = listOfFilteringNulls unspecified;

107+ default = [];

108+ };

109+

110+ extraSrcFiles = mkOption {

111+ type = listOfFilteringNulls unspecified;

112+ default = [];

113+ };

114+

115+ extraTmpFiles = mkOption {

116+ type = listOfFilteringNulls unspecified;

117+ default = [];

118+ };

119+

120+ extraDocFiles = mkOption {

121+ type = listOfFilteringNulls unspecified;

122+ default = [];

123+ };

124+

125+ cleanHpack = mkOption {

126+ type = bool;

127+ default = false;

128+ };

85129 };

86130

87131 components = let

@@ -108,6 +152,50 @@ with types;

108152 type = listOfFilteringNulls unspecified;

109153 default = [];

110154 };

155+ modules = mkOption {

156+ type = listOfFilteringNulls unspecified;

157+ default = [];

158+ };

159+ asmSources = mkOption {

160+ type = listOfFilteringNulls unspecified;

161+ default = [];

162+ };

163+ cmmSources = mkOption {

164+ type = listOfFilteringNulls unspecified;

165+ default = [];

166+ };

167+ cSources = mkOption {

168+ type = listOfFilteringNulls unspecified;

169+ default = [];

170+ };

171+ cxxSources = mkOption {

172+ type = listOfFilteringNulls unspecified;

173+ default = [];

174+ };

175+ jsSources = mkOption {

176+ type = listOfFilteringNulls unspecified;

177+ default = [];

178+ };

179+ hsSourceDirs = mkOption {

180+ type = listOfFilteringNulls unspecified;

181+ default = [];

182+ };

183+ includeDirs = mkOption {

184+ type = listOfFilteringNulls unspecified;

185+ default = [];

186+ };

187+ includes = mkOption {

188+ type = listOfFilteringNulls unspecified;

189+ default = [];

190+ };

191+ mainPath = mkOption {

192+ type = listOfFilteringNulls unspecified;

193+ default = [];

194+ };

195+ extraSrcFiles = mkOption {

196+ type = listOfFilteringNulls unspecified;

197+ default = [];

198+ };

111199 };

112200 };

113201 in {

nix-tools/nix-tools-src.json+3−3

@@ -1,7 +1,7 @@

11{

22 "url": "https://github.com/input-output-hk/nix-tools",

3− "rev": "3d021291ddf148220050adef350ac51bf27d415f",

4− "date": "2019-05-31T11:38:58+08:00",

5− "sha256": "1zgkmm1a3xhclwvnfcbcsqp9zl91g20csiirk2rqyn8k8vy8wwkb",

3+ "rev": "a9f227d9238102fb424157156712ef9b64413096",

4+ "date": "2019-07-18T12:21:16+12:00",

5+ "sha256": "19r5an3hrjn1a4mm2n0fhaa5b1fi975pyjcisbnrhwfajlqb362f",

66 "fetchSubmodules": false

77}

nix-tools/pkgs.nix+7−7

@@ -7,7 +7,7 @@

77 "test-framework-hunit".revision = (((hackage."test-framework-hunit")."0.3.0.2").revisions).default;

88 "test-framework-hunit".flags.base4 = true;

99 "test-framework-hunit".flags.base3 = false;

10− "http-client".revision = (((hackage."http-client")."0.5.14").revisions).default;

10+ "http-client".revision = (((hackage."http-client")."0.6.4").revisions).default;

1111 "http-client".flags.network-uri = true;

1212 "cookie".revision = (((hackage."cookie")."0.4.4").revisions).default;

1313 "void".revision = (((hackage."void")."0.7.3").revisions).default;

@@ -21,7 +21,6 @@

2121 "semigroupoids".flags.containers = true;

2222 "semigroupoids".flags.contravariant = true;

2323 "free".revision = (((hackage."free")."5.1.1").revisions).default;

24− "tf-random".revision = (((hackage."tf-random")."0.5").revisions).default;

2524 "Only".revision = (((hackage."Only")."0.1").revisions).default;

2625 "cereal".revision = (((hackage."cereal")."0.5.8.0").revisions).default;

2726 "cereal".flags.bytestring-builder = false;

@@ -106,7 +105,7 @@

106105 "distributive".flags.tagged = true;

107106 "asn1-encoding".revision = (((hackage."asn1-encoding")."0.9.5").revisions).default;

108107 "vector-th-unbox".revision = (((hackage."vector-th-unbox")."0.2.1.6").revisions).default;

109− "QuickCheck".revision = (((hackage."QuickCheck")."2.12.6.1").revisions).default;

108+ "QuickCheck".revision = (((hackage."QuickCheck")."2.13.1").revisions).default;

110109 "QuickCheck".flags.templatehaskell = true;

111110 "scientific".revision = (((hackage."scientific")."0.3.6.2").revisions).default;

112111 "scientific".flags.integer-simple = false;

@@ -124,6 +123,7 @@

124123 "optparse-applicative".revision = (((hackage."optparse-applicative")."0.14.3.0").revisions).default;

125124 "network".revision = (((hackage."network")."3.1.0.0").revisions).default;

126125 "connection".revision = (((hackage."connection")."0.3.0").revisions).default;

126+ "splitmix".revision = (((hackage."splitmix")."0.0.2").revisions).default;

127127 "async".revision = (((hackage."async")."2.2.1").revisions).default;

128128 "async".flags.bench = false;

129129 "dlist".revision = (((hackage."dlist")."0.8.0.6").revisions).default;

@@ -216,9 +216,9 @@

216216 "reflection".revision = (((hackage."reflection")."2.1.4").revisions).default;

217217 "reflection".flags.slow = false;

218218 "reflection".flags.template-haskell = true;

219− "these".revision = (((hackage."these")."0.7.6").revisions).default;

219+ "these".revision = (((hackage."these")."0.8").revisions).default;

220220 "regex-tdfa-text".revision = (((hackage."regex-tdfa-text")."1.0.0.3").revisions).default;

221− "dependent-sum".revision = (((hackage."dependent-sum")."0.4").revisions).default;

221+ "dependent-sum".revision = (((hackage."dependent-sum")."0.5").revisions).default;

222222 "socks".revision = (((hackage."socks")."0.6.0").revisions).default;

223223 "streaming-commons".revision = (((hackage."streaming-commons")."0.2.1.0").revisions).default;

224224 "streaming-commons".flags.use-bytestring-builder = false;

@@ -266,6 +266,7 @@

266266 "deriving-compat".flags.new-functor-classes = true;

267267 "text".revision = (((hackage."text")."1.2.3.1").revisions).default;

268268 "Cabal".revision = (((hackage."Cabal")."2.4.0.1").revisions).default;

269+ "assoc".revision = (((hackage."assoc")."1").revisions).default;

269270 "Diff".revision = (((hackage."Diff")."0.3.4").revisions).default;

270271 "unordered-containers".revision = (((hackage."unordered-containers")."0.2.10.0").revisions).default;

271272 "unordered-containers".flags.debug = false;

@@ -292,7 +293,7 @@

292293 "prettyprinter".flags.buildreadme = false;

293294 "cryptohash-sha512".revision = (((hackage."cryptohash-sha512")."0.11.100.1").revisions).default;

294295 "tasty-hunit".revision = (((hackage."tasty-hunit")."0.10.0.2").revisions).default;

295− "pretty-show".revision = (((hackage."pretty-show")."1.9.5").revisions).default;

296+ "pretty-show".revision = (((hackage."pretty-show")."1.8.2").revisions).default;

296297 "transformers".revision = (((hackage."transformers")."0.5.6.2").revisions).default;

297298 "hashable".revision = (((hackage."hashable")."1.2.7.0").revisions).default;

298299 "hashable".flags.sse2 = true;

@@ -354,7 +355,6 @@

354355 "array".revision = (((hackage."array")."0.5.3.0").revisions).default;

355356 "repline".revision = (((hackage."repline")."0.2.1.0").revisions).default;

356357 "xml".revision = (((hackage."xml")."1.3.14").revisions).default;

357− "erf".revision = (((hackage."erf")."2.0.0.0").revisions).default;

358358 "lens-family-core".revision = (((hackage."lens-family-core")."1.2.3").revisions).default;

359359 "integer-gmp".revision = (((hackage."integer-gmp")."1.0.2.0").revisions).default;

360360 };

test/cabal-source-repo/default.nix+3−3

@@ -1,14 +1,14 @@

1−{ stdenv, mkCabalProjectPkgSet, callCabalProjectToNix }:

1+{ stdenv, mkCabalProjectPkgSet, callCabalProjectToNix, importAndFilterProject }:

33with stdenv.lib;

55let

66 pkgSet = mkCabalProjectPkgSet {

7− plan-pkgs = import (callCabalProjectToNix {

7+ plan-pkgs = (importAndFilterProject (callCabalProjectToNix {

88 index-state = "2019-04-30T00:00:00Z";

99 # reuse the cabal-simple test project

1010 src = ./.;

11− });

11+ })).pkgs;

1212 };

1313 packages = pkgSet.config.hsPkgs;

1414in

test/call-cabal-project-to-nix/default.nix+3−3

@@ -1,14 +1,14 @@

1−{ stdenv, mkCabalProjectPkgSet, callCabalProjectToNix }:

1+{ stdenv, mkCabalProjectPkgSet, callCabalProjectToNix, importAndFilterProject }:

33with stdenv.lib;

55let

66 pkgSet = mkCabalProjectPkgSet {

7− plan-pkgs = import (callCabalProjectToNix {

7+ plan-pkgs = (importAndFilterProject (callCabalProjectToNix {

88 index-state = "2019-04-30T00:00:00Z";

99 # reuse the cabal-simple test project

1010 src = ../cabal-simple;

11− });

11+ })).pkgs;

1212 };

1313 packages = pkgSet.config.hsPkgs;

1414in

test/call-stack-to-nix/default.nix+3−3

@@ -1,12 +1,12 @@

1−{ stdenv, mkStackPkgSet, callStackToNix }:

1+{ stdenv, mkStackPkgSet, callStackToNix, importAndFilterProject }:

33with stdenv.lib;

55let

66 pkgSet = mkStackPkgSet {

7− stack-pkgs = import (callStackToNix {

7+ stack-pkgs = (importAndFilterProject (callStackToNix {

88 src = ../stack-simple;

9− });

9+ })).pkgs;

1010 pkg-def-extras = [];

1111 modules = [];

1212 };