on iko/ghcjs-patch-master help sign in
Move envDeps and exactDeps into lib and ghc (#360)

This change moves the work done to in `envDeps` and `exactDeps` from `make-config-files.nix` into the library components and the `ghc` compiler component (well `passthru` derivations on the `ghc` component). This greatly reduces the number of times `runCommand` is used while nix is working out what to build. This fixes a performance regression in #323 for builds where little or no work needs to be done. While maintaining the performance improvement in #323 that resulted from reducing the number of calls to `ghc-pkg` when significant work does need to be done.

commit300fff437e08bbd67be6200d841a2ada7f347715 authorHamish Mackenzie <Hamish.Mackenzie@iohk.io> dateFri 13 Dec 2019, 14:44 (7 years ago) parent5d1b05f
4 files changed+111−54
builder/comp-builder.nix+25−1 builder/make-config-files.nix+30−43 compiler/ghc/default.nix+10−2 overlays/bootstrap.nix+46−8
builder/comp-builder.nix+25−1
@@ -259,7 +259,9 @@ stdenv.mkDerivation ({
259259 #
260260 # Note 3: if a package has no libs SETUP will not generate anything. This can
261261 # happen when building the `all` component of a package.
262− installPhase = ''
262+ installPhase = let
263+ target-pkg-and-db = "${ghc.targetPrefix}ghc-pkg -v0 --package-db $out/package.conf.d";
264+ in ''
263265 runHook preInstall
264266 $SETUP_HS copy ${lib.concatStringsSep " " component.setupInstallFlags}
265267 ${lib.optionalString (haskellLib.isLibrary componentId || haskellLib.isAll componentId) ''
@@ -272,6 +274,28 @@ stdenv.mkDerivation ({
272274 elif [ -e "${name}.conf" ]; then
273275 ${ghc.targetPrefix}ghc-pkg -v0 --package-db ${configFiles}/package.conf.d -f $out/package.conf.d register ${name}.conf
274276 fi
277+
278+ mkdir -p $out/exactDep
279+ touch $out/exactDep/configure-flags
280+ touch $out/exactDep/cabal.config
281+
282+ if id=$(${target-pkg-and-db} field ${package.identifier.name} id --simple-output); then
283+ echo "--dependency=${package.identifier.name}=$id" >> $out/exactDep/configure-flags
284+ elif id=$(${target-pkg-and-db} field "z-${package.identifier.name}-z-*" id --simple-output); then
285+ name=$(${target-pkg-and-db} field "z-${package.identifier.name}-z-*" name --simple-output)
286+ # so we are dealing with a sublib. As we build sublibs separately, the above
287+ # query should be safe.
288+ echo "--dependency=''${name#z-${package.identifier.name}-z-}=$id" >> $out/exactDep/configure-flags
289+ fi
290+ if ver=$(${target-pkg-and-db} field ${package.identifier.name} version --simple-output); then
291+ echo "constraint: ${package.identifier.name} == $ver" >> $out/exactDep/cabal.config
292+ echo "constraint: ${package.identifier.name} installed" >> $out/exactDep/cabal.config
293+ fi
294+
295+ touch $out/envDep
296+ if id=$(${target-pkg-and-db} field ${package.identifier.name} id --simple-output); then
297+ echo "package-id $id" >> $out/envDep
298+ fi
275299 ''}
276300 ${(lib.optionalString (haskellLib.isTest componentId || haskellLib.isBenchmark componentId || haskellLib.isAll componentId) ''
277301 mkdir -p $out/bin
builder/make-config-files.nix+30−43
@@ -9,12 +9,17 @@ let
99 target-pkg = "${ghc.targetPrefix}ghc-pkg";
1010
1111 # This is a bit of a hack. So we'll have a slightly longer explaination here:
12+
13+ # Every library component built with `comp-builder.nix` includes an `exactDep`
14+ # and `envDep` directory with precomputed values used here.
15+ # GHC derivations include `exactDep` and `envDep` derivations that have
16+ # the same information for each of the built in packages.
17+
1218 # exactDep will pass --exact-configuration to the `SETUP_HS confiugre` command.
1319 # This requires us to pass --dependency={dep name}={pkg id}. The dependency
14− # name will usually be the name of the package `p`, which we can locate in the
15− # package-db, passed in via `pdbArg`. Thus querying the package-db for the
16− # id field for package `p`, will unsually provide is with the right value. Sublibs
17− # need a bit of special handling:
20+ # name will usually be the name of the package `p`, that will have been located
21+ # in the suitable package db when the dependency (along with `exactDep` and `envDep`)
22+ # was built. Sublibs need a bit of special handling:
1823 #
1924 # - Sublibs: if the dependency is a sublibrary of a package, we need to use
2025 # the sublibrary's name for the dep name, and lookup the sublibraries
@@ -22,52 +27,33 @@ let
2227 # sublib name to exactDep, as we don't have access to it at the call-site,
2328 # we resort to a bit of globbing, which (as pkg db's should contain only
2429 # a single package) work.
25− exactDep = pdbArg: p: nativeBuildInputs: runCommand "${p}-exactdep" { inherit nativeBuildInputs; } ''
26− mkdir -p $out
27− touch $out/configure-flags
28− touch $out/cabal.config
29−
30− if id=$(${target-pkg} ${pdbArg} field ${p} id --simple-output); then
31− echo "--dependency=${p}=$id" >> $out/configure-flags
32− elif id=$(${target-pkg} ${pdbArg} field "z-${p}-z-*" id --simple-output); then
33− name=$(${target-pkg} ${pdbArg} field "z-${p}-z-*" name --simple-output)
34− # so we are dealing with a sublib. As we build sublibs separately, the above
35− # query should be safe.
36− echo "--dependency=''${name#z-${p}-z-}=$id" >> $out/configure-flags
37− fi
38− if ver=$(${target-pkg} ${pdbArg} field ${p} version --simple-output); then
39− echo "constraint: ${p} == $ver" >> $out/cabal.config
40− echo "constraint: ${p} installed" >> $out/cabal.config
41− fi
42− '';
4330
44− catExactDep = dep: ''
45− cat ${dep}/configure-flags >> $out/configure-flags
46− cat ${dep}/cabal.config >> $out/cabal.config
31+ getLibComponent = dep:
32+ dep.components.library # Regular package dependency
33+ or dep; # or a sublib
34+
35+ catPkgExactDep = p: ''
36+ cat ${getLibComponent p}/exactDep/configure-flags >> $out/configure-flags
37+ cat ${getLibComponent p}/exactDep/cabal.config >> $out/cabal.config
4738 '';
4839
49− catPkgExactDep = p:
50− catExactDep (exactDep (packageDb p) p.identifier.name [ghc (p.components.library or p)]);
51−
52− catGhcPkgExactDep = p: catExactDep (exactDep "" p [ghc]);
53−
54− envDep = pdbArg: p: nativeBuildInputs: runCommand "${p}-envdep" { inherit nativeBuildInputs; } ''
55− touch $out
56− if id=$(${target-pkg} ${pdbArg} field ${p} id --simple-output); then
57− echo "package-id $id" >> $out
40+ catGhcPkgExactDep = p: ''
41+ if [ -e ${ghc.exactDeps}/${p} ]; then
42+ cat ${ghc.exactDeps}/${p}/configure-flags >> $out/configure-flags
43+ cat ${ghc.exactDeps}/${p}/cabal.config >> $out/cabal.config
5844 fi
5945 '';
6046
61− catEnvDep = ghcEnv: ''
62− cat ${ghcEnv} >> $out/ghc-environment
47+ catPkgEnvDep = p: ''
48+ cat ${getLibComponent p}/envDep >> $out/ghc-environment
6349 '';
6450
65− catPkgEnvDep = p:
66− catEnvDep (envDep (packageDb p) p.identifier.name [ghc (p.components.library or p)]);
67−
68− catGhcPkgEnvDep = p: catEnvDep (envDep "" p [ghc]);
51+ catGhcPkgEnvDep = p: ''
52+ if [ -e ${ghc.envDeps}/${p} ]; then
53+ cat ${ghc.envDeps}/${p} >> $out/ghc-environment
54+ fi
55+ '';
6956
70− packageDb = p: "--package-db ${p.components.library or p}/package.conf.d";
7157in { identifier, component, fullName, flags ? {} }:
7258 runCommand "${fullName}-config" { nativeBuildInputs = [ghc]; } (''
7359 mkdir -p $out
@@ -86,8 +72,9 @@ in { identifier, component, fullName, flags ? {} }:
8672 '') nonReinstallablePkgs}
8773
8874 ${lib.concatMapStringsSep "\n" (p: ''
89− cp -f "${p}/package.conf.d/"*.conf $out/package.conf.d
90− '') (haskellLib.flatLibDepends component)}
75+ cp -f "${(getLibComponent p).configFiles}/package.conf.d/"*.conf $out/package.conf.d
76+ cp -f "${getLibComponent p}/package.conf.d/"*.conf $out/package.conf.d
77+ '') component.depends}
9178
9279 # Note: we pass `clear` first to ensure that we never consult the implicit global package db.
9380 ${flagsAndConfig "package-db" ["clear" "$out/package.conf.d"]}
compiler/ghc/default.nix+10−2
@@ -13,6 +13,8 @@
1313
1414, libiconv ? null, ncurses
1515
16+, exactDeps, envDeps
17+
1618, # GHC can be built with system libffi or a bundled one.
1719 libffi ? null
1820
@@ -204,7 +206,7 @@ in let configured-src = stdenv.mkDerivation (rec {
204206 installPhase = "cp -r . $out";
205207 });
206208
207−in stdenv.mkDerivation (rec {
209+ drv =stdenv.mkDerivation (rec {
208210 version = ghc-version;
209211 name = "${targetPrefix}ghc-${version}";
210212
@@ -313,6 +315,10 @@ in stdenv.mkDerivation (rec {
313315
314316 # Used to detect non haskell-nix compilers (accedental use of nixpkgs compilers can lead to unexpected errors)
315317 isHaskellNixCompiler = true;
318+
319+ # These are used in make-config-files.nix
320+ exactDeps = exactDeps drv;
321+ envDeps = envDeps drv;
316322 } // extra-passthru;
317323
318324 meta = {
@@ -326,4 +332,6 @@ in stdenv.mkDerivation (rec {
326332 dontStrip = true;
327333 dontPatchELF = true;
328334 noAuditTmpdir = true;
329−})
335+});
336+
337+in drv
overlays/bootstrap.nix+46−8
@@ -1,4 +1,36 @@
1−self: super: {
1+self: super:
2+let
3+ exactDeps = ghc: self.runCommand "ghc-exactdep-${ghc.version}" { nativeBuildInputs = [ ghc ]; } ''
4+ for P in $(${ghc.targetPrefix}ghc-pkg list --simple-output | sed 's/-[0-9][0-9.]*//g'); do
5+ mkdir -p $out/$P
6+ touch $out/$P/configure-flags
7+ touch $out/$P/cabal.config
8+
9+ if id=$(${ghc.targetPrefix}ghc-pkg field $P id --simple-output); then
10+ echo "--dependency=$P=$id" >> $out/$P/configure-flags
11+ elif id=$(${ghc.targetPrefix}ghc-pkg field "z-$P-z-*" id --simple-output); then
12+ name=$(${ghc.targetPrefix}ghc-pkg field "z-$P-z-*" name --simple-output)
13+ # so we are dealing with a sublib. As we build sublibs separately, the above
14+ # query should be safe.
15+ echo "--dependency=''${name#z-$P-z-}=$id" >> $out/$P/configure-flags
16+ fi
17+ if ver=$(${ghc.targetPrefix}ghc-pkg field $P version --simple-output); then
18+ echo "constraint: $P == $ver" >> $out/$P/cabal.config
19+ echo "constraint: $P installed" >> $out/$P/cabal.config
20+ fi
21+ done
22+ '';
23+
24+ envDeps = ghc: self.runCommand "ghc-envdep-${ghc.version}" { nativeBuildInputs = [ ghc ]; } ''
25+ mkdir -p $out
26+ for P in $(${ghc.targetPrefix}ghc-pkg list --simple-output | sed 's/-[0-9.]*//g'); do
27+ touch $out/$P
28+ if id=$(${ghc.targetPrefix}ghc-pkg field $P id --simple-output); then
29+ echo "package-id $id" >> $out/$P
30+ fi
31+ done
32+ '';
33+in {
234 haskell-nix = super.haskell-nix // {
335 # Use this to disable the existing haskell infra structure for testing purposes
436 compiler =
@@ -73,7 +105,7 @@ self: super: {
73105 ghc844 = self.callPackage ../compiler/ghc {
74106 extra-passthru = { buildGHC = self.buildPackages.haskell-nix.compiler.ghc844; };
75107
76− inherit bootPkgs sphinx;
108+ inherit bootPkgs sphinx exactDeps envDeps;
77109
78110 buildLlvmPackages = self.buildPackages.llvmPackages_5;
79111 llvmPackages = self.llvmPackages_5;
@@ -91,7 +123,7 @@ self: super: {
91123 ghc861 = self.callPackage ../compiler/ghc {
92124 extra-passthru = { buildGHC = self.buildPackages.haskell-nix.compiler.ghc861; };
93125
94− inherit bootPkgs sphinx;
126+ inherit bootPkgs sphinx exactDeps envDeps;
95127
96128 buildLlvmPackages = self.buildPackages.llvmPackages_5;
97129 llvmPackages = self.llvmPackages_5;
@@ -107,7 +139,7 @@ self: super: {
107139 ghc862 = self.callPackage ../compiler/ghc {
108140 extra-passthru = { buildGHC = self.buildPackages.haskell-nix.compiler.ghc862; };
109141
110− inherit bootPkgs sphinx;
142+ inherit bootPkgs sphinx exactDeps envDeps;
111143
112144 buildLlvmPackages = self.buildPackages.llvmPackages_5;
113145 llvmPackages = self.llvmPackages_5;
@@ -124,7 +156,7 @@ self: super: {
124156 ghc863 = self.callPackage ../compiler/ghc {
125157 extra-passthru = { buildGHC = self.buildPackages.haskell-nix.compiler.ghc863; };
126158
127− inherit bootPkgs sphinx;
159+ inherit bootPkgs sphinx exactDeps envDeps;
128160
129161 buildLlvmPackages = self.buildPackages.llvmPackages_5;
130162 llvmPackages = self.llvmPackages_5;
@@ -141,7 +173,7 @@ self: super: {
141173 ghc864 = self.callPackage ../compiler/ghc {
142174 extra-passthru = { buildGHC = self.buildPackages.haskell-nix.compiler.ghc864; };
143175
144− inherit bootPkgs sphinx;
176+ inherit bootPkgs sphinx exactDeps envDeps;
145177
146178 buildLlvmPackages = self.buildPackages.llvmPackages_5;
147179 llvmPackages = self.llvmPackages_5;
@@ -158,7 +190,7 @@ self: super: {
158190 ghc865 = self.callPackage ../compiler/ghc {
159191 extra-passthru = { buildGHC = self.buildPackages.haskell-nix.compiler.ghc865; };
160192
161− inherit bootPkgs sphinx;
193+ inherit bootPkgs sphinx exactDeps envDeps;
162194
163195 buildLlvmPackages = self.buildPackages.llvmPackages_5;
164196 llvmPackages = self.llvmPackages_5;
@@ -231,7 +263,13 @@ self: super: {
231263 # compilers into the same place where nix expects them.
232264 # We mark these compilers as boot compilers to make sure they are only used
233265 # where a boot compiler is expected.
234− compiler = builtins.mapAttrs (_: v: v // { isHaskellNixBootCompiler = true; })
266+ compiler = builtins.mapAttrs (_: v:
267+ v // {
268+ isHaskellNixBootCompiler = true;
269+ exactDeps = exactDeps v;
270+ envDeps = envDeps v;
271+ }
272+ )
235273 (import ../compiler/old-ghc-nix { pkgs = self; });
236274
237275 packages = {