on iko/ghcjs-patch-master help sign in
improve speed of make-config-files (#318) (#323)

caches `configure-flags`, `cabal.config`, and `ghc-environment` files per library in the nix store, so the `make-config-files` script just has to append them into the same named files for the package being built instead of using `ghc-pkg dump` and `ghc-pkg register` for each dependency, copies the `.conf` files from the dependencies `package.conf.d` directory and then uses a single `ghc-pkg recache` eliminate the call of `ghc-pkg check` suppress some output of `ghc-pkg` (not related to speed improvements, I'm flexible on rolling this bit back) in my tests, the make-config-files scripts now only take seconds, even for packages that have many many dependencies - prior to this change, this script took more than a minute for such packages

commit24669a42085aa18867ad0623e445e44f84bbb230 authorRichard Wallace <rwallace@thewallacepack.net> dateSat 23 Nov 2019, 05:48 (7 years ago) parentff62e14
1 file changed+44−27
builder/make-config-files.nix+44−27
builder/make-config-files.nix+44−27
@@ -1,4 +1,4 @@
1−{ stdenv, lib, haskellLib, ghc, nonReinstallablePkgs, runCommand }:
1+{ stdenv, lib, haskellLib, ghc, nonReinstallablePkgs, runCommand, writeText, writeScript }:
22
33let
44 flagsAndConfig = field: xs: lib.optionalString (xs != []) ''
@@ -6,6 +6,8 @@ let
66 echo "${field}: ${lib.concatStringsSep " " xs}" >> $out/cabal.config
77 '';
88
9+ target-pkg = "${ghc.targetPrefix}ghc-pkg";
10+
911 # This is a bit of a hack. So we'll have a slightly longer explaination here:
1012 # exactDep will pass --exact-configuration to the `SETUP_HS confiugre` command.
1113 # This requires us to pass --dependency={dep name}={pkg id}. The dependency
@@ -20,38 +22,57 @@ let
2022 # sublib name to exactDep, as we don't have access to it at the call-site,
2123 # we resort to a bit of globbing, which (as pkg db's should contain only
2224 # a single package) work.
23− exactDep = pdbArg: p: ''
24− if id=$(target-pkg ${pdbArg} field ${p} id --simple-output); then
25+ exactDep = pdbArg: p: nativeBuildInputs: runCommand "${p}-exactdep" { inherit nativeBuildInputs; } ''
26+ mkdir -p $out
27+ touch $out/configure-flags
28+ touch $out/cabal.config
29+
30+ if id=$(${target-pkg} ${pdbArg} field ${p} id --simple-output); then
2531 echo "--dependency=${p}=$id" >> $out/configure-flags
26− elif id=$(target-pkg ${pdbArg} field "z-${p}-z-*" id --simple-output); then
27− name=$(target-pkg ${pdbArg} field "z-${p}-z-*" name --simple-output)
32+ elif id=$(${target-pkg} ${pdbArg} field "z-${p}-z-*" id --simple-output); then
33+ name=$(${target-pkg} ${pdbArg} field "z-${p}-z-*" name --simple-output)
2834 # so we are dealing with a sublib. As we build sublibs separately, the above
2935 # query should be safe.
3036 echo "--dependency=''${name#z-${p}-z-}=$id" >> $out/configure-flags
3137 fi
32− if ver=$(target-pkg ${pdbArg} field ${p} version --simple-output); then
38+ if ver=$(${target-pkg} ${pdbArg} field ${p} version --simple-output); then
3339 echo "constraint: ${p} == $ver" >> $out/cabal.config
3440 echo "constraint: ${p} installed" >> $out/cabal.config
3541 fi
3642 '';
3743
38− envDep = pdbArg: p: ''
39− if id=$(target-pkg ${pdbArg} field ${p} id --simple-output); then
40− echo "package-id $id" >> $out/ghc-environment
44+ catExactDep = dep: ''
45+ cat ${dep}/configure-flags >> $out/configure-flags
46+ cat ${dep}/cabal.config >> $out/cabal.config
47+ '';
48+
49+ catPkgExactDep = p:
50+ catExactDep (exactDep (packageDb p) p.identifier.name [ghc (p.components.library or p)]);
51+
52+ catGhcPkgExactDep = p: catExactDep (exactDep "" p [ghc]);
53+
54+ envDep = pdbArg: p: nativeBuildInputs: runCommand "${p}-envdep" { inherit nativeBuildInputs; } ''
55+ touch $out
56+ if id=$(${target-pkg} ${pdbArg} field ${p} id --simple-output); then
57+ echo "package-id $id" >> $out
4158 fi
4259 '';
4360
44−in { identifier, component, fullName, flags ? {} }:
61+ catEnvDep = ghcEnv: ''
62+ cat ${ghcEnv} >> $out/ghc-environment
63+ '';
64+
65+ catPkgEnvDep = p:
66+ catEnvDep (envDep (packageDb p) p.identifier.name [ghc (p.components.library or p)]);
67+
68+ catGhcPkgEnvDep = p: catEnvDep (envDep "" p [ghc]);
4569
70+ packageDb = p: "--package-db ${p.components.library or p}/package.conf.d";
71+in { identifier, component, fullName, flags ? {} }:
4672 runCommand "${fullName}-config" { nativeBuildInputs = [ghc]; } (''
4773 mkdir -p $out
4874
49− # Calls ghc-pkg for the target platform
50− target-pkg() {
51− ${ghc.targetPrefix}ghc-pkg "$@"
52− }
53−
54− target-pkg init $out/package.conf.d
75+ ${target-pkg} init $out/package.conf.d
5576
5677 ${lib.concatStringsSep "\n" (lib.mapAttrsToList flagsAndConfig {
5778 "extra-lib-dirs" = map (p: "${lib.getLib p}/lib") component.libs;
@@ -60,14 +81,12 @@ in { identifier, component, fullName, flags ? {} }:
6081 })}
6182
6283 # Copy over the nonReinstallablePkgs from the global package db.
63− # Note: we need to use --global-package-db with ghc-pkg to prevent it
64− # from looking into the implicit global package db when registering the package.
6584 ${lib.concatMapStringsSep "\n" (p: ''
66− target-pkg describe ${p} | target-pkg --force --global-package-db $out/package.conf.d register - || true
85+ find ${ghc}/lib/${ghc.name}/package.conf.d -name '${p}*.conf' -exec cp -f {} $out/package.conf.d \;
6786 '') nonReinstallablePkgs}
6887
6988 ${lib.concatMapStringsSep "\n" (p: ''
70− target-pkg --package-db ${p}/package.conf.d dump | target-pkg --force --package-db $out/package.conf.d register -
89+ cp -f "${p}/package.conf.d/"*.conf $out/package.conf.d
7190 '') (haskellLib.flatLibDepends component)}
7291
7392 # Note: we pass `clear` first to ensure that we never consult the implicit global package db.
@@ -82,16 +101,16 @@ in { identifier, component, fullName, flags ? {} }:
82101 cat > $out/ghc-environment <<EOF
83102 package-db $out/package.conf.d
84103 EOF
85− ${lib.concatMapStringsSep "\n" (p: envDep "--package-db ${p.components.library or p}/package.conf.d" p.identifier.name) component.depends}
86− ${lib.concatMapStringsSep "\n" (envDep "") (lib.remove "ghc" nonReinstallablePkgs)}
87104
105+ ${lib.concatMapStringsSep "\n" catPkgEnvDep component.depends}
106+ ${lib.concatMapStringsSep "\n" catGhcPkgEnvDep (lib.remove "ghc" nonReinstallablePkgs)}
88107 '' + lib.optionalString component.doExactConfig ''
89108 echo "--exact-configuration" >> $out/configure-flags
90109 echo "allow-newer: ${identifier.name}:*" >> $out/cabal.config
91110 echo "allow-older: ${identifier.name}:*" >> $out/cabal.config
92111
93− ${lib.concatMapStringsSep "\n" (p: exactDep "--package-db ${p.components.library or p}/package.conf.d" p.identifier.name) component.depends}
94− ${lib.concatMapStringsSep "\n" (exactDep "") nonReinstallablePkgs}
112+ ${lib.concatMapStringsSep "\n" catPkgExactDep component.depends}
113+ ${lib.concatMapStringsSep "\n" catGhcPkgExactDep nonReinstallablePkgs}
95114
96115 ''
97116 # This code originates in the `generic-builder.nix` from nixpkgs. However GHC has been fixed
@@ -124,7 +143,5 @@ in { identifier, component, fullName, flags ? {} }:
124143 sed -i "s,dynamic-library-dirs: .*,dynamic-library-dirs: $dynamicLinksDir," $f
125144 done
126145 '' + ''
127− target-pkg --package-db $out/package.conf.d recache
128− '' + ''
129− target-pkg --package-db $out/package.conf.d check
146+ ${target-pkg} -v0 --package-db $out/package.conf.d recache
130147 '')