git.iko.soy/haskell.nix
on iko/ghcjs-patch-master help sign in
Supporting `ca-derivations` experimental feature (#1494)
2c1701bAndrea Ciceri
builder/comp-builder.nix+19−2
compare with iko/ghcjs-patch-master
unifiedsplit
@@ -50,6 +50,7 @@ let self =
50, writeHieFiles ? component.writeHieFiles
50, writeHieFiles ? component.writeHieFiles
51
51
52, ghcOptions ? component.ghcOptions
52, ghcOptions ? component.ghcOptions
53+, contentAddressed ? component.contentAddressed
53
54
54# Options for Haddock generation
55# Options for Haddock generation
55, doHaddock ? component.doHaddock # Enable haddock and hoogle generation
56, doHaddock ? component.doHaddock # Enable haddock and hoogle generation
@@ -316,7 +317,13 @@ let
316 componentDrv = drv;
317 componentDrv = drv;
317 };
318 };
318
319
320+ contentAddressedAttrs = lib.optionalAttrs contentAddressed {
321+ __contentAddressed = true;
322+ outputHashMode = "recursive";
323+ outputHashAlgo = "sha256";
324+ };
325+
319− drv = stdenv.mkDerivation (commonAttrs // {
326+ drv = stdenv.mkDerivation (commonAttrs // contentAddressedAttrs // {
320 pname = nameOnly;
327 pname = nameOnly;
321 inherit (package.identifier) version;
328 inherit (package.identifier) version;
322
329
@@ -405,7 +412,17 @@ let
405 (lib.optionalString stdenv.hostPlatform.isWindows ''
412 (lib.optionalString stdenv.hostPlatform.isWindows ''
406 export pkgsHostTargetAsString="''${pkgsHostTarget[@]}"
413 export pkgsHostTargetAsString="''${pkgsHostTarget[@]}"
407 '') +
414 '') +
415+ # The following could be refactored but would lead to many rebuilds
416+
417+ # In case of content addressed components we need avoid parallel building (passing -j1)
418+ # in order to have a deterministic output and therefore avoid potential situations
419+ # where the binary cache becomes useless
420+ # See also https://gitlab.haskell.org/ghc/ghc/-/issues/12935
421+ (if contentAddressed then ''
422+ runHook preBuild
423+ $SETUP_HS build ${haskellLib.componentTarget componentId} -j1 ${lib.concatStringsSep " " setupBuildFlags}
424+ runHook postBuild
408− (if stdenv.hostPlatform.isGhcjs then ''
425+ '' else if stdenv.hostPlatform.isGhcjs then ''
409 runHook preBuild
426 runHook preBuild
410 # https://gitlab.haskell.org/ghc/ghc/issues/9221
427 # https://gitlab.haskell.org/ghc/ghc/issues/9221
411 $SETUP_HS build ${haskellLib.componentTarget componentId} ${lib.concatStringsSep " " setupBuildFlags}
428 $SETUP_HS build ${haskellLib.componentTarget componentId} ${lib.concatStringsSep " " setupBuildFlags}
docs/tutorials/ca-derivations.mdnew file+65
@@ -0,0 +1,65 @@
1+# Content addressed derivations
2+## Introduction
3+
4+Floating content addressed derivations (from now *CA derivations*) is an experimental feature which substantially change how the hashes in the store paths are calculated.
5+Indeed, normally derivations are input addressed i.e. the outputs store paths depends only on the derivation inputs, instead with CA derivations they depend on the content of the outputs.
6+
7+This has two main advantages:
8+
9+- The so-called "early cutoff", namely the ability of Nix to stop a build if the build outputs would be something already built.
10+For example suppose you add a comment in an Haskell source, at this point Nix will rebuild the component depending on this source but since the output will be the same (adding a comment is an "output-invariant" change for `ghc`) every other component that depends on that will not be rebuilt.
11+- Users of the same Nix store does not need to trust each other when using substituters.
12+
13+You can find more information in the [ca-derivations page on the wiki](https://nixos.wiki/wiki/Ca-derivations) (and in the other resources linked there).
14+
15+## Usage
16+### Enable CA derivations in your system
17+First of all your Nix installation must support the `ca-derivations` experimental feature, this can done by adding the following in your `nix.conf`:
18+
19+```
20+experimental-features = ca-derivations
21+```
22+
23+Or if you use NixOS:
24+```
25+nix.extraOptions = ''
26+ experimental-features = ca-derivations
27+'';
28+```
29+
30+## Enable CA derivations in your project
31+At this point you can pass a new module to `project'` that tells `haskell.nix` to build every component in the project as CA derivation.
32+
33+```
34+haskell-nix.project' {
35+ # ...
36+
37+ modules = [{
38+ contentAddressed = true;
39+ # packages.project-name.components.exes.executable.contentAddressed = true;
40+ }];
41+};
42+```
43+
44+Optionally you can also specify which components you don't want to be content addressed.
45+
46+## Known problems
47+### Limitation of the current CA derivations implementation
48+
49+As explained in the [RFC 62](https://github.com/tweag/rfcs/blob/cas-rfc/rfcs/0062-content-addressed-paths.md)
50+
51+> The current implementation has a naive approach that just forbids fetching a path if the local system has a different realisation for the same drv output. This approach is simple and correct, but it's possible that it might not be good-enough in practice as it can result in a totally useless binary cache in some pathological cases.
52+
53+For example, suppose that your machine builds a derivation `A` producing an output `A.out` in your store and that after that a CI machine builds the same derivation `A` but producing a different output `A.out'` and populating a cache with this output.
54+At this point, if you need to build a derivation `B` that depends on `A`, since you already have the realisation `A.out` in your local store and you can't get `B.out` from the cache and you will end up building `B` even if one of its realisation is in the cache.
55+
56+This means that, in some cases, enabling CA derivations would lead to more rebuilds than not having it.
57+
58+### Hydra
59+Hydra currently doesn't support CA derivations, efforts are being made in this direction.
60+
61+
62+### GHC is not deterministic
63+Currently `ghc` is determinstic only disabling the parallel building i.e. passing `-j1`. [Here](https://gitlab.haskell.org/ghc/ghc/-/issues/12935) the upstream issue.
64+
65+Having a deterministic `ghc` would be a dream since it will automatically fix all the pathological cases about substituters discussed above and would allow `haskell.nix` to parallel build even when using CA derivations.
modules/plan.nix+8
@@ -244,6 +244,14 @@ let
244 type = listOfFilteringNulls str;
244 type = listOfFilteringNulls str;
245 default = def.ghcOptions or [];
245 default = def.ghcOptions or [];
246 };
246 };
247+ contentAddressed = mkOption {
248+ type = bool;
249+ default = (def.contentAddressed or false);
250+ description = ''
251+ Build content addressed derivation, requires Nix to have experimental feature
252+ `ca-derivations` enabled.
253+ '';
254+ };
247 planned = mkOption {
255 planned = mkOption {
248 description = "Set to true by `plan-to-nix` for any component that was included in the `plan.json` file.";
256 description = "Set to true by `plan-to-nix` for any component that was included in the `plan.json` file.";
249 # This is here so that (rather than in componentOptions) so it can be set project wide for stack projects
257 # This is here so that (rather than in componentOptions) so it can be set project wide for stack projects
test/ca-derivations-include/default.nixnew file+57
@@ -0,0 +1,57 @@
1+# Build a project enabling content addressed derivations for
2+# only a subset of the components
3+{ stdenv, pkgs, lib, mkCabalProjectPkgSet, project', haskellLib, recurseIntoAttrs, testSrc, compiler-nix-name, CADerivationsEnabled }:
4+
5+with lib;
6+
7+let
8+
9+ cabalProject = ''
10+ packages: .
11+ allow-newer: aeson:*
12+ '';
13+
14+ src = testSrc "cabal-simple";
15+
16+ # each derivation is content addressed
17+ projectA = project' {
18+ inherit compiler-nix-name src cabalProject;
19+ modules = [{ contentAddressed = true; }];
20+ };
21+
22+ # each derivation but one (the executable) is content addressed
23+ projectB = project' {
24+ inherit compiler-nix-name src cabalProject;
25+ modules = [{
26+ contentAddressed = true;
27+ packages.cabal-simple.components.exes.cabal-simple.contentAddressed = false;
28+ }];
29+ };
30+
31+ exeA = projectA.hsPkgs.cabal-simple.components.exes.cabal-simple.exePath;
32+ exeB = projectB.hsPkgs.cabal-simple.components.exes.cabal-simple.exePath;
33+
34+in
35+recurseIntoAttrs {
36+
37+ meta.disabled = !CADerivationsEnabled;
38+
39+ # check if the built executables are different (one is content addressed)
40+ # the other components are all content addressed (same output paths then)
41+ run = stdenv.mkDerivation {
42+ name = "ca-derivations-include-test";
43+
44+ buildCommand = ''
45+ [ "${exeA}" == "${exeB}" ] && exit 1
46+ touch $out
47+ '';
48+
49+ meta.platforms = platforms.all;
50+
51+ passthru = {
52+ # Used for debugging with nix repl
53+ inherit projectB projectA;
54+ };
55+ };
56+
57+}
test/ca-derivations/default.nixnew file+59
@@ -0,0 +1,59 @@
1+# Test if derivations are content addressed building two derivations producing
2+# the same outputs and checking if the path stores are equals
3+{ stdenv, pkgs, lib, mkCabalProjectPkgSet, project', haskellLib, recurseIntoAttrs, testSrc, compiler-nix-name, CADerivationsEnabled }:
4+
5+with lib;
6+
7+let
8+
9+ cabalProject = ''
10+ packages: .
11+ allow-newer: aeson:*
12+ '';
13+
14+ srcPlain = testSrc "cabal-simple";
15+
16+ # we alter the source adding an Haskell comment since they are ignored by ghc
17+ srcWithComment = pkgs.runCommand "src-with-comment" { } ''
18+ mkdir $out
19+ install ${srcPlain}/* $out
20+ echo " -- Altering source without altering executable..." >> $out/Main.hs
21+ '';
22+
23+ projectPlain = project' {
24+ inherit compiler-nix-name cabalProject;
25+ src = srcPlain;
26+ modules = [{ contentAddressed = true; }];
27+ };
28+
29+ projectWithComment = project' {
30+ inherit compiler-nix-name cabalProject;
31+ src = srcWithComment;
32+ modules = [{ contentAddressed = true; }];
33+ };
34+
35+ exe-plain = projectPlain.hsPkgs.cabal-simple.components.exes.cabal-simple.exePath;
36+ exe-withComment = projectWithComment.hsPkgs.cabal-simple.components.exes.cabal-simple.exePath;
37+
38+in
39+recurseIntoAttrs {
40+
41+ meta.disabled = !CADerivationsEnabled;
42+
43+ run = stdenv.mkDerivation {
44+ name = "ca-derivations-test";
45+
46+ buildCommand = ''
47+ [ "${exe-plain}" != "${exe-withComment}" ] && exit 1
48+ touch $out
49+ '';
50+
51+ meta.platforms = platforms.all;
52+
53+ passthru = {
54+ # Used for debugging with nix repl
55+ inherit projectWithComment projectPlain;
56+ };
57+ };
58+
59+}
test/default.nix+3
@@ -4,6 +4,7 @@
4, nixpkgsArgs ? haskellNix.nixpkgsArgs
4, nixpkgsArgs ? haskellNix.nixpkgsArgs
5, ifdLevel ? 1000
5, ifdLevel ? 1000
6, compiler-nix-name
6, compiler-nix-name
7+, CADerivationsEnabled ? false
7, checkMaterialization ? false
8, checkMaterialization ? false
8}:
9}:
9
10
@@ -206,6 +207,8 @@ let
206 external-static-plugin = callTest ./external-static-plugin { inherit compiler-nix-name; };
207 external-static-plugin = callTest ./external-static-plugin { inherit compiler-nix-name; };
207 exe-dlls = callTest ./exe-dlls { inherit util compiler-nix-name; };
208 exe-dlls = callTest ./exe-dlls { inherit util compiler-nix-name; };
208 exe-lib-dlls = callTest ./exe-lib-dlls { inherit util compiler-nix-name; };
209 exe-lib-dlls = callTest ./exe-lib-dlls { inherit util compiler-nix-name; };
210+ ca-derivations = callTest ./ca-derivations { inherit compiler-nix-name CADerivationsEnabled; };
211+ ca-derivations-include = callTest ./ca-derivations-include { inherit compiler-nix-name CADerivationsEnabled; };
209
212
210 unit = unitTests;
213 unit = unitTests;
211 };
214 };
test/tests.sh+4−1
@@ -3,6 +3,8 @@
3
3
4set -euo pipefail
4set -euo pipefail
5
5
6+# check if Nix has the `ca-derivations` experimental features (code 0) is enabled
7+NIX_CA_DERIVATIONS=$(jq -e '."experimental-features".value|any(. == 0)' <<< $(nix show-config --json)) || true
6NIX_BUILD_ARGS="${NIX_BUILD_ARGS:-}"
8NIX_BUILD_ARGS="${NIX_BUILD_ARGS:-}"
7
9
8cd $(dirname $0)
10cd $(dirname $0)
@@ -24,7 +26,8 @@ nix build $NIX_BUILD_ARGS \
24 --option restrict-eval true \
26 --option restrict-eval true \
25 --option allowed-uris "https://github.com/NixOS https://github.com/input-output-hk" \
27 --option allowed-uris "https://github.com/NixOS https://github.com/input-output-hk" \
26 --no-link --keep-going -f default.nix \
28 --no-link --keep-going -f default.nix \
27− --argstr compiler-nix-name $GHC
29+ --argstr compiler-nix-name $GHC \
30+ --arg CADerivationsEnabled $NIX_CA_DERIVATIONS
28echo >& 2
31echo >& 2
29
32
30printf "*** Running the unit tests... " >& 2
33printf "*** Running the unit tests... " >& 2