on iko/ghcjs-patch-master help sign in
Supporting `ca-derivations` experimental feature (#1494)

* Supporting `ca-derivations` experimental feature * Improved `contentAddressed.include` option description * Ensure deterministic output for content addressed components * Comment transformed into a Nix comment To avoid possibly useless rebuilds * Use `modules` arg `contentAddressed` flag * Tutorial about CA derivations Co-authored-by: Hamish Mackenzie <Hamish.K.Mackenzie@gmail.com>

commit2c1701b68122f44911872e95c0f0e1fe37375ebb authorAndrea Ciceri <andrea.ciceri@autistici.org> dateThu 23 Jun 2022, 02:39 (4 years ago) parent2a89c68
7 files changed+215−3
builder/comp-builder.nix+19−2 docs/tutorials/ca-derivations.md+65 modules/plan.nix+8 test/ca-derivations-include/default.nix+57 test/ca-derivations/default.nix+59 test/default.nix+3 test/tests.sh+4−1
builder/comp-builder.nix+19−2
@@ -50,6 +50,7 @@ let self =
5050, writeHieFiles ? component.writeHieFiles
5151
5252, ghcOptions ? component.ghcOptions
53+, contentAddressed ? component.contentAddressed
5354
5455# Options for Haddock generation
5556, doHaddock ? component.doHaddock # Enable haddock and hoogle generation
@@ -316,7 +317,13 @@ let
316317 componentDrv = drv;
317318 };
318319
319− drv = stdenv.mkDerivation (commonAttrs // {
320+ contentAddressedAttrs = lib.optionalAttrs contentAddressed {
321+ __contentAddressed = true;
322+ outputHashMode = "recursive";
323+ outputHashAlgo = "sha256";
324+ };
325+
326+ drv = stdenv.mkDerivation (commonAttrs // contentAddressedAttrs // {
320327 pname = nameOnly;
321328 inherit (package.identifier) version;
322329
@@ -405,7 +412,17 @@ let
405412 (lib.optionalString stdenv.hostPlatform.isWindows ''
406413 export pkgsHostTargetAsString="''${pkgsHostTarget[@]}"
407414 '') +
408− (if stdenv.hostPlatform.isGhcjs then ''
415+ # The following could be refactored but would lead to many rebuilds
416+
417+ # In case of content addressed components we need avoid parallel building (passing -j1)
418+ # in order to have a deterministic output and therefore avoid potential situations
419+ # where the binary cache becomes useless
420+ # See also https://gitlab.haskell.org/ghc/ghc/-/issues/12935
421+ (if contentAddressed then ''
422+ runHook preBuild
423+ $SETUP_HS build ${haskellLib.componentTarget componentId} -j1 ${lib.concatStringsSep " " setupBuildFlags}
424+ runHook postBuild
425+ '' else if stdenv.hostPlatform.isGhcjs then ''
409426 runHook preBuild
410427 # https://gitlab.haskell.org/ghc/ghc/issues/9221
411428 $SETUP_HS build ${haskellLib.componentTarget componentId} ${lib.concatStringsSep " " setupBuildFlags}
docs/tutorials/ca-derivations.mdnew file+65
@@ -0,0 +1,65 @@
1+# Content addressed derivations
2+## Introduction
3+
4+Floating content addressed derivations (from now *CA derivations*) is an experimental feature which substantially change how the hashes in the store paths are calculated.
5+Indeed, normally derivations are input addressed i.e. the outputs store paths depends only on the derivation inputs, instead with CA derivations they depend on the content of the outputs.
6+
7+This has two main advantages:
8+
9+- The so-called "early cutoff", namely the ability of Nix to stop a build if the build outputs would be something already built.
10+For example suppose you add a comment in an Haskell source, at this point Nix will rebuild the component depending on this source but since the output will be the same (adding a comment is an "output-invariant" change for `ghc`) every other component that depends on that will not be rebuilt.
11+- Users of the same Nix store does not need to trust each other when using substituters.
12+
13+You can find more information in the [ca-derivations page on the wiki](https://nixos.wiki/wiki/Ca-derivations) (and in the other resources linked there).
14+
15+## Usage
16+### Enable CA derivations in your system
17+First of all your Nix installation must support the `ca-derivations` experimental feature, this can done by adding the following in your `nix.conf`:
18+
19+```
20+experimental-features = ca-derivations
21+```
22+
23+Or if you use NixOS:
24+```
25+nix.extraOptions = ''
26+ experimental-features = ca-derivations
27+'';
28+```
29+
30+## Enable CA derivations in your project
31+At this point you can pass a new module to `project'` that tells `haskell.nix` to build every component in the project as CA derivation.
32+
33+```
34+haskell-nix.project' {
35+ # ...
36+
37+ modules = [{
38+ contentAddressed = true;
39+ # packages.project-name.components.exes.executable.contentAddressed = true;
40+ }];
41+};
42+```
43+
44+Optionally you can also specify which components you don't want to be content addressed.
45+
46+## Known problems
47+### Limitation of the current CA derivations implementation
48+
49+As explained in the [RFC 62](https://github.com/tweag/rfcs/blob/cas-rfc/rfcs/0062-content-addressed-paths.md)
50+
51+> The current implementation has a naive approach that just forbids fetching a path if the local system has a different realisation for the same drv output. This approach is simple and correct, but it's possible that it might not be good-enough in practice as it can result in a totally useless binary cache in some pathological cases.
52+
53+For example, suppose that your machine builds a derivation `A` producing an output `A.out` in your store and that after that a CI machine builds the same derivation `A` but producing a different output `A.out'` and populating a cache with this output.
54+At this point, if you need to build a derivation `B` that depends on `A`, since you already have the realisation `A.out` in your local store and you can't get `B.out` from the cache and you will end up building `B` even if one of its realisation is in the cache.
55+
56+This means that, in some cases, enabling CA derivations would lead to more rebuilds than not having it.
57+
58+### Hydra
59+Hydra currently doesn't support CA derivations, efforts are being made in this direction.
60+
61+
62+### GHC is not deterministic
63+Currently `ghc` is determinstic only disabling the parallel building i.e. passing `-j1`. [Here](https://gitlab.haskell.org/ghc/ghc/-/issues/12935) the upstream issue.
64+
65+Having a deterministic `ghc` would be a dream since it will automatically fix all the pathological cases about substituters discussed above and would allow `haskell.nix` to parallel build even when using CA derivations.
modules/plan.nix+8
@@ -244,6 +244,14 @@ let
244244 type = listOfFilteringNulls str;
245245 default = def.ghcOptions or [];
246246 };
247+ contentAddressed = mkOption {
248+ type = bool;
249+ default = (def.contentAddressed or false);
250+ description = ''
251+ Build content addressed derivation, requires Nix to have experimental feature
252+ `ca-derivations` enabled.
253+ '';
254+ };
247255 planned = mkOption {
248256 description = "Set to true by `plan-to-nix` for any component that was included in the `plan.json` file.";
249257 # This is here so that (rather than in componentOptions) so it can be set project wide for stack projects
test/ca-derivations-include/default.nixnew file+57
@@ -0,0 +1,57 @@
1+# Build a project enabling content addressed derivations for
2+# only a subset of the components
3+{ stdenv, pkgs, lib, mkCabalProjectPkgSet, project', haskellLib, recurseIntoAttrs, testSrc, compiler-nix-name, CADerivationsEnabled }:
4+
5+with lib;
6+
7+let
8+
9+ cabalProject = ''
10+ packages: .
11+ allow-newer: aeson:*
12+ '';
13+
14+ src = testSrc "cabal-simple";
15+
16+ # each derivation is content addressed
17+ projectA = project' {
18+ inherit compiler-nix-name src cabalProject;
19+ modules = [{ contentAddressed = true; }];
20+ };
21+
22+ # each derivation but one (the executable) is content addressed
23+ projectB = project' {
24+ inherit compiler-nix-name src cabalProject;
25+ modules = [{
26+ contentAddressed = true;
27+ packages.cabal-simple.components.exes.cabal-simple.contentAddressed = false;
28+ }];
29+ };
30+
31+ exeA = projectA.hsPkgs.cabal-simple.components.exes.cabal-simple.exePath;
32+ exeB = projectB.hsPkgs.cabal-simple.components.exes.cabal-simple.exePath;
33+
34+in
35+recurseIntoAttrs {
36+
37+ meta.disabled = !CADerivationsEnabled;
38+
39+ # check if the built executables are different (one is content addressed)
40+ # the other components are all content addressed (same output paths then)
41+ run = stdenv.mkDerivation {
42+ name = "ca-derivations-include-test";
43+
44+ buildCommand = ''
45+ [ "${exeA}" == "${exeB}" ] && exit 1
46+ touch $out
47+ '';
48+
49+ meta.platforms = platforms.all;
50+
51+ passthru = {
52+ # Used for debugging with nix repl
53+ inherit projectB projectA;
54+ };
55+ };
56+
57+}
test/ca-derivations/default.nixnew file+59
@@ -0,0 +1,59 @@
1+# Test if derivations are content addressed building two derivations producing
2+# the same outputs and checking if the path stores are equals
3+{ stdenv, pkgs, lib, mkCabalProjectPkgSet, project', haskellLib, recurseIntoAttrs, testSrc, compiler-nix-name, CADerivationsEnabled }:
4+
5+with lib;
6+
7+let
8+
9+ cabalProject = ''
10+ packages: .
11+ allow-newer: aeson:*
12+ '';
13+
14+ srcPlain = testSrc "cabal-simple";
15+
16+ # we alter the source adding an Haskell comment since they are ignored by ghc
17+ srcWithComment = pkgs.runCommand "src-with-comment" { } ''
18+ mkdir $out
19+ install ${srcPlain}/* $out
20+ echo " -- Altering source without altering executable..." >> $out/Main.hs
21+ '';
22+
23+ projectPlain = project' {
24+ inherit compiler-nix-name cabalProject;
25+ src = srcPlain;
26+ modules = [{ contentAddressed = true; }];
27+ };
28+
29+ projectWithComment = project' {
30+ inherit compiler-nix-name cabalProject;
31+ src = srcWithComment;
32+ modules = [{ contentAddressed = true; }];
33+ };
34+
35+ exe-plain = projectPlain.hsPkgs.cabal-simple.components.exes.cabal-simple.exePath;
36+ exe-withComment = projectWithComment.hsPkgs.cabal-simple.components.exes.cabal-simple.exePath;
37+
38+in
39+recurseIntoAttrs {
40+
41+ meta.disabled = !CADerivationsEnabled;
42+
43+ run = stdenv.mkDerivation {
44+ name = "ca-derivations-test";
45+
46+ buildCommand = ''
47+ [ "${exe-plain}" != "${exe-withComment}" ] && exit 1
48+ touch $out
49+ '';
50+
51+ meta.platforms = platforms.all;
52+
53+ passthru = {
54+ # Used for debugging with nix repl
55+ inherit projectWithComment projectPlain;
56+ };
57+ };
58+
59+}
test/default.nix+3
@@ -4,6 +4,7 @@
44, nixpkgsArgs ? haskellNix.nixpkgsArgs
55, ifdLevel ? 1000
66, compiler-nix-name
7+, CADerivationsEnabled ? false
78, checkMaterialization ? false
89}:
910
@@ -206,6 +207,8 @@ let
206207 external-static-plugin = callTest ./external-static-plugin { inherit compiler-nix-name; };
207208 exe-dlls = callTest ./exe-dlls { inherit util compiler-nix-name; };
208209 exe-lib-dlls = callTest ./exe-lib-dlls { inherit util compiler-nix-name; };
210+ ca-derivations = callTest ./ca-derivations { inherit compiler-nix-name CADerivationsEnabled; };
211+ ca-derivations-include = callTest ./ca-derivations-include { inherit compiler-nix-name CADerivationsEnabled; };
209212
210213 unit = unitTests;
211214 };
test/tests.sh+4−1
@@ -3,6 +3,8 @@
33
44set -euo pipefail
55
6+# check if Nix has the `ca-derivations` experimental features (code 0) is enabled
7+NIX_CA_DERIVATIONS=$(jq -e '."experimental-features".value|any(. == 0)' <<< $(nix show-config --json)) || true
68NIX_BUILD_ARGS="${NIX_BUILD_ARGS:-}"
79
810cd $(dirname $0)
@@ -24,7 +26,8 @@ nix build $NIX_BUILD_ARGS \
2426 --option restrict-eval true \
2527 --option allowed-uris "https://github.com/NixOS https://github.com/input-output-hk" \
2628 --no-link --keep-going -f default.nix \
27− --argstr compiler-nix-name $GHC
29+ --argstr compiler-nix-name $GHC \
30+ --arg CADerivationsEnabled $NIX_CA_DERIVATIONS
2831echo >& 2
2932
3033printf "*** Running the unit tests... " >& 2